DPDK  25.03.0
rte_mempool.h
Go to the documentation of this file.
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2014 Intel Corporation.
3  * Copyright(c) 2016 6WIND S.A.
4  * Copyright(c) 2022 SmartShare Systems
5  */
6 
7 #ifndef _RTE_MEMPOOL_H_
8 #define _RTE_MEMPOOL_H_
9 
37 #include <stdalign.h>
38 #include <stdio.h>
39 #include <stdint.h>
40 #include <inttypes.h>
41 
42 #include <rte_compat.h>
43 #include <rte_config.h>
44 #include <rte_spinlock.h>
45 #include <rte_debug.h>
46 #include <rte_lcore.h>
47 #include <rte_log.h>
48 #include <rte_branch_prediction.h>
49 #include <rte_ring.h>
50 #include <rte_memcpy.h>
51 #include <rte_common.h>
52 
53 #include "rte_mempool_trace_fp.h"
54 
55 #ifdef __cplusplus
56 extern "C" {
57 #endif
58 
59 #define RTE_MEMPOOL_HEADER_COOKIE1 0xbadbadbadadd2e55ULL
60 #define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL
61 #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL
63 #ifdef RTE_LIBRTE_MEMPOOL_STATS
64 
70 struct __rte_cache_aligned rte_mempool_debug_stats {
71  uint64_t put_bulk;
72  uint64_t put_objs;
73  uint64_t put_common_pool_bulk;
74  uint64_t put_common_pool_objs;
75  uint64_t get_common_pool_bulk;
76  uint64_t get_common_pool_objs;
77  uint64_t get_success_bulk;
78  uint64_t get_success_objs;
79  uint64_t get_fail_bulk;
80  uint64_t get_fail_objs;
81  uint64_t get_success_blks;
82  uint64_t get_fail_blks;
84 };
85 #endif
86 
91  uint32_t size;
92  uint32_t flushthresh;
93  uint32_t len;
94 #ifdef RTE_LIBRTE_MEMPOOL_STATS
95  uint32_t unused;
96  /*
97  * Alternative location for the most frequently updated mempool statistics (per-lcore),
98  * providing faster update access when using a mempool cache.
99  */
100  struct {
101  uint64_t put_bulk;
102  uint64_t put_objs;
103  uint64_t get_success_bulk;
104  uint64_t get_success_objs;
105  } stats;
106 #endif
107 
113  alignas(RTE_CACHE_LINE_SIZE) void *objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 2];
114 };
115 
120  uint32_t elt_size;
121  uint32_t header_size;
122  uint32_t trailer_size;
123  uint32_t total_size;
125 };
126 
128 #define RTE_MEMPOOL_NAMESIZE (RTE_RING_NAMESIZE - \
129  sizeof(RTE_MEMPOOL_MZ_PREFIX) + 1)
130 #define RTE_MEMPOOL_MZ_PREFIX "MP_"
131 
132 /* "MP_<name>" */
133 #define RTE_MEMPOOL_MZ_FORMAT RTE_MEMPOOL_MZ_PREFIX "%s"
134 
135 #ifndef RTE_MEMPOOL_ALIGN
136 
139 #define RTE_MEMPOOL_ALIGN RTE_CACHE_LINE_SIZE
140 #endif
141 
142 #define RTE_MEMPOOL_ALIGN_MASK (RTE_MEMPOOL_ALIGN - 1)
143 
155  struct rte_mempool *mp;
157 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
158  uint64_t cookie;
159 #endif
160 };
161 
165 RTE_STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr);
166 
167 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
168 
175 struct rte_mempool_objtlr {
176  uint64_t cookie;
177 };
178 
179 #endif
180 
184 extern int rte_mempool_logtype;
185 #define RTE_LOGTYPE_MEMPOOL rte_mempool_logtype
186 #define RTE_MEMPOOL_LOG(level, ...) \
187  RTE_LOG_LINE(level, MEMPOOL, "" __VA_ARGS__)
188 
192 RTE_STAILQ_HEAD(rte_mempool_memhdr_list, rte_mempool_memhdr);
193 
198  void *opaque);
199 
208  struct rte_mempool *mp;
209  void *addr;
211  size_t len;
213  void *opaque;
214 };
215 
224  unsigned int contig_block_size;
225 };
226 
231  char name[RTE_MEMPOOL_NAMESIZE];
232  union {
233  void *pool_data;
234  uint64_t pool_id;
235  };
236  void *pool_config;
237  const struct rte_memzone *mz;
238  unsigned int flags;
239  int socket_id;
240  uint32_t size;
241  uint32_t cache_size;
244  uint32_t elt_size;
245  uint32_t header_size;
246  uint32_t trailer_size;
248  unsigned private_data_size;
256  int32_t ops_index;
257 
260  uint32_t populated_size;
261  struct rte_mempool_objhdr_list elt_list;
262  uint32_t nb_mem_chunks;
263  struct rte_mempool_memhdr_list mem_list;
265 #ifdef RTE_LIBRTE_MEMPOOL_STATS
266 
270  struct rte_mempool_debug_stats stats[RTE_MAX_LCORE + 1];
271 #endif
272 };
273 
275 #define RTE_MEMPOOL_F_NO_SPREAD 0x0001
276 
280 #define MEMPOOL_F_NO_SPREAD RTE_MEMPOOL_F_NO_SPREAD
281 
282 #define RTE_MEMPOOL_F_NO_CACHE_ALIGN 0x0002
283 
287 #define MEMPOOL_F_NO_CACHE_ALIGN RTE_MEMPOOL_F_NO_CACHE_ALIGN
288 
289 #define RTE_MEMPOOL_F_SP_PUT 0x0004
290 
294 #define MEMPOOL_F_SP_PUT RTE_MEMPOOL_F_SP_PUT
295 
296 #define RTE_MEMPOOL_F_SC_GET 0x0008
297 
301 #define MEMPOOL_F_SC_GET RTE_MEMPOOL_F_SC_GET
302 
303 #define RTE_MEMPOOL_F_POOL_CREATED 0x0010
304 
305 #define RTE_MEMPOOL_F_NO_IOVA_CONTIG 0x0020
306 
310 #define MEMPOOL_F_NO_IOVA_CONTIG RTE_MEMPOOL_F_NO_IOVA_CONTIG
311 
312 #define RTE_MEMPOOL_F_NON_IO 0x0040
313 
317 #define RTE_MEMPOOL_VALID_USER_FLAGS (RTE_MEMPOOL_F_NO_SPREAD \
318  | RTE_MEMPOOL_F_NO_CACHE_ALIGN \
319  | RTE_MEMPOOL_F_SP_PUT \
320  | RTE_MEMPOOL_F_SC_GET \
321  | RTE_MEMPOOL_F_NO_IOVA_CONTIG \
322  )
323 
334 #ifdef RTE_LIBRTE_MEMPOOL_STATS
335 #define RTE_MEMPOOL_STAT_ADD(mp, name, n) do { \
336  unsigned int __lcore_id = rte_lcore_id(); \
337  if (likely(__lcore_id < RTE_MAX_LCORE)) \
338  (mp)->stats[__lcore_id].name += (n); \
339  else \
340  rte_atomic_fetch_add_explicit(&((mp)->stats[RTE_MAX_LCORE].name), \
341  (n), rte_memory_order_relaxed); \
342  } while (0)
343 #else
344 #define RTE_MEMPOOL_STAT_ADD(mp, name, n) do {} while (0)
345 #endif
346 
357 #ifdef RTE_LIBRTE_MEMPOOL_STATS
358 #define RTE_MEMPOOL_CACHE_STAT_ADD(cache, name, n) ((cache)->stats.name += (n))
359 #else
360 #define RTE_MEMPOOL_CACHE_STAT_ADD(cache, name, n) do {} while (0)
361 #endif
362 
371 #define RTE_MEMPOOL_HEADER_SIZE(mp, cs) \
372  (sizeof(*(mp)) + (((cs) == 0) ? 0 : \
373  (sizeof(struct rte_mempool_cache) * RTE_MAX_LCORE)))
374 
375 /* return the header of a mempool object (internal) */
376 static inline struct rte_mempool_objhdr *
377 rte_mempool_get_header(void *obj)
378 {
379  return (struct rte_mempool_objhdr *)RTE_PTR_SUB(obj,
380  sizeof(struct rte_mempool_objhdr));
381 }
382 
392 static inline struct rte_mempool *rte_mempool_from_obj(void *obj)
393 {
394  struct rte_mempool_objhdr *hdr = rte_mempool_get_header(obj);
395  return hdr->mp;
396 }
397 
398 /* return the trailer of a mempool object (internal) */
399 static inline struct rte_mempool_objtlr *rte_mempool_get_trailer(void *obj)
400 {
401  struct rte_mempool *mp = rte_mempool_from_obj(obj);
402  return (struct rte_mempool_objtlr *)RTE_PTR_ADD(obj, mp->elt_size);
403 }
404 
419 void rte_mempool_check_cookies(const struct rte_mempool *mp,
420  void * const *obj_table_const, unsigned n, int free);
421 
422 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
423 #define RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table_const, n, free) \
424  rte_mempool_check_cookies(mp, obj_table_const, n, free)
425 #else
426 #define RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table_const, n, free) do {} while (0)
427 #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
428 
444 void rte_mempool_contig_blocks_check_cookies(const struct rte_mempool *mp,
445  void * const *first_obj_table_const, unsigned int n, int free);
446 
447 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
448 #define RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table_const, n, \
449  free) \
450  rte_mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
451  free)
452 #else
453 #define RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table_const, n, \
454  free) \
455  do {} while (0)
456 #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
457 
458 #define RTE_MEMPOOL_OPS_NAMESIZE 32
470 typedef int (*rte_mempool_alloc_t)(struct rte_mempool *mp);
471 
475 typedef void (*rte_mempool_free_t)(struct rte_mempool *mp);
476 
483 typedef int (*rte_mempool_enqueue_t)(struct rte_mempool *mp,
484  void * const *obj_table, unsigned int n);
485 
492 typedef int (*rte_mempool_dequeue_t)(struct rte_mempool *mp,
493  void **obj_table, unsigned int n);
494 
499  void **first_obj_table, unsigned int n);
500 
504 typedef unsigned (*rte_mempool_get_count)(const struct rte_mempool *mp);
505 
529 typedef ssize_t (*rte_mempool_calc_mem_size_t)(const struct rte_mempool *mp,
530  uint32_t obj_num, uint32_t pg_shift,
531  size_t *min_chunk_size, size_t *align);
532 
568 ssize_t rte_mempool_op_calc_mem_size_helper(const struct rte_mempool *mp,
569  uint32_t obj_num, uint32_t pg_shift, size_t chunk_reserve,
570  size_t *min_chunk_size, size_t *align);
571 
579 ssize_t rte_mempool_op_calc_mem_size_default(const struct rte_mempool *mp,
580  uint32_t obj_num, uint32_t pg_shift,
581  size_t *min_chunk_size, size_t *align);
582 
595 typedef void (rte_mempool_populate_obj_cb_t)(struct rte_mempool *mp,
596  void *opaque, void *vaddr, rte_iova_t iova);
597 
626 typedef int (*rte_mempool_populate_t)(struct rte_mempool *mp,
627  unsigned int max_objs,
628  void *vaddr, rte_iova_t iova, size_t len,
629  rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
630 
634 #define RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ 0x0001
635 
668 int rte_mempool_op_populate_helper(struct rte_mempool *mp,
669  unsigned int flags, unsigned int max_objs,
670  void *vaddr, rte_iova_t iova, size_t len,
671  rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
672 
680  unsigned int max_objs,
681  void *vaddr, rte_iova_t iova, size_t len,
682  rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
683 
687 typedef int (*rte_mempool_get_info_t)(const struct rte_mempool *mp,
688  struct rte_mempool_info *info);
689 
690 
717 };
718 
719 #define RTE_MEMPOOL_MAX_OPS_IDX 16
730 struct __rte_cache_aligned rte_mempool_ops_table {
732  uint32_t num_ops;
737 };
738 
741 
751 static inline struct rte_mempool_ops *
752 rte_mempool_get_ops(int ops_index)
753 {
754  RTE_VERIFY((ops_index >= 0) && (ops_index < RTE_MEMPOOL_MAX_OPS_IDX));
755 
756  return &rte_mempool_ops_table.ops[ops_index];
757 }
758 
768 int
769 rte_mempool_ops_alloc(struct rte_mempool *mp);
770 
784 static inline int
785 rte_mempool_ops_dequeue_bulk(struct rte_mempool *mp,
786  void **obj_table, unsigned n)
787 {
788  struct rte_mempool_ops *ops;
789  int ret;
790 
791  rte_mempool_trace_ops_dequeue_bulk(mp, obj_table, n);
792  ops = rte_mempool_get_ops(mp->ops_index);
793  ret = ops->dequeue(mp, obj_table, n);
794  if (ret == 0) {
795  RTE_MEMPOOL_STAT_ADD(mp, get_common_pool_bulk, 1);
796  RTE_MEMPOOL_STAT_ADD(mp, get_common_pool_objs, n);
797  }
798  return ret;
799 }
800 
814 static inline int
815 rte_mempool_ops_dequeue_contig_blocks(struct rte_mempool *mp,
816  void **first_obj_table, unsigned int n)
817 {
818  struct rte_mempool_ops *ops;
819 
820  ops = rte_mempool_get_ops(mp->ops_index);
821  RTE_ASSERT(ops->dequeue_contig_blocks != NULL);
822  rte_mempool_trace_ops_dequeue_contig_blocks(mp, first_obj_table, n);
823  return ops->dequeue_contig_blocks(mp, first_obj_table, n);
824 }
825 
839 static inline int
840 rte_mempool_ops_enqueue_bulk(struct rte_mempool *mp, void * const *obj_table,
841  unsigned n)
842 {
843  struct rte_mempool_ops *ops;
844  int ret;
845 
846  RTE_MEMPOOL_STAT_ADD(mp, put_common_pool_bulk, 1);
847  RTE_MEMPOOL_STAT_ADD(mp, put_common_pool_objs, n);
848  rte_mempool_trace_ops_enqueue_bulk(mp, obj_table, n);
849  ops = rte_mempool_get_ops(mp->ops_index);
850  ret = ops->enqueue(mp, obj_table, n);
851 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
852  if (unlikely(ret < 0))
853  RTE_MEMPOOL_LOG(CRIT, "cannot enqueue %u objects to mempool %s",
854  n, mp->name);
855 #endif
856  return ret;
857 }
858 
867 unsigned
868 rte_mempool_ops_get_count(const struct rte_mempool *mp);
869 
889 ssize_t rte_mempool_ops_calc_mem_size(const struct rte_mempool *mp,
890  uint32_t obj_num, uint32_t pg_shift,
891  size_t *min_chunk_size, size_t *align);
892 
916 int rte_mempool_ops_populate(struct rte_mempool *mp, unsigned int max_objs,
917  void *vaddr, rte_iova_t iova, size_t len,
919  void *obj_cb_arg);
920 
933 int rte_mempool_ops_get_info(const struct rte_mempool *mp,
934  struct rte_mempool_info *info);
935 
942 void
943 rte_mempool_ops_free(struct rte_mempool *mp);
944 
962 int
963 rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name,
964  void *pool_config);
965 
976 int rte_mempool_register_ops(const struct rte_mempool_ops *ops);
977 
983 #define RTE_MEMPOOL_REGISTER_OPS(ops) \
984  RTE_INIT(mp_hdlr_init_##ops) \
985  { \
986  rte_mempool_register_ops(&ops); \
987  }
988 
994 typedef void (rte_mempool_obj_cb_t)(struct rte_mempool *mp,
995  void *opaque, void *obj, unsigned obj_idx);
996 typedef rte_mempool_obj_cb_t rte_mempool_obj_ctor_t; /* compat */
997 
1003 typedef void (rte_mempool_mem_cb_t)(struct rte_mempool *mp,
1004  void *opaque, struct rte_mempool_memhdr *memhdr,
1005  unsigned mem_idx);
1006 
1013 typedef void (rte_mempool_ctor_t)(struct rte_mempool *, void *);
1014 
1026 void
1027 rte_mempool_free(struct rte_mempool *mp);
1028 
1107 struct rte_mempool *
1108 rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
1109  unsigned cache_size, unsigned private_data_size,
1110  rte_mempool_ctor_t *mp_init, void *mp_init_arg,
1111  rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
1112  int socket_id, unsigned int flags)
1114 
1149 struct rte_mempool *
1150 rte_mempool_create_empty(const char *name, unsigned int n, unsigned int elt_size,
1151  unsigned int cache_size, unsigned int private_data_size,
1152  int socket_id, unsigned int flags)
1154 
1185 int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
1186  rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
1187  void *opaque);
1188 
1215 int
1216 rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
1217  size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb,
1218  void *opaque);
1219 
1234 
1248 int rte_mempool_populate_anon(struct rte_mempool *mp);
1249 
1265 uint32_t rte_mempool_obj_iter(struct rte_mempool *mp,
1266  rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg);
1267 
1283 uint32_t rte_mempool_mem_iter(struct rte_mempool *mp,
1284  rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg);
1285 
1294 void rte_mempool_dump(FILE *f, struct rte_mempool *mp);
1295 
1310 struct rte_mempool_cache *
1311 rte_mempool_cache_create(uint32_t size, int socket_id);
1312 
1319 void
1321 
1333 static __rte_always_inline struct rte_mempool_cache *
1334 rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
1335 {
1336  if (mp->cache_size == 0)
1337  return NULL;
1338 
1339  if (lcore_id >= RTE_MAX_LCORE)
1340  return NULL;
1341 
1342  rte_mempool_trace_default_cache(mp, lcore_id,
1343  &mp->local_cache[lcore_id]);
1344  return &mp->local_cache[lcore_id];
1345 }
1346 
1355 static __rte_always_inline void
1357  struct rte_mempool *mp)
1358 {
1359  if (cache == NULL)
1360  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1361  if (cache == NULL || cache->len == 0)
1362  return;
1363  rte_mempool_trace_cache_flush(cache, mp);
1364  rte_mempool_ops_enqueue_bulk(mp, cache->objs, cache->len);
1365  cache->len = 0;
1366 }
1367 
1380 static __rte_always_inline void
1381 rte_mempool_do_generic_put(struct rte_mempool *mp, void * const *obj_table,
1382  unsigned int n, struct rte_mempool_cache *cache)
1383 {
1384  void **cache_objs;
1385 
1386  /* No cache provided */
1387  if (unlikely(cache == NULL))
1388  goto driver_enqueue;
1389 
1390  /* increment stat now, adding in mempool always success */
1391  RTE_MEMPOOL_CACHE_STAT_ADD(cache, put_bulk, 1);
1392  RTE_MEMPOOL_CACHE_STAT_ADD(cache, put_objs, n);
1393 
1394  /* The request itself is too big for the cache */
1395  if (unlikely(n > cache->flushthresh))
1396  goto driver_enqueue_stats_incremented;
1397 
1398  /*
1399  * The cache follows the following algorithm:
1400  * 1. If the objects cannot be added to the cache without crossing
1401  * the flush threshold, flush the cache to the backend.
1402  * 2. Add the objects to the cache.
1403  */
1404 
1405  if (cache->len + n <= cache->flushthresh) {
1406  cache_objs = &cache->objs[cache->len];
1407  cache->len += n;
1408  } else {
1409  cache_objs = &cache->objs[0];
1410  rte_mempool_ops_enqueue_bulk(mp, cache_objs, cache->len);
1411  cache->len = n;
1412  }
1413 
1414  /* Add the objects to the cache. */
1415  rte_memcpy(cache_objs, obj_table, sizeof(void *) * n);
1416 
1417  return;
1418 
1419 driver_enqueue:
1420 
1421  /* increment stat now, adding in mempool always success */
1422  RTE_MEMPOOL_STAT_ADD(mp, put_bulk, 1);
1423  RTE_MEMPOOL_STAT_ADD(mp, put_objs, n);
1424 
1425 driver_enqueue_stats_incremented:
1426 
1427  /* push objects to the backend */
1428  rte_mempool_ops_enqueue_bulk(mp, obj_table, n);
1429 }
1430 
1431 
1444 static __rte_always_inline void
1445 rte_mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
1446  unsigned int n, struct rte_mempool_cache *cache)
1447 {
1448  rte_mempool_trace_generic_put(mp, obj_table, n, cache);
1449  RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table, n, 0);
1450  rte_mempool_do_generic_put(mp, obj_table, n, cache);
1451 }
1452 
1467 static __rte_always_inline void
1468 rte_mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,
1469  unsigned int n)
1470 {
1471  struct rte_mempool_cache *cache;
1472  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1473  rte_mempool_trace_put_bulk(mp, obj_table, n, cache);
1474  rte_mempool_generic_put(mp, obj_table, n, cache);
1475 }
1476 
1489 static __rte_always_inline void
1490 rte_mempool_put(struct rte_mempool *mp, void *obj)
1491 {
1492  rte_mempool_put_bulk(mp, &obj, 1);
1493 }
1494 
1509 static __rte_always_inline int
1510 rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table,
1511  unsigned int n, struct rte_mempool_cache *cache)
1512 {
1513  int ret;
1514  unsigned int remaining;
1515  uint32_t index, len;
1516  void **cache_objs;
1517 
1518  /* No cache provided */
1519  if (unlikely(cache == NULL)) {
1520  remaining = n;
1521  goto driver_dequeue;
1522  }
1523 
1524  /* The cache is a stack, so copy will be in reverse order. */
1525  cache_objs = &cache->objs[cache->len];
1526 
1527  if (__rte_constant(n) && n <= cache->len) {
1528  /*
1529  * The request size is known at build time, and
1530  * the entire request can be satisfied from the cache,
1531  * so let the compiler unroll the fixed length copy loop.
1532  */
1533  cache->len -= n;
1534  for (index = 0; index < n; index++)
1535  *obj_table++ = *--cache_objs;
1536 
1537  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1538  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1539 
1540  return 0;
1541  }
1542 
1543  /*
1544  * Use the cache as much as we have to return hot objects first.
1545  * If the request size 'n' is known at build time, the above comparison
1546  * ensures that n > cache->len here, so omit RTE_MIN().
1547  */
1548  len = __rte_constant(n) ? cache->len : RTE_MIN(n, cache->len);
1549  cache->len -= len;
1550  remaining = n - len;
1551  for (index = 0; index < len; index++)
1552  *obj_table++ = *--cache_objs;
1553 
1554  /*
1555  * If the request size 'n' is known at build time, the case
1556  * where the entire request can be satisfied from the cache
1557  * has already been handled above, so omit handling it here.
1558  */
1559  if (!__rte_constant(n) && remaining == 0) {
1560  /* The entire request is satisfied from the cache. */
1561 
1562  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1563  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1564 
1565  return 0;
1566  }
1567 
1568  /* if dequeue below would overflow mem allocated for cache */
1569  if (unlikely(remaining > RTE_MEMPOOL_CACHE_MAX_SIZE))
1570  goto driver_dequeue;
1571 
1572  /* Fill the cache from the backend; fetch size + remaining objects. */
1573  ret = rte_mempool_ops_dequeue_bulk(mp, cache->objs,
1574  cache->size + remaining);
1575  if (unlikely(ret < 0)) {
1576  /*
1577  * We are buffer constrained, and not able to allocate
1578  * cache + remaining.
1579  * Do not fill the cache, just satisfy the remaining part of
1580  * the request directly from the backend.
1581  */
1582  goto driver_dequeue;
1583  }
1584 
1585  /* Satisfy the remaining part of the request from the filled cache. */
1586  cache_objs = &cache->objs[cache->size + remaining];
1587  for (index = 0; index < remaining; index++)
1588  *obj_table++ = *--cache_objs;
1589 
1590  cache->len = cache->size;
1591 
1592  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1593  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1594 
1595  return 0;
1596 
1597 driver_dequeue:
1598 
1599  /* Get remaining objects directly from the backend. */
1600  ret = rte_mempool_ops_dequeue_bulk(mp, obj_table, remaining);
1601 
1602  if (ret < 0) {
1603  if (likely(cache != NULL)) {
1604  cache->len = n - remaining;
1605  /*
1606  * No further action is required to roll the first part
1607  * of the request back into the cache, as objects in
1608  * the cache are intact.
1609  */
1610  }
1611 
1612  RTE_MEMPOOL_STAT_ADD(mp, get_fail_bulk, 1);
1613  RTE_MEMPOOL_STAT_ADD(mp, get_fail_objs, n);
1614  } else {
1615  if (likely(cache != NULL)) {
1616  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1617  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1618  } else {
1619  RTE_MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
1620  RTE_MEMPOOL_STAT_ADD(mp, get_success_objs, n);
1621  }
1622  }
1623 
1624  return ret;
1625 }
1626 
1647 static __rte_always_inline int
1648 rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table,
1649  unsigned int n, struct rte_mempool_cache *cache)
1650 {
1651  int ret;
1652  ret = rte_mempool_do_generic_get(mp, obj_table, n, cache);
1653  if (ret == 0)
1654  RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table, n, 1);
1655  rte_mempool_trace_generic_get(mp, obj_table, n, cache);
1656  return ret;
1657 }
1658 
1681 static __rte_always_inline int
1682 rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
1683 {
1684  struct rte_mempool_cache *cache;
1685  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1686  rte_mempool_trace_get_bulk(mp, obj_table, n, cache);
1687  return rte_mempool_generic_get(mp, obj_table, n, cache);
1688 }
1689 
1710 static __rte_always_inline int
1711 rte_mempool_get(struct rte_mempool *mp, void **obj_p)
1712 {
1713  return rte_mempool_get_bulk(mp, obj_p, 1);
1714 }
1715 
1737 static __rte_always_inline int
1739  void **first_obj_table, unsigned int n)
1740 {
1741  int ret;
1742 
1743  ret = rte_mempool_ops_dequeue_contig_blocks(mp, first_obj_table, n);
1744  if (ret == 0) {
1745  RTE_MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
1746  RTE_MEMPOOL_STAT_ADD(mp, get_success_blks, n);
1747  RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table, n,
1748  1);
1749  } else {
1750  RTE_MEMPOOL_STAT_ADD(mp, get_fail_bulk, 1);
1751  RTE_MEMPOOL_STAT_ADD(mp, get_fail_blks, n);
1752  }
1753 
1754  rte_mempool_trace_get_contig_blocks(mp, first_obj_table, n);
1755  return ret;
1756 }
1757 
1770 unsigned int rte_mempool_avail_count(const struct rte_mempool *mp);
1771 
1784 unsigned int
1785 rte_mempool_in_use_count(const struct rte_mempool *mp);
1786 
1800 static inline int
1802 {
1803  return rte_mempool_avail_count(mp) == mp->size;
1804 }
1805 
1819 static inline int
1821 {
1822  return rte_mempool_avail_count(mp) == 0;
1823 }
1824 
1835 static inline rte_iova_t
1836 rte_mempool_virt2iova(const void *elt)
1837 {
1838  const struct rte_mempool_objhdr *hdr;
1839  hdr = (const struct rte_mempool_objhdr *)RTE_PTR_SUB(elt,
1840  sizeof(*hdr));
1841  return hdr->iova;
1842 }
1843 
1854 void rte_mempool_audit(struct rte_mempool *mp);
1855 
1864 static inline void *rte_mempool_get_priv(struct rte_mempool *mp)
1865 {
1866  return (char *)mp +
1867  RTE_MEMPOOL_HEADER_SIZE(mp, mp->cache_size);
1868 }
1869 
1876 void rte_mempool_list_dump(FILE *f);
1877 
1889 struct rte_mempool *rte_mempool_lookup(const char *name);
1890 
1908 uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
1909  struct rte_mempool_objsz *sz);
1910 
1919 void rte_mempool_walk(void (*func)(struct rte_mempool *, void *arg),
1920  void *arg);
1921 
1928  void *start;
1930  size_t length;
1933 };
1934 
1950 __rte_experimental
1951 int
1952 rte_mempool_get_mem_range(const struct rte_mempool *mp,
1953  struct rte_mempool_mem_range_info *mem_range);
1954 
1967 __rte_experimental
1968 size_t
1969 rte_mempool_get_obj_alignment(const struct rte_mempool *mp);
1970 
1975 int
1976 rte_mempool_get_page_size(struct rte_mempool *mp, size_t *pg_sz);
1977 
1987 };
1988 
1998 typedef void (rte_mempool_event_callback)(
1999  enum rte_mempool_event event,
2000  struct rte_mempool *mp,
2001  void *user_data);
2002 
2019 __rte_internal
2020 int
2021 rte_mempool_event_callback_register(rte_mempool_event_callback *func,
2022  void *user_data);
2023 
2037 __rte_internal
2038 int
2039 rte_mempool_event_callback_unregister(rte_mempool_event_callback *func,
2040  void *user_data);
2041 
2042 #ifdef __cplusplus
2043 }
2044 #endif
2045 
2046 #endif /* _RTE_MEMPOOL_H_ */
uint64_t pool_id
Definition: rte_mempool.h:234
#define __rte_always_inline
Definition: rte_common.h:456
struct rte_mempool * rte_mempool_lookup(const char *name)
__rte_experimental size_t rte_mempool_get_obj_alignment(const struct rte_mempool *mp)
struct rte_mempool_cache * rte_mempool_cache_create(uint32_t size, int socket_id)
#define RTE_CACHE_GUARD
Definition: rte_common.h:721
uint32_t rte_mempool_mem_iter(struct rte_mempool *mp, rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg)
uint32_t header_size
Definition: rte_mempool.h:245
#define likely(x)
const struct rte_memzone * mz
Definition: rte_mempool.h:237
uint64_t rte_iova_t
Definition: rte_common.h:736
static __rte_always_inline void rte_mempool_put_bulk(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
Definition: rte_mempool.h:1468
struct rte_mempool * mp
Definition: rte_mempool.h:208
void rte_mempool_list_dump(FILE *f)
int(* rte_mempool_dequeue_contig_blocks_t)(struct rte_mempool *mp, void **first_obj_table, unsigned int n)
Definition: rte_mempool.h:498
rte_mempool_alloc_t alloc
Definition: rte_mempool.h:694
void( rte_mempool_memchunk_free_cb_t)(struct rte_mempool_memhdr *memhdr, void *opaque)
Definition: rte_mempool.h:197
unsigned int flags
Definition: rte_mempool.h:238
void( rte_mempool_obj_cb_t)(struct rte_mempool *mp, void *opaque, void *obj, unsigned obj_idx)
Definition: rte_mempool.h:994
static __rte_always_inline void rte_mempool_generic_put(struct rte_mempool *mp, void *const *obj_table, unsigned int n, struct rte_mempool_cache *cache)
Definition: rte_mempool.h:1445
int(* rte_mempool_alloc_t)(struct rte_mempool *mp)
Definition: rte_mempool.h:470
static int rte_mempool_empty(const struct rte_mempool *mp)
Definition: rte_mempool.h:1820
RTE_STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr)
rte_mempool_memchunk_free_cb_t * free_cb
Definition: rte_mempool.h:212
char name[RTE_MEMPOOL_OPS_NAMESIZE]
Definition: rte_mempool.h:693
int(* rte_mempool_dequeue_t)(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:492
rte_mempool_get_info_t get_info
Definition: rte_mempool.h:712
rte_mempool_dequeue_contig_blocks_t dequeue_contig_blocks
Definition: rte_mempool.h:716
#define __rte_cache_aligned
Definition: rte_common.h:705
uint32_t cache_size
Definition: rte_mempool.h:241
char name[RTE_MEMPOOL_NAMESIZE]
Definition: rte_mempool.h:231
unsigned int rte_mempool_avail_count(const struct rte_mempool *mp)
uint32_t size
Definition: rte_mempool.h:240
#define RTE_PTR_ADD(ptr, x)
Definition: rte_common.h:520
void rte_mempool_cache_free(struct rte_mempool_cache *cache)
uint32_t total_size
Definition: rte_mempool.h:123
uint32_t rte_mempool_obj_iter(struct rte_mempool *mp, rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg)
int rte_mempool_ops_get_info(const struct rte_mempool *mp, struct rte_mempool_info *info)
int rte_mempool_register_ops(const struct rte_mempool_ops *ops)
#define unlikely(x)
static __rte_always_inline int rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table, unsigned int n, struct rte_mempool_cache *cache)
Definition: rte_mempool.h:1648
int rte_mempool_populate_default(struct rte_mempool *mp)
#define RTE_MIN(a, b)
Definition: rte_common.h:761
static __rte_always_inline void rte_mempool_cache_flush(struct rte_mempool_cache *cache, struct rte_mempool *mp)
Definition: rte_mempool.h:1356
#define RTE_MEMPOOL_OPS_NAMESIZE
Definition: rte_mempool.h:458
static __rte_always_inline int rte_mempool_get(struct rte_mempool *mp, void **obj_p)
Definition: rte_mempool.h:1711
uint32_t nb_mem_chunks
Definition: rte_mempool.h:262
struct rte_mempool * mp
Definition: rte_mempool.h:155
struct rte_mempool_ops ops[RTE_MEMPOOL_MAX_OPS_IDX]
Definition: rte_mempool.h:736
rte_mempool_get_count get_count
Definition: rte_mempool.h:698
ssize_t(* rte_mempool_calc_mem_size_t)(const struct rte_mempool *mp, uint32_t obj_num, uint32_t pg_shift, size_t *min_chunk_size, size_t *align)
Definition: rte_mempool.h:529
static int rte_mempool_full(const struct rte_mempool *mp)
Definition: rte_mempool.h:1801
static unsigned rte_lcore_id(void)
Definition: rte_lcore.h:78
#define __rte_dealloc(dealloc, argno)
Definition: rte_common.h:305
void * pool_config
Definition: rte_mempool.h:236
unsigned int rte_mempool_in_use_count(const struct rte_mempool *mp)
void( rte_mempool_mem_cb_t)(struct rte_mempool *mp, void *opaque, struct rte_mempool_memhdr *memhdr, unsigned mem_idx)
Definition: rte_mempool.h:1003
static __rte_always_inline int rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:1682
int rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name, void *pool_config)
rte_mempool_calc_mem_size_t calc_mem_size
Definition: rte_mempool.h:703
#define __rte_malloc
Definition: rte_common.h:294
unsigned(* rte_mempool_get_count)(const struct rte_mempool *mp)
Definition: rte_mempool.h:504
uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, struct rte_mempool_objsz *sz)
ssize_t rte_mempool_op_calc_mem_size_default(const struct rte_mempool *mp, uint32_t obj_num, uint32_t pg_shift, size_t *min_chunk_size, size_t *align)
void * pool_data
Definition: rte_mempool.h:233
unsigned int contig_block_size
Definition: rte_mempool.h:224
int rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque)
int(* rte_mempool_populate_t)(struct rte_mempool *mp, unsigned int max_objs, void *vaddr, rte_iova_t iova, size_t len, rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
Definition: rte_mempool.h:626
uint32_t elt_size
Definition: rte_mempool.h:244
void rte_mempool_audit(struct rte_mempool *mp)
int(* rte_mempool_get_info_t)(const struct rte_mempool *mp, struct rte_mempool_info *info)
Definition: rte_mempool.h:687
RTE_STAILQ_ENTRY(rte_mempool_objhdr) next
unsigned private_data_size
Definition: rte_mempool.h:248
void( rte_mempool_ctor_t)(struct rte_mempool *, void *)
Definition: rte_mempool.h:1013
rte_spinlock_t sl
Definition: rte_mempool.h:731
static __rte_always_inline struct rte_mempool_cache * rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
Definition: rte_mempool.h:1334
void rte_mempool_dump(FILE *f, struct rte_mempool *mp)
int rte_mempool_populate_anon(struct rte_mempool *mp)
__rte_experimental int rte_mempool_get_mem_range(const struct rte_mempool *mp, struct rte_mempool_mem_range_info *mem_range)
struct rte_mempool_cache * local_cache
Definition: rte_mempool.h:258
uint32_t trailer_size
Definition: rte_mempool.h:246
rte_mempool_dequeue_t dequeue
Definition: rte_mempool.h:697
int(* rte_mempool_enqueue_t)(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
Definition: rte_mempool.h:483
uint32_t flushthresh
Definition: rte_mempool.h:92
int rte_mempool_op_populate_default(struct rte_mempool *mp, unsigned int max_objs, void *vaddr, rte_iova_t iova, size_t len, rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
#define RTE_PTR_SUB(ptr, x)
Definition: rte_common.h:525
rte_mempool_event
Definition: rte_mempool.h:1982
static struct rte_mempool * rte_mempool_from_obj(void *obj)
Definition: rte_mempool.h:392
uint32_t header_size
Definition: rte_mempool.h:121
int32_t ops_index
Definition: rte_mempool.h:256
static void * rte_memcpy(void *dst, const void *src, size_t n)
uint32_t populated_size
Definition: rte_mempool.h:260
static rte_iova_t rte_mempool_virt2iova(const void *elt)
Definition: rte_mempool.h:1836
struct rte_mempool struct rte_mempool int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr, rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque)
static __rte_always_inline void rte_mempool_put(struct rte_mempool *mp, void *obj)
Definition: rte_mempool.h:1490
rte_mempool_populate_t populate
Definition: rte_mempool.h:708
struct rte_mempool * rte_mempool_create(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, rte_mempool_ctor_t *mp_init, void *mp_init_arg, rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, int socket_id, unsigned int flags) __rte_malloc __rte_dealloc(rte_mempool_free
void( rte_mempool_populate_obj_cb_t)(struct rte_mempool *mp, void *opaque, void *vaddr, rte_iova_t iova)
Definition: rte_mempool.h:595
void rte_mempool_free(struct rte_mempool *mp)
void rte_mempool_walk(void(*func)(struct rte_mempool *, void *arg), void *arg)
void * objs[RTE_MEMPOOL_CACHE_MAX_SIZE *2]
Definition: rte_mempool.h:113
static void * rte_mempool_get_priv(struct rte_mempool *mp)
Definition: rte_mempool.h:1864
uint32_t trailer_size
Definition: rte_mempool.h:122
struct rte_mempool struct rte_mempool * rte_mempool_create_empty(const char *name, unsigned int n, unsigned int elt_size, unsigned int cache_size, unsigned int private_data_size, int socket_id, unsigned int flags) __rte_malloc __rte_dealloc(rte_mempool_free
#define RTE_MEMPOOL_MAX_OPS_IDX
Definition: rte_mempool.h:719
void(* rte_mempool_free_t)(struct rte_mempool *mp)
Definition: rte_mempool.h:475
rte_mempool_free_t free
Definition: rte_mempool.h:695
static __rte_always_inline int rte_mempool_get_contig_blocks(struct rte_mempool *mp, void **first_obj_table, unsigned int n)
Definition: rte_mempool.h:1738
RTE_STAILQ_ENTRY(rte_mempool_memhdr) next
rte_mempool_enqueue_t enqueue
Definition: rte_mempool.h:696