DPDK 23.11.0
Loading...
Searching...
No Matches
rte_mbuf.h
Go to the documentation of this file.
1/* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation.
3 * Copyright 2014 6WIND S.A.
4 */
5
6#ifndef _RTE_MBUF_H_
7#define _RTE_MBUF_H_
8
34#include <stdint.h>
35
36#include <rte_common.h>
37#include <rte_config.h>
38#include <rte_mempool.h>
39#include <rte_prefetch.h>
41#include <rte_mbuf_ptype.h>
42#include <rte_mbuf_core.h>
43
44#ifdef __cplusplus
45extern "C" {
46#endif
47
56const char *rte_get_rx_ol_flag_name(uint64_t mask);
57
70int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);
71
82const char *rte_get_tx_ol_flag_name(uint64_t mask);
83
96int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);
97
108static inline void
110{
111 rte_prefetch0(&m->cacheline0);
112}
113
125static inline void
127{
128#if RTE_CACHE_LINE_SIZE == 64
129 rte_prefetch0(&m->cacheline1);
130#else
131 RTE_SET_USED(m);
132#endif
133}
134
135
136static inline uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp);
137
146static inline rte_iova_t
148{
149#if RTE_IOVA_IN_MBUF
150 return m->buf_iova;
151#else
152 return (rte_iova_t)m->buf_addr;
153#endif
154}
155
164static inline void
166{
167#if RTE_IOVA_IN_MBUF
168 m->buf_iova = iova;
169#else
170 RTE_SET_USED(m);
171 RTE_SET_USED(iova);
172#endif
173}
174
183static inline rte_iova_t
185{
186 return rte_mbuf_iova_get(mb) + mb->data_off;
187}
188
201static inline rte_iova_t
203{
204 return rte_mbuf_iova_get(mb) + RTE_PKTMBUF_HEADROOM;
205}
206
215static inline struct rte_mbuf *
217{
218 return (struct rte_mbuf *)RTE_PTR_SUB(mi->buf_addr, sizeof(*mi) + mi->priv_size);
219}
220
236static inline char *
238{
239 return (char *)mb + sizeof(*mb) + rte_pktmbuf_priv_size(mp);
240}
241
250static inline char *
252{
253 return rte_mbuf_buf_addr(mb, mb->pool) + RTE_PKTMBUF_HEADROOM;
254}
255
269static inline char *
271{
272 return rte_mbuf_buf_addr(md, md->pool);
273}
274
287static inline void *
289{
290 return RTE_PTR_ADD(m, sizeof(struct rte_mbuf));
291}
292
304
313static inline uint32_t
315{
316 struct rte_pktmbuf_pool_private *mbp_priv;
317
318 mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
319 return mbp_priv->flags;
320}
321
328#define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF (1 << 0)
329
337#define RTE_MBUF_HAS_PINNED_EXTBUF(mb) \
338 (rte_pktmbuf_priv_flags(mb->pool) & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF)
339
340#ifdef RTE_LIBRTE_MBUF_DEBUG
341
343#define __rte_mbuf_sanity_check(m, is_h) rte_mbuf_sanity_check(m, is_h)
344
345#else /* RTE_LIBRTE_MBUF_DEBUG */
346
348#define __rte_mbuf_sanity_check(m, is_h) do { } while (0)
349
350#endif /* RTE_LIBRTE_MBUF_DEBUG */
351
352#ifdef RTE_MBUF_REFCNT_ATOMIC
353
361static inline uint16_t
362rte_mbuf_refcnt_read(const struct rte_mbuf *m)
363{
364 return rte_atomic_load_explicit(&m->refcnt, rte_memory_order_relaxed);
365}
366
374static inline void
375rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
376{
377 rte_atomic_store_explicit(&m->refcnt, new_value, rte_memory_order_relaxed);
378}
379
380/* internal */
381static inline uint16_t
382__rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
383{
384 return rte_atomic_fetch_add_explicit(&m->refcnt, value,
385 rte_memory_order_acq_rel) + value;
386}
387
397static inline uint16_t
398rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
399{
400 /*
401 * The atomic_add is an expensive operation, so we don't want to
402 * call it in the case where we know we are the unique holder of
403 * this mbuf (i.e. ref_cnt == 1). Otherwise, an atomic
404 * operation has to be used because concurrent accesses on the
405 * reference counter can occur.
406 */
407 if (likely(rte_mbuf_refcnt_read(m) == 1)) {
408 ++value;
409 rte_mbuf_refcnt_set(m, (uint16_t)value);
410 return (uint16_t)value;
411 }
412
413 return __rte_mbuf_refcnt_update(m, value);
414}
415
416#else /* ! RTE_MBUF_REFCNT_ATOMIC */
417
418/* internal */
419static inline uint16_t
420__rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
421{
422 m->refcnt = (uint16_t)(m->refcnt + value);
423 return m->refcnt;
424}
425
429static inline uint16_t
430rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
431{
432 return __rte_mbuf_refcnt_update(m, value);
433}
434
438static inline uint16_t
440{
441 return m->refcnt;
442}
443
447static inline void
448rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
449{
450 m->refcnt = new_value;
451}
452
453#endif /* RTE_MBUF_REFCNT_ATOMIC */
454
463static inline uint16_t
465{
466 return rte_atomic_load_explicit(&shinfo->refcnt, rte_memory_order_relaxed);
467}
468
477static inline void
479 uint16_t new_value)
480{
481 rte_atomic_store_explicit(&shinfo->refcnt, new_value, rte_memory_order_relaxed);
482}
483
495static inline uint16_t
497 int16_t value)
498{
499 if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1)) {
500 ++value;
501 rte_mbuf_ext_refcnt_set(shinfo, (uint16_t)value);
502 return (uint16_t)value;
503 }
504
505 return rte_atomic_fetch_add_explicit(&shinfo->refcnt, value,
506 rte_memory_order_acq_rel) + value;
507}
508
510#define RTE_MBUF_PREFETCH_TO_FREE(m) do { \
511 if ((m) != NULL) \
512 rte_prefetch0(m); \
513} while (0)
514
515
528void
529rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header);
530
550int rte_mbuf_check(const struct rte_mbuf *m, int is_header,
551 const char **reason);
552
565static __rte_always_inline void
567{
568 RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1);
569 RTE_ASSERT(m->next == NULL);
570 RTE_ASSERT(m->nb_segs == 1);
572}
573
575#define MBUF_RAW_ALLOC_CHECK(m) __rte_mbuf_raw_sanity_check(m)
576
596static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)
597{
598 struct rte_mbuf *m;
599
600 if (rte_mempool_get(mp, (void **)&m) < 0)
601 return NULL;
603 return m;
604}
605
620static __rte_always_inline void
628
651void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg,
652 void *m, unsigned i);
653
674void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg);
675
709struct rte_mempool *
710rte_pktmbuf_pool_create(const char *name, unsigned n,
711 unsigned cache_size, uint16_t priv_size, uint16_t data_room_size,
712 int socket_id);
713
750struct rte_mempool *
751rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n,
752 unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size,
753 int socket_id, const char *ops_name);
754
762
803struct rte_mempool *
804rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n,
805 unsigned int cache_size, uint16_t priv_size,
806 uint16_t data_room_size, int socket_id,
807 const struct rte_pktmbuf_extmem *ext_mem,
808 unsigned int ext_num);
809
821static inline uint16_t
823{
824 struct rte_pktmbuf_pool_private *mbp_priv;
825
826 mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
827 return mbp_priv->mbuf_data_room_size;
828}
829
842static inline uint16_t
844{
845 struct rte_pktmbuf_pool_private *mbp_priv;
846
847 mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
848 return mbp_priv->mbuf_priv_size;
849}
850
859static inline void rte_pktmbuf_reset_headroom(struct rte_mbuf *m)
860{
861 m->data_off = (uint16_t)RTE_MIN((uint16_t)RTE_PKTMBUF_HEADROOM,
862 (uint16_t)m->buf_len);
863}
864
873static inline void rte_pktmbuf_reset(struct rte_mbuf *m)
874{
875 m->next = NULL;
876 m->pkt_len = 0;
877 m->tx_offload = 0;
878 m->vlan_tci = 0;
879 m->vlan_tci_outer = 0;
880 m->nb_segs = 1;
882
884 m->packet_type = 0;
886
887 m->data_len = 0;
889}
890
904static inline struct rte_mbuf *rte_pktmbuf_alloc(struct rte_mempool *mp)
905{
906 struct rte_mbuf *m;
907 if ((m = rte_mbuf_raw_alloc(mp)) != NULL)
909 return m;
910}
911
926static inline int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool,
927 struct rte_mbuf **mbufs, unsigned count)
928{
929 unsigned idx = 0;
930 int rc;
931
932 rc = rte_mempool_get_bulk(pool, (void **)mbufs, count);
933 if (unlikely(rc))
934 return rc;
935
936 /* To understand duff's device on loop unwinding optimization, see
937 * https://en.wikipedia.org/wiki/Duff's_device.
938 * Here while() loop is used rather than do() while{} to avoid extra
939 * check if count is zero.
940 */
941 switch (count % 4) {
942 case 0:
943 while (idx != count) {
944 __rte_mbuf_raw_sanity_check(mbufs[idx]);
945 rte_pktmbuf_reset(mbufs[idx]);
946 idx++;
947 /* fall-through */
948 case 3:
949 __rte_mbuf_raw_sanity_check(mbufs[idx]);
950 rte_pktmbuf_reset(mbufs[idx]);
951 idx++;
952 /* fall-through */
953 case 2:
954 __rte_mbuf_raw_sanity_check(mbufs[idx]);
955 rte_pktmbuf_reset(mbufs[idx]);
956 idx++;
957 /* fall-through */
958 case 1:
959 __rte_mbuf_raw_sanity_check(mbufs[idx]);
960 rte_pktmbuf_reset(mbufs[idx]);
961 idx++;
962 /* fall-through */
963 }
964 }
965 return 0;
966}
967
1000static inline struct rte_mbuf_ext_shared_info *
1001rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len,
1003{
1004 struct rte_mbuf_ext_shared_info *shinfo;
1005 void *buf_end = RTE_PTR_ADD(buf_addr, *buf_len);
1006 void *addr;
1007
1008 addr = RTE_PTR_ALIGN_FLOOR(RTE_PTR_SUB(buf_end, sizeof(*shinfo)),
1009 sizeof(uintptr_t));
1010 if (addr <= buf_addr)
1011 return NULL;
1012
1013 shinfo = (struct rte_mbuf_ext_shared_info *)addr;
1014 shinfo->free_cb = free_cb;
1015 shinfo->fcb_opaque = fcb_opaque;
1016 rte_mbuf_ext_refcnt_set(shinfo, 1);
1017
1018 *buf_len = (uint16_t)RTE_PTR_DIFF(shinfo, buf_addr);
1019 return shinfo;
1020}
1021
1082static inline void
1083rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr,
1084 rte_iova_t buf_iova, uint16_t buf_len,
1085 struct rte_mbuf_ext_shared_info *shinfo)
1086{
1087 /* mbuf should not be read-only */
1088 RTE_ASSERT(RTE_MBUF_DIRECT(m) && rte_mbuf_refcnt_read(m) == 1);
1089 RTE_ASSERT(shinfo->free_cb != NULL);
1090
1091 m->buf_addr = buf_addr;
1092 rte_mbuf_iova_set(m, buf_iova);
1093 m->buf_len = buf_len;
1094
1095 m->data_len = 0;
1096 m->data_off = 0;
1097
1099 m->shinfo = shinfo;
1100}
1101
1109#define rte_pktmbuf_detach_extbuf(m) rte_pktmbuf_detach(m)
1110
1119static inline void
1120rte_mbuf_dynfield_copy(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
1121{
1122 memcpy(&mdst->dynfield1, msrc->dynfield1, sizeof(mdst->dynfield1));
1123}
1124
1125/* internal */
1126static inline void
1127__rte_pktmbuf_copy_hdr(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
1128{
1129 mdst->port = msrc->port;
1130 mdst->vlan_tci = msrc->vlan_tci;
1131 mdst->vlan_tci_outer = msrc->vlan_tci_outer;
1132 mdst->tx_offload = msrc->tx_offload;
1133 mdst->hash = msrc->hash;
1134 mdst->packet_type = msrc->packet_type;
1135 rte_mbuf_dynfield_copy(mdst, msrc);
1136}
1137
1159static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
1160{
1161 RTE_ASSERT(RTE_MBUF_DIRECT(mi) &&
1162 rte_mbuf_refcnt_read(mi) == 1);
1163
1164 if (RTE_MBUF_HAS_EXTBUF(m)) {
1166 mi->ol_flags = m->ol_flags;
1167 mi->shinfo = m->shinfo;
1168 } else {
1169 /* if m is not direct, get the mbuf that embeds the data */
1171 mi->priv_size = m->priv_size;
1173 }
1174
1175 __rte_pktmbuf_copy_hdr(mi, m);
1176
1177 mi->data_off = m->data_off;
1178 mi->data_len = m->data_len;
1180 mi->buf_addr = m->buf_addr;
1181 mi->buf_len = m->buf_len;
1182
1183 mi->next = NULL;
1184 mi->pkt_len = mi->data_len;
1185 mi->nb_segs = 1;
1186
1189}
1190
1198static inline void
1199__rte_pktmbuf_free_extbuf(struct rte_mbuf *m)
1200{
1201 RTE_ASSERT(RTE_MBUF_HAS_EXTBUF(m));
1202 RTE_ASSERT(m->shinfo != NULL);
1203
1204 if (rte_mbuf_ext_refcnt_update(m->shinfo, -1) == 0)
1206}
1207
1214static inline void
1215__rte_pktmbuf_free_direct(struct rte_mbuf *m)
1216{
1217 struct rte_mbuf *md;
1218
1219 RTE_ASSERT(RTE_MBUF_CLONED(m));
1220
1221 md = rte_mbuf_from_indirect(m);
1222
1223 if (rte_mbuf_refcnt_update(md, -1) == 0) {
1224 md->next = NULL;
1225 md->nb_segs = 1;
1226 rte_mbuf_refcnt_set(md, 1);
1228 }
1229}
1230
1249static inline void rte_pktmbuf_detach(struct rte_mbuf *m)
1250{
1251 struct rte_mempool *mp = m->pool;
1252 uint32_t mbuf_size, buf_len;
1253 uint16_t priv_size;
1254
1255 if (RTE_MBUF_HAS_EXTBUF(m)) {
1256 /*
1257 * The mbuf has the external attached buffer,
1258 * we should check the type of the memory pool where
1259 * the mbuf was allocated from to detect the pinned
1260 * external buffer.
1261 */
1262 uint32_t flags = rte_pktmbuf_priv_flags(mp);
1263
1265 /*
1266 * The pinned external buffer should not be
1267 * detached from its backing mbuf, just exit.
1268 */
1269 return;
1270 }
1271 __rte_pktmbuf_free_extbuf(m);
1272 } else {
1273 __rte_pktmbuf_free_direct(m);
1274 }
1275 priv_size = rte_pktmbuf_priv_size(mp);
1276 mbuf_size = (uint32_t)(sizeof(struct rte_mbuf) + priv_size);
1277 buf_len = rte_pktmbuf_data_room_size(mp);
1278
1279 m->priv_size = priv_size;
1280 m->buf_addr = (char *)m + mbuf_size;
1281 rte_mbuf_iova_set(m, rte_mempool_virt2iova(m) + mbuf_size);
1282 m->buf_len = (uint16_t)buf_len;
1284 m->data_len = 0;
1285 m->ol_flags = 0;
1286}
1287
1301static inline int __rte_pktmbuf_pinned_extbuf_decref(struct rte_mbuf *m)
1302{
1303 struct rte_mbuf_ext_shared_info *shinfo;
1304
1305 /* Clear flags, mbuf is being freed. */
1307 shinfo = m->shinfo;
1308
1309 /* Optimize for performance - do not dec/reinit */
1310 if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1))
1311 return 0;
1312
1313 /*
1314 * Direct usage of add primitive to avoid
1315 * duplication of comparing with one.
1316 */
1317 if (likely(rte_atomic_fetch_add_explicit(&shinfo->refcnt, -1,
1318 rte_memory_order_acq_rel) - 1))
1319 return 1;
1320
1321 /* Reinitialize counter before mbuf freeing. */
1322 rte_mbuf_ext_refcnt_set(shinfo, 1);
1323 return 0;
1324}
1325
1340static __rte_always_inline struct rte_mbuf *
1342{
1344
1345 if (likely(rte_mbuf_refcnt_read(m) == 1)) {
1346
1347 if (!RTE_MBUF_DIRECT(m)) {
1349 if (RTE_MBUF_HAS_EXTBUF(m) &&
1351 __rte_pktmbuf_pinned_extbuf_decref(m))
1352 return NULL;
1353 }
1354
1355 if (m->next != NULL)
1356 m->next = NULL;
1357 if (m->nb_segs != 1)
1358 m->nb_segs = 1;
1359
1360 return m;
1361
1362 } else if (__rte_mbuf_refcnt_update(m, -1) == 0) {
1363
1364 if (!RTE_MBUF_DIRECT(m)) {
1366 if (RTE_MBUF_HAS_EXTBUF(m) &&
1368 __rte_pktmbuf_pinned_extbuf_decref(m))
1369 return NULL;
1370 }
1371
1372 if (m->next != NULL)
1373 m->next = NULL;
1374 if (m->nb_segs != 1)
1375 m->nb_segs = 1;
1376 rte_mbuf_refcnt_set(m, 1);
1377
1378 return m;
1379 }
1380 return NULL;
1381}
1382
1392static __rte_always_inline void
1394{
1396 if (likely(m != NULL))
1398}
1399
1409static inline void rte_pktmbuf_free(struct rte_mbuf *m)
1410{
1411 struct rte_mbuf *m_next;
1412
1413 if (m != NULL)
1415
1416 while (m != NULL) {
1417 m_next = m->next;
1419 m = m_next;
1420 }
1421}
1422
1435void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count);
1436
1454struct rte_mbuf *
1456
1478struct rte_mbuf *
1479rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp,
1480 uint32_t offset, uint32_t length);
1481
1493static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
1494{
1496
1497 do {
1499 } while ((m = m->next) != NULL);
1500}
1501
1510static inline uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
1511{
1513 return m->data_off;
1514}
1515
1524static inline uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
1525{
1527 return (uint16_t)(m->buf_len - rte_pktmbuf_headroom(m) -
1528 m->data_len);
1529}
1530
1539static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m)
1540{
1542 while (m->next != NULL)
1543 m = m->next;
1544 return m;
1545}
1546
1555#define rte_pktmbuf_pkt_len(m) ((m)->pkt_len)
1556
1565#define rte_pktmbuf_data_len(m) ((m)->data_len)
1566
1582static inline char *rte_pktmbuf_prepend(struct rte_mbuf *m,
1583 uint16_t len)
1584{
1586
1587 if (unlikely(len > rte_pktmbuf_headroom(m)))
1588 return NULL;
1589
1590 /* NB: elaborating the subtraction like this instead of using
1591 * -= allows us to ensure the result type is uint16_t
1592 * avoiding compiler warnings on gcc 8.1 at least */
1593 m->data_off = (uint16_t)(m->data_off - len);
1594 m->data_len = (uint16_t)(m->data_len + len);
1595 m->pkt_len = (m->pkt_len + len);
1596
1597 return (char *)m->buf_addr + m->data_off;
1598}
1599
1615static inline char *rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
1616{
1617 void *tail;
1618 struct rte_mbuf *m_last;
1619
1621
1622 m_last = rte_pktmbuf_lastseg(m);
1623 if (unlikely(len > rte_pktmbuf_tailroom(m_last)))
1624 return NULL;
1625
1626 tail = (char *)m_last->buf_addr + m_last->data_off + m_last->data_len;
1627 m_last->data_len = (uint16_t)(m_last->data_len + len);
1628 m->pkt_len = (m->pkt_len + len);
1629 return (char*) tail;
1630}
1631
1646static inline char *rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
1647{
1649
1650 if (unlikely(len > m->data_len))
1651 return NULL;
1652
1653 /* NB: elaborating the addition like this instead of using
1654 * += allows us to ensure the result type is uint16_t
1655 * avoiding compiler warnings on gcc 8.1 at least */
1656 m->data_len = (uint16_t)(m->data_len - len);
1657 m->data_off = (uint16_t)(m->data_off + len);
1658 m->pkt_len = (m->pkt_len - len);
1659 return (char *)m->buf_addr + m->data_off;
1660}
1661
1676static inline int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
1677{
1678 struct rte_mbuf *m_last;
1679
1681
1682 m_last = rte_pktmbuf_lastseg(m);
1683 if (unlikely(len > m_last->data_len))
1684 return -1;
1685
1686 m_last->data_len = (uint16_t)(m_last->data_len - len);
1687 m->pkt_len = (m->pkt_len - len);
1688 return 0;
1689}
1690
1700static inline int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
1701{
1703 return m->nb_segs == 1;
1704}
1705
1709const void *__rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off,
1710 uint32_t len, void *buf);
1711
1732static inline const void *rte_pktmbuf_read(const struct rte_mbuf *m,
1733 uint32_t off, uint32_t len, void *buf)
1734{
1735 if (likely(off + len <= rte_pktmbuf_data_len(m)))
1736 return rte_pktmbuf_mtod_offset(m, char *, off);
1737 else
1738 return __rte_pktmbuf_read(m, off, len, buf);
1739}
1740
1757static inline int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
1758{
1759 struct rte_mbuf *cur_tail;
1760
1761 /* Check for number-of-segments-overflow */
1762 if (head->nb_segs + tail->nb_segs > RTE_MBUF_MAX_NB_SEGS)
1763 return -EOVERFLOW;
1764
1765 /* Chain 'tail' onto the old tail */
1766 cur_tail = rte_pktmbuf_lastseg(head);
1767 cur_tail->next = tail;
1768
1769 /* accumulate number of segments and total length.
1770 * NB: elaborating the addition like this instead of using
1771 * -= allows us to ensure the result type is uint16_t
1772 * avoiding compiler warnings on gcc 8.1 at least */
1773 head->nb_segs = (uint16_t)(head->nb_segs + tail->nb_segs);
1774 head->pkt_len += tail->pkt_len;
1775
1776 /* pkt_len is only set in the head */
1777 tail->pkt_len = tail->data_len;
1778
1779 return 0;
1780}
1781
1803static __rte_always_inline uint64_t
1804rte_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso,
1805 uint64_t ol3, uint64_t ol2, uint64_t unused)
1806{
1807 return il2 << RTE_MBUF_L2_LEN_OFS |
1808 il3 << RTE_MBUF_L3_LEN_OFS |
1809 il4 << RTE_MBUF_L4_LEN_OFS |
1810 tso << RTE_MBUF_TSO_SEGSZ_OFS |
1811 ol3 << RTE_MBUF_OUTL3_LEN_OFS |
1812 ol2 << RTE_MBUF_OUTL2_LEN_OFS |
1813 unused << RTE_MBUF_TXOFLD_UNUSED_OFS;
1814}
1815
1826static inline int
1828{
1829 uint64_t ol_flags = m->ol_flags;
1830
1831 /* Does packet set any of available offloads? */
1833 return 0;
1834
1835 /* IP checksum can be counted only for IPv4 packet */
1837 return -EINVAL;
1838
1839 /* IP type not set when required */
1842 return -EINVAL;
1843
1844 /* Check requirements for TSO packet */
1846 if ((m->tso_segsz == 0) ||
1849 return -EINVAL;
1850
1851 /* RTE_MBUF_F_TX_OUTER_IP_CKSUM set for non outer IPv4 packet. */
1854 return -EINVAL;
1855
1856 return 0;
1857}
1858
1862int __rte_pktmbuf_linearize(struct rte_mbuf *mbuf);
1863
1876static inline int
1878{
1879 if (rte_pktmbuf_is_contiguous(mbuf))
1880 return 0;
1881 return __rte_pktmbuf_linearize(mbuf);
1882}
1883
1898void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len);
1899
1903static inline uint32_t
1905{
1906 return m->hash.sched.queue_id;
1907}
1908
1912static inline uint8_t
1914{
1915 return m->hash.sched.traffic_class;
1916}
1917
1921static inline uint8_t
1923{
1924 return m->hash.sched.color;
1925}
1926
1939static inline void
1940rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id,
1941 uint8_t *traffic_class,
1942 uint8_t *color)
1943{
1944 struct rte_mbuf_sched sched = m->hash.sched;
1945
1946 *queue_id = sched.queue_id;
1948 *color = sched.color;
1949}
1950
1954static inline void
1956{
1957 m->hash.sched.queue_id = queue_id;
1958}
1959
1963static inline void
1965{
1966 m->hash.sched.traffic_class = traffic_class;
1967}
1968
1972static inline void
1974{
1975 m->hash.sched.color = color;
1976}
1977
1990static inline void
1992 uint8_t traffic_class,
1993 uint8_t color)
1994{
1995 m->hash.sched = (struct rte_mbuf_sched){
1996 .queue_id = queue_id,
1997 .traffic_class = traffic_class,
1998 .color = color,
1999 .reserved = 0,
2000 };
2001}
2002
2003#ifdef __cplusplus
2004}
2005#endif
2006
2007#endif /* _RTE_MBUF_H_ */
#define likely(x)
#define unlikely(x)
#define RTE_PTR_ALIGN_FLOOR(ptr, align)
Definition rte_common.h:400
#define RTE_MIN(a, b)
Definition rte_common.h:581
#define RTE_PTR_DIFF(ptr1, ptr2)
Definition rte_common.h:383
#define RTE_PTR_SUB(ptr, x)
Definition rte_common.h:376
uint64_t rte_iova_t
Definition rte_common.h:556
#define RTE_PTR_ADD(ptr, x)
Definition rte_common.h:371
#define RTE_SET_USED(x)
Definition rte_common.h:159
#define __rte_unused
Definition rte_common.h:143
#define __rte_always_inline
Definition rte_common.h:331
static __rte_always_inline void rte_pktmbuf_free_seg(struct rte_mbuf *m)
Definition rte_mbuf.h:1393
int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen)
static void rte_mbuf_prefetch_part2(struct rte_mbuf *m)
Definition rte_mbuf.h:126
static __rte_always_inline uint64_t rte_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso, uint64_t ol3, uint64_t ol2, uint64_t unused)
Definition rte_mbuf.h:1804
static void rte_pktmbuf_free(struct rte_mbuf *m)
Definition rte_mbuf.h:1409
static rte_iova_t rte_mbuf_data_iova_default(const struct rte_mbuf *mb)
Definition rte_mbuf.h:202
static rte_iova_t rte_mbuf_data_iova(const struct rte_mbuf *mb)
Definition rte_mbuf.h:184
static uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
Definition rte_mbuf.h:1524
int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen)
struct rte_mbuf * rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp)
#define rte_pktmbuf_data_len(m)
Definition rte_mbuf.h:1565
static uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
Definition rte_mbuf.h:1510
static uint32_t rte_mbuf_sched_queue_get(const struct rte_mbuf *m)
Definition rte_mbuf.h:1904
static void rte_mbuf_prefetch_part1(struct rte_mbuf *m)
Definition rte_mbuf.h:109
static char * rte_pktmbuf_prepend(struct rte_mbuf *m, uint16_t len)
Definition rte_mbuf.h:1582
static char * rte_mbuf_data_addr_default(struct rte_mbuf *mb)
Definition rte_mbuf.h:251
struct rte_mempool * rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n, unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id, const char *ops_name)
struct rte_mempool * rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n, unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id, const struct rte_pktmbuf_extmem *ext_mem, unsigned int ext_num)
static void rte_mbuf_sched_set(struct rte_mbuf *m, uint32_t queue_id, uint8_t traffic_class, uint8_t color)
Definition rte_mbuf.h:1991
static void rte_mbuf_dynfield_copy(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
Definition rte_mbuf.h:1120
static char * rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
Definition rte_mbuf.h:1646
static void rte_mbuf_sched_color_set(struct rte_mbuf *m, uint8_t color)
Definition rte_mbuf.h:1973
static int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
Definition rte_mbuf.h:1700
int rte_mbuf_check(const struct rte_mbuf *m, int is_header, const char **reason)
static char * rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
Definition rte_mbuf.h:1615
const char * rte_get_tx_ol_flag_name(uint64_t mask)
static void rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id, uint8_t *traffic_class, uint8_t *color)
Definition rte_mbuf.h:1940
static void rte_pktmbuf_detach(struct rte_mbuf *m)
Definition rte_mbuf.h:1249
static char * rte_mbuf_to_baddr(struct rte_mbuf *md)
Definition rte_mbuf.h:270
static struct rte_mbuf_ext_shared_info * rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len, rte_mbuf_extbuf_free_callback_t free_cb, void *fcb_opaque)
Definition rte_mbuf.h:1001
static uint32_t rte_pktmbuf_priv_flags(struct rte_mempool *mp)
Definition rte_mbuf.h:314
static void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
Definition rte_mbuf.h:1493
void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg, void *m, unsigned i)
static void rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
Definition rte_mbuf.h:448
static struct rte_mbuf * rte_mbuf_raw_alloc(struct rte_mempool *mp)
Definition rte_mbuf.h:596
static uint16_t rte_mbuf_ext_refcnt_update(struct rte_mbuf_ext_shared_info *shinfo, int16_t value)
Definition rte_mbuf.h:496
struct rte_mempool * rte_pktmbuf_pool_create(const char *name, unsigned n, unsigned cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id)
void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count)
void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len)
#define __rte_mbuf_sanity_check(m, is_h)
Definition rte_mbuf.h:348
static const void * rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off, uint32_t len, void *buf)
Definition rte_mbuf.h:1732
static uint16_t rte_mbuf_refcnt_read(const struct rte_mbuf *m)
Definition rte_mbuf.h:439
static int rte_pktmbuf_linearize(struct rte_mbuf *mbuf)
Definition rte_mbuf.h:1877
static __rte_always_inline void rte_mbuf_raw_free(struct rte_mbuf *m)
Definition rte_mbuf.h:621
struct rte_mbuf * rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp, uint32_t offset, uint32_t length)
static struct rte_mbuf * rte_pktmbuf_alloc(struct rte_mempool *mp)
Definition rte_mbuf.h:904
static char * rte_mbuf_buf_addr(struct rte_mbuf *mb, struct rte_mempool *mp)
Definition rte_mbuf.h:237
static void rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo, uint16_t new_value)
Definition rte_mbuf.h:478
void rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header)
static uint8_t rte_mbuf_sched_traffic_class_get(const struct rte_mbuf *m)
Definition rte_mbuf.h:1913
static uint16_t rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
Definition rte_mbuf.h:464
static uint16_t rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
Definition rte_mbuf.h:430
static int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
Definition rte_mbuf.h:1676
#define RTE_MBUF_HAS_PINNED_EXTBUF(mb)
Definition rte_mbuf.h:337
static rte_iova_t rte_mbuf_iova_get(const struct rte_mbuf *m)
Definition rte_mbuf.h:147
static void rte_mbuf_sched_queue_set(struct rte_mbuf *m, uint32_t queue_id)
Definition rte_mbuf.h:1955
static uint16_t rte_pktmbuf_data_room_size(struct rte_mempool *mp)
Definition rte_mbuf.h:822
static struct rte_mbuf * rte_pktmbuf_lastseg(struct rte_mbuf *m)
Definition rte_mbuf.h:1539
static void rte_mbuf_sched_traffic_class_set(struct rte_mbuf *m, uint8_t traffic_class)
Definition rte_mbuf.h:1964
#define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF
Definition rte_mbuf.h:328
static __rte_always_inline struct rte_mbuf * rte_pktmbuf_prefree_seg(struct rte_mbuf *m)
Definition rte_mbuf.h:1341
static int rte_validate_tx_offload(const struct rte_mbuf *m)
Definition rte_mbuf.h:1827
static void rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr, rte_iova_t buf_iova, uint16_t buf_len, struct rte_mbuf_ext_shared_info *shinfo)
Definition rte_mbuf.h:1083
static struct rte_mbuf * rte_mbuf_from_indirect(struct rte_mbuf *mi)
Definition rte_mbuf.h:216
static int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool, struct rte_mbuf **mbufs, unsigned count)
Definition rte_mbuf.h:926
static void rte_mbuf_iova_set(struct rte_mbuf *m, rte_iova_t iova)
Definition rte_mbuf.h:165
static void rte_pktmbuf_reset_headroom(struct rte_mbuf *m)
Definition rte_mbuf.h:859
static void * rte_mbuf_to_priv(struct rte_mbuf *m)
Definition rte_mbuf.h:288
static uint8_t rte_mbuf_sched_color_get(const struct rte_mbuf *m)
Definition rte_mbuf.h:1922
static int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
Definition rte_mbuf.h:1757
static __rte_always_inline void __rte_mbuf_raw_sanity_check(__rte_unused const struct rte_mbuf *m)
Definition rte_mbuf.h:566
const char * rte_get_rx_ol_flag_name(uint64_t mask)
void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg)
static uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp)
Definition rte_mbuf.h:843
static void rte_pktmbuf_reset(struct rte_mbuf *m)
Definition rte_mbuf.h:873
static void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
Definition rte_mbuf.h:1159
#define RTE_MBUF_F_TX_OUTER_IP_CKSUM
#define RTE_MBUF_F_EXTERNAL
#define RTE_MBUF_F_TX_IP_CKSUM
#define RTE_MBUF_MAX_NB_SEGS
#define RTE_MBUF_F_TX_TCP_SEG
#define RTE_MBUF_F_TX_L4_MASK
#define RTE_MBUF_F_TX_OUTER_IPV4
#define RTE_MBUF_HAS_EXTBUF(mb)
#define RTE_MBUF_F_TX_OFFLOAD_MASK
#define RTE_MBUF_CLONED(mb)
#define RTE_MBUF_DIRECT(mb)
#define RTE_MBUF_F_TX_IPV4
void(* rte_mbuf_extbuf_free_callback_t)(void *addr, void *opaque)
#define rte_pktmbuf_mtod_offset(m, t, o)
#define RTE_MBUF_F_TX_IPV6
#define RTE_MBUF_F_INDIRECT
#define RTE_MBUF_PORT_INVALID
static __rte_always_inline int rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
static rte_iova_t rte_mempool_virt2iova(const void *elt)
static __rte_always_inline int rte_mempool_get(struct rte_mempool *mp, void **obj_p)
static __rte_always_inline void rte_mempool_put(struct rte_mempool *mp, void *obj)
static void * rte_mempool_get_priv(struct rte_mempool *mp)
static void rte_prefetch0(const volatile void *p)
rte_mbuf_extbuf_free_callback_t free_cb
uint8_t traffic_class
uint64_t ol_flags
uint16_t nb_segs
uint16_t vlan_tci
uint16_t priv_size
uint32_t pkt_len
uint16_t buf_len
struct rte_mbuf_ext_shared_info * shinfo
uint32_t packet_type
uint16_t port
void * buf_addr
struct rte_mempool * pool
uint32_t dynfield1[9]
uint16_t vlan_tci_outer
struct rte_mbuf * next
uint64_t tx_offload
uint16_t data_len
uint64_t tso_segsz
uint16_t refcnt
char name[RTE_MEMPOOL_NAMESIZE]
uint32_t cache_size
unsigned int flags
uint16_t elt_size
Definition rte_mbuf.h:760
rte_iova_t buf_iova
Definition rte_mbuf.h:758