DPDK 23.11.0
Loading...
Searching...
No Matches
rte_graph_worker_common.h
Go to the documentation of this file.
1/* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2020 Marvell International Ltd.
3 */
4
5#ifndef _RTE_GRAPH_WORKER_COMMON_H_
6#define _RTE_GRAPH_WORKER_COMMON_H_
7
15#include <rte_common.h>
16#include <rte_cycles.h>
17#include <rte_prefetch.h>
18#include <rte_memcpy.h>
19#include <rte_memory.h>
20
21#include "rte_graph.h"
22
23#ifdef __cplusplus
24extern "C" {
25#endif
26
28/* When adding a new graph model entry, update rte_graph_model_is_valid() implementation. */
29#define RTE_GRAPH_MODEL_RTC 0
30#define RTE_GRAPH_MODEL_MCORE_DISPATCH 1
32#define RTE_GRAPH_MODEL_DEFAULT RTE_GRAPH_MODEL_RTC
39SLIST_HEAD(rte_graph_rq_head, rte_graph);
40
46struct rte_graph {
47 /* Fast path area. */
48 uint32_t tail;
49 uint32_t head;
50 uint32_t cir_mask;
51 rte_node_t nb_nodes;
52 rte_graph_off_t *cir_start;
53 rte_graph_off_t nodes_start;
54 uint8_t model;
55 uint8_t reserved1;
56 uint16_t reserved2;
57 union {
58 /* Fast schedule area for mcore dispatch model */
59 struct {
60 struct rte_graph_rq_head *rq __rte_cache_aligned; /* The run-queue */
61 struct rte_graph_rq_head rq_head; /* The head for run-queue list */
62
63 unsigned int lcore_id;
64 struct rte_ring *wq;
65 struct rte_mempool *mp;
66 } dispatch;
67 };
68 SLIST_ENTRY(rte_graph) next; /* The next for rte_graph list */
69 /* End of Fast path area.*/
70 rte_graph_t id;
71 int socket;
72 char name[RTE_GRAPH_NAMESIZE];
73 bool pcap_enable;
75 uint64_t nb_pkt_captured;
77 uint64_t nb_pkt_to_capture;
78 char pcap_filename[RTE_GRAPH_PCAP_FILE_SZ];
79 uint64_t fence;
81
87struct rte_node {
88 /* Slow path area */
89 uint64_t fence;
90 rte_graph_off_t next;
91 rte_node_t id;
92 rte_node_t parent_id;
93 rte_edge_t nb_edges;
94 uint32_t realloc_count;
96 char parent[RTE_NODE_NAMESIZE];
97 char name[RTE_NODE_NAMESIZE];
100 rte_node_process_t original_process;
101
102 union {
103 /* Fast schedule area for mcore dispatch model */
104 struct {
105 unsigned int lcore_id;
106 uint64_t total_sched_objs;
107 uint64_t total_sched_fail;
108 } dispatch;
109 };
110 /* Fast path area */
111#define RTE_NODE_CTX_SZ 16
112 uint8_t ctx[RTE_NODE_CTX_SZ] __rte_cache_aligned;
113 uint16_t size;
114 uint16_t idx;
115 rte_graph_off_t off;
116 uint64_t total_cycles;
117 uint64_t total_calls;
118 uint64_t total_objs;
119 union {
120 void **objs;
121 uint64_t objs_u64;
122 };
123 union {
124 rte_node_process_t process;
125 uint64_t process_u64;
126 };
127 struct rte_node *nodes[] __rte_cache_min_aligned;
129
142void __rte_node_stream_alloc(struct rte_graph *graph, struct rte_node *node);
143
158void __rte_node_stream_alloc_size(struct rte_graph *graph,
159 struct rte_node *node, uint16_t req_size);
160
161/* Fast path helper functions */
162
173static __rte_always_inline void
174__rte_node_process(struct rte_graph *graph, struct rte_node *node)
175{
176 uint64_t start;
177 uint16_t rc;
178 void **objs;
179
180 RTE_ASSERT(node->fence == RTE_GRAPH_FENCE);
181 objs = node->objs;
182 rte_prefetch0(objs);
183
185 start = rte_rdtsc();
186 rc = node->process(graph, node, objs, node->idx);
187 node->total_cycles += rte_rdtsc() - start;
188 node->total_calls++;
189 node->total_objs += rc;
190 } else {
191 node->process(graph, node, objs, node->idx);
192 }
193 node->idx = 0;
194}
195
206static __rte_always_inline void
207__rte_node_enqueue_tail_update(struct rte_graph *graph, struct rte_node *node)
208{
209 uint32_t tail;
210
211 tail = graph->tail;
212 graph->cir_start[tail++] = node->off;
213 graph->tail = tail & graph->cir_mask;
214}
215
233static __rte_always_inline void
234__rte_node_enqueue_prologue(struct rte_graph *graph, struct rte_node *node,
235 const uint16_t idx, const uint16_t space)
236{
237
238 /* Add to the pending stream list if the node is new */
239 if (idx == 0)
240 __rte_node_enqueue_tail_update(graph, node);
241
242 if (unlikely(node->size < (idx + space)))
243 __rte_node_stream_alloc_size(graph, node, node->size + space);
244}
245
259static __rte_always_inline struct rte_node *
260__rte_node_next_node_get(struct rte_node *node, rte_edge_t next)
261{
262 RTE_ASSERT(next < node->nb_edges);
263 RTE_ASSERT(node->fence == RTE_GRAPH_FENCE);
264 node = node->nodes[next];
265 RTE_ASSERT(node->fence == RTE_GRAPH_FENCE);
266
267 return node;
268}
269
285static inline void
286rte_node_enqueue(struct rte_graph *graph, struct rte_node *node,
287 rte_edge_t next, void **objs, uint16_t nb_objs)
288{
289 node = __rte_node_next_node_get(node, next);
290 const uint16_t idx = node->idx;
291
292 __rte_node_enqueue_prologue(graph, node, idx, nb_objs);
293
294 rte_memcpy(&node->objs[idx], objs, nb_objs * sizeof(void *));
295 node->idx = idx + nb_objs;
296}
297
311static inline void
312rte_node_enqueue_x1(struct rte_graph *graph, struct rte_node *node,
313 rte_edge_t next, void *obj)
314{
315 node = __rte_node_next_node_get(node, next);
316 uint16_t idx = node->idx;
317
318 __rte_node_enqueue_prologue(graph, node, idx, 1);
319
320 node->objs[idx++] = obj;
321 node->idx = idx;
322}
323
340static inline void
341rte_node_enqueue_x2(struct rte_graph *graph, struct rte_node *node,
342 rte_edge_t next, void *obj0, void *obj1)
343{
344 node = __rte_node_next_node_get(node, next);
345 uint16_t idx = node->idx;
346
347 __rte_node_enqueue_prologue(graph, node, idx, 2);
348
349 node->objs[idx++] = obj0;
350 node->objs[idx++] = obj1;
351 node->idx = idx;
352}
353
374static inline void
375rte_node_enqueue_x4(struct rte_graph *graph, struct rte_node *node,
376 rte_edge_t next, void *obj0, void *obj1, void *obj2,
377 void *obj3)
378{
379 node = __rte_node_next_node_get(node, next);
380 uint16_t idx = node->idx;
381
382 __rte_node_enqueue_prologue(graph, node, idx, 4);
383
384 node->objs[idx++] = obj0;
385 node->objs[idx++] = obj1;
386 node->objs[idx++] = obj2;
387 node->objs[idx++] = obj3;
388 node->idx = idx;
389}
390
407static inline void
408rte_node_enqueue_next(struct rte_graph *graph, struct rte_node *node,
409 rte_edge_t *nexts, void **objs, uint16_t nb_objs)
410{
411 uint16_t i;
412
413 for (i = 0; i < nb_objs; i++)
414 rte_node_enqueue_x1(graph, node, nexts[i], objs[i]);
415}
416
436static inline void **
437rte_node_next_stream_get(struct rte_graph *graph, struct rte_node *node,
438 rte_edge_t next, uint16_t nb_objs)
439{
440 node = __rte_node_next_node_get(node, next);
441 const uint16_t idx = node->idx;
442 uint16_t free_space = node->size - idx;
443
444 if (unlikely(free_space < nb_objs))
445 __rte_node_stream_alloc_size(graph, node, node->size + nb_objs);
446
447 return &node->objs[idx];
448}
449
466static inline void
467rte_node_next_stream_put(struct rte_graph *graph, struct rte_node *node,
468 rte_edge_t next, uint16_t idx)
469{
470 if (unlikely(!idx))
471 return;
472
473 node = __rte_node_next_node_get(node, next);
474 if (node->idx == 0)
475 __rte_node_enqueue_tail_update(graph, node);
476
477 node->idx += idx;
478}
479
494static inline void
495rte_node_next_stream_move(struct rte_graph *graph, struct rte_node *src,
496 rte_edge_t next)
497{
498 struct rte_node *dst = __rte_node_next_node_get(src, next);
499
500 /* Let swap the pointers if dst don't have valid objs */
501 if (likely(dst->idx == 0)) {
502 void **dobjs = dst->objs;
503 uint16_t dsz = dst->size;
504 dst->objs = src->objs;
505 dst->size = src->size;
506 src->objs = dobjs;
507 src->size = dsz;
508 dst->idx = src->idx;
509 __rte_node_enqueue_tail_update(graph, dst);
510 } else { /* Move the objects from src node to dst node */
511 rte_node_enqueue(graph, src, next, src->objs, src->idx);
512 }
513}
514
524bool
526
537int rte_graph_worker_model_set(uint8_t model);
538
551uint8_t rte_graph_worker_model_get(struct rte_graph *graph);
552
566uint8_t rte_graph_worker_model_no_check_get(struct rte_graph *graph)
567{
568 return graph->model;
569}
570
571#ifdef __cplusplus
572}
573#endif
574
575#endif /* _RTE_GRAPH_WORKER_COIMMON_H_ */
#define likely(x)
#define unlikely(x)
#define __rte_cache_aligned
Definition rte_common.h:524
#define __rte_cache_min_aligned
Definition rte_common.h:528
#define __rte_always_inline
Definition rte_common.h:331
uint16_t(* rte_node_process_t)(struct rte_graph *graph, struct rte_node *node, void **objs, uint16_t nb_objs)
Definition rte_graph.h:93
uint32_t rte_node_t
Definition rte_graph.h:40
#define RTE_GRAPH_FENCE
Definition rte_graph.h:37
uint16_t rte_edge_t
Definition rte_graph.h:41
#define RTE_NODE_NAMESIZE
Definition rte_graph.h:31
#define RTE_GRAPH_PCAP_FILE_SZ
Definition rte_graph.h:32
uint32_t rte_graph_off_t
Definition rte_graph.h:39
uint16_t rte_graph_t
Definition rte_graph.h:42
#define RTE_GRAPH_NAMESIZE
Definition rte_graph.h:30
static __rte_always_inline int rte_graph_has_stats_feature(void)
Definition rte_graph.h:686
static __rte_always_inline uint8_t rte_graph_worker_model_no_check_get(struct rte_graph *graph)
static void rte_node_enqueue_x4(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, void *obj0, void *obj1, void *obj2, void *obj3)
bool rte_graph_model_is_valid(uint8_t model)
static void rte_node_next_stream_put(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, uint16_t idx)
static void rte_node_enqueue(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, void **objs, uint16_t nb_objs)
static void rte_node_next_stream_move(struct rte_graph *graph, struct rte_node *src, rte_edge_t next)
int rte_graph_worker_model_set(uint8_t model)
static void rte_node_enqueue_next(struct rte_graph *graph, struct rte_node *node, rte_edge_t *nexts, void **objs, uint16_t nb_objs)
static void rte_node_enqueue_x1(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, void *obj)
static void ** rte_node_next_stream_get(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, uint16_t nb_objs)
static void rte_node_enqueue_x2(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, void *obj0, void *obj1)
uint8_t rte_graph_worker_model_get(struct rte_graph *graph)
static void * rte_memcpy(void *dst, const void *src, size_t n)
static void rte_prefetch0(const volatile void *p)