> -----Original Message----- > From: Jerin Jacob <jerinjac...@gmail.com> > Sent: Monday, February 20, 2023 10:20 PM > To: Yan, Zhirun <zhirun....@intel.com> > Cc: dev@dpdk.org; jer...@marvell.com; kirankum...@marvell.com; > ndabilpu...@marvell.com; Liang, Cunming <cunming.li...@intel.com>; Wang, > Haiyue <haiyue.w...@intel.com> > Subject: Re: [PATCH v1 13/13] examples/l3fwd-graph: introduce generic worker > model > > On Thu, Nov 17, 2022 at 10:41 AM Zhirun Yan <zhirun....@intel.com> wrote: > > > > Add new parameter "model" to choose generic or rtc worker model. > > And in generic model, the node will affinity to worker core successively. > > > > Note: > > only support one RX node for remote model in current implementation. > > > > ./dpdk-l3fwd-graph -l 8,9,10,11 -n 4 -- -p 0x1 --config="(0,0,9)" -P > > --model="generic" > > Patch apply issue, please rebase with main. > See https://patches.dpdk.org/project/dpdk/patch/20221117050926.136974-14- > zhirun....@intel.com/ > Will fix in next version. Thanks for your comments. > > > > Signed-off-by: Haiyue Wang <haiyue.w...@intel.com> > > Signed-off-by: Cunming Liang <cunming.li...@intel.com> > > Signed-off-by: Zhirun Yan <zhirun....@intel.com> > > --- > > examples/l3fwd-graph/main.c | 218 > > +++++++++++++++++++++++++++++------- > > 1 file changed, 179 insertions(+), 39 deletions(-) > > > > diff --git a/examples/l3fwd-graph/main.c b/examples/l3fwd-graph/main.c > > index 6dcb6ee92b..c145a3e3e8 100644 > > --- a/examples/l3fwd-graph/main.c > > +++ b/examples/l3fwd-graph/main.c > > @@ -147,6 +147,19 @@ static struct ipv4_l3fwd_lpm_route > ipv4_l3fwd_lpm_route_array[] = { > > {RTE_IPV4(198, 18, 6, 0), 24, 6}, {RTE_IPV4(198, 18, 7, 0), > > 24, 7}, }; > > > > +static int > > +check_worker_model_params(void) > > +{ > > + if (rte_graph_worker_model_get() == RTE_GRAPH_MODEL_GENERIC && > > + nb_lcore_params > 1) { > > + printf("Exceeded max number of lcore params for remote > model: %hu\n", > > + nb_lcore_params); > > + return -1; > > + } > > + > > + return 0; > > +} > > + > > static int > > check_lcore_params(void) > > { > > @@ -291,6 +304,20 @@ parse_max_pkt_len(const char *pktlen) > > return len; > > } > > > > +static int > > +parse_worker_model(const char *model) { > > + if (strcmp(model, WORKER_MODEL_DEFAULT) == 0) > > + return RTE_GRAPH_MODEL_DEFAULT; > > + else if (strcmp(model, WORKER_MODEL_GENERIC) == 0) { > > + rte_graph_worker_model_set(RTE_GRAPH_MODEL_GENERIC); > > + return RTE_GRAPH_MODEL_GENERIC; > > + } > > + rte_exit(EXIT_FAILURE, "Invalid worker model: %s", model); > > + > > + return RTE_GRAPH_MODEL_MAX; > > +} > > + > > static int > > parse_portmask(const char *portmask) > > { > > @@ -404,6 +431,7 @@ static const char short_options[] = "p:" /* portmask */ > > #define CMD_LINE_OPT_NO_NUMA "no-numa" > > #define CMD_LINE_OPT_MAX_PKT_LEN "max-pkt-len" > > #define CMD_LINE_OPT_PER_PORT_POOL "per-port-pool" > > +#define CMD_LINE_OPT_WORKER_MODEL "model" > > enum { > > /* Long options mapped to a short option */ > > > > @@ -416,6 +444,7 @@ enum { > > CMD_LINE_OPT_NO_NUMA_NUM, > > CMD_LINE_OPT_MAX_PKT_LEN_NUM, > > CMD_LINE_OPT_PARSE_PER_PORT_POOL, > > + CMD_LINE_OPT_WORKER_MODEL_TYPE, > > }; > > > > static const struct option lgopts[] = { @@ -424,6 +453,7 @@ static > > const struct option lgopts[] = { > > {CMD_LINE_OPT_NO_NUMA, 0, 0, CMD_LINE_OPT_NO_NUMA_NUM}, > > {CMD_LINE_OPT_MAX_PKT_LEN, 1, 0, > CMD_LINE_OPT_MAX_PKT_LEN_NUM}, > > {CMD_LINE_OPT_PER_PORT_POOL, 0, 0, > > CMD_LINE_OPT_PARSE_PER_PORT_POOL}, > > + {CMD_LINE_OPT_WORKER_MODEL, 1, 0, > > + CMD_LINE_OPT_WORKER_MODEL_TYPE}, > > {NULL, 0, 0, 0}, > > }; > > > > @@ -498,6 +528,11 @@ parse_args(int argc, char **argv) > > per_port_pool = 1; > > break; > > > > + case CMD_LINE_OPT_WORKER_MODEL_TYPE: > > + printf("Use new worker model: %s\n", optarg); > > + parse_worker_model(optarg); > > + break; > > + > > default: > > print_usage(prgname); > > return -1; > > @@ -735,6 +770,140 @@ config_port_max_pkt_len(struct rte_eth_conf > *conf, > > return 0; > > } > > > > +static void > > +graph_config_generic(struct rte_graph_param graph_conf) { > > + uint16_t nb_patterns = graph_conf.nb_node_patterns; > > + int worker_count = rte_lcore_count() - 1; > > + int main_lcore_id = rte_get_main_lcore(); > > + int worker_lcore = main_lcore_id; > > + rte_graph_t main_graph_id = 0; > > + struct rte_node *node_tmp; > > + struct lcore_conf *qconf; > > + struct rte_graph *graph; > > + rte_graph_t graph_id; > > + rte_graph_off_t off; > > + int n_rx_node = 0; > > + rte_node_t count; > > + rte_edge_t i; > > + int ret; > > + > > + for (int j = 0; j < nb_lcore_params; j++) { > > + qconf = &lcore_conf[lcore_params[j].lcore_id]; > > + /* Add rx node patterns of all lcore */ > > + for (i = 0; i < qconf->n_rx_queue; i++) { > > + char *node_name = > > + qconf->rx_queue_list[i].node_name; > > + > > + graph_conf.node_patterns[nb_patterns + n_rx_node + > > i] = > node_name; > > + n_rx_node++; > > + ret = > > rte_node_model_generic_set_lcore_affinity(node_name, > > + > > lcore_params[j].lcore_id); > > + if (ret == 0) > > + printf("Set node %s affinity to lcore > > %u\n", node_name, > > + lcore_params[j].lcore_id); > > + } > > + } > > + > > + graph_conf.nb_node_patterns = nb_patterns + n_rx_node; > > + graph_conf.socket_id = rte_lcore_to_socket_id(main_lcore_id); > > + > > + snprintf(qconf->name, sizeof(qconf->name), "worker_%u", > > + main_lcore_id); > > + > > + /* create main graph */ > > + main_graph_id = rte_graph_create(qconf->name, &graph_conf); > > + if (main_graph_id == RTE_GRAPH_ID_INVALID) > > + rte_exit(EXIT_FAILURE, > > + "rte_graph_create(): main_graph_id invalid for > > lcore %u\n", > > + main_lcore_id); > > + > > + qconf->graph_id = main_graph_id; > > + qconf->graph = rte_graph_lookup(qconf->name); > > + /* >8 End of graph initialization. */ > > + if (!qconf->graph) > > + rte_exit(EXIT_FAILURE, > > + "rte_graph_lookup(): graph %s not found\n", > > + qconf->name); > > + > > + graph = qconf->graph; > > + rte_graph_foreach_node(count, off, graph, node_tmp) { > > + worker_lcore = rte_get_next_lcore(worker_lcore, true, > > + 1); > > + > > + /* Need to set the node Lcore affinity before clone graph > > for each > lcore */ > > + if (node_tmp->lcore_id == RTE_MAX_LCORE) { > > + ret = > > rte_node_model_generic_set_lcore_affinity(node_tmp- > >name, > > + > > worker_lcore); > > + if (ret == 0) > > + printf("Set node %s affinity to lcore %u\n", > > + node_tmp->name, worker_lcore); > > + } > > + } > > + > > + worker_lcore = main_lcore_id; > > + for (int i = 0; i < worker_count; i++) { > > + worker_lcore = rte_get_next_lcore(worker_lcore, true, > > + 1); > > + > > + qconf = &lcore_conf[worker_lcore]; > > + snprintf(qconf->name, sizeof(qconf->name), "cloned-%u", > worker_lcore); > > + graph_id = rte_graph_clone(main_graph_id, qconf->name); > > + ret = rte_graph_bind_core(graph_id, worker_lcore); > > + if (ret == 0) > > + printf("bind graph %d to lcore %u\n", > > + graph_id, worker_lcore); > > + > > + /* full cloned graph name */ > > + snprintf(qconf->name, sizeof(qconf->name), "%s", > > + rte_graph_id_to_name(graph_id)); > > + qconf->graph_id = graph_id; > > + qconf->graph = rte_graph_lookup(qconf->name); > > + if (!qconf->graph) > > + rte_exit(EXIT_FAILURE, > > + "Failed to lookup graph %s\n", > > + qconf->name); > > + continue; > > + } > > +} > > + > > +static void > > +graph_config_rtc(struct rte_graph_param graph_conf) { > > + uint16_t nb_patterns = graph_conf.nb_node_patterns; > > + struct lcore_conf *qconf; > > + rte_graph_t graph_id; > > + uint32_t lcore_id; > > + rte_edge_t i; > > + > > + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { > > + if (rte_lcore_is_enabled(lcore_id) == 0) > > + continue; > > + > > + qconf = &lcore_conf[lcore_id]; > > + /* Skip graph creation if no source exists */ > > + if (!qconf->n_rx_queue) > > + continue; > > + /* Add rx node patterns of this lcore */ > > + for (i = 0; i < qconf->n_rx_queue; i++) { > > + graph_conf.node_patterns[nb_patterns + i] = > > + qconf->rx_queue_list[i].node_name; > > + } > > + graph_conf.nb_node_patterns = nb_patterns + i; > > + graph_conf.socket_id = rte_lcore_to_socket_id(lcore_id); > > + snprintf(qconf->name, sizeof(qconf->name), "worker_%u", > > + lcore_id); > > + graph_id = rte_graph_create(qconf->name, &graph_conf); > > + if (graph_id == RTE_GRAPH_ID_INVALID) > > + rte_exit(EXIT_FAILURE, > > + "rte_graph_create(): graph_id invalid for > > lcore %u\n", > > + lcore_id); > > + qconf->graph_id = graph_id; > > + qconf->graph = rte_graph_lookup(qconf->name); > > + /* >8 End of graph initialization. */ > > + if (!qconf->graph) > > + rte_exit(EXIT_FAILURE, > > + "rte_graph_lookup(): graph %s not found\n", > > + qconf->name); > > + } > > +} > > + > > int > > main(int argc, char **argv) > > { > > @@ -759,6 +928,7 @@ main(int argc, char **argv) > > uint16_t nb_patterns; > > uint8_t rewrite_len; > > uint32_t lcore_id; > > + uint16_t model; > > int ret; > > > > /* Init EAL */ > > @@ -787,6 +957,9 @@ main(int argc, char **argv) > > if (check_lcore_params() < 0) > > rte_exit(EXIT_FAILURE, "check_lcore_params() > > failed\n"); > > > > + if (check_worker_model_params() < 0) > > + rte_exit(EXIT_FAILURE, "check_worker_model_params() > > + failed\n"); > > + > > ret = init_lcore_rx_queues(); > > if (ret < 0) > > rte_exit(EXIT_FAILURE, "init_lcore_rx_queues() > > failed\n"); @@ -1026,46 +1199,13 @@ main(int argc, char **argv) > > > > memset(&graph_conf, 0, sizeof(graph_conf)); > > graph_conf.node_patterns = node_patterns; > > + graph_conf.nb_node_patterns = nb_patterns; > > > > - for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { > > - rte_graph_t graph_id; > > - rte_edge_t i; > > - > > - if (rte_lcore_is_enabled(lcore_id) == 0) > > - continue; > > - > > - qconf = &lcore_conf[lcore_id]; > > - > > - /* Skip graph creation if no source exists */ > > - if (!qconf->n_rx_queue) > > - continue; > > - > > - /* Add rx node patterns of this lcore */ > > - for (i = 0; i < qconf->n_rx_queue; i++) { > > - graph_conf.node_patterns[nb_patterns + i] = > > - qconf->rx_queue_list[i].node_name; > > - } > > - > > - graph_conf.nb_node_patterns = nb_patterns + i; > > - graph_conf.socket_id = rte_lcore_to_socket_id(lcore_id); > > - > > - snprintf(qconf->name, sizeof(qconf->name), "worker_%u", > > - lcore_id); > > - > > - graph_id = rte_graph_create(qconf->name, &graph_conf); > > - if (graph_id == RTE_GRAPH_ID_INVALID) > > - rte_exit(EXIT_FAILURE, > > - "rte_graph_create(): graph_id invalid" > > - " for lcore %u\n", lcore_id); > > - > > - qconf->graph_id = graph_id; > > - qconf->graph = rte_graph_lookup(qconf->name); > > - /* >8 End of graph initialization. */ > > - if (!qconf->graph) > > - rte_exit(EXIT_FAILURE, > > - "rte_graph_lookup(): graph %s not found\n", > > - qconf->name); > > - } > > + model = rte_graph_worker_model_get(); > > + if (model == RTE_GRAPH_MODEL_DEFAULT) > > + graph_config_rtc(graph_conf); > > + else if (model == RTE_GRAPH_MODEL_GENERIC) > > + graph_config_generic(graph_conf); > > > > memset(&rewrite_data, 0, sizeof(rewrite_data)); > > rewrite_len = sizeof(rewrite_data); > > -- > > 2.25.1 > >
