Hi Lukasz,
> Make number of buffers in a pool nb_mbuf_in_pool dependent on number
> of ports, cores and crypto queues. Add command line option -s which
> when used overrides dynamic calculation of number of buffers in a pool.
>
> Signed-off-by: Anoob Joseph <ano...@marvell.com>
> Signed-off-by: Lukasz Bartosik <lbarto...@marvell.com>
> ---
> examples/ipsec-secgw/ipsec-secgw.c | 59
> +++++++++++++++++++++++++++++++-------
> 1 file changed, 48 insertions(+), 11 deletions(-)
>
> diff --git a/examples/ipsec-secgw/ipsec-secgw.c
> b/examples/ipsec-secgw/ipsec-secgw.c
> index 862a7f0..f7acb52 100644
> --- a/examples/ipsec-secgw/ipsec-secgw.c
> +++ b/examples/ipsec-secgw/ipsec-secgw.c
> @@ -59,8 +59,6 @@ volatile bool force_quit;
>
> #define MEMPOOL_CACHE_SIZE 256
>
> -#define NB_MBUF (32000)
> -
> #define CDEV_QUEUE_DESC 2048
> #define CDEV_MAP_ENTRIES 16384
> #define CDEV_MP_NB_OBJS 1024
> @@ -163,6 +161,7 @@ static int32_t promiscuous_on = 1;
> static int32_t numa_on = 1; /**< NUMA is enabled by default. */
> static uint32_t nb_lcores;
> static uint32_t single_sa;
> +static uint32_t nb_bufs_in_pool;
>
> /*
> * RX/TX HW offload capabilities to enable/use on ethernet ports.
> @@ -1259,6 +1258,7 @@ print_usage(const char *prgname)
> " [-w REPLAY_WINDOW_SIZE]"
> " [-e]"
> " [-a]"
> + " [-s NUMBER_OF_MBUFS_IN_PKT_POOL]"
> " -f CONFIG_FILE"
> " --config (port,queue,lcore)[,(port,queue,lcore)]"
> " [--single-sa SAIDX]"
> @@ -1280,6 +1280,7 @@ print_usage(const char *prgname)
> " size for each SA\n"
> " -e enables ESN\n"
> " -a enables SA SQN atomic behaviour\n"
> + " -s number of mbufs in packet pool (default 8192)\n"
> " -f CONFIG_FILE: Configuration file\n"
> " --config (port,queue,lcore): Rx queue configuration\n"
> " --single-sa SAIDX: In poll mode use single SA index for\n"
> @@ -1479,7 +1480,7 @@ parse_args(int32_t argc, char **argv, struct eh_conf
> *eh_conf)
>
> argvopt = argv;
>
> - while ((opt = getopt_long(argc, argvopt, "aelp:Pu:f:j:w:",
> + while ((opt = getopt_long(argc, argvopt, "aelp:Pu:f:j:w:s:",
> lgopts, &option_index)) != EOF) {
>
> switch (opt) {
> @@ -1513,6 +1514,19 @@ parse_args(int32_t argc, char **argv, struct eh_conf
> *eh_conf)
> cfgfile = optarg;
> f_present = 1;
> break;
> +
> + case 's':
> + ret = parse_decimal(optarg);
> + if (ret < 0) {
> + printf("Invalid number of buffers in a pool: "
> + "%s\n", optarg);
> + print_usage(prgname);
> + return -1;
> + }
> +
> + nb_bufs_in_pool = ret;
> + break;
> +
> case 'j':
> ret = parse_decimal(optarg);
> if (ret < RTE_MBUF_DEFAULT_BUF_SIZE ||
> @@ -1876,12 +1890,12 @@ check_cryptodev_mask(uint8_t cdev_id)
> return -1;
> }
>
> -static int32_t
> +static uint16_t
> cryptodevs_init(void)
> {
> struct rte_cryptodev_config dev_conf;
> struct rte_cryptodev_qp_conf qp_conf;
> - uint16_t idx, max_nb_qps, qp, i;
> + uint16_t idx, max_nb_qps, qp, total_nb_qps, i;
> int16_t cdev_id;
> struct rte_hash_parameters params = { 0 };
>
> @@ -1909,6 +1923,7 @@ cryptodevs_init(void)
> printf("lcore/cryptodev/qp mappings:\n");
>
> idx = 0;
> + total_nb_qps = 0;
> for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
> struct rte_cryptodev_info cdev_info;
>
> @@ -1942,6 +1957,7 @@ cryptodevs_init(void)
> if (qp == 0)
> continue;
>
> + total_nb_qps += qp;
> dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
> dev_conf.nb_queue_pairs = qp;
> dev_conf.ff_disable = RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO;
> @@ -1974,7 +1990,7 @@ cryptodevs_init(void)
>
> printf("\n");
>
> - return 0;
> + return total_nb_qps;
> }
>
> static void
> @@ -2607,16 +2623,18 @@ int32_t
> main(int32_t argc, char **argv)
> {
> int32_t ret;
> - uint32_t lcore_id;
> + uint32_t lcore_id, nb_txq, nb_rxq = 0;
> uint32_t cdev_id;
> uint32_t i;
> uint8_t socket_id;
> - uint16_t portid;
> + uint16_t portid, nb_crypto_qp, nb_ports = 0;
> uint64_t req_rx_offloads[RTE_MAX_ETHPORTS];
> uint64_t req_tx_offloads[RTE_MAX_ETHPORTS];
> struct eh_conf *eh_conf = NULL;
> size_t sess_sz;
>
> + nb_bufs_in_pool = 0;
> +
> /* init EAL */
> ret = rte_eal_init(argc, argv);
> if (ret < 0)
> @@ -2665,6 +2683,26 @@ main(int32_t argc, char **argv)
>
> sess_sz = max_session_size();
>
> + nb_crypto_qp = cryptodevs_init();
> +
> + if (nb_bufs_in_pool == 0) {
> + RTE_ETH_FOREACH_DEV(portid) {
> + if ((enabled_port_mask & (1 << portid)) == 0)
> + continue;
> + nb_ports++;
> + nb_rxq += get_port_nb_rx_queues(portid);
> + }
> +
> + nb_txq = nb_lcores;
> +
> + nb_bufs_in_pool = RTE_MAX((nb_rxq*nb_rxd +
> + nb_ports*nb_lcores*MAX_PKT_BURST +
> + nb_ports*nb_txq*nb_txd +
> + nb_lcores*MEMPOOL_CACHE_SIZE +
> + nb_crypto_qp*CDEV_QUEUE_DESC),
I think you forgot to take into account possible reassemble table:
@@ -2699,7 +2699,9 @@ main(int32_t argc, char **argv)
nb_ports*nb_lcores*MAX_PKT_BURST +
nb_ports*nb_txq*nb_txd +
nb_lcores*MEMPOOL_CACHE_SIZE +
- nb_crypto_qp*CDEV_QUEUE_DESC),
+ nb_crypto_qp*CDEV_QUEUE_DESC +
+ nb_lcores * frag_tbl_sz *
+ FRAG_TBL_BUCKET_ENTRIES),
8192U);
Also it might be worth for better readability to put code for nb_bufs_in_pool
calculation
in a separate function (and add spaces between '*' and its' operands).
Apart from that - whole series LGTM.
Konstantin
> + 8192U);
> + }
> +
> for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
> if (rte_lcore_is_enabled(lcore_id) == 0)
> continue;
> @@ -2678,11 +2716,12 @@ main(int32_t argc, char **argv)
> if (socket_ctx[socket_id].mbuf_pool)
> continue;
>
> - pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
> + pool_init(&socket_ctx[socket_id], socket_id, nb_bufs_in_pool);
> session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
> session_priv_pool_init(&socket_ctx[socket_id], socket_id,
> sess_sz);
> }
> + printf("Number of mbufs in packet pool %d\n", nb_bufs_in_pool);
>
> RTE_ETH_FOREACH_DEV(portid) {
> if ((enabled_port_mask & (1 << portid)) == 0)
> @@ -2694,8 +2733,6 @@ main(int32_t argc, char **argv)
> req_tx_offloads[portid]);
> }
>
> - cryptodevs_init();
> -
> /*
> * Set the enabled port mask in helper config for use by helper
> * sub-system. This will be used while initializing devices using
> --
> 2.7.4
