Hi Konstantin,
Please see inline.
Thanks,
Lukasz
On 05.02.2020 14:42, Ananyev, Konstantin wrote:
> External Email
>
> ----------------------------------------------------------------------
>
> Hi Lukasz,
>
>> Make number of buffers in a pool nb_mbuf_in_pool dependent on number
>> of ports, cores and crypto queues. Add command line option -s which
>> when used overrides dynamic calculation of number of buffers in a pool.
>>
>> Signed-off-by: Anoob Joseph <ano...@marvell.com>
>> Signed-off-by: Lukasz Bartosik <lbarto...@marvell.com>
>> ---
>> examples/ipsec-secgw/ipsec-secgw.c | 59
>> +++++++++++++++++++++++++++++++-------
>> 1 file changed, 48 insertions(+), 11 deletions(-)
>>
>> diff --git a/examples/ipsec-secgw/ipsec-secgw.c
>> b/examples/ipsec-secgw/ipsec-secgw.c
>> index 862a7f0..f7acb52 100644
>> --- a/examples/ipsec-secgw/ipsec-secgw.c
>> +++ b/examples/ipsec-secgw/ipsec-secgw.c
>> @@ -59,8 +59,6 @@ volatile bool force_quit;
>>
>> #define MEMPOOL_CACHE_SIZE 256
>>
>> -#define NB_MBUF (32000)
>> -
>> #define CDEV_QUEUE_DESC 2048
>> #define CDEV_MAP_ENTRIES 16384
>> #define CDEV_MP_NB_OBJS 1024
>> @@ -163,6 +161,7 @@ static int32_t promiscuous_on = 1;
>> static int32_t numa_on = 1; /**< NUMA is enabled by default. */
>> static uint32_t nb_lcores;
>> static uint32_t single_sa;
>> +static uint32_t nb_bufs_in_pool;
>>
>> /*
>> * RX/TX HW offload capabilities to enable/use on ethernet ports.
>> @@ -1259,6 +1258,7 @@ print_usage(const char *prgname)
>> " [-w REPLAY_WINDOW_SIZE]"
>> " [-e]"
>> " [-a]"
>> + " [-s NUMBER_OF_MBUFS_IN_PKT_POOL]"
>> " -f CONFIG_FILE"
>> " --config (port,queue,lcore)[,(port,queue,lcore)]"
>> " [--single-sa SAIDX]"
>> @@ -1280,6 +1280,7 @@ print_usage(const char *prgname)
>> " size for each SA\n"
>> " -e enables ESN\n"
>> " -a enables SA SQN atomic behaviour\n"
>> + " -s number of mbufs in packet pool (default 8192)\n"
>> " -f CONFIG_FILE: Configuration file\n"
>> " --config (port,queue,lcore): Rx queue configuration\n"
>> " --single-sa SAIDX: In poll mode use single SA index for\n"
>> @@ -1479,7 +1480,7 @@ parse_args(int32_t argc, char **argv, struct eh_conf
>> *eh_conf)
>>
>> argvopt = argv;
>>
>> - while ((opt = getopt_long(argc, argvopt, "aelp:Pu:f:j:w:",
>> + while ((opt = getopt_long(argc, argvopt, "aelp:Pu:f:j:w:s:",
>> lgopts, &option_index)) != EOF) {
>>
>> switch (opt) {
>> @@ -1513,6 +1514,19 @@ parse_args(int32_t argc, char **argv, struct eh_conf
>> *eh_conf)
>> cfgfile = optarg;
>> f_present = 1;
>> break;
>> +
>> + case 's':
>> + ret = parse_decimal(optarg);
>> + if (ret < 0) {
>> + printf("Invalid number of buffers in a pool: "
>> + "%s\n", optarg);
>> + print_usage(prgname);
>> + return -1;
>> + }
>> +
>> + nb_bufs_in_pool = ret;
>> + break;
>> +
>> case 'j':
>> ret = parse_decimal(optarg);
>> if (ret < RTE_MBUF_DEFAULT_BUF_SIZE ||
>> @@ -1876,12 +1890,12 @@ check_cryptodev_mask(uint8_t cdev_id)
>> return -1;
>> }
>>
>> -static int32_t
>> +static uint16_t
>> cryptodevs_init(void)
>> {
>> struct rte_cryptodev_config dev_conf;
>> struct rte_cryptodev_qp_conf qp_conf;
>> - uint16_t idx, max_nb_qps, qp, i;
>> + uint16_t idx, max_nb_qps, qp, total_nb_qps, i;
>> int16_t cdev_id;
>> struct rte_hash_parameters params = { 0 };
>>
>> @@ -1909,6 +1923,7 @@ cryptodevs_init(void)
>> printf("lcore/cryptodev/qp mappings:\n");
>>
>> idx = 0;
>> + total_nb_qps = 0;
>> for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
>> struct rte_cryptodev_info cdev_info;
>>
>> @@ -1942,6 +1957,7 @@ cryptodevs_init(void)
>> if (qp == 0)
>> continue;
>>
>> + total_nb_qps += qp;
>> dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
>> dev_conf.nb_queue_pairs = qp;
>> dev_conf.ff_disable = RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO;
>> @@ -1974,7 +1990,7 @@ cryptodevs_init(void)
>>
>> printf("\n");
>>
>> - return 0;
>> + return total_nb_qps;
>> }
>>
>> static void
>> @@ -2607,16 +2623,18 @@ int32_t
>> main(int32_t argc, char **argv)
>> {
>> int32_t ret;
>> - uint32_t lcore_id;
>> + uint32_t lcore_id, nb_txq, nb_rxq = 0;
>> uint32_t cdev_id;
>> uint32_t i;
>> uint8_t socket_id;
>> - uint16_t portid;
>> + uint16_t portid, nb_crypto_qp, nb_ports = 0;
>> uint64_t req_rx_offloads[RTE_MAX_ETHPORTS];
>> uint64_t req_tx_offloads[RTE_MAX_ETHPORTS];
>> struct eh_conf *eh_conf = NULL;
>> size_t sess_sz;
>>
>> + nb_bufs_in_pool = 0;
>> +
>> /* init EAL */
>> ret = rte_eal_init(argc, argv);
>> if (ret < 0)
>> @@ -2665,6 +2683,26 @@ main(int32_t argc, char **argv)
>>
>> sess_sz = max_session_size();
>>
>> + nb_crypto_qp = cryptodevs_init();
>> +
>> + if (nb_bufs_in_pool == 0) {
>> + RTE_ETH_FOREACH_DEV(portid) {
>> + if ((enabled_port_mask & (1 << portid)) == 0)
>> + continue;
>> + nb_ports++;
>> + nb_rxq += get_port_nb_rx_queues(portid);
>> + }
>> +
>> + nb_txq = nb_lcores;
>> +
>> + nb_bufs_in_pool = RTE_MAX((nb_rxq*nb_rxd +
>> + nb_ports*nb_lcores*MAX_PKT_BURST +
>> + nb_ports*nb_txq*nb_txd +
>> + nb_lcores*MEMPOOL_CACHE_SIZE +
>> + nb_crypto_qp*CDEV_QUEUE_DESC),
>
> I think you forgot to take into account possible reassemble table:
> @@ -2699,7 +2699,9 @@ main(int32_t argc, char **argv)
> nb_ports*nb_lcores*MAX_PKT_BURST +
> nb_ports*nb_txq*nb_txd +
> nb_lcores*MEMPOOL_CACHE_SIZE +
> - nb_crypto_qp*CDEV_QUEUE_DESC),
> + nb_crypto_qp*CDEV_QUEUE_DESC +
> + nb_lcores * frag_tbl_sz *
> + FRAG_TBL_BUCKET_ENTRIES),
> 8192U);
[Lukasz] I will add it in V4.
>
>
> Also it might be worth for better readability to put code for nb_bufs_in_pool
> calculation
> in a separate function (and add spaces between '*' and its' operands).
> Apart from that - whole series LGTM.
> Konstantin
[Lukasz] Thank you for reviewing the changes. I will resolve your comment in V4.
>
>
>> + 8192U);
>> + }
>> +
>> for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
>> if (rte_lcore_is_enabled(lcore_id) == 0)
>> continue;
>> @@ -2678,11 +2716,12 @@ main(int32_t argc, char **argv)
>> if (socket_ctx[socket_id].mbuf_pool)
>> continue;
>>
>> - pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
>> + pool_init(&socket_ctx[socket_id], socket_id, nb_bufs_in_pool);
>> session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
>> session_priv_pool_init(&socket_ctx[socket_id], socket_id,
>> sess_sz);
>> }
>> + printf("Number of mbufs in packet pool %d\n", nb_bufs_in_pool);
>>
>> RTE_ETH_FOREACH_DEV(portid) {
>> if ((enabled_port_mask & (1 << portid)) == 0)
>> @@ -2694,8 +2733,6 @@ main(int32_t argc, char **argv)
>> req_tx_offloads[portid]);
>> }
>>
>> - cryptodevs_init();
>> -
>> /*
>> * Set the enabled port mask in helper config for use by helper
>> * sub-system. This will be used while initializing devices using
>> --
>> 2.7.4
>
