Hi Chuanyu,
> -----Original Message-----
> From: Chuanyu Xue <chuanyu....@uconn.edu>
> Sent: Monday, December 18, 2023 4:21 AM
> To: Lu, Wenzhuo <wenzhuo...@intel.com>; Zhang, Qi Z
> <qi.z.zh...@intel.com>; Xing, Beilei <beilei.x...@intel.com>
> Cc: dev@dpdk.org; Chuanyu Xue <chuanyu....@uconn.edu>
> Subject: [PATCH] net/e1000: support launchtime feature
>
> Enable the time-based scheduled Tx of packets based on the
> RTE_ETH_TX_OFFLOAD_SEND_ON_TIMESTAMP flag. The launchtime defines
> the packet transmission time based on PTP clock at MAC layer, which should
> be set to the advanced transmit descriptor.
>
> Signed-off-by: Chuanyu Xue <chuanyu....@uconn.edu>
> ---
> drivers/net/e1000/base/e1000_regs.h | 1 +
> drivers/net/e1000/e1000_ethdev.h | 3 ++
> drivers/net/e1000/igb_ethdev.c | 28 ++++++++++++++++++
> drivers/net/e1000/igb_rxtx.c | 44 ++++++++++++++++++++++++-----
> 4 files changed, 69 insertions(+), 7 deletions(-)
>
> diff --git a/drivers/net/e1000/base/e1000_regs.h
> b/drivers/net/e1000/base/e1000_regs.h
> index d44de59c29..092d9d71e6 100644
> --- a/drivers/net/e1000/base/e1000_regs.h
> +++ b/drivers/net/e1000/base/e1000_regs.h
> @@ -162,6 +162,7 @@
>
> /* QAV Tx mode control register */
> #define E1000_I210_TQAVCTRL 0x3570
> +#define E1000_I210_LAUNCH_OS0 0x3578
What does this register mean?
>
> /* QAV Tx mode control register bitfields masks */
> /* QAV enable */
> diff --git a/drivers/net/e1000/e1000_ethdev.h
> b/drivers/net/e1000/e1000_ethdev.h
> index 718a9746ed..174f7aaf52 100644
> --- a/drivers/net/e1000/e1000_ethdev.h
> +++ b/drivers/net/e1000/e1000_ethdev.h
> @@ -382,6 +382,9 @@ extern struct igb_rss_filter_list igb_filter_rss_list;
> TAILQ_HEAD(igb_flow_mem_list, igb_flow_mem); extern struct
> igb_flow_mem_list igb_flow_list;
>
> +extern uint64_t igb_tx_timestamp_dynflag; extern int
> +igb_tx_timestamp_dynfield_offset;
> +
> extern const struct rte_flow_ops igb_flow_ops;
>
> /*
> diff --git a/drivers/net/e1000/igb_ethdev.c b/drivers/net/e1000/igb_ethdev.c
> index 8858f975f8..4d3d8ae30a 100644
> --- a/drivers/net/e1000/igb_ethdev.c
> +++ b/drivers/net/e1000/igb_ethdev.c
> @@ -223,6 +223,7 @@ static int igb_timesync_read_time(struct rte_eth_dev
> *dev,
> struct timespec *timestamp);
> static int igb_timesync_write_time(struct rte_eth_dev *dev,
> const struct timespec *timestamp);
> +static int eth_igb_read_clock(__rte_unused struct rte_eth_dev *dev,
> +uint64_t *clock);
> static int eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev,
> uint16_t queue_id);
> static int eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, @@ -313,6
> +314,9 @@ static const struct rte_pci_id pci_id_igbvf_map[] = {
> { .vendor_id = 0, /* sentinel */ },
> };
>
> +uint64_t igb_tx_timestamp_dynflag;
> +int igb_tx_timestamp_dynfield_offset = -1;
> +
> static const struct rte_eth_desc_lim rx_desc_lim = {
> .nb_max = E1000_MAX_RING_DESC,
> .nb_min = E1000_MIN_RING_DESC,
> @@ -389,6 +393,7 @@ static const struct eth_dev_ops eth_igb_ops = {
> .timesync_adjust_time = igb_timesync_adjust_time,
> .timesync_read_time = igb_timesync_read_time,
> .timesync_write_time = igb_timesync_write_time,
> + .read_clock = eth_igb_read_clock,
> };
>
> /*
> @@ -1198,6 +1203,7 @@ eth_igb_start(struct rte_eth_dev *dev)
> struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
> struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
> int ret, mask;
> + uint32_t tqavctrl;
> uint32_t intr_vector = 0;
> uint32_t ctrl_ext;
> uint32_t *speeds;
> @@ -1281,6 +1287,15 @@ eth_igb_start(struct rte_eth_dev *dev)
> return ret;
> }
>
> + if (igb_tx_timestamp_dynflag > 0) {
> + tqavctrl = E1000_READ_REG(hw, E1000_I210_TQAVCTRL);
> + tqavctrl |= E1000_TQAVCTRL_MODE;
> + tqavctrl |= E1000_TQAVCTRL_FETCH_ARB; /* Fetch the queue most
> empty, no Round Robin*/
> + tqavctrl |= E1000_TQAVCTRL_LAUNCH_TIMER_ENABLE; /* Enable
> launch time */
In kernel driver, "E1000_TQAVCTRL_DATATRANTIM (BIT(9))" and
"E1000_TQAVCTRL_FETCHTIME_DELTA (0xFFFF << 16)" are set, does it have some
other intention here?
> + E1000_WRITE_REG(hw, E1000_I210_TQAVCTRL, tqavctrl);
> + E1000_WRITE_REG(hw, E1000_I210_LAUNCH_OS0, 1ULL << 31); /*
> Set launch offset to default */
> + }
> +
> e1000_clear_hw_cntrs_base_generic(hw);
>
> /*
> @@ -4882,6 +4897,19 @@ igb_timesync_read_tx_timestamp(struct
> rte_eth_dev *dev,
> return 0;
> }
>
> +static int
> +eth_igb_read_clock(__rte_unused struct rte_eth_dev *dev, uint64_t
> +*clock) {
> + uint64_t systime_cycles;
> + struct e1000_adapter *adapter = dev->data->dev_private;
> +
> + systime_cycles = igb_read_systime_cyclecounter(dev);
> + uint64_t ns = rte_timecounter_update(&adapter->systime_tc,
> systime_cycles);
Do you also run "ptp timesync" when testing this launchtime feature?
> + *clock = ns;
> +
> + return 0;
> +}
> +
> static int
> eth_igb_get_reg_length(struct rte_eth_dev *dev __rte_unused) { diff --git
> a/drivers/net/e1000/igb_rxtx.c b/drivers/net/e1000/igb_rxtx.c index
> 448c4b7d9d..e5da8e250d 100644
> --- a/drivers/net/e1000/igb_rxtx.c
> +++ b/drivers/net/e1000/igb_rxtx.c
> @@ -212,6 +212,9 @@ struct igb_tx_queue {
> #define IGB_TSO_MAX_HDRLEN (512)
> #define IGB_TSO_MAX_MSS (9216)
>
> +/* Macro to compensate latency in launch time offloading*/
> +#define E1000_I210_LT_LATENCY 0x41F9
What does this value depend on?
> +
>
> /******************************************************************
> ***
> *
> * TX function
> @@ -244,12 +247,13 @@ check_tso_para(uint64_t ol_req, union
> igb_tx_offload ol_para) static inline void igbe_set_xmit_ctx(struct
> igb_tx_queue* txq,
> volatile struct e1000_adv_tx_context_desc *ctx_txd,
> - uint64_t ol_flags, union igb_tx_offload tx_offload)
> + uint64_t ol_flags, union igb_tx_offload tx_offload, uint64_t
> txtime)
> {
> uint32_t type_tucmd_mlhl;
> uint32_t mss_l4len_idx;
> uint32_t ctx_idx, ctx_curr;
> uint32_t vlan_macip_lens;
> + uint32_t launch_time;
> union igb_tx_offload tx_offload_mask;
>
> ctx_curr = txq->ctx_curr;
> @@ -312,16 +316,25 @@ igbe_set_xmit_ctx(struct igb_tx_queue* txq,
> }
> }
>
> - txq->ctx_cache[ctx_curr].flags = ol_flags;
> - txq->ctx_cache[ctx_curr].tx_offload.data =
> - tx_offload_mask.data & tx_offload.data;
> - txq->ctx_cache[ctx_curr].tx_offload_mask = tx_offload_mask;
> + if (!txtime) {
> + txq->ctx_cache[ctx_curr].flags = ol_flags;
> + txq->ctx_cache[ctx_curr].tx_offload.data =
> + tx_offload_mask.data & tx_offload.data;
> + txq->ctx_cache[ctx_curr].tx_offload_mask = tx_offload_mask;
> + }
>
> ctx_txd->type_tucmd_mlhl = rte_cpu_to_le_32(type_tucmd_mlhl);
> vlan_macip_lens = (uint32_t)tx_offload.data;
> ctx_txd->vlan_macip_lens = rte_cpu_to_le_32(vlan_macip_lens);
> ctx_txd->mss_l4len_idx = rte_cpu_to_le_32(mss_l4len_idx);
> ctx_txd->u.seqnum_seed = 0;
> +
> + if (txtime) {
> + launch_time = (txtime - E1000_I210_LT_LATENCY) % NSEC_PER_SEC;
> + ctx_txd->u.launch_time = rte_cpu_to_le_32(launch_time / 32);
> + } else {
> + ctx_txd->u.launch_time = 0;
> + }
> }
>
> /*
> @@ -400,6 +413,7 @@ eth_igb_xmit_pkts(void *tx_queue, struct rte_mbuf
> **tx_pkts,
> uint32_t new_ctx = 0;
> uint32_t ctx = 0;
> union igb_tx_offload tx_offload = {0};
> + uint64_t ts;
>
> txq = tx_queue;
> sw_ring = txq->sw_ring;
> @@ -552,7 +566,12 @@ eth_igb_xmit_pkts(void *tx_queue, struct rte_mbuf
> **tx_pkts,
> txe->mbuf = NULL;
> }
>
> - igbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req,
> tx_offload);
> + if (igb_tx_timestamp_dynflag > 0) {
> + ts = *RTE_MBUF_DYNFIELD(tx_pkt,
> igb_tx_timestamp_dynfield_offset, uint64_t *);
> + igbe_set_xmit_ctx(txq, ctx_txd,
> tx_ol_req, tx_offload,
> ts);
> + } else {
> + igbe_set_xmit_ctx(txq, ctx_txd,
> tx_ol_req, tx_offload,
> 0);
> + }
>
> txe->last_id = tx_last;
> tx_id = txe->next_id;
> @@ -1464,7 +1483,8 @@ igb_get_tx_port_offloads_capa(struct rte_eth_dev
> *dev)
> RTE_ETH_TX_OFFLOAD_TCP_CKSUM |
> RTE_ETH_TX_OFFLOAD_SCTP_CKSUM |
> RTE_ETH_TX_OFFLOAD_TCP_TSO |
> - RTE_ETH_TX_OFFLOAD_MULTI_SEGS;
> + RTE_ETH_TX_OFFLOAD_MULTI_SEGS |
> + RTE_ETH_TX_OFFLOAD_SEND_ON_TIMESTAMP;
>
> return tx_offload_capa;
> }
> @@ -2579,9 +2599,11 @@ eth_igb_tx_init(struct rte_eth_dev *dev) {
> struct e1000_hw *hw;
> struct igb_tx_queue *txq;
> + uint64_t offloads = dev->data->dev_conf.txmode.offloads;
> uint32_t tctl;
> uint32_t txdctl;
> uint16_t i;
> + int err;
>
> hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
>
> @@ -2612,6 +2634,14 @@ eth_igb_tx_init(struct rte_eth_dev *dev)
> dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
> }
>
> + if (offloads & RTE_ETH_TX_OFFLOAD_SEND_ON_TIMESTAMP) {
> + err = rte_mbuf_dyn_tx_timestamp_register(
> + &igb_tx_timestamp_dynfield_offset,
> + &igb_tx_timestamp_dynflag);
> + if (err)
> + PMD_DRV_LOG(ERR, "Failed to register tx timestamp
> dynamic
> field");
> + }
> +
> /* Program the Transmit Control Register. */
> tctl = E1000_READ_REG(hw, E1000_TCTL);
> tctl &= ~E1000_TCTL_CT;
> --
> 2.25.1
