Packet-split descriptors are used by Linux VF driver for MTU values from 2048 upwards.
Signed-off-by: Tomasz Dzieciol <t.dziec...@partner.samsung.com> --- hw/net/e1000x_regs.h | 1 + hw/net/igb_core.c | 701 ++++++++++++++++++++++++++++++----------- hw/net/igb_regs.h | 18 ++ hw/net/trace-events | 4 +- tests/qtest/igb-test.c | 4 + 5 files changed, 542 insertions(+), 186 deletions(-) diff --git a/hw/net/e1000x_regs.h b/hw/net/e1000x_regs.h index 6d3c4c6d3a..82b14e12e0 100644 --- a/hw/net/e1000x_regs.h +++ b/hw/net/e1000x_regs.h @@ -827,6 +827,7 @@ union e1000_rx_desc_packet_split { /* Receive Checksum Control bits */ #define E1000_RXCSUM_IPOFLD 0x100 /* IP Checksum Offload Enable */ #define E1000_RXCSUM_TUOFLD 0x200 /* TCP/UDP Checksum Offload Enable */ +#define E1000_RXCSUM_IPPCSE 0x1000 /* IP Payload Checksum enable */ #define E1000_RXCSUM_PCSD 0x2000 /* Packet Checksum Disable */ #define E1000_RING_DESC_LEN (16) diff --git a/hw/net/igb_core.c b/hw/net/igb_core.c index d733fed6cf..4e14175486 100644 --- a/hw/net/igb_core.c +++ b/hw/net/igb_core.c @@ -239,6 +239,20 @@ igb_rx_use_legacy_descriptor(IGBCore *core) return false; } +static uint32_t +igb_rx_queue_desctyp_get(IGBCore *core, int qidx) +{ + return (core->mac[E1000_SRRCTL(qidx) >> 2] & E1000_SRRCTL_DESCTYPE_MASK) >> + E1000_SRRCTL_DESCTYPE_OFFSET; +} + +static inline bool +igb_rx_use_ps_descriptor(IGBCore *core, int qidx) +{ + uint32_t desctyp = igb_rx_queue_desctyp_get(core, qidx); + return desctyp == 2 || desctyp == 5; +} + static inline bool igb_rss_enabled(IGBCore *core) { @@ -876,6 +890,21 @@ igb_rxbufsize(IGBCore *core, const E1000E_RingInfo *r) return e1000x_rxbufsize(core->mac[RCTL]); } +static size_t +igb_calc_rxdesclen(IGBCore *core) +{ + size_t rxdesc_size; + + if (igb_rx_use_legacy_descriptor(core)) { + rxdesc_size = sizeof(struct e1000_rx_desc); + } else { + rxdesc_size = sizeof(union e1000_adv_rx_desc); + } + + trace_e1000e_rx_desc_len(rxdesc_size); + return rxdesc_size; +} + static bool igb_has_rxbufs(IGBCore *core, const E1000E_RingInfo *r, size_t total_size) { @@ -883,9 +912,8 @@ igb_has_rxbufs(IGBCore *core, const E1000E_RingInfo *r, size_t total_size) uint32_t bufsize = igb_rxbufsize(core, r); trace_e1000e_rx_has_buffers(r->idx, bufs, total_size, bufsize); - - return total_size <= bufs / (core->rx_desc_len / E1000_MIN_RX_DESC_LEN) * - bufsize; + return total_size <= bufs / (igb_calc_rxdesclen(core) / + E1000_MIN_RX_DESC_LEN) * bufsize; } void @@ -1158,21 +1186,43 @@ igb_read_lgcy_rx_descr(IGBCore *core, struct e1000_rx_desc *desc, } static inline void -igb_read_adv_rx_descr(IGBCore *core, union e1000_adv_rx_desc *desc, - hwaddr *buff_addr) +igb_read_adv_rx_single_buf_descr(IGBCore *core, union e1000_adv_rx_desc *desc, + hwaddr *buff_addr) { *buff_addr = le64_to_cpu(desc->read.pkt_addr); } +static inline void +igb_read_adv_rx_split_buf_descr(IGBCore *core, union e1000_adv_rx_desc *desc, + hwaddr *buff_addr) +{ + buff_addr[0] = le64_to_cpu(desc->read.hdr_addr); + buff_addr[1] = le64_to_cpu(desc->read.pkt_addr); +} + static inline void igb_read_rx_descr(IGBCore *core, union e1000_rx_desc_union *desc, - hwaddr *buff_addr) + hwaddr (*buff_addr)[MAX_PS_BUFFERS], + int qidx) { if (igb_rx_use_legacy_descriptor(core)) { - igb_read_lgcy_rx_descr(core, &desc->legacy, buff_addr); - } else { - igb_read_adv_rx_descr(core, &desc->adv, buff_addr); + igb_read_lgcy_rx_descr(core, &desc->legacy, &(*buff_addr)[1]); + (*buff_addr)[0] = 0; + return; } + + /* advanced header split descriptor */ + if (igb_rx_use_ps_descriptor(core, qidx)) { + igb_read_adv_rx_split_buf_descr(core, &desc->adv, &(*buff_addr)[0]); + return; + } + + /* descriptor replication modes not supported */ + assert(igb_rx_queue_desctyp_get(core, qidx) == 1); + + /* advanced single buffer descriptor */ + igb_read_adv_rx_single_buf_descr(core, &desc->adv, &(*buff_addr)[1]); + (*buff_addr)[0] = 0; } static void @@ -1214,25 +1264,19 @@ igb_verify_csum_in_sw(IGBCore *core, } static void -igb_build_rx_metadata(IGBCore *core, - struct NetRxPkt *pkt, - bool is_eop, - const E1000E_RSSInfo *rss_info, - uint16_t *pkt_info, uint16_t *hdr_info, - uint32_t *rss, - uint32_t *status_flags, - uint16_t *ip_id, - uint16_t *vlan_tag) +igb_build_rx_metadata_common(IGBCore *core, + struct NetRxPkt *pkt, + bool is_eop, + uint32_t *status_flags, + uint16_t *vlan_tag) { struct virtio_net_hdr *vhdr; bool hasip4, hasip6; EthL4HdrProto l4hdr_proto; - uint32_t pkt_type; *status_flags = E1000_RXD_STAT_DD; /* No additional metadata needed for non-EOP descriptors */ - /* TODO: EOP apply only to status so don't skip whole function. */ if (!is_eop) { goto func_exit; } @@ -1249,51 +1293,6 @@ igb_build_rx_metadata(IGBCore *core, trace_e1000e_rx_metadata_vlan(*vlan_tag); } - /* Packet parsing results */ - if ((core->mac[RXCSUM] & E1000_RXCSUM_PCSD) != 0) { - if (rss_info->enabled) { - *rss = cpu_to_le32(rss_info->hash); - trace_igb_rx_metadata_rss(*rss); - } - } else if (hasip4) { - *status_flags |= E1000_RXD_STAT_IPIDV; - *ip_id = cpu_to_le16(net_rx_pkt_get_ip_id(pkt)); - trace_e1000e_rx_metadata_ip_id(*ip_id); - } - - if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP && net_rx_pkt_is_tcp_ack(pkt)) { - *status_flags |= E1000_RXD_STAT_ACK; - trace_e1000e_rx_metadata_ack(); - } - - if (hasip6 && (core->mac[RFCTL] & E1000_RFCTL_IPV6_DIS)) { - trace_e1000e_rx_metadata_ipv6_filtering_disabled(); - pkt_type = E1000_RXD_PKT_MAC; - } else if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP || - l4hdr_proto == ETH_L4_HDR_PROTO_UDP) { - pkt_type = hasip4 ? E1000_RXD_PKT_IP4_XDP : E1000_RXD_PKT_IP6_XDP; - } else if (hasip4 || hasip6) { - pkt_type = hasip4 ? E1000_RXD_PKT_IP4 : E1000_RXD_PKT_IP6; - } else { - pkt_type = E1000_RXD_PKT_MAC; - } - - trace_e1000e_rx_metadata_pkt_type(pkt_type); - - if (pkt_info) { - if (rss_info->enabled) { - *pkt_info = rss_info->type; - } - - *pkt_info |= (pkt_type << 4); - } else { - *status_flags |= E1000_RXD_PKT_TYPE(pkt_type); - } - - if (hdr_info) { - *hdr_info = 0; - } - /* RX CSO information */ if (hasip6 && (core->mac[RFCTL] & E1000_RFCTL_IPV6_XSUM_DIS)) { trace_e1000e_rx_metadata_ipv6_sum_disabled(); @@ -1332,62 +1331,171 @@ igb_build_rx_metadata(IGBCore *core, trace_e1000e_rx_metadata_l4_cso_disabled(); } - trace_e1000e_rx_metadata_status_flags(*status_flags); - func_exit: + trace_e1000e_rx_metadata_status_flags(*status_flags); *status_flags = cpu_to_le32(*status_flags); } static inline void -igb_write_lgcy_rx_descr(IGBCore *core, struct e1000_rx_desc *desc, +igb_write_lgcy_rx_descr(IGBCore *core, + union e1000_rx_desc_union *desc, struct NetRxPkt *pkt, const E1000E_RSSInfo *rss_info, uint16_t length) { - uint32_t status_flags, rss; - uint16_t ip_id; + uint32_t status_flags; + struct e1000_rx_desc *d = &desc->legacy; assert(!rss_info->enabled); - desc->length = cpu_to_le16(length); - desc->csum = 0; - igb_build_rx_metadata(core, pkt, pkt != NULL, - rss_info, - NULL, NULL, &rss, - &status_flags, &ip_id, - &desc->special); - desc->errors = (uint8_t) (le32_to_cpu(status_flags) >> 24); - desc->status = (uint8_t) le32_to_cpu(status_flags); + memset(d, 0, sizeof(*d)); + d->length = cpu_to_le16(length); + igb_build_rx_metadata_common(core, pkt, pkt != NULL, + &status_flags, + &d->special); + + d->errors = (uint8_t) (le32_to_cpu(status_flags) >> 24); + d->status = (uint8_t) le32_to_cpu(status_flags); +} + +typedef struct { + bool sph; + bool hbo; + size_t descriptor_hdr_len; +} igb_split_descriptor_data; + +static inline bool +igb_rx_ps_descriptor_split_always(IGBCore *core, int qidx) +{ + uint32_t desctyp = igb_rx_queue_desctyp_get(core, qidx); + return desctyp == 5; +} + +static uint16_t +igb_rx_desc_get_packet_type(IGBCore *core, struct NetRxPkt *pkt) +{ + uint16_t pkt_type = 0; + bool hasip4, hasip6, issctp = false; + EthL4HdrProto l4hdr_proto; + + net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto); + + if (hasip6 && !(core->mac[RFCTL] & E1000_RFCTL_IPV6_DIS)) { + eth_ip6_hdr_info *ip6hdr_info = net_rx_pkt_get_ip6_info(pkt); + pkt_type |= ip6hdr_info->has_ext_hdrs ? IGB_RX_DESC_PKT_TYPE_IPV6E : + IGB_RX_DESC_PKT_TYPE_IPV6; + issctp = ip6hdr_info->l4proto == IPPROTO_SCTP; + } else if (hasip4) { + pkt_type = IGB_RX_DESC_PKT_TYPE_IPV4; + eth_ip4_hdr_info *ip4hdr_info = net_rx_pkt_get_ip4_info(pkt); + issctp = ip4hdr_info->ip4_hdr.ip_p == IPPROTO_SCTP; + } + + if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP) { + pkt_type |= IGB_RX_DESC_PKT_TYPE_TCP; + } else if (l4hdr_proto == ETH_L4_HDR_PROTO_UDP) { + pkt_type |= IGB_RX_DESC_PKT_TYPE_UDP; + } else if (issctp) { + pkt_type |= IGB_RX_DESC_PKT_TYPE_SCTP; + } + + return pkt_type; } static inline void -igb_write_adv_rx_descr(IGBCore *core, union e1000_adv_rx_desc *desc, +igb_write_adv_rx_descr(IGBCore *core, + union e1000_rx_desc_union *desc, struct NetRxPkt *pkt, const E1000E_RSSInfo *rss_info, uint16_t length) { - memset(&desc->wb, 0, sizeof(desc->wb)); + bool hasip4, hasip6; + EthL4HdrProto l4hdr_proto; + uint16_t rss_type = 0, pkt_type; + bool eop = (pkt != NULL); + union e1000_adv_rx_desc *d = &desc->adv; + memset(&d->wb, 0, sizeof(d->wb)); - desc->wb.upper.length = cpu_to_le16(length); + d->wb.upper.length = cpu_to_le16(length); + igb_build_rx_metadata_common(core, pkt, eop, + &d->wb.upper.status_error, + &d->wb.upper.vlan); - igb_build_rx_metadata(core, pkt, pkt != NULL, - rss_info, - &desc->wb.lower.lo_dword.pkt_info, - &desc->wb.lower.lo_dword.hdr_info, - &desc->wb.lower.hi_dword.rss, - &desc->wb.upper.status_error, - &desc->wb.lower.hi_dword.csum_ip.ip_id, - &desc->wb.upper.vlan); + if (!eop) { + return; + } + + net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto); + + if ((core->mac[RXCSUM] & E1000_RXCSUM_PCSD) && rss_info->enabled) { + d->wb.lower.hi_dword.rss = cpu_to_le32(rss_info->hash); + rss_type = rss_info->type; + trace_igb_rx_metadata_rss(d->wb.lower.hi_dword.rss, rss_type); + } else if (!(core->mac[RXCSUM] & E1000_RXCSUM_PCSD) && + core->mac[RXCSUM] & E1000_RXCSUM_IPPCSE && + hasip4) { + d->wb.lower.hi_dword.csum_ip.ip_id = cpu_to_le16(net_rx_pkt_get_ip_id(pkt)); + trace_e1000e_rx_metadata_ip_id(d->wb.lower.hi_dword.csum_ip.ip_id); + } + + pkt_type = igb_rx_desc_get_packet_type(core, pkt); + trace_e1000e_rx_metadata_pkt_type(pkt_type); + d->wb.lower.lo_dword.pkt_info = cpu_to_le16(rss_type | (pkt_type << 4)); } static inline void -igb_write_rx_descr(IGBCore *core, union e1000_rx_desc_union *desc, -struct NetRxPkt *pkt, const E1000E_RSSInfo *rss_info, uint16_t length) +igb_write_adv_ps_split_rx_descr(IGBCore *core, + union e1000_rx_desc_union *desc, + struct NetRxPkt *pkt, + const E1000E_RSSInfo *rss_info, + igb_split_descriptor_data *ps_desc_data, + uint16_t(*written)[MAX_PS_BUFFERS]) +{ + size_t pkt_len; + bool split_always; + + size_t hdr_len = ps_desc_data->descriptor_hdr_len; + union e1000_adv_rx_desc *d = &desc->adv; + + split_always = igb_rx_ps_descriptor_split_always(core, rss_info->queue); + if (!split_always) { + if ((!ps_desc_data->sph && !ps_desc_data->hbo) || + ( ps_desc_data->sph && ps_desc_data->hbo)) { + pkt_len = (*written)[0] + (*written)[1]; + } else { + assert(!ps_desc_data->hbo); + pkt_len = (*written)[1]; + } + } else { + pkt_len = (*written)[1]; + } + + igb_write_adv_rx_descr(core, desc, pkt, rss_info, pkt_len); + + d->wb.lower.lo_dword.hdr_info = (hdr_len << RX_DESC_ADV_HDR_LEN_OFFSET) & + RX_DESC_ADV_HDR_LEN_MASK; + d->wb.lower.lo_dword.hdr_info |= ps_desc_data->sph ? RX_DESC_ADV_HDR_SPH + : 0; + d->wb.upper.status_error |= ps_desc_data->hbo ? + RX_DESC_ADV_ST_ERR_HBO_OFFSET : 0; +} + +static inline void +igb_write_rx_descr(IGBCore *core, + union e1000_rx_desc_union *desc, + struct NetRxPkt *pkt, + const E1000E_RSSInfo *rss_info, + igb_split_descriptor_data *ps_desc_data, + uint16_t(*written)[MAX_PS_BUFFERS], + int qidx) { if (igb_rx_use_legacy_descriptor(core)) { - igb_write_lgcy_rx_descr(core, &desc->legacy, pkt, rss_info, length); + igb_write_lgcy_rx_descr(core, desc, pkt, rss_info, (*written)[1]); + } else if (igb_rx_use_ps_descriptor(core, qidx)) { + igb_write_adv_ps_split_rx_descr(core, desc, pkt, rss_info, + ps_desc_data, written); } else { - igb_write_adv_rx_descr(core, &desc->adv, pkt, rss_info, length); + igb_write_adv_rx_descr(core, desc, pkt, rss_info, (*written)[1]); } } @@ -1423,19 +1531,6 @@ igb_pci_dma_write_rx_desc(IGBCore *core, PCIDevice *dev, dma_addr_t addr, } } -static void -igb_write_to_rx_buffers(IGBCore *core, - PCIDevice *d, - hwaddr ba, - uint16_t *written, - const char *data, - dma_addr_t data_len) -{ - trace_igb_rx_desc_buff_write(ba, *written, data, data_len); - pci_dma_write(d, ba + *written, data, data_len); - *written += data_len; -} - static void igb_update_rx_stats(IGBCore *core, const E1000E_RingInfo *rxi, size_t data_size, size_t data_fcs_size) @@ -1473,6 +1568,283 @@ igb_rx_descr_threshold_hit(IGBCore *core, const E1000E_RingInfo *rxi) ((core->mac[E1000_SRRCTL(rxi->idx) >> 2] >> 20) & 31) * 16; } +static uint32_t +igb_get_queue_rx_desc_header_buf_size(IGBCore *core, int qidx) +{ + uint32_t srr_size = (core->mac[E1000_SRRCTL(qidx) >> 2] & + E1000_SRRCTL_BSIZEHDR_MASK) >> 8; + return srr_size * 64; +} + +static bool +igb_do_ps(IGBCore *core, + int qidx, + struct NetRxPkt *pkt, + size_t *hdr_len, + igb_split_descriptor_data *ps_desc_data) +{ + bool hasip4, hasip6; + EthL4HdrProto l4hdr_proto; + bool fragment; + bool split_always; + size_t bheader_size; + size_t total_pkt_len; + + if (!igb_rx_use_ps_descriptor(core, qidx)) { + return false; + } + + memset(ps_desc_data, 0, sizeof(igb_split_descriptor_data)); + + total_pkt_len = net_rx_pkt_get_total_len(pkt); + bheader_size = igb_get_queue_rx_desc_header_buf_size(core, qidx); + split_always = igb_rx_ps_descriptor_split_always(core, qidx); + if (split_always && total_pkt_len <= bheader_size) { + *hdr_len = total_pkt_len; + ps_desc_data->descriptor_hdr_len = total_pkt_len; + return true; + } + + net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto); + + if (hasip4) { + fragment = net_rx_pkt_get_ip4_info(pkt)->fragment; + } else if (hasip6) { + fragment = net_rx_pkt_get_ip6_info(pkt)->fragment; + } else { + ps_desc_data->descriptor_hdr_len = bheader_size; + goto header_cant_be_decoded_or_bigger_than_header_buffer; + } + + if (fragment && (core->mac[RFCTL] & E1000_RFCTL_IPFRSP_DIS)) { + ps_desc_data->descriptor_hdr_len = bheader_size; + goto header_cant_be_decoded_or_bigger_than_header_buffer; + } + + /* no header splitting for SCTP */ + if (!fragment && (l4hdr_proto == ETH_L4_HDR_PROTO_UDP || + l4hdr_proto == ETH_L4_HDR_PROTO_TCP)) { + *hdr_len = net_rx_pkt_get_l5_hdr_offset(pkt); + } else { + *hdr_len = net_rx_pkt_get_l4_hdr_offset(pkt); + } + + ps_desc_data->sph = true; + ps_desc_data->descriptor_hdr_len = *hdr_len; + + if (*hdr_len > bheader_size) { + ps_desc_data->hbo = true; + goto header_cant_be_decoded_or_bigger_than_header_buffer; + } + + return true; + +header_cant_be_decoded_or_bigger_than_header_buffer: + if (split_always) { + *hdr_len = bheader_size; + return true; + } + + return false; +} + +typedef struct igb_ba_state_st { + uint16_t written[MAX_PS_BUFFERS]; + uint8_t cur_idx; +} igb_ba_state; + +typedef struct { + size_t size; + size_t total_size; + size_t ps_hdr_len; + size_t desc_size; + size_t desc_offset; + uint32_t rx_desc_packet_buf_size; + uint32_t rx_desc_header_buf_size; + size_t iov_ofs; + bool do_ps; + bool is_first; + struct iovec *iov; + igb_ba_state bastate; + hwaddr ba[MAX_PS_BUFFERS]; +} igb_packet_dma_state; + +static inline void +igb_truncate_to_descriptor_size(igb_packet_dma_state *pdma_st, size_t *size) +{ + if (pdma_st->do_ps && pdma_st->is_first) { + if (*size > pdma_st->rx_desc_packet_buf_size + pdma_st->ps_hdr_len) { + *size = pdma_st->rx_desc_packet_buf_size + pdma_st->ps_hdr_len; + } + } else { + if (*size > pdma_st->rx_desc_packet_buf_size) { + *size = pdma_st->rx_desc_packet_buf_size; + } + } +} + +static inline void +igb_write_hdr_to_rx_buffers(IGBCore *core, + PCIDevice *d, + hwaddr (*ba)[MAX_PS_BUFFERS], + igb_ba_state *bastate, + uint32_t rx_desc_header_buf_size, + const char *data, + dma_addr_t data_len) +{ + assert(data_len <= rx_desc_header_buf_size - bastate->written[0]); + pci_dma_write(d, (*ba)[0] + bastate->written[0], data, data_len); + bastate->written[0] += data_len; + bastate->cur_idx = 1; +} + +static void +igb_write_packet_hdr_to_descr_addr(IGBCore *core, + struct NetRxPkt *pkt, + PCIDevice *d, + igb_packet_dma_state *pdma_st, + size_t *copy_size) +{ + size_t iov_copy; + size_t ps_hdr_copied = 0; + + if (!pdma_st->is_first) { + /* Leave buffer 0 of each descriptor except first */ + /* empty */ + igb_write_hdr_to_rx_buffers(core, d, &pdma_st->ba, &pdma_st->bastate, + pdma_st->rx_desc_header_buf_size, + NULL, 0); + return; + } + + do { + iov_copy = MIN(pdma_st->ps_hdr_len - ps_hdr_copied, + pdma_st->iov->iov_len - pdma_st->iov_ofs); + + igb_write_hdr_to_rx_buffers(core, d, &pdma_st->ba, + &pdma_st->bastate, + pdma_st->rx_desc_header_buf_size, + pdma_st->iov->iov_base, + iov_copy); + + *copy_size -= iov_copy; + ps_hdr_copied += iov_copy; + + pdma_st->iov_ofs += iov_copy; + if (pdma_st->iov_ofs == pdma_st->iov->iov_len) { + pdma_st->iov++; + pdma_st->iov_ofs = 0; + } + } while (ps_hdr_copied < pdma_st->ps_hdr_len); + + pdma_st->is_first = false; +} + +static void +igb_write_to_rx_buffers(IGBCore *core, + PCIDevice *d, + hwaddr (*ba)[MAX_PS_BUFFERS], + igb_ba_state *bastate, + uint32_t cur_buf_len, + const char *data, + dma_addr_t data_len) +{ + while (data_len > 0) { + assert(bastate->cur_idx < MAX_PS_BUFFERS); + + uint32_t cur_buf_bytes_left = cur_buf_len - + bastate->written[bastate->cur_idx]; + uint32_t bytes_to_write = MIN(data_len, cur_buf_bytes_left); + + trace_igb_rx_desc_buff_write(bastate->cur_idx, + (*ba)[bastate->cur_idx], + bastate->written[bastate->cur_idx], + data, + bytes_to_write); + + pci_dma_write(d, + (*ba)[bastate->cur_idx] + + bastate->written[bastate->cur_idx], + data, bytes_to_write); + + bastate->written[bastate->cur_idx] += bytes_to_write; + data += bytes_to_write; + data_len -= bytes_to_write; + + if (bastate->written[bastate->cur_idx] == cur_buf_len) { + bastate->cur_idx++; + } + } +} + +static void +igb_write_packet_payload_to_descr_addr(IGBCore *core, + struct NetRxPkt *pkt, + PCIDevice *d, + igb_packet_dma_state *pdma_st, + size_t *copy_size) +{ + static const uint32_t fcs_pad; + size_t iov_copy; + + /* Copy packet payload */ + while (*copy_size) { + iov_copy = MIN(*copy_size, pdma_st->iov->iov_len - pdma_st->iov_ofs); + igb_write_to_rx_buffers(core, d, &pdma_st->ba, &pdma_st->bastate, + pdma_st->rx_desc_packet_buf_size, + pdma_st->iov->iov_base + pdma_st->iov_ofs, + iov_copy); + + *copy_size -= iov_copy; + pdma_st->iov_ofs += iov_copy; + if (pdma_st->iov_ofs == pdma_st->iov->iov_len) { + pdma_st->iov++; + pdma_st->iov_ofs = 0; + } + } + + if (pdma_st->desc_offset + pdma_st->desc_size >= pdma_st->total_size) { + /* Simulate FCS checksum presence in the last descriptor */ + igb_write_to_rx_buffers(core, d, &pdma_st->ba, &pdma_st->bastate, + pdma_st->rx_desc_packet_buf_size, + (const char *) &fcs_pad, + e1000x_fcs_len(core->mac)); + } +} + +static void +igb_write_packet_to_descr_addr(IGBCore *core, + struct NetRxPkt *pkt, + PCIDevice *d, + igb_packet_dma_state *pdma_st) +{ + size_t copy_size; + + if (!(pdma_st->ba)[1]) { + /* as per intel docs; skip descriptors with null buf addr */ + trace_e1000e_rx_null_descriptor(); + return; + } + + if (pdma_st->desc_offset >= pdma_st->size) { + return; + } + + pdma_st->desc_size = pdma_st->total_size - pdma_st->desc_offset; + igb_truncate_to_descriptor_size(pdma_st, &pdma_st->desc_size); + copy_size = pdma_st->size - pdma_st->desc_offset; + igb_truncate_to_descriptor_size(pdma_st, ©_size); + + /* For PS mode copy the packet header first */ + if (pdma_st->do_ps) { + igb_write_packet_hdr_to_descr_addr(core, pkt, d, pdma_st, ©_size); + } else { + pdma_st->bastate.cur_idx = 1; + } + + igb_write_packet_payload_to_descr_addr(core, pkt, d, pdma_st, ©_size); +} + static void igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt, const E1000E_RxRing *rxr, @@ -1481,91 +1853,63 @@ igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt, PCIDevice *d; dma_addr_t base; union e1000_rx_desc_union desc; - size_t desc_size; - size_t desc_offset = 0; - size_t iov_ofs = 0; - - struct iovec *iov = net_rx_pkt_get_iovec(pkt); - size_t size = net_rx_pkt_get_total_len(pkt); - size_t total_size = size + e1000x_fcs_len(core->mac); - const E1000E_RingInfo *rxi = rxr->i; - size_t bufsize = igb_rxbufsize(core, rxi); - + const E1000E_RingInfo *rxi; + size_t rx_desc_len; + igb_split_descriptor_data ps_desc_data; + + igb_packet_dma_state pdma_st = {0}; + pdma_st.is_first = true; + pdma_st.size = net_rx_pkt_get_total_len(pkt); + pdma_st.total_size = pdma_st.size + e1000x_fcs_len(core->mac); + + rxi = rxr->i; + rx_desc_len = igb_calc_rxdesclen(core); + pdma_st.rx_desc_packet_buf_size = + igb_rxbufsize(core, rxi); + pdma_st.rx_desc_header_buf_size = + igb_get_queue_rx_desc_header_buf_size(core, rxi->idx); + pdma_st.iov = net_rx_pkt_get_iovec(pkt); d = pcie_sriov_get_vf_at_index(core->owner, rxi->idx % 8); if (!d) { d = core->owner; } + pdma_st.do_ps = igb_do_ps(core, rxi->idx, pkt, + &pdma_st.ps_hdr_len, + &ps_desc_data); + do { - hwaddr ba; - uint16_t written = 0; + memset(&pdma_st.bastate, 0, sizeof(igb_ba_state)); bool is_last = false; - desc_size = total_size - desc_offset; - - if (desc_size > bufsize) { - desc_size = bufsize; - } - if (igb_ring_empty(core, rxi)) { return; } base = igb_ring_head_descr(core, rxi); + pci_dma_read(d, base, &desc, rx_desc_len); + trace_e1000e_rx_descr(rxi->idx, base, rx_desc_len); - pci_dma_read(d, base, &desc, core->rx_desc_len); - - trace_e1000e_rx_descr(rxi->idx, base, core->rx_desc_len); - - igb_read_rx_descr(core, &desc, &ba); - - if (ba) { - if (desc_offset < size) { - static const uint32_t fcs_pad; - size_t iov_copy; - size_t copy_size = size - desc_offset; - if (copy_size > bufsize) { - copy_size = bufsize; - } - - /* Copy packet payload */ - while (copy_size) { - iov_copy = MIN(copy_size, iov->iov_len - iov_ofs); - - igb_write_to_rx_buffers(core, d, ba, &written, - iov->iov_base + iov_ofs, iov_copy); - - copy_size -= iov_copy; - iov_ofs += iov_copy; - if (iov_ofs == iov->iov_len) { - iov++; - iov_ofs = 0; - } - } + igb_read_rx_descr(core, &desc, &pdma_st.ba, rxi->idx); - if (desc_offset + desc_size >= total_size) { - /* Simulate FCS checksum presence in the last descriptor */ - igb_write_to_rx_buffers(core, d, ba, &written, - (const char *) &fcs_pad, e1000x_fcs_len(core->mac)); - } - } - } else { /* as per intel docs; skip descriptors with null buf addr */ - trace_e1000e_rx_null_descriptor(); - } - desc_offset += desc_size; - if (desc_offset >= total_size) { + igb_write_packet_to_descr_addr(core, pkt, d, &pdma_st); + pdma_st.desc_offset += pdma_st.desc_size; + if (pdma_st.desc_offset >= pdma_st.total_size) { is_last = true; } - igb_write_rx_descr(core, &desc, is_last ? core->rx_pkt : NULL, - rss_info, written); - igb_pci_dma_write_rx_desc(core, d, base, &desc, core->rx_desc_len); + igb_write_rx_descr(core, &desc, + is_last ? pkt : NULL, + rss_info, + &ps_desc_data, + &pdma_st.bastate.written, + rxi->idx); + pci_dma_write(d, base, &desc, rx_desc_len); + igb_ring_advance(core, rxi, + rx_desc_len / E1000_MIN_RX_DESC_LEN); + } while (pdma_st.desc_offset < pdma_st.total_size); - igb_ring_advance(core, rxi, core->rx_desc_len / E1000_MIN_RX_DESC_LEN); - - } while (desc_offset < total_size); - - igb_update_rx_stats(core, rxi, size, total_size); + igb_update_rx_stats(core, rxi, pdma_st.size, pdma_st.total_size); } static bool @@ -1819,17 +2163,6 @@ igb_set_rfctl(IGBCore *core, int index, uint32_t val) core->mac[RFCTL] = val; } -static void -igb_calc_rxdesclen(IGBCore *core) -{ - if (igb_rx_use_legacy_descriptor(core)) { - core->rx_desc_len = sizeof(struct e1000_rx_desc); - } else { - core->rx_desc_len = sizeof(union e1000_adv_rx_desc); - } - trace_e1000e_rx_desc_len(core->rx_desc_len); -} - static void igb_set_rx_control(IGBCore *core, int index, uint32_t val) { diff --git a/hw/net/igb_regs.h b/hw/net/igb_regs.h index c5c5b3c3b8..0099794c8d 100644 --- a/hw/net/igb_regs.h +++ b/hw/net/igb_regs.h @@ -81,6 +81,23 @@ union e1000_adv_rx_desc { } wb; /* writeback */ }; +#define IGB_MAX_PS_BUFFERS 2 +#define IGB_MAX_RX_DESC_LEN (sizeof(uint64_t) * MAX_PS_BUFFERS) + +#define IGB_RX_DESC_PKT_TYPE_IPV4 BIT(0) +#define IGB_RX_DESC_PKT_TYPE_IPV4E BIT(1) +#define IGB_RX_DESC_PKT_TYPE_IPV6 BIT(2) +#define IGB_RX_DESC_PKT_TYPE_IPV6E BIT(3) +#define IGB_RX_DESC_PKT_TYPE_TCP BIT(4) +#define IGB_RX_DESC_PKT_TYPE_UDP BIT(5) +#define IGB_RX_DESC_PKT_TYPE_SCTP BIT(6) + +#define RX_DESC_ADV_HDR_LEN_OFFSET (21 - 16) +#define RX_DESC_ADV_HDR_LEN_MASK ((BIT(10) - 1) << \ + RX_DESC_ADV_HDR_LEN_OFFSET) +#define RX_DESC_ADV_HDR_SPH BIT(15) +#define RX_DESC_ADV_ST_ERR_HBO_OFFSET BIT(3 + 20) + /* from igb/e1000_phy.h */ /* IGP01E1000 Specific Registers */ @@ -406,6 +423,7 @@ union e1000_adv_rx_desc { #define E1000_SRRCTL_DESCTYPE_ADV_ONEBUF 0x02000000 #define E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS 0x0A000000 #define E1000_SRRCTL_DESCTYPE_MASK 0x0E000000 +#define E1000_SRRCTL_DESCTYPE_OFFSET 25 #define E1000_SRRCTL_DROP_EN 0x80000000 #define E1000_SRRCTL_BSIZEPKT_MASK 0x0000007F diff --git a/hw/net/trace-events b/hw/net/trace-events index d35554fce8..82ae89f2e5 100644 --- a/hw/net/trace-events +++ b/hw/net/trace-events @@ -283,9 +283,9 @@ igb_core_mdic_write_unhandled(uint32_t addr) "MDIC WRITE: PHY[%u] UNHANDLED" igb_link_set_ext_params(bool asd_check, bool speed_select_bypass, bool pfrstd) "Set extended link params: ASD check: %d, Speed select bypass: %d, PF reset done: %d" igb_rx_desc_buff_size(uint32_t b) "buffer size: %u" -igb_rx_desc_buff_write(uint64_t addr, uint16_t offset, const void* source, uint32_t len) "addr: 0x%"PRIx64", offset: %u, from: %p, length: %u" +igb_rx_desc_buff_write(uint8_t idx, uint64_t addr, uint16_t offset, const void* source, uint32_t len) "buffer #%u, addr: 0x%"PRIx64", offset: %u, from: %p, length: %u" -igb_rx_metadata_rss(uint32_t rss) "RSS data: 0x%X" +igb_rx_metadata_rss(uint32_t rss, uint16_t rss_pkt_type) "RSS data: rss: 0x%X, rss_pkt_type: 0x%X" igb_irq_icr_clear_gpie_nsicr(void) "Clearing ICR on read due to GPIE.NSICR enabled" igb_irq_icr_write(uint32_t bits, uint32_t old_icr, uint32_t new_icr) "Clearing ICR bits 0x%x: 0x%x --> 0x%x" diff --git a/tests/qtest/igb-test.c b/tests/qtest/igb-test.c index 3d397ea697..d33d324871 100644 --- a/tests/qtest/igb-test.c +++ b/tests/qtest/igb-test.c @@ -104,6 +104,10 @@ static void igb_receive_verify(QE1000E *d, int *test_sockets, QGuestAllocator *a char buffer[64]; int ret; + /* Set supported descriptor mode */ + uint32_t srrctl0 = e1000e_macreg_read(d, E1000_SRRCTL(0)); + e1000e_macreg_write(d, E1000_SRRCTL(0), srrctl0 | BIT(25)); + /* Send a dummy packet to device's socket*/ ret = iov_send(test_sockets[0], iov, 2, 0, sizeof(len) + sizeof(packet)); g_assert_cmpint(ret, == , sizeof(packet) + sizeof(len)); -- 2.25.1