This patch provides a rte_memcpy* call with temporal stores.
Use -Dcpu_instruction_set=znverX with build to enable this API.
Signed-off-by: Aman Kumar <aman.ku...@vvdntech.in>
---
config/x86/meson.build | 2 +
lib/eal/x86/include/rte_memcpy.h | 114 +++++++++++++++++++++++++++++++
2 files changed, 116 insertions(+)
diff --git a/config/x86/meson.build b/config/x86/meson.build
index 21cda6fd33..56dae4aca7 100644
--- a/config/x86/meson.build
+++ b/config/x86/meson.build
@@ -78,6 +78,8 @@ if get_option('cpu_instruction_set') == 'znver1'
dpdk_conf.set('RTE_MAX_LCORE', 256)
elif get_option('cpu_instruction_set') == 'znver2'
dpdk_conf.set('RTE_MAX_LCORE', 512)
+ dpdk_conf.set('RTE_MEMCPY_AMDEPYC', 1)
elif get_option('cpu_instruction_set') == 'znver3'
dpdk_conf.set('RTE_MAX_LCORE', 512)
+ dpdk_conf.set('RTE_MEMCPY_AMDEPYC', 1)
endif
diff --git a/lib/eal/x86/include/rte_memcpy.h b/lib/eal/x86/include/rte_memcpy.h
index 1b6c6e585f..8fe7822cb4 100644
--- a/lib/eal/x86/include/rte_memcpy.h
+++ b/lib/eal/x86/include/rte_memcpy.h
@@ -376,6 +376,120 @@ rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t
n)
}
}
+#if defined RTE_MEMCPY_AMDEPYC
+
+/**
+ * Copy 16 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy16_ts(uint8_t *dst, uint8_t *src)
+{
+ __m128i var128;
+
+ var128 = _mm_stream_load_si128((__m128i *)src);
+ _mm_storeu_si128((__m128i *)dst, var128);
+}
+
+/**
+ * Copy 32 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy32_ts(uint8_t *dst, uint8_t *src)
+{
+ __m256i ymm0;
+
+ ymm0 = _mm256_stream_load_si256((const __m256i *)src);
+ _mm256_storeu_si256((__m256i *)dst, ymm0);
+}
+
+/**
+ * Copy 64 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy64_ts(uint8_t *dst, uint8_t *src)
+{
+ rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
+ rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
+}
+
+/**
+ * Copy 128 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy128_ts(uint8_t *dst, uint8_t *src)
+{
+ rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
+ rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
+ rte_copy32_ts(dst + 2 * 32, src + 2 * 32);
+ rte_copy32_ts(dst + 3 * 32, src + 3 * 32);
+}
+
+/**
+ * Copy len bytes from one location to another,
+ * with temporal stores 16B aligned
+ */
+static __rte_always_inline void *
+rte_memcpy_aligned_tstore16_generic(void *dst, void *src, int len)
+{
+ void *dest = dst;
+
+ while (len >= 128) {
+ rte_copy128_ts((uint8_t *)dst, (uint8_t *)src);
+ dst = (uint8_t *)dst + 128;
+ src = (uint8_t *)src + 128;
+ len -= 128;
+ }
+ while (len >= 64) {
+ rte_copy64_ts((uint8_t *)dst, (uint8_t *)src);
+ dst = (uint8_t *)dst + 64;
+ src = (uint8_t *)src + 64;
+ len -= 64;
+ }
+ while (len >= 32) {
+ rte_copy32_ts((uint8_t *)dst, (uint8_t *)src);
+ dst = (uint8_t *)dst + 32;
+ src = (uint8_t *)src + 32;
+ len -= 32;
+ }
+ if (len >= 16) {
+ rte_copy16_ts((uint8_t *)dst, (uint8_t *)src);
+ dst = (uint8_t *)dst + 16;
+ src = (uint8_t *)src + 16;
+ len -= 16;
+ }
+ if (len >= 8) {
+ *(uint64_t *)dst = *(const uint64_t *)src;
+ dst = (uint8_t *)dst + 8;
+ src = (uint8_t *)src + 8;
+ len -= 8;
+ }
+ if (len >= 4) {
+ *(uint32_t *)dst = *(const uint32_t *)src;
+ dst = (uint8_t *)dst + 4;
+ src = (uint8_t *)src + 4;
+ len -= 4;
+ }
+ if (len != 0) {
+ dst = (uint8_t *)dst - (4 - len);
+ src = (uint8_t *)src - (4 - len);
+ *(uint32_t *)dst = *(const uint32_t *)src;
+ }
+
+ return dest;
+}
+
+static __rte_always_inline void *
+rte_memcpy_aligned_tstore16(void *dst, void *src, int len)
+{
+ return rte_memcpy_aligned_tstore16_generic(dst, src, len);
+}
+
+#endif /* RTE_MEMCPY_AMDEPYC */
+
static __rte_always_inline void *
rte_memcpy_generic(void *dst, const void *src, size_t n)
{
--
2.25.1
