Author: Guillaume Chatelet Date: 2021-01-15T15:32:02Z New Revision: 5bf47e142b6ebe1baf0cab257800c27a1a3bbde7
URL: https://github.com/llvm/llvm-project/commit/5bf47e142b6ebe1baf0cab257800c27a1a3bbde7 DIFF: https://github.com/llvm/llvm-project/commit/5bf47e142b6ebe1baf0cab257800c27a1a3bbde7.diff LOG: [libc] CopyAlignedBlocks can now specify alignment on top of block size This has been requested in D92236 Differential Revision: https://reviews.llvm.org/D94770 Added: Modified: libc/src/string/memory_utils/memcpy_utils.h libc/src/string/memory_utils/utils.h libc/test/src/string/memory_utils/memcpy_utils_test.cpp Removed: ################################################################################ diff --git a/libc/src/string/memory_utils/memcpy_utils.h b/libc/src/string/memory_utils/memcpy_utils.h index 1e7d907d2333..44b88374e20f 100644 --- a/libc/src/string/memory_utils/memcpy_utils.h +++ b/libc/src/string/memory_utils/memcpy_utils.h @@ -72,28 +72,35 @@ static void CopyBlockOverlap(char *__restrict dst, const char *__restrict src, // Copies `count` bytes by blocks of `kBlockSize` bytes. // Copies at the start and end of the buffer are unaligned. -// Copies in the middle of the buffer are aligned to `kBlockSize`. +// Copies in the middle of the buffer are aligned to `kAlignment`. // // e.g. with // [12345678123456781234567812345678] -// [__XXXXXXXXXXXXXXXXXXXXXXXXXXX___] -// [__XXXXXXXX______________________] -// [________XXXXXXXX________________] -// [________________XXXXXXXX________] -// [_____________________XXXXXXXX___] +// [__XXXXXXXXXXXXXXXXXXXXXXXXXXXX___] +// [__XXXX___________________________] +// [_____XXXXXXXX____________________] +// [_____________XXXXXXXX____________] +// [_____________________XXXXXXXX____] +// [______________________XXXXXXXX___] // -// Precondition: `count > 2 * kBlockSize` for efficiency. -// `count >= kBlockSize` for correctness. -template <size_t kBlockSize> +// Precondition: `kAlignment <= kBlockSize` +// `count > 2 * kBlockSize` for efficiency. +// `count >= kAlignment` for correctness. +template <size_t kBlockSize, size_t kAlignment = kBlockSize> static void CopyAlignedBlocks(char *__restrict dst, const char *__restrict src, size_t count) { - CopyBlock<kBlockSize>(dst, src); // Copy first block + static_assert(is_power2(kAlignment), "kAlignment must be a power of two"); + static_assert(is_power2(kBlockSize), "kBlockSize must be a power of two"); + static_assert(kAlignment <= kBlockSize, + "kAlignment must be less or equal to block size"); + CopyBlock<kAlignment>(dst, src); // Copy first block // Copy aligned blocks - const size_t ofla = offset_from_last_aligned<kBlockSize>(src); + const size_t ofla = offset_from_last_aligned<kAlignment>(src); const size_t limit = count + ofla - kBlockSize; - for (size_t offset = kBlockSize; offset < limit; offset += kBlockSize) - CopyBlock<kBlockSize>(dst - ofla + offset, src - ofla + offset); + for (size_t offset = kAlignment; offset < limit; offset += kBlockSize) + CopyBlock<kBlockSize>(dst - ofla + offset, + assume_aligned<kAlignment>(src - ofla + offset)); CopyLastBlock<kBlockSize>(dst, src, count); // Copy last block } diff --git a/libc/src/string/memory_utils/utils.h b/libc/src/string/memory_utils/utils.h index 401c7767a59e..c2472c20ab78 100644 --- a/libc/src/string/memory_utils/utils.h +++ b/libc/src/string/memory_utils/utils.h @@ -60,6 +60,10 @@ static inline intptr_t offset_to_next_cache_line(const void *ptr) { return offset_to_next_aligned<LLVM_LIBC_CACHELINE_SIZE>(ptr); } +template <size_t alignment, typename T> static T *assume_aligned(T *ptr) { + return reinterpret_cast<T *>(__builtin_assume_aligned(ptr, alignment)); +} + } // namespace __llvm_libc #endif // LLVM_LIBC_SRC_MEMORY_UTILS_H diff --git a/libc/test/src/string/memory_utils/memcpy_utils_test.cpp b/libc/test/src/string/memory_utils/memcpy_utils_test.cpp index d466495357c2..d45794710862 100644 --- a/libc/test/src/string/memory_utils/memcpy_utils_test.cpp +++ b/libc/test/src/string/memory_utils/memcpy_utils_test.cpp @@ -211,7 +211,24 @@ TEST(MemcpyUtilsTest, CopyAlignedBlocks) { EXPECT_STREQ(trace.Read(), "011121111111"); } -TEST(MemcpyUtilsTest, MaxReloads) { +TEST(MemcpyUtilsTest, CopyAlignedBlocksWithAlignment) { + auto &trace = GetTrace(); + // Source is aligned and multiple of alignment. + // "11111111" + trace.Clear(); + CopyAlignedBlocks<8, 4>(I(0), I(0), 8); + EXPECT_STREQ(trace.Write(), "22221111"); + EXPECT_STREQ(trace.Read(), "22221111"); + + // Source is aligned and multiple of alignment. + // "111111111" + trace.Clear(); + CopyAlignedBlocks<8, 4>(I(0), I(0), 9); + EXPECT_STREQ(trace.Write(), "122211111"); + EXPECT_STREQ(trace.Read(), "122211111"); +} + +TEST(MemcpyUtilsTest, CopyAlignedBlocksMaxReloads) { auto &trace = GetTrace(); for (size_t alignment = 0; alignment < 32; ++alignment) { for (size_t count = 64; count < 768; ++count) { @@ -231,4 +248,24 @@ TEST(MemcpyUtilsTest, MaxReloads) { } } +TEST(MemcpyUtilsTest, CopyAlignedBlocksWithAlignmentMaxReloads) { + auto &trace = GetTrace(); + for (size_t alignment = 0; alignment < 32; ++alignment) { + for (size_t count = 64; count < 768; ++count) { + trace.Clear(); + // We should never reload more than twice when copying from count = 2x32. + CopyAlignedBlocks<32, 16>(I(alignment), I(0), count); + const char *const written = trace.Write(); + // First bytes are untouched. + for (size_t i = 0; i < alignment; ++i) + EXPECT_EQ(written[i], '0'); + // Next bytes are loaded once or twice but no more. + for (size_t i = alignment; i < count; ++i) { + EXPECT_GE(written[i], '1'); + EXPECT_LE(written[i], '2'); + } + } + } +} + } // namespace __llvm_libc _______________________________________________ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
