ckissane updated this revision to Diff 440288.
ckissane added a comment.
Fix missing cmake findZstd, remove unused crc32 from compression namespaces and
simplify some code
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D128465/new/
https://reviews.llvm.org/D128465
Files:
clang-tools-extra/clangd/CMakeLists.txt
clang-tools-extra/clangd/index/Serialization.cpp
clang-tools-extra/clangd/test/lit.site.cfg.py.in
clang-tools-extra/clangd/unittests/SerializationTests.cpp
clang/cmake/caches/Apple-stage1.cmake
clang/cmake/caches/Apple-stage2.cmake
clang/cmake/caches/Fuchsia-stage2.cmake
clang/cmake/caches/Fuchsia.cmake
clang/lib/Driver/ToolChains/Clang.cpp
clang/lib/Serialization/ASTReader.cpp
clang/lib/Serialization/ASTWriter.cpp
clang/test/CMakeLists.txt
clang/test/lit.site.cfg.py.in
compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
compiler-rt/test/lit.common.configured.in
flang/CMakeLists.txt
lld/ELF/CMakeLists.txt
lld/ELF/Driver.cpp
lld/ELF/InputSection.cpp
lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
llvm/CMakeLists.txt
llvm/cmake/config-ix.cmake
llvm/cmake/modules/FindZSTD.cmake
llvm/cmake/modules/LLVMConfig.cmake.in
llvm/include/llvm/Config/llvm-config.h.cmake
llvm/include/llvm/Support/Compression.h
llvm/lib/MC/ELFObjectWriter.cpp
llvm/lib/ObjCopy/ELF/ELFObject.cpp
llvm/lib/Object/Decompressor.cpp
llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
llvm/lib/ProfileData/InstrProf.cpp
llvm/lib/ProfileData/SampleProfReader.cpp
llvm/lib/ProfileData/SampleProfWriter.cpp
llvm/lib/Support/CMakeLists.txt
llvm/lib/Support/Compression.cpp
llvm/tools/llvm-mc/llvm-mc.cpp
llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
llvm/unittests/ProfileData/InstrProfTest.cpp
llvm/unittests/Support/CompressionTest.cpp
utils/bazel/llvm_configs/llvm-config.h.cmake
Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===================================================================
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -92,6 +92,9 @@
/* Define if zlib compression is available */
#cmakedefine01 LLVM_ENABLE_ZLIB
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
/* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
#cmakedefine LLVM_HAVE_TF_API
Index: llvm/unittests/Support/CompressionTest.cpp
===================================================================
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -11,13 +11,16 @@
//===----------------------------------------------------------------------===//
#include "llvm/Support/Compression.h"
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Config/config.h"
+#include "llvm/Support/CRC.h"
#include "llvm/Support/Error.h"
#include "gtest/gtest.h"
using namespace llvm;
+using namespace compression;
namespace {
@@ -63,9 +66,54 @@
}
TEST(CompressionTest, ZlibCRC32) {
+ const char* StringString = strdup("The quick brown fox jumps over the lazy dog");
+ const ArrayRef<uint8_t> StringByteArray = ArrayRef<uint8_t>(*StringString);
EXPECT_EQ(
0x414FA339U,
- zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
+ llvm::crc32(StringByteArray));
+}
+
+#endif
+
+#if LLVM_ENABLE_ZSTD
+
+void TestZstdCompression(StringRef Input, int Level) {
+ SmallString<32> Compressed;
+ SmallString<32> Uncompressed;
+
+ zstd::compress(Input, Compressed, Level);
+
+ // Check that uncompressed buffer is the same as original.
+ Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+ consumeError(std::move(E));
+
+ EXPECT_EQ(Input, Uncompressed);
+ if (Input.size() > 0) {
+ // Uncompression fails if expected length is too short.
+ E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+ EXPECT_EQ("zstd error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+ }
+}
+
+TEST(CompressionTest, Zstd) {
+ TestZstdCompression("", zstd::DefaultCompression);
+
+ TestZstdCompression("hello, world!", zstd::NoCompression);
+ TestZstdCompression("hello, world!", zstd::BestSizeCompression);
+ TestZstdCompression("hello, world!", zstd::BestSpeedCompression);
+ TestZstdCompression("hello, world!", zstd::DefaultCompression);
+
+ const size_t kSize = 1024;
+ char BinaryData[kSize];
+ for (size_t i = 0; i < kSize; ++i) {
+ BinaryData[i] = i & 255;
+ }
+ StringRef BinaryDataStr(BinaryData, kSize);
+
+ TestZstdCompression(BinaryDataStr, zstd::NoCompression);
+ TestZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+ TestZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+ TestZstdCompression(BinaryDataStr, zstd::DefaultCompression);
}
#endif
Index: llvm/unittests/ProfileData/InstrProfTest.cpp
===================================================================
--- llvm/unittests/ProfileData/InstrProfTest.cpp
+++ llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -1147,14 +1147,14 @@
// Compressing:
std::string FuncNameStrings1;
EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
- FuncNames1, (DoCompression && zlib::isAvailable()),
+ FuncNames1, (DoCompression && compression::internal::isAvailable()),
FuncNameStrings1),
Succeeded());
// Compressing:
std::string FuncNameStrings2;
EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
- FuncNames2, (DoCompression && zlib::isAvailable()),
+ FuncNames2, (DoCompression && compression::internal::isAvailable()),
FuncNameStrings2),
Succeeded());
Index: llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
===================================================================
--- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -740,7 +740,7 @@
.str()
.c_str());
}
- if (!zlib::isAvailable())
+ if (!compression::elf::isAvailable())
return createStringError(
errc::invalid_argument,
"LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
@@ -999,7 +999,7 @@
"--decompress-debug-sections");
}
- if (Config.DecompressDebugSections && !zlib::isAvailable())
+ if (Config.DecompressDebugSections && !compression::elf::isAvailable())
return createStringError(
errc::invalid_argument,
"LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress");
Index: llvm/tools/llvm-mc/llvm-mc.cpp
===================================================================
--- llvm/tools/llvm-mc/llvm-mc.cpp
+++ llvm/tools/llvm-mc/llvm-mc.cpp
@@ -403,7 +403,7 @@
MAI->setRelaxELFRelocations(RelaxELFRel);
if (CompressDebugSections != DebugCompressionType::None) {
- if (!zlib::isAvailable()) {
+ if (!compression::zlib::isAvailable()) {
WithColor::error(errs(), ProgName)
<< "build tools with zlib to enable -compress-debug-sections";
return 1;
Index: llvm/lib/Support/Compression.cpp
===================================================================
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,14 +20,20 @@
#if LLVM_ENABLE_ZLIB
#include <zlib.h>
#endif
+#if LLVM_ENABLE_ZSTD
+#include <zstd.h>
+#endif
using namespace llvm;
-#if LLVM_ENABLE_ZLIB
+using namespace compression;
+
static Error createError(StringRef Err) {
return make_error<StringError>(Err, inconvertibleErrorCode());
}
+#if LLVM_ENABLE_ZLIB
+
static StringRef convertZlibCodeToString(int Code) {
switch (Code) {
case Z_MEM_ERROR:
@@ -78,15 +84,11 @@
size_t UncompressedSize) {
UncompressedBuffer.resize_for_overwrite(UncompressedSize);
Error E =
- uncompress(InputBuffer, UncompressedBuffer.data(), UncompressedSize);
+ zlib::uncompress(InputBuffer, UncompressedBuffer.data(), UncompressedSize);
UncompressedBuffer.truncate(UncompressedSize);
return E;
}
-uint32_t zlib::crc32(StringRef Buffer) {
- return ::crc32(0, (const Bytef *)Buffer.data(), Buffer.size());
-}
-
#else
bool zlib::isAvailable() { return false; }
void zlib::compress(StringRef InputBuffer,
@@ -102,7 +104,64 @@
size_t UncompressedSize) {
llvm_unreachable("zlib::uncompress is unavailable");
}
-uint32_t zlib::crc32(StringRef Buffer) {
- llvm_unreachable("zlib::crc32 is unavailable");
+#endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(StringRef InputBuffer,
+ SmallVectorImpl<char> &CompressedBuffer, int Level) {
+ unsigned long CompressedBufferSize = ::ZSTD_compressBound(InputBuffer.size());
+ CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+ unsigned long CompressedSize =
+ ::ZSTD_compress((Bytef *)CompressedBuffer.data(), CompressedBufferSize,
+ (const Bytef *)InputBuffer.data(), InputBuffer.size(), Level);
+ if (ZSTD_isError(CompressedSize))
+ report_bad_alloc_error("Allocation failed");
+ // assert(Res == Z_OK);
+ // Tell MemorySanitizer that zstd output buffer is fully initialized.
+ // This avoids a false report when running LLVM with uninstrumented ZLib.
+ __msan_unpoison(CompressedBuffer.data(), CompressedSize);
+ CompressedBuffer.truncate(CompressedSize);
+}
+
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+ size_t &UncompressedSize) {
+ unsigned long long const rSize = ZSTD_getFrameContentSize((const Bytef *)InputBuffer.data(), InputBuffer.size());
+ size_t const Res =
+ ::ZSTD_decompress((Bytef *)UncompressedBuffer, rSize,
+ (const Bytef *)InputBuffer.data(), InputBuffer.size());
+ UncompressedSize=Res;
+ // Tell MemorySanitizer that zstd output buffer is fully initialized.
+ // This avoids a false report when running LLVM with uninstrumented ZLib.
+ __msan_unpoison(UncompressedBuffer, UncompressedSize);
+ return Res!=rSize ? createError(ZSTD_getErrorName(Res)) : Error::success();
+}
+
+Error zstd::uncompress(StringRef InputBuffer,
+ SmallVectorImpl<char> &UncompressedBuffer,
+ size_t UncompressedSize) {
+ UncompressedBuffer.resize_for_overwrite(UncompressedSize);
+ Error E =
+ zstd::uncompress(InputBuffer, UncompressedBuffer.data(), UncompressedSize);
+ UncompressedBuffer.truncate(UncompressedSize);
+ return E;
+}
+
+#else
+bool zstd::isAvailable() { return false; }
+void zstd::compress(StringRef InputBuffer,
+ SmallVectorImpl<char> &CompressedBuffer, int Level) {
+ llvm_unreachable("zstd::compress is unavailable");
+}
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+ size_t &UncompressedSize) {
+ llvm_unreachable("zstd::uncompress is unavailable");
+}
+Error zstd::uncompress(StringRef InputBuffer,
+ SmallVectorImpl<char> &UncompressedBuffer,
+ size_t UncompressedSize) {
+ llvm_unreachable("zstd::uncompress is unavailable");
}
#endif
Index: llvm/lib/Support/CMakeLists.txt
===================================================================
--- llvm/lib/Support/CMakeLists.txt
+++ llvm/lib/Support/CMakeLists.txt
@@ -22,7 +22,11 @@
endif()
if(LLVM_ENABLE_ZLIB)
- set(imported_libs ZLIB::ZLIB)
+ list(APPEND imported_libs ZLIB::ZLIB)
+endif()
+
+if(LLVM_ENABLE_ZSTD)
+ list(APPEND imported_libs zstd)
endif()
if( MSVC OR MINGW )
Index: llvm/lib/ProfileData/SampleProfWriter.cpp
===================================================================
--- llvm/lib/ProfileData/SampleProfWriter.cpp
+++ llvm/lib/ProfileData/SampleProfWriter.cpp
@@ -78,7 +78,7 @@
}
std::error_code SampleProfileWriterExtBinaryBase::compressAndOutput() {
- if (!llvm::zlib::isAvailable())
+ if (!llvm::compression::internal::isAvailable())
return sampleprof_error::zlib_unavailable;
std::string &UncompressedStrings =
static_cast<raw_string_ostream *>(LocalBufStream.get())->str();
@@ -86,8 +86,8 @@
return sampleprof_error::success;
auto &OS = *OutputStream;
SmallString<128> CompressedStrings;
- zlib::compress(UncompressedStrings, CompressedStrings,
- zlib::BestSizeCompression);
+ compression::internal::compress(UncompressedStrings, CompressedStrings,
+ compression::internal::BestSizeCompression);
encodeULEB128(UncompressedStrings.size(), OS);
encodeULEB128(CompressedStrings.size(), OS);
OS << CompressedStrings.str();
Index: llvm/lib/ProfileData/SampleProfReader.cpp
===================================================================
--- llvm/lib/ProfileData/SampleProfReader.cpp
+++ llvm/lib/ProfileData/SampleProfReader.cpp
@@ -877,7 +877,7 @@
if (std::error_code EC = CompressSize.getError())
return EC;
- if (!llvm::zlib::isAvailable())
+ if (!llvm::compression::profile::isAvailable())
return sampleprof_error::zlib_unavailable;
StringRef CompressedStrings(reinterpret_cast<const char *>(Data),
@@ -885,7 +885,7 @@
char *Buffer = Allocator.Allocate<char>(DecompressBufSize);
size_t UCSize = DecompressBufSize;
llvm::Error E =
- zlib::uncompress(CompressedStrings, Buffer, UCSize);
+ compression::profile::uncompress(CompressedStrings, Buffer, UCSize);
if (E)
return sampleprof_error::uncompress_failed;
DecompressBuf = reinterpret_cast<const uint8_t *>(Buffer);
Index: llvm/lib/ProfileData/InstrProf.cpp
===================================================================
--- llvm/lib/ProfileData/InstrProf.cpp
+++ llvm/lib/ProfileData/InstrProf.cpp
@@ -467,8 +467,8 @@
}
SmallString<128> CompressedNameStrings;
- zlib::compress(StringRef(UncompressedNameStrings), CompressedNameStrings,
- zlib::BestSizeCompression);
+ compression::profile::compress(StringRef(UncompressedNameStrings), CompressedNameStrings,
+ compression::profile::BestSizeCompression);
return WriteStringToResult(CompressedNameStrings.size(),
CompressedNameStrings);
@@ -488,7 +488,7 @@
NameStrs.push_back(std::string(getPGOFuncNameVarInitializer(NameVar)));
}
return collectPGOFuncNameStrings(
- NameStrs, zlib::isAvailable() && doCompression, Result);
+ NameStrs, compression::profile::isAvailable() && doCompression, Result);
}
Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
@@ -504,13 +504,13 @@
SmallString<128> UncompressedNameStrings;
StringRef NameStrings;
if (isCompressed) {
- if (!llvm::zlib::isAvailable())
+ if (!llvm::compression::profile::isAvailable())
return make_error<InstrProfError>(instrprof_error::zlib_unavailable);
StringRef CompressedNameStrings(reinterpret_cast<const char *>(P),
CompressedSize);
if (Error E =
- zlib::uncompress(CompressedNameStrings, UncompressedNameStrings,
+ compression::profile::uncompress(CompressedNameStrings, UncompressedNameStrings,
UncompressedSize)) {
consumeError(std::move(E));
return make_error<InstrProfError>(instrprof_error::uncompress_failed);
Index: llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
===================================================================
--- llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
+++ llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
@@ -48,9 +48,9 @@
SmallString<128> CompressedStr;
bool doCompression =
- Compress && zlib::isAvailable() && DoInstrProfNameCompression;
+ Compress && compression::internal::isAvailable() && DoInstrProfNameCompression;
if (doCompression)
- zlib::compress(FilenamesStr, CompressedStr, zlib::BestSizeCompression);
+ compression::internal::compress(FilenamesStr, CompressedStr, compression::internal::BestSizeCompression);
// ::= <num-filenames>
// <uncompressed-len>
Index: llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
===================================================================
--- llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
+++ llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
@@ -119,7 +119,7 @@
return Err;
if (CompressedLen > 0) {
- if (!zlib::isAvailable())
+ if (!compression::internal::isAvailable())
return make_error<CoverageMapError>(
coveragemap_error::decompression_failed);
@@ -130,7 +130,7 @@
StringRef CompressedFilenames = Data.substr(0, CompressedLen);
Data = Data.substr(CompressedLen);
auto Err =
- zlib::uncompress(CompressedFilenames, StorageBuf, UncompressedLen);
+ compression::internal::uncompress(CompressedFilenames, StorageBuf, UncompressedLen);
if (Err) {
consumeError(std::move(Err));
return make_error<CoverageMapError>(
Index: llvm/lib/Object/Decompressor.cpp
===================================================================
--- llvm/lib/Object/Decompressor.cpp
+++ llvm/lib/Object/Decompressor.cpp
@@ -19,7 +19,7 @@
Expected<Decompressor> Decompressor::create(StringRef Name, StringRef Data,
bool IsLE, bool Is64Bit) {
- if (!zlib::isAvailable())
+ if (!compression::elf::isAvailable())
return createError("zlib is not available");
Decompressor D(Data);
@@ -94,5 +94,5 @@
Error Decompressor::decompress(MutableArrayRef<char> Buffer) {
size_t Size = Buffer.size();
- return zlib::uncompress(SectionData, Buffer.data(), Size);
+ return compression::elf::uncompress(SectionData, Buffer.data(), Size);
}
Index: llvm/lib/ObjCopy/ELF/ELFObject.cpp
===================================================================
--- llvm/lib/ObjCopy/ELF/ELFObject.cpp
+++ llvm/lib/ObjCopy/ELF/ELFObject.cpp
@@ -468,7 +468,7 @@
Sec.OriginalData.size() - DataOffset);
SmallVector<char, 128> DecompressedContent;
- if (Error Err = zlib::uncompress(CompressedContent, DecompressedContent,
+ if (Error Err = compression::elf::uncompress(CompressedContent, DecompressedContent,
static_cast<size_t>(Sec.Size)))
return createStringError(errc::invalid_argument,
"'" + Sec.Name + "': " + toString(std::move(Err)));
@@ -549,7 +549,7 @@
DebugCompressionType CompressionType)
: SectionBase(Sec), CompressionType(CompressionType),
DecompressedSize(Sec.OriginalData.size()), DecompressedAlign(Sec.Align) {
- zlib::compress(StringRef(reinterpret_cast<const char *>(OriginalData.data()),
+ compression::elf::compress(StringRef(reinterpret_cast<const char *>(OriginalData.data()),
OriginalData.size()),
CompressedData);
Index: llvm/lib/MC/ELFObjectWriter.cpp
===================================================================
--- llvm/lib/MC/ELFObjectWriter.cpp
+++ llvm/lib/MC/ELFObjectWriter.cpp
@@ -876,7 +876,7 @@
Asm.writeSectionData(VecOS, &Section, Layout);
SmallVector<char, 128> CompressedContents;
- zlib::compress(StringRef(UncompressedData.data(), UncompressedData.size()),
+ compression::elf::compress(StringRef(UncompressedData.data(), UncompressedData.size()),
CompressedContents);
bool ZlibStyle = MAI->compressDebugSections() == DebugCompressionType::Z;
Index: llvm/include/llvm/Support/Compression.h
===================================================================
--- llvm/include/llvm/Support/Compression.h
+++ llvm/include/llvm/Support/Compression.h
@@ -20,6 +20,8 @@
class Error;
class StringRef;
+namespace compression {
+
namespace zlib {
static constexpr int NoCompression = 0;
@@ -39,9 +41,52 @@
SmallVectorImpl<char> &UncompressedBuffer,
size_t UncompressedSize);
-uint32_t crc32(StringRef Buffer);
+} // End of namespace zlib
+
+namespace zstd {
+
+static constexpr int NoCompression = -5;
+static constexpr int BestSpeedCompression = 1;
+static constexpr int DefaultCompression = 2;
+static constexpr int BestSizeCompression = 12;
+
+bool isAvailable();
+
+void compress(StringRef InputBuffer, SmallVectorImpl<char> &CompressedBuffer,
+ int Level = DefaultCompression);
+
+Error uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+ size_t &UncompressedSize);
+
+Error uncompress(StringRef InputBuffer,
+ SmallVectorImpl<char> &UncompressedBuffer,
+ size_t UncompressedSize);
+
+} // End of namespace zstd
+
+#if LLVM_ENABLE_ZSTD
+namespace internal = llvm::compression::zstd;
+#else
+namespace internal = llvm::compression::zlib;
+#endif
+
+namespace elf = llvm::compression::zlib;
+
+// TODO: COLE: should this be always zlib? if not make profile just internal, if so make profile zlib
+// #if
+// namespace profile = llvm::compression::zlib;
+// #else
+namespace profile = llvm::compression::internal;
+// #endif
+
+// TODO: COLE: should this be always zlib? if not make serialize just internal, if so make serialize zlib
+// #if
+// namespace serialize = llvm::compression::zlib;
+// #else
+namespace serialize = llvm::compression::internal;
+// #endif
-} // End of namespace zlib
+} // End of namespace compression
} // End of namespace llvm
Index: llvm/include/llvm/Config/llvm-config.h.cmake
===================================================================
--- llvm/include/llvm/Config/llvm-config.h.cmake
+++ llvm/include/llvm/Config/llvm-config.h.cmake
@@ -92,6 +92,9 @@
/* Define if zlib compression is available */
#cmakedefine01 LLVM_ENABLE_ZLIB
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
/* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
#cmakedefine LLVM_HAVE_TF_API
Index: llvm/cmake/modules/LLVMConfig.cmake.in
===================================================================
--- llvm/cmake/modules/LLVMConfig.cmake.in
+++ llvm/cmake/modules/LLVMConfig.cmake.in
@@ -73,6 +73,12 @@
find_package(ZLIB)
endif()
+set(LLVM_ENABLE_ZSTD @LLVM_ENABLE_ZSTD@)
+if(LLVM_ENABLE_ZSTD)
+ set(ZSTD_ROOT @ZSTD_ROOT@)
+ find_package(ZSTD)
+endif()
+
set(LLVM_ENABLE_LIBXML2 @LLVM_ENABLE_LIBXML2@)
if(LLVM_ENABLE_LIBXML2)
find_package(LibXml2)
Index: llvm/cmake/modules/FindZSTD.cmake
===================================================================
--- /dev/null
+++ llvm/cmake/modules/FindZSTD.cmake
@@ -0,0 +1,41 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# - Try to find Facebook zstd library
+# This will define
+# ZSTD_FOUND
+# ZSTD_INCLUDE_DIR
+# ZSTD_LIBRARY
+#
+
+find_path(ZSTD_INCLUDE_DIR NAMES zstd.h)
+
+find_library(ZSTD_LIBRARY_DEBUG NAMES zstdd zstd_staticd)
+find_library(ZSTD_LIBRARY_RELEASE NAMES zstd zstd_static)
+
+include(SelectLibraryConfigurations)
+SELECT_LIBRARY_CONFIGURATIONS(ZSTD)
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(
+ ZSTD DEFAULT_MSG
+ ZSTD_LIBRARY ZSTD_INCLUDE_DIR
+)
+
+if (ZSTD_FOUND)
+ message(STATUS "Found Zstd: ${ZSTD_LIBRARY}")
+endif()
+
+mark_as_advanced(ZSTD_INCLUDE_DIR ZSTD_LIBRARY)
\ No newline at end of file
Index: llvm/cmake/config-ix.cmake
===================================================================
--- llvm/cmake/config-ix.cmake
+++ llvm/cmake/config-ix.cmake
@@ -135,6 +135,27 @@
set(LLVM_ENABLE_ZLIB "${HAVE_ZLIB}")
endif()
+if(LLVM_ENABLE_ZSTD)
+ if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON)
+ find_package(ZSTD REQUIRED)
+ elseif(NOT LLVM_USE_SANITIZER MATCHES "Memory.*")
+ find_package(ZSTD)
+ endif()
+ if(ZSTD_FOUND)
+ # Check if zstd we found is usable; for example, we may have found a 32-bit
+ # library on a 64-bit system which would result in a link-time failure.
+ cmake_push_check_state()
+ list(APPEND CMAKE_REQUIRED_INCLUDES ${ZSTD_INCLUDE_DIRS})
+ list(APPEND CMAKE_REQUIRED_LIBRARIES ${ZSTD_LIBRARY})
+ check_symbol_exists(ZSTD_compress zstd.h HAVE_ZSTD)
+ cmake_pop_check_state()
+ if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON AND NOT HAVE_ZSTD)
+ message(FATAL_ERROR "Failed to configure zstd")
+ endif()
+ endif()
+ set(LLVM_ENABLE_ZSTD "${HAVE_ZSTD}")
+endif()
+
if(LLVM_ENABLE_LIBXML2)
if(LLVM_ENABLE_LIBXML2 STREQUAL FORCE_ON)
find_package(LibXml2 REQUIRED)
Index: llvm/CMakeLists.txt
===================================================================
--- llvm/CMakeLists.txt
+++ llvm/CMakeLists.txt
@@ -438,6 +438,8 @@
set(LLVM_ENABLE_ZLIB "ON" CACHE STRING "Use zlib for compression/decompression if available. Can be ON, OFF, or FORCE_ON")
+set(LLVM_ENABLE_ZSTD "ON" CACHE STRING "Use zstd for compression/decompression if available. Can be ON, OFF, or FORCE_ON")
+
set(LLVM_ENABLE_CURL "OFF" CACHE STRING "Use libcurl for the HTTP client if available. Can be ON, OFF, or FORCE_ON")
set(LLVM_Z3_INSTALL_DIR "" CACHE STRING "Install directory of the Z3 solver.")
Index: lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
===================================================================
--- lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
+++ lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
@@ -18,6 +18,10 @@
list(APPEND LLDB_SYSTEM_LIBS ZLIB::ZLIB)
endif()
+if(LLVM_ENABLE_ZSTD)
+ list(APPEND LLDB_SYSTEM_LIBS zstd)
+endif()
+
add_lldb_library(lldbPluginProcessGDBRemote PLUGIN
GDBRemoteClientBase.cpp
GDBRemoteCommunication.cpp
Index: lld/ELF/InputSection.cpp
===================================================================
--- lld/ELF/InputSection.cpp
+++ lld/ELF/InputSection.cpp
@@ -73,7 +73,7 @@
// If SHF_COMPRESSED is set, parse the header. The legacy .zdebug format is no
// longer supported.
if (flags & SHF_COMPRESSED) {
- if (!zlib::isAvailable())
+ if (!compression::elf::isAvailable())
error(toString(file) + ": contains a compressed section, " +
"but zlib is not available");
invokeELFT(parseCompressedHeader);
@@ -122,7 +122,7 @@
uncompressedBuf = bAlloc().Allocate<char>(size);
}
- if (Error e = zlib::uncompress(toStringRef(rawData), uncompressedBuf, size))
+ if (Error e = compression::elf::uncompress(toStringRef(rawData), uncompressedBuf, size))
fatal(toString(this) +
": uncompress failed: " + llvm::toString(std::move(e)));
rawData = makeArrayRef((uint8_t *)uncompressedBuf, size);
@@ -1213,7 +1213,7 @@
// to the buffer.
if (uncompressedSize >= 0) {
size_t size = uncompressedSize;
- if (Error e = zlib::uncompress(toStringRef(rawData), (char *)buf, size))
+ if (Error e = compression::elf::uncompress(toStringRef(rawData), (char *)buf, size))
fatal(toString(this) +
": uncompress failed: " + llvm::toString(std::move(e)));
uint8_t *bufEnd = buf + size;
Index: lld/ELF/Driver.cpp
===================================================================
--- lld/ELF/Driver.cpp
+++ lld/ELF/Driver.cpp
@@ -953,7 +953,7 @@
return false;
if (s != "zlib")
error("unknown --compress-debug-sections value: " + s);
- if (!zlib::isAvailable())
+ if (!compression::elf::isAvailable())
error("--compress-debug-sections: zlib is not available");
return true;
}
Index: lld/ELF/CMakeLists.txt
===================================================================
--- lld/ELF/CMakeLists.txt
+++ lld/ELF/CMakeLists.txt
@@ -3,7 +3,11 @@
add_public_tablegen_target(ELFOptionsTableGen)
if(LLVM_ENABLE_ZLIB)
- set(imported_libs ZLIB::ZLIB)
+ list(APPEND imported_libs ZLIB::ZLIB)
+endif()
+
+if(LLVM_ENABLE_ZSTD)
+ list(APPEND imported_libs zstd)
endif()
add_lld_library(lldELF
Index: flang/CMakeLists.txt
===================================================================
--- flang/CMakeLists.txt
+++ flang/CMakeLists.txt
@@ -70,6 +70,10 @@
if(LLVM_ENABLE_ZLIB)
find_package(ZLIB REQUIRED)
endif()
+ # If LLVM links to zstd we need the imported targets so we can too.
+ if(LLVM_ENABLE_ZSTD)
+ find_package(ZSTD REQUIRED)
+ endif()
option(LLVM_ENABLE_PEDANTIC "Compile with pedantic enabled." ON)
if(CMAKE_COMPILER_IS_GNUCXX)
set(USE_NO_MAYBE_UNINITIALIZED 1)
Index: compiler-rt/test/lit.common.configured.in
===================================================================
--- compiler-rt/test/lit.common.configured.in
+++ compiler-rt/test/lit.common.configured.in
@@ -65,6 +65,7 @@
set_default("target_suffix", "-%s" % config.target_arch)
set_default("have_zlib", "@LLVM_ENABLE_ZLIB@")
+set_default("have_zstd", "@LLVM_ENABLE_ZSTD@")
set_default("libcxx_used", "@LLVM_LIBCXX_USED@")
# LLVM tools dir can be passed in lit parameters, so try to
Index: compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
===================================================================
--- compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
+++ compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
@@ -138,6 +138,7 @@
-DCMAKE_CXX_FLAGS_RELEASE="${LLVM_FLAGS}" \
-DLLVM_TABLEGEN=$TBLGEN \
-DLLVM_ENABLE_ZLIB=ON \
+ -DLLVM_ENABLE_ZSTD=ON \
-DLLVM_ENABLE_TERMINFO=OFF \
-DLLVM_ENABLE_THREADS=OFF \
$LLVM_SRC
Index: clang/test/lit.site.cfg.py.in
===================================================================
--- clang/test/lit.site.cfg.py.in
+++ clang/test/lit.site.cfg.py.in
@@ -21,6 +21,7 @@
config.host_cxx = "@CMAKE_CXX_COMPILER@"
config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
config.clang_arcmt = @CLANG_ENABLE_ARCMT@
config.clang_default_pie_on_linux = @CLANG_DEFAULT_PIE_ON_LINUX@
config.clang_enable_opaque_pointers = @CLANG_ENABLE_OPAQUE_POINTERS_INTERNAL@
Index: clang/test/CMakeLists.txt
===================================================================
--- clang/test/CMakeLists.txt
+++ clang/test/CMakeLists.txt
@@ -11,6 +11,7 @@
CLANG_SPAWN_CC1
ENABLE_BACKTRACES
LLVM_ENABLE_ZLIB
+ LLVM_ENABLE_ZSTD
LLVM_ENABLE_PER_TARGET_RUNTIME_DIR
LLVM_ENABLE_THREADS
LLVM_WITH_Z3
Index: clang/lib/Serialization/ASTWriter.cpp
===================================================================
--- clang/lib/Serialization/ASTWriter.cpp
+++ clang/lib/Serialization/ASTWriter.cpp
@@ -2001,8 +2001,8 @@
// Compress the buffer if possible. We expect that almost all PCM
// consumers will not want its contents.
SmallString<0> CompressedBuffer;
- if (llvm::zlib::isAvailable()) {
- llvm::zlib::compress(Blob.drop_back(1), CompressedBuffer);
+ if (llvm::compression::serialize::isAvailable()) {
+ llvm::compression::serialize::compress(Blob.drop_back(1), CompressedBuffer);
RecordDataType Record[] = {SM_SLOC_BUFFER_BLOB_COMPRESSED, Blob.size() - 1};
Stream.EmitRecordWithBlob(SLocBufferBlobCompressedAbbrv, Record,
CompressedBuffer);
Index: clang/lib/Serialization/ASTReader.cpp
===================================================================
--- clang/lib/Serialization/ASTReader.cpp
+++ clang/lib/Serialization/ASTReader.cpp
@@ -1462,13 +1462,13 @@
unsigned RecCode = MaybeRecCode.get();
if (RecCode == SM_SLOC_BUFFER_BLOB_COMPRESSED) {
- if (!llvm::zlib::isAvailable()) {
+ if (!llvm::compression::serialize::isAvailable()) {
Error("zlib is not available");
return nullptr;
}
SmallString<0> Uncompressed;
if (llvm::Error E =
- llvm::zlib::uncompress(Blob, Uncompressed, Record[0])) {
+ llvm::compression::serialize::uncompress(Blob, Uncompressed, Record[0])) {
Error("could not decompress embedded file contents: " +
llvm::toString(std::move(E)));
return nullptr;
Index: clang/lib/Driver/ToolChains/Clang.cpp
===================================================================
--- clang/lib/Driver/ToolChains/Clang.cpp
+++ clang/lib/Driver/ToolChains/Clang.cpp
@@ -1144,7 +1144,7 @@
if (Value == "none") {
CmdArgs.push_back("--compress-debug-sections=none");
} else if (Value == "zlib") {
- if (llvm::zlib::isAvailable()) {
+ if (llvm::compression::elf::isAvailable()) {
CmdArgs.push_back(
Args.MakeArgString("--compress-debug-sections=" + Twine(Value)));
} else {
Index: clang/cmake/caches/Fuchsia.cmake
===================================================================
--- clang/cmake/caches/Fuchsia.cmake
+++ clang/cmake/caches/Fuchsia.cmake
@@ -14,6 +14,7 @@
set(LLVM_ENABLE_UNWIND_TABLES OFF CACHE BOOL "")
set(LLVM_ENABLE_Z3_SOLVER OFF CACHE BOOL "")
set(LLVM_ENABLE_ZLIB OFF CACHE BOOL "")
+set(LLVM_ENABLE_ZSTD OFF CACHE BOOL "")
set(LLVM_INCLUDE_DOCS OFF CACHE BOOL "")
set(LLVM_INCLUDE_EXAMPLES OFF CACHE BOOL "")
set(LLVM_INCLUDE_GO_TESTS OFF CACHE BOOL "")
Index: clang/cmake/caches/Fuchsia-stage2.cmake
===================================================================
--- clang/cmake/caches/Fuchsia-stage2.cmake
+++ clang/cmake/caches/Fuchsia-stage2.cmake
@@ -18,6 +18,7 @@
set(LLVM_ENABLE_UNWIND_TABLES OFF CACHE BOOL "")
set(LLVM_ENABLE_Z3_SOLVER OFF CACHE BOOL "")
set(LLVM_ENABLE_ZLIB ON CACHE BOOL "")
+set(LLVM_ENABLE_ZSTD ON CACHE BOOL "")
set(LLVM_INCLUDE_DOCS OFF CACHE BOOL "")
set(LLVM_INCLUDE_EXAMPLES OFF CACHE BOOL "")
set(LLVM_INCLUDE_GO_TESTS OFF CACHE BOOL "")
Index: clang/cmake/caches/Apple-stage2.cmake
===================================================================
--- clang/cmake/caches/Apple-stage2.cmake
+++ clang/cmake/caches/Apple-stage2.cmake
@@ -12,6 +12,7 @@
set(CLANG_LINKS_TO_CREATE clang++ cc c++ CACHE STRING "")
set(CMAKE_MACOSX_RPATH ON CACHE BOOL "")
set(LLVM_ENABLE_ZLIB ON CACHE BOOL "")
+set(LLVM_ENABLE_ZSTD ON CACHE BOOL "")
set(LLVM_ENABLE_BACKTRACES OFF CACHE BOOL "")
set(LLVM_ENABLE_MODULES ON CACHE BOOL "")
set(LLVM_EXTERNALIZE_DEBUGINFO ON CACHE BOOL "")
Index: clang/cmake/caches/Apple-stage1.cmake
===================================================================
--- clang/cmake/caches/Apple-stage1.cmake
+++ clang/cmake/caches/Apple-stage1.cmake
@@ -18,6 +18,7 @@
set(COMPILER_RT_BUILD_SANITIZERS OFF CACHE BOOL "")
set(CMAKE_MACOSX_RPATH ON CACHE BOOL "")
set(LLVM_ENABLE_ZLIB OFF CACHE BOOL "")
+set(LLVM_ENABLE_ZSTD OFF CACHE BOOL "")
set(LLVM_ENABLE_BACKTRACES OFF CACHE BOOL "")
set(CLANG_PLUGIN_SUPPORT OFF CACHE BOOL "")
set(CLANG_SPAWN_CC1 ON CACHE BOOL "")
Index: clang-tools-extra/clangd/unittests/SerializationTests.cpp
===================================================================
--- clang-tools-extra/clangd/unittests/SerializationTests.cpp
+++ clang-tools-extra/clangd/unittests/SerializationTests.cpp
@@ -391,8 +391,8 @@
// Check we detect invalid string table size size without allocating it first.
// If this detection fails, the test should allocate a huge array and crash.
TEST(SerializationTest, NoCrashOnBadStringTableSize) {
- if (!llvm::zlib::isAvailable()) {
- log("skipping test, no zlib");
+ if (!llvm::compression::serialize::isAvailable()) {
+ log("skipping test, no llvm::compression");
return;
}
Index: clang-tools-extra/clangd/test/lit.site.cfg.py.in
===================================================================
--- clang-tools-extra/clangd/test/lit.site.cfg.py.in
+++ clang-tools-extra/clangd/test/lit.site.cfg.py.in
@@ -17,6 +17,7 @@
config.clangd_enable_remote = @CLANGD_ENABLE_REMOTE@
config.clangd_tidy_checks = @CLANGD_TIDY_CHECKS@
config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
# Delegate logic to lit.cfg.py.
lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg.py")
Index: clang-tools-extra/clangd/index/Serialization.cpp
===================================================================
--- clang-tools-extra/clangd/index/Serialization.cpp
+++ clang-tools-extra/clangd/index/Serialization.cpp
@@ -190,9 +190,9 @@
RawTable.append(std::string(S));
RawTable.push_back(0);
}
- if (llvm::zlib::isAvailable()) {
+ if (llvm::compression::serialize::isAvailable()) {
llvm::SmallString<1> Compressed;
- llvm::zlib::compress(RawTable, Compressed);
+ llvm::compression::serialize::compress(RawTable, Compressed);
write32(RawTable.size(), OS);
OS << Compressed;
} else {
@@ -223,7 +223,7 @@
llvm::SmallString<1> UncompressedStorage;
if (UncompressedSize == 0) // No compression
Uncompressed = R.rest();
- else if (llvm::zlib::isAvailable()) {
+ else if (llvm::compression::serialize::isAvailable()) {
// Don't allocate a massive buffer if UncompressedSize was corrupted
// This is effective for sharded index, but not big monolithic ones, as
// once compressed size reaches 4MB nothing can be ruled out.
@@ -233,7 +233,7 @@
return error("Bad stri table: uncompress {0} -> {1} bytes is implausible",
R.rest().size(), UncompressedSize);
- if (llvm::Error E = llvm::zlib::uncompress(R.rest(), UncompressedStorage,
+ if (llvm::Error E = llvm::compression::serialize::uncompress(R.rest(), UncompressedStorage,
UncompressedSize))
return std::move(E);
Uncompressed = UncompressedStorage;
Index: clang-tools-extra/clangd/CMakeLists.txt
===================================================================
--- clang-tools-extra/clangd/CMakeLists.txt
+++ clang-tools-extra/clangd/CMakeLists.txt
@@ -29,6 +29,7 @@
CLANGD_MALLOC_TRIM
CLANGD_TIDY_CHECKS
LLVM_ENABLE_ZLIB
+ LLVM_ENABLE_ZSTD
)
configure_file(
_______________________________________________
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
