[PATCH] D128465: Zstandard as a second compression method to LLVM

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 440288.
ckissane added a comment.

Fix missing cmake findZstd, remove unused crc32 from compression namespaces and 
simplify some code


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/cmake/caches/Apple-stage1.cmake
  clang/cmake/caches/Apple-stage2.cmake
  clang/cmake/caches/Fuchsia-stage2.cmake
  clang/cmake/caches/Fuchsia.cmake
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -92,6 +92,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -11,13 +11,16 @@
 //===--===//
 
 #include "llvm/Support/Compression.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Config/config.h"
+#include "llvm/Support/CRC.h"
 #include "llvm/Support/Error.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
+using namespace compression;
 
 namespace {
 
@@ -63,9 +66,54 @@
 }
 
 TEST(CompressionTest, ZlibCRC32) {
+  const char* StringString = strdup("The quick brown fox jumps over the lazy dog");
+  const ArrayRef StringByteArray = ArrayRef(*StringString);
   EXPECT_EQ(
   0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
+  llvm::crc32(StringByteArray));
+}
+
+#endif
+
+#if LLVM_ENABLE_ZSTD
+
+void TestZstdCompression(StringRef Input, int Level) {
+  SmallString<32> Compressed;
+  SmallString<32> Uncompressed;
+
+  zstd::compress(Input, Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, Uncompressed);
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("zstd error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  TestZstdCompression("", zstd::DefaultCompression);
+
+  TestZstdCompression("hello, world!", zstd::NoCompression);
+  TestZstdCompression("hello, world!", zstd::BestSizeCompression);
+  TestZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  TestZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i) {
+BinaryData[i] = i & 255;
+  }
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  TestZstdCompression(BinaryDataStr, zstd::NoCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  TestZstdCompression(BinaryDataStr, zstd::DefaultCompression);
 }
 
 #endif
Index: llvm/unittests/Profile

[PATCH] D128667: [WIP] Add Zstd ELF support

[Cole Kissane via Phabricator via cfe-commits]

ckissane created this revision.
Herald added subscribers: hiraditya, arichardson, emaste.
Herald added a reviewer: alexander-shaposhnikov.
Herald added a reviewer: rupprecht.
Herald added a reviewer: jhenderson.
Herald added a reviewer: MaskRay.
Herald added a project: All.
ckissane requested review of this revision.
Herald added subscribers: llvm-commits, cfe-commits, StephenFan.
Herald added projects: clang, LLVM.

Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D128667

Files:
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Driver/ToolChains/CommonArgs.cpp
  clang/lib/Driver/ToolChains/Gnu.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  lld/ELF/InputSection.h
  lld/ELF/OutputSections.h
  llvm/include/llvm/BinaryFormat/ELF.h
  llvm/include/llvm/MC/MCTargetOptions.h
  llvm/include/llvm/Object/Decompressor.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.h
  llvm/lib/Object/Decompressor.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp

Index: llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
===
--- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -731,6 +731,7 @@
   InputArgs.getLastArgValue(OBJCOPY_compress_debug_sections_eq))
   .Case("zlib-gnu", DebugCompressionType::GNU)
   .Case("zlib", DebugCompressionType::Z)
+  .Case("zstd", DebugCompressionType::Zstd)
   .Default(DebugCompressionType::None);
   if (Config.CompressionType == DebugCompressionType::None)
 return createStringError(
@@ -740,10 +741,17 @@
 .str()
 .c_str());
 }
-if (!compression::elf::isAvailable())
-  return createStringError(
-  errc::invalid_argument,
-  "LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
+if(Config.CompressionType == DebugCompressionType::Zstd) {
+  if (!compression::zstd::isAvailable())
+return createStringError(
+errc::invalid_argument,
+"LLVM was not compiled with LLVM_ENABLE_ZSTD: can not compress");
+} else {
+  if (!compression::zlib::isAvailable())
+return createStringError(
+errc::invalid_argument,
+"LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
+}
   }
 
   Config.AddGnuDebugLink = InputArgs.getLastArgValue(OBJCOPY_add_gnu_debuglink);
@@ -998,11 +1006,17 @@
 "cannot specify both --compress-debug-sections and "
 "--decompress-debug-sections");
   }
-
-  if (Config.DecompressDebugSections && !compression::elf::isAvailable())
+if(Config.CompressionType != DebugCompressionType::Zstd) {
+  if (Config.DecompressDebugSections && !compression::zlib::isAvailable())
 return createStringError(
 errc::invalid_argument,
 "LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress");
+}else{
+  if (Config.DecompressDebugSections && !compression::zstd::isAvailable())
+return createStringError(
+errc::invalid_argument,
+"LLVM was not compiled with LLVM_ENABLE_ZSTD: cannot decompress");
+}
 
   if (Config.ExtractPartition && Config.ExtractMainPartition)
 return createStringError(errc::invalid_argument,
Index: llvm/lib/Object/Decompressor.cpp
===
--- llvm/lib/Object/Decompressor.cpp
+++ llvm/lib/Object/Decompressor.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/Object/Decompressor.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/MC/MCTargetOptions.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/DataExtractor.h"
@@ -19,19 +20,43 @@
 
 Expected Decompressor::create(StringRef Name, StringRef Data,
 bool IsLE, bool Is64Bit) {
-  if (!compression::elf::isAvailable())
+  DebugCompressionType CType = DebugCompressionType::Z;
+  if(isGnuStyle(Name)){
+CType = DebugCompressionType::GNU;
+  }else{
+if(Is64Bit){
+  auto *chdr = reinterpret_cast(Data.bytes_begin());
+  if(chdr->ch_type == ELF::ELFCOMPRESS_ZLIB){
+CType=DebugCompressionType::Z;
+  } else if(chdr->ch_type == ELF::ELFCOMPRESS_ZSTD){
+CType = DebugCompressionType::Zstd;
+  } else {
+return createError("unknown elf compress type in compressed section header");
+  }
+} else {
+  auto *chdr = reinterpret_cast(Data.bytes_begin());
+  if(chdr->ch_type == ELF::ELFCOMPRESS_ZLIB){
+CType = DebugCompressionType::Z;
+  } else if(chdr->ch_type == ELF::ELFCOMPRESS_ZSTD){
+CType = DebugCompressionType::Zstd;
+  } else {
+return createError("unknown elf compress type in compressed section header");
+  }
+}
+  }
+ 
+  if ((CType == DebugCompressionType::Z || CType==DebugCompressionType::GNU) && !co

[PATCH] D128667: [WIP] Add Zstd ELF support

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 440296.
ckissane added a comment.

simplifed codeflow


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128667/new/

https://reviews.llvm.org/D128667

Files:
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Driver/ToolChains/CommonArgs.cpp
  clang/lib/Driver/ToolChains/Gnu.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  lld/ELF/InputSection.h
  lld/ELF/OutputSections.h
  llvm/include/llvm/BinaryFormat/ELF.h
  llvm/include/llvm/MC/MCTargetOptions.h
  llvm/include/llvm/Object/Decompressor.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.h
  llvm/lib/Object/Decompressor.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp

Index: llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
===
--- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -731,6 +731,7 @@
   InputArgs.getLastArgValue(OBJCOPY_compress_debug_sections_eq))
   .Case("zlib-gnu", DebugCompressionType::GNU)
   .Case("zlib", DebugCompressionType::Z)
+  .Case("zstd", DebugCompressionType::Zstd)
   .Default(DebugCompressionType::None);
   if (Config.CompressionType == DebugCompressionType::None)
 return createStringError(
@@ -740,10 +741,17 @@
 .str()
 .c_str());
 }
-if (!compression::elf::isAvailable())
-  return createStringError(
-  errc::invalid_argument,
-  "LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
+if(Config.CompressionType == DebugCompressionType::Zstd) {
+  if (!compression::zstd::isAvailable())
+return createStringError(
+errc::invalid_argument,
+"LLVM was not compiled with LLVM_ENABLE_ZSTD: can not compress");
+} else {
+  if (!compression::zlib::isAvailable())
+return createStringError(
+errc::invalid_argument,
+"LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
+}
   }
 
   Config.AddGnuDebugLink = InputArgs.getLastArgValue(OBJCOPY_add_gnu_debuglink);
@@ -998,11 +1006,17 @@
 "cannot specify both --compress-debug-sections and "
 "--decompress-debug-sections");
   }
-
-  if (Config.DecompressDebugSections && !compression::elf::isAvailable())
+if(Config.CompressionType != DebugCompressionType::Zstd) {
+  if (Config.DecompressDebugSections && !compression::zlib::isAvailable())
 return createStringError(
 errc::invalid_argument,
 "LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress");
+}else{
+  if (Config.DecompressDebugSections && !compression::zstd::isAvailable())
+return createStringError(
+errc::invalid_argument,
+"LLVM was not compiled with LLVM_ENABLE_ZSTD: cannot decompress");
+}
 
   if (Config.ExtractPartition && Config.ExtractMainPartition)
 return createStringError(errc::invalid_argument,
Index: llvm/lib/Object/Decompressor.cpp
===
--- llvm/lib/Object/Decompressor.cpp
+++ llvm/lib/Object/Decompressor.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/Object/Decompressor.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/MC/MCTargetOptions.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/DataExtractor.h"
@@ -19,19 +20,43 @@
 
 Expected Decompressor::create(StringRef Name, StringRef Data,
 bool IsLE, bool Is64Bit) {
-  if (!compression::elf::isAvailable())
+  DebugCompressionType CType = DebugCompressionType::Z;
+  if(isGnuStyle(Name)){
+CType = DebugCompressionType::GNU;
+  }else{
+if(Is64Bit){
+  auto *chdr = reinterpret_cast(Data.bytes_begin());
+  if(chdr->ch_type == ELF::ELFCOMPRESS_ZLIB){
+CType=DebugCompressionType::Z;
+  } else if(chdr->ch_type == ELF::ELFCOMPRESS_ZSTD){
+CType = DebugCompressionType::Zstd;
+  } else {
+return createError("unknown elf compress type in compressed section header");
+  }
+} else {
+  auto *chdr = reinterpret_cast(Data.bytes_begin());
+  if(chdr->ch_type == ELF::ELFCOMPRESS_ZLIB){
+CType = DebugCompressionType::Z;
+  } else if(chdr->ch_type == ELF::ELFCOMPRESS_ZSTD){
+CType = DebugCompressionType::Zstd;
+  } else {
+return createError("unknown elf compress type in compressed section header");
+  }
+}
+  }
+ 
+  if ((CType == DebugCompressionType::Z || CType==DebugCompressionType::GNU) && !compression::zlib::isAvailable())
 return createError("zlib is not available");
-
-  Decompressor D(Data);
-  Error Err = isGnuStyle(Name) ? D.consumeCompressedGnuHeader()
-   : D.consumeCompressedZLibHeader(Is64Bit, IsLE);
+  De

[PATCH] D128465: Zstandard as a second compression method to LLVM

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 440298.
ckissane added a comment.

fix type for FindZstd cmake usage


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/cmake/caches/Apple-stage1.cmake
  clang/cmake/caches/Apple-stage2.cmake
  clang/cmake/caches/Fuchsia-stage2.cmake
  clang/cmake/caches/Fuchsia.cmake
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -92,6 +92,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -11,13 +11,16 @@
 //===--===//
 
 #include "llvm/Support/Compression.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Config/config.h"
+#include "llvm/Support/CRC.h"
 #include "llvm/Support/Error.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
+using namespace compression;
 
 namespace {
 
@@ -63,9 +66,54 @@
 }
 
 TEST(CompressionTest, ZlibCRC32) {
+  const char* StringString = strdup("The quick brown fox jumps over the lazy dog");
+  const ArrayRef StringByteArray = ArrayRef(*StringString);
   EXPECT_EQ(
   0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
+  llvm::crc32(StringByteArray));
+}
+
+#endif
+
+#if LLVM_ENABLE_ZSTD
+
+void TestZstdCompression(StringRef Input, int Level) {
+  SmallString<32> Compressed;
+  SmallString<32> Uncompressed;
+
+  zstd::compress(Input, Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, Uncompressed);
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("zstd error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  TestZstdCompression("", zstd::DefaultCompression);
+
+  TestZstdCompression("hello, world!", zstd::NoCompression);
+  TestZstdCompression("hello, world!", zstd::BestSizeCompression);
+  TestZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  TestZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i) {
+BinaryData[i] = i & 255;
+  }
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  TestZstdCompression(BinaryDataStr, zstd::NoCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  TestZstdCompression(BinaryDataStr, zstd::DefaultCompression);
 }
 
 #endif
Index: llvm/unittests/ProfileData/InstrProfTest.cpp
===

[PATCH] D128667: [WIP] Add Zstd ELF support

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 440321.
ckissane added a comment.

remove extra struct member


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128667/new/

https://reviews.llvm.org/D128667

Files:
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Driver/ToolChains/CommonArgs.cpp
  clang/lib/Driver/ToolChains/Gnu.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  lld/ELF/InputSection.h
  lld/ELF/OutputSections.h
  llvm/include/llvm/BinaryFormat/ELF.h
  llvm/include/llvm/MC/MCTargetOptions.h
  llvm/include/llvm/Object/Decompressor.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.h
  llvm/lib/Object/Decompressor.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp

Index: llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
===
--- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -731,6 +731,7 @@
   InputArgs.getLastArgValue(OBJCOPY_compress_debug_sections_eq))
   .Case("zlib-gnu", DebugCompressionType::GNU)
   .Case("zlib", DebugCompressionType::Z)
+  .Case("zstd", DebugCompressionType::Zstd)
   .Default(DebugCompressionType::None);
   if (Config.CompressionType == DebugCompressionType::None)
 return createStringError(
@@ -740,10 +741,17 @@
 .str()
 .c_str());
 }
-if (!compression::elf::isAvailable())
-  return createStringError(
-  errc::invalid_argument,
-  "LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
+if(Config.CompressionType == DebugCompressionType::Zstd) {
+  if (!compression::zstd::isAvailable())
+return createStringError(
+errc::invalid_argument,
+"LLVM was not compiled with LLVM_ENABLE_ZSTD: can not compress");
+} else {
+  if (!compression::zlib::isAvailable())
+return createStringError(
+errc::invalid_argument,
+"LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
+}
   }
 
   Config.AddGnuDebugLink = InputArgs.getLastArgValue(OBJCOPY_add_gnu_debuglink);
@@ -998,11 +1006,17 @@
 "cannot specify both --compress-debug-sections and "
 "--decompress-debug-sections");
   }
-
-  if (Config.DecompressDebugSections && !compression::elf::isAvailable())
+if(Config.CompressionType != DebugCompressionType::Zstd) {
+  if (Config.DecompressDebugSections && !compression::zlib::isAvailable())
 return createStringError(
 errc::invalid_argument,
 "LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress");
+}else{
+  if (Config.DecompressDebugSections && !compression::zstd::isAvailable())
+return createStringError(
+errc::invalid_argument,
+"LLVM was not compiled with LLVM_ENABLE_ZSTD: cannot decompress");
+}
 
   if (Config.ExtractPartition && Config.ExtractMainPartition)
 return createStringError(errc::invalid_argument,
Index: llvm/lib/Object/Decompressor.cpp
===
--- llvm/lib/Object/Decompressor.cpp
+++ llvm/lib/Object/Decompressor.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/Object/Decompressor.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/MC/MCTargetOptions.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/DataExtractor.h"
@@ -19,19 +20,43 @@
 
 Expected Decompressor::create(StringRef Name, StringRef Data,
 bool IsLE, bool Is64Bit) {
-  if (!compression::elf::isAvailable())
+  DebugCompressionType CType = DebugCompressionType::Z;
+  if(isGnuStyle(Name)){
+CType = DebugCompressionType::GNU;
+  }else{
+if(Is64Bit){
+  auto *chdr = reinterpret_cast(Data.bytes_begin());
+  if(chdr->ch_type == ELF::ELFCOMPRESS_ZLIB){
+CType=DebugCompressionType::Z;
+  } else if(chdr->ch_type == ELF::ELFCOMPRESS_ZSTD){
+CType = DebugCompressionType::Zstd;
+  } else {
+return createError("unknown elf compress type in compressed section header");
+  }
+} else {
+  auto *chdr = reinterpret_cast(Data.bytes_begin());
+  if(chdr->ch_type == ELF::ELFCOMPRESS_ZLIB){
+CType = DebugCompressionType::Z;
+  } else if(chdr->ch_type == ELF::ELFCOMPRESS_ZSTD){
+CType = DebugCompressionType::Zstd;
+  } else {
+return createError("unknown elf compress type in compressed section header");
+  }
+}
+  }
+ 
+  if ((CType == DebugCompressionType::Z || CType==DebugCompressionType::GNU) && !compression::zlib::isAvailable())
 return createError("zlib is not available");
-
-  Decompressor D(Data);
-  Error Err = isGnuStyle(Name) ? D.consumeCompressedGnuHeader()
-   : D.consumeCompressedZLibHeader(Is64Bit, IsLE

[PATCH] D128667: [WIP] Add Zstd ELF support

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 440323.
ckissane added a comment.

add more diff context


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128667/new/

https://reviews.llvm.org/D128667

Files:
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Driver/ToolChains/CommonArgs.cpp
  clang/lib/Driver/ToolChains/Gnu.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  lld/ELF/InputSection.h
  lld/ELF/OutputSections.h
  llvm/include/llvm/BinaryFormat/ELF.h
  llvm/include/llvm/MC/MCTargetOptions.h
  llvm/include/llvm/Object/Decompressor.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.h
  llvm/lib/Object/Decompressor.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp

Index: llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
===
--- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -731,6 +731,7 @@
   InputArgs.getLastArgValue(OBJCOPY_compress_debug_sections_eq))
   .Case("zlib-gnu", DebugCompressionType::GNU)
   .Case("zlib", DebugCompressionType::Z)
+  .Case("zstd", DebugCompressionType::Zstd)
   .Default(DebugCompressionType::None);
   if (Config.CompressionType == DebugCompressionType::None)
 return createStringError(
@@ -740,10 +741,17 @@
 .str()
 .c_str());
 }
-if (!compression::elf::isAvailable())
-  return createStringError(
-  errc::invalid_argument,
-  "LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
+if(Config.CompressionType == DebugCompressionType::Zstd) {
+  if (!compression::zstd::isAvailable())
+return createStringError(
+errc::invalid_argument,
+"LLVM was not compiled with LLVM_ENABLE_ZSTD: can not compress");
+} else {
+  if (!compression::zlib::isAvailable())
+return createStringError(
+errc::invalid_argument,
+"LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
+}
   }
 
   Config.AddGnuDebugLink = InputArgs.getLastArgValue(OBJCOPY_add_gnu_debuglink);
@@ -998,11 +1006,17 @@
 "cannot specify both --compress-debug-sections and "
 "--decompress-debug-sections");
   }
-
-  if (Config.DecompressDebugSections && !compression::elf::isAvailable())
+if(Config.CompressionType != DebugCompressionType::Zstd) {
+  if (Config.DecompressDebugSections && !compression::zlib::isAvailable())
 return createStringError(
 errc::invalid_argument,
 "LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress");
+}else{
+  if (Config.DecompressDebugSections && !compression::zstd::isAvailable())
+return createStringError(
+errc::invalid_argument,
+"LLVM was not compiled with LLVM_ENABLE_ZSTD: cannot decompress");
+}
 
   if (Config.ExtractPartition && Config.ExtractMainPartition)
 return createStringError(errc::invalid_argument,
Index: llvm/lib/Object/Decompressor.cpp
===
--- llvm/lib/Object/Decompressor.cpp
+++ llvm/lib/Object/Decompressor.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/Object/Decompressor.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/MC/MCTargetOptions.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/DataExtractor.h"
@@ -19,19 +20,43 @@
 
 Expected Decompressor::create(StringRef Name, StringRef Data,
 bool IsLE, bool Is64Bit) {
-  if (!compression::elf::isAvailable())
+  DebugCompressionType CType = DebugCompressionType::Z;
+  if(isGnuStyle(Name)){
+CType = DebugCompressionType::GNU;
+  }else{
+if(Is64Bit){
+  auto *chdr = reinterpret_cast(Data.bytes_begin());
+  if(chdr->ch_type == ELF::ELFCOMPRESS_ZLIB){
+CType=DebugCompressionType::Z;
+  } else if(chdr->ch_type == ELF::ELFCOMPRESS_ZSTD){
+CType = DebugCompressionType::Zstd;
+  } else {
+return createError("unknown elf compress type in compressed section header");
+  }
+} else {
+  auto *chdr = reinterpret_cast(Data.bytes_begin());
+  if(chdr->ch_type == ELF::ELFCOMPRESS_ZLIB){
+CType = DebugCompressionType::Z;
+  } else if(chdr->ch_type == ELF::ELFCOMPRESS_ZSTD){
+CType = DebugCompressionType::Zstd;
+  } else {
+return createError("unknown elf compress type in compressed section header");
+  }
+}
+  }
+ 
+  if ((CType == DebugCompressionType::Z || CType==DebugCompressionType::GNU) && !compression::zlib::isAvailable())
 return createError("zlib is not available");
-
-  Decompressor D(Data);
-  Error Err = isGnuStyle(Name) ? D.consumeCompressedGnuHeader()
-   : D.consumeCompressedZLibHeader(Is64Bit, IsLE);
+  Decompressor D(Data, CType);
+  Error 

[PATCH] D128667: [WIP] Add Zstd ELF support

[Cole Kissane via Phabricator via cfe-commits]

ckissane added a comment.

In D128667#3612956 , @MaskRay wrote:

> The lld/ change should be separate. And it needs tests. We should have 
> several tests such as mixed zlib and zstd input, malformed zstd input, 
> compressed level, etc. `OutputSections` should not have a new member.

Got it, I have also removed the extra member.

In D128667#3612956 , @MaskRay wrote:

> The parallel output part will be tricky, too (see D117853 
> ). I can handle the lld/ELF part and leave 
> other stuff to you.

Sounds good, and thank you for your offer to help!

In D128667#3612956 , @MaskRay wrote:

> llvm-objcopy should probably be a separate patch as well.

got it.

> - clang (Driver, cc1as) and MC (including `llvm-mc 
> --compress-debug-sections`) in one patch
> - lld/ELF
> - llvm-objcopy
>
>> Context not available.
>
> See 
> https://llvm.org/docs/Phabricator.html#requesting-a-review-via-the-web-interface
>  "To make reviews easier, please always include as much context as possible 
> with your diff!"

Doing so now


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128667/new/

https://reviews.llvm.org/D128667

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128465: Zstandard as a second compression method to LLVM

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 440325.
ckissane added a comment.

more context


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/cmake/caches/Apple-stage1.cmake
  clang/cmake/caches/Apple-stage2.cmake
  clang/cmake/caches/Fuchsia-stage2.cmake
  clang/cmake/caches/Fuchsia.cmake
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -92,6 +92,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -11,13 +11,16 @@
 //===--===//
 
 #include "llvm/Support/Compression.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Config/config.h"
+#include "llvm/Support/CRC.h"
 #include "llvm/Support/Error.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
+using namespace compression;
 
 namespace {
 
@@ -63,9 +66,54 @@
 }
 
 TEST(CompressionTest, ZlibCRC32) {
+  const char* StringString = strdup("The quick brown fox jumps over the lazy dog");
+  const ArrayRef StringByteArray = ArrayRef(*StringString);
   EXPECT_EQ(
   0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
+  llvm::crc32(StringByteArray));
+}
+
+#endif
+
+#if LLVM_ENABLE_ZSTD
+
+void TestZstdCompression(StringRef Input, int Level) {
+  SmallString<32> Compressed;
+  SmallString<32> Uncompressed;
+
+  zstd::compress(Input, Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, Uncompressed);
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("zstd error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  TestZstdCompression("", zstd::DefaultCompression);
+
+  TestZstdCompression("hello, world!", zstd::NoCompression);
+  TestZstdCompression("hello, world!", zstd::BestSizeCompression);
+  TestZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  TestZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i) {
+BinaryData[i] = i & 255;
+  }
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  TestZstdCompression(BinaryDataStr, zstd::NoCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  TestZstdCompression(BinaryDataStr, zstd::DefaultCompression);
 }
 
 #endif
Index: llvm/unittests/ProfileData/InstrProfTest.cpp
===
--- llvm/unittests/ProfileData/Inst

[PATCH] D128465: Zstandard as a second compression method to LLVM

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 440327.
ckissane added a comment.

format


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/cmake/caches/Apple-stage1.cmake
  clang/cmake/caches/Apple-stage2.cmake
  clang/cmake/caches/Fuchsia-stage2.cmake
  clang/cmake/caches/Fuchsia.cmake
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -92,6 +92,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -11,13 +11,16 @@
 //===--===//
 
 #include "llvm/Support/Compression.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Config/config.h"
+#include "llvm/Support/CRC.h"
 #include "llvm/Support/Error.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
+using namespace compression;
 
 namespace {
 
@@ -63,9 +66,53 @@
 }
 
 TEST(CompressionTest, ZlibCRC32) {
-  EXPECT_EQ(
-  0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
+  const char *StringString =
+  strdup("The quick brown fox jumps over the lazy dog");
+  const ArrayRef StringByteArray = ArrayRef(*StringString);
+  EXPECT_EQ(0x414FA339U, llvm::crc32(StringByteArray));
+}
+
+#endif
+
+#if LLVM_ENABLE_ZSTD
+
+void TestZstdCompression(StringRef Input, int Level) {
+  SmallString<32> Compressed;
+  SmallString<32> Uncompressed;
+
+  zstd::compress(Input, Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, Uncompressed);
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("zstd error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  TestZstdCompression("", zstd::DefaultCompression);
+
+  TestZstdCompression("hello, world!", zstd::NoCompression);
+  TestZstdCompression("hello, world!", zstd::BestSizeCompression);
+  TestZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  TestZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i) {
+BinaryData[i] = i & 255;
+  }
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  TestZstdCompression(BinaryDataStr, zstd::NoCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  TestZstdCompression(BinaryDataStr, zstd::DefaultCompression);
 }
 
 #endif
Index: llvm/unittests/ProfileData/InstrProfTest.cpp
===
--- llvm/unitte

[PATCH] D128667: [WIP] Add Zstd ELF support

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 440330.
ckissane added a comment.

format


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128667/new/

https://reviews.llvm.org/D128667

Files:
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Driver/ToolChains/CommonArgs.cpp
  clang/lib/Driver/ToolChains/Gnu.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  lld/ELF/InputSection.h
  lld/ELF/OutputSections.h
  llvm/include/llvm/BinaryFormat/ELF.h
  llvm/include/llvm/MC/MCTargetOptions.h
  llvm/include/llvm/Object/Decompressor.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.h
  llvm/lib/Object/Decompressor.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp

Index: llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
===
--- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -731,6 +731,7 @@
   InputArgs.getLastArgValue(OBJCOPY_compress_debug_sections_eq))
   .Case("zlib-gnu", DebugCompressionType::GNU)
   .Case("zlib", DebugCompressionType::Z)
+  .Case("zstd", DebugCompressionType::Zstd)
   .Default(DebugCompressionType::None);
   if (Config.CompressionType == DebugCompressionType::None)
 return createStringError(
@@ -740,10 +741,17 @@
 .str()
 .c_str());
 }
-if (!compression::elf::isAvailable())
-  return createStringError(
-  errc::invalid_argument,
-  "LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
+if (Config.CompressionType == DebugCompressionType::Zstd) {
+  if (!compression::zstd::isAvailable())
+return createStringError(
+errc::invalid_argument,
+"LLVM was not compiled with LLVM_ENABLE_ZSTD: can not compress");
+} else {
+  if (!compression::zlib::isAvailable())
+return createStringError(
+errc::invalid_argument,
+"LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
+}
   }
 
   Config.AddGnuDebugLink = InputArgs.getLastArgValue(OBJCOPY_add_gnu_debuglink);
@@ -998,11 +1006,17 @@
 "cannot specify both --compress-debug-sections and "
 "--decompress-debug-sections");
   }
-
-  if (Config.DecompressDebugSections && !compression::elf::isAvailable())
-return createStringError(
-errc::invalid_argument,
-"LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress");
+  if (Config.CompressionType != DebugCompressionType::Zstd) {
+if (Config.DecompressDebugSections && !compression::zlib::isAvailable())
+  return createStringError(
+  errc::invalid_argument,
+  "LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress");
+  } else {
+if (Config.DecompressDebugSections && !compression::zstd::isAvailable())
+  return createStringError(
+  errc::invalid_argument,
+  "LLVM was not compiled with LLVM_ENABLE_ZSTD: cannot decompress");
+  }
 
   if (Config.ExtractPartition && Config.ExtractMainPartition)
 return createStringError(errc::invalid_argument,
Index: llvm/lib/Object/Decompressor.cpp
===
--- llvm/lib/Object/Decompressor.cpp
+++ llvm/lib/Object/Decompressor.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/Object/Decompressor.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/MC/MCTargetOptions.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/DataExtractor.h"
@@ -19,19 +20,52 @@
 
 Expected Decompressor::create(StringRef Name, StringRef Data,
 bool IsLE, bool Is64Bit) {
-  if (!compression::elf::isAvailable())
+  DebugCompressionType CType = DebugCompressionType::Z;
+  if (isGnuStyle(Name)) {
+CType = DebugCompressionType::GNU;
+  } else {
+if (Is64Bit) {
+  auto *chdr =
+  reinterpret_cast(Data.bytes_begin());
+  if (chdr->ch_type == ELF::ELFCOMPRESS_ZLIB) {
+CType = DebugCompressionType::Z;
+  } else if (chdr->ch_type == ELF::ELFCOMPRESS_ZSTD) {
+CType = DebugCompressionType::Zstd;
+  } else {
+return createError(
+"unknown elf compress type in compressed section header");
+  }
+} else {
+  auto *chdr =
+  reinterpret_cast(Data.bytes_begin());
+  if (chdr->ch_type == ELF::ELFCOMPRESS_ZLIB) {
+CType = DebugCompressionType::Z;
+  } else if (chdr->ch_type == ELF::ELFCOMPRESS_ZSTD) {
+CType = DebugCompressionType::Zstd;
+  } else {
+return createError(
+"unknown elf compress type in compressed section header");
+  }
+}
+  }
+
+  if ((CType == DebugCompressionType::Z ||
+   CType == DebugCompressionType::GNU) &&
+  !compression::zlib::isAvailable())
 return createError("zlib is not available");
-
-  Decompressor D(Data

[PATCH] D128667: [WIP] Add Zstd ELF support

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 440343.
ckissane added a comment.

improve style on ELFWriter::maybeWriteCompression
fix typo in comment on zstd value for elf enum


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128667/new/

https://reviews.llvm.org/D128667

Files:
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Driver/ToolChains/CommonArgs.cpp
  clang/lib/Driver/ToolChains/Gnu.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  lld/ELF/InputSection.h
  lld/ELF/OutputSections.h
  llvm/include/llvm/BinaryFormat/ELF.h
  llvm/include/llvm/MC/MCTargetOptions.h
  llvm/include/llvm/Object/Decompressor.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.h
  llvm/lib/Object/Decompressor.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp

Index: llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
===
--- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -731,6 +731,7 @@
   InputArgs.getLastArgValue(OBJCOPY_compress_debug_sections_eq))
   .Case("zlib-gnu", DebugCompressionType::GNU)
   .Case("zlib", DebugCompressionType::Z)
+  .Case("zstd", DebugCompressionType::Zstd)
   .Default(DebugCompressionType::None);
   if (Config.CompressionType == DebugCompressionType::None)
 return createStringError(
@@ -740,10 +741,17 @@
 .str()
 .c_str());
 }
-if (!compression::elf::isAvailable())
-  return createStringError(
-  errc::invalid_argument,
-  "LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
+if (Config.CompressionType == DebugCompressionType::Zstd) {
+  if (!compression::zstd::isAvailable())
+return createStringError(
+errc::invalid_argument,
+"LLVM was not compiled with LLVM_ENABLE_ZSTD: can not compress");
+} else {
+  if (!compression::zlib::isAvailable())
+return createStringError(
+errc::invalid_argument,
+"LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
+}
   }
 
   Config.AddGnuDebugLink = InputArgs.getLastArgValue(OBJCOPY_add_gnu_debuglink);
@@ -998,11 +1006,17 @@
 "cannot specify both --compress-debug-sections and "
 "--decompress-debug-sections");
   }
-
-  if (Config.DecompressDebugSections && !compression::elf::isAvailable())
-return createStringError(
-errc::invalid_argument,
-"LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress");
+  if (Config.CompressionType != DebugCompressionType::Zstd) {
+if (Config.DecompressDebugSections && !compression::zlib::isAvailable())
+  return createStringError(
+  errc::invalid_argument,
+  "LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress");
+  } else {
+if (Config.DecompressDebugSections && !compression::zstd::isAvailable())
+  return createStringError(
+  errc::invalid_argument,
+  "LLVM was not compiled with LLVM_ENABLE_ZSTD: cannot decompress");
+  }
 
   if (Config.ExtractPartition && Config.ExtractMainPartition)
 return createStringError(errc::invalid_argument,
Index: llvm/lib/Object/Decompressor.cpp
===
--- llvm/lib/Object/Decompressor.cpp
+++ llvm/lib/Object/Decompressor.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/Object/Decompressor.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/MC/MCTargetOptions.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/DataExtractor.h"
@@ -19,19 +20,52 @@
 
 Expected Decompressor::create(StringRef Name, StringRef Data,
 bool IsLE, bool Is64Bit) {
-  if (!compression::elf::isAvailable())
+  DebugCompressionType CType = DebugCompressionType::Z;
+  if (isGnuStyle(Name)) {
+CType = DebugCompressionType::GNU;
+  } else {
+if (Is64Bit) {
+  auto *chdr =
+  reinterpret_cast(Data.bytes_begin());
+  if (chdr->ch_type == ELF::ELFCOMPRESS_ZLIB) {
+CType = DebugCompressionType::Z;
+  } else if (chdr->ch_type == ELF::ELFCOMPRESS_ZSTD) {
+CType = DebugCompressionType::Zstd;
+  } else {
+return createError(
+"unknown elf compress type in compressed section header");
+  }
+} else {
+  auto *chdr =
+  reinterpret_cast(Data.bytes_begin());
+  if (chdr->ch_type == ELF::ELFCOMPRESS_ZLIB) {
+CType = DebugCompressionType::Z;
+  } else if (chdr->ch_type == ELF::ELFCOMPRESS_ZSTD) {
+CType = DebugCompressionType::Zstd;
+  } else {
+return createError(
+"unknown elf compress type in compressed section header");
+  }
+}
+  }
+
+  if ((CType == DebugCompressionType::Z ||
+   CType == DebugCompressionType::GNU) &&
+  !compression::zlib:

[PATCH] D128667: [WIP] Add Zstd ELF support

[Cole Kissane via Phabricator via cfe-commits]

ckissane added a comment.

In D128667#3613222 , @MaskRay wrote:

> `llvm-objcopy --decompress-debug-sections clang.zstd clang.zstd.uncompress` 
> doesn't work.
>
>   % /tmp/out/custom1/bin/llvm-objcopy --decompress-debug-sections clang.zstd 
> clang.zstd.uncompress
>   /tmp/out/custom1/bin/llvm-objcopy: error: 'clang.zstd': '.debug_loc': zlib 
> error: Z_DATA_ERROR
>
> Add a new test 
> `llvm/test/tools/llvm-objcopy/compress-debug-sections-zstd.test` and check 
> existing tests for zlib (SHF_COMPRESSED).
> I'll create a patch to remove .zdebug support to simplify the code.
>
> Tip: you can nullify a compression/uncompression place, then run `ninja 
> check-llvm` to find all places in llvm/ where a test may be relevant.

it seems that --decompress-debug-sections assumes always Z type compression. I 
can update the patch to fix this


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128667/new/

https://reviews.llvm.org/D128667

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128754: Refactor LLVM compression namespaces

[Cole Kissane via Phabricator via cfe-commits]

ckissane created this revision.
ckissane added a reviewer: MaskRay.
ckissane added a project: LLVM.
Herald added subscribers: StephenFan, wenlei, usaxena95, kadircet, arphaman, 
hiraditya, arichardson, mgorny, emaste.
Herald added a reviewer: alexander-shaposhnikov.
Herald added a reviewer: rupprecht.
Herald added a reviewer: jhenderson.
Herald added a project: All.
ckissane requested review of this revision.
Herald added projects: clang, clang-tools-extra.
Herald added subscribers: cfe-commits, llvm-commits.

Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D128754

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/CMakeLists.txt
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/docs/ReleaseNotes.rst
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/ProfileData/SampleProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -19,6 +19,8 @@
 
 using namespace llvm;
 
+using namespace compression;
+
 namespace {
 
 #if LLVM_ENABLE_ZLIB
@@ -62,12 +64,6 @@
   TestZlibCompression(BinaryDataStr, zlib::DefaultCompression);
 }
 
-TEST(CompressionTest, ZlibCRC32) {
-  EXPECT_EQ(
-  0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
-}
-
 #endif
 
 }
Index: llvm/unittests/ProfileData/InstrProfTest.cpp
===
--- llvm/unittests/ProfileData/InstrProfTest.cpp
+++ llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -1146,17 +1146,19 @@
   for (bool DoCompression : {false, true}) {
 // Compressing:
 std::string FuncNameStrings1;
-EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames1, (DoCompression && zlib::isAvailable()),
-  FuncNameStrings1),
-  Succeeded());
+EXPECT_THAT_ERROR(
+collectPGOFuncNameStrings(
+FuncNames1, (DoCompression && compression::profile::isAvailable()),
+FuncNameStrings1),
+Succeeded());
 
 // Compressing:
 std::string FuncNameStrings2;
-EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames2, (DoCompression && zlib::isAvailable()),
-  FuncNameStrings2),
-  Succeeded());
+EXPECT_THAT_ERROR(
+collectPGOFuncNameStrings(
+FuncNames2, (DoCompression && compression::profile::isAvailable()),
+FuncNameStrings2),
+Succeeded());
 
 for (int Padding = 0; Padding < 2; Padding++) {
   // Join with paddings :
Index: llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
===
--- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -740,7 +740,7 @@
 .str()
 .c_str());
 }
-if (!zlib::isAvailable())
+if (!compression::zlib::isAvailable())
   return createStringError(
   errc::invalid_argument,
   "LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
@@ -999,7 +999,7 @@
 "--decompress-debug-sections");
   }
 
-  if (Config.DecompressDebugSections && !zlib::isAvailable())
+  if (Config.DecompressDebugSections && !compression::zlib::isAvailable())
 return createStringError(
 errc::invalid_argument,
 "LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress");
Index: llvm/tools/llvm-mc/llvm-mc.cpp
===
--- llvm/tools/llvm-mc/llvm-mc.cpp
+++ llvm/tools/llvm-mc/llvm-mc.cpp
@@ -403,7 +403,7 @@
   MAI->setRelaxELFRelocations(RelaxELFRel);
 
   if (CompressDebugSections != DebugCompressionType::None) {
-if (!zlib::isAvailable()) {
+if (!compression::zlib::isAvailable()) {
   WithColor::error(errs(), ProgName)
   << "build tools with zlib to enable -compress-debug-sections";
   return 1;
Index: llvm/lib/Support/Compression.cpp
=

[PATCH] D128754: Refactor LLVM compression namespaces

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 440736.
ckissane added a comment.

up release notes


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128754/new/

https://reviews.llvm.org/D128754

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/CMakeLists.txt
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/docs/ReleaseNotes.rst
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/ProfileData/SampleProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -19,6 +19,8 @@
 
 using namespace llvm;
 
+using namespace compression;
+
 namespace {
 
 #if LLVM_ENABLE_ZLIB
@@ -62,12 +64,6 @@
   TestZlibCompression(BinaryDataStr, zlib::DefaultCompression);
 }
 
-TEST(CompressionTest, ZlibCRC32) {
-  EXPECT_EQ(
-  0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
-}
-
 #endif
 
 }
Index: llvm/unittests/ProfileData/InstrProfTest.cpp
===
--- llvm/unittests/ProfileData/InstrProfTest.cpp
+++ llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -1146,17 +1146,19 @@
   for (bool DoCompression : {false, true}) {
 // Compressing:
 std::string FuncNameStrings1;
-EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames1, (DoCompression && zlib::isAvailable()),
-  FuncNameStrings1),
-  Succeeded());
+EXPECT_THAT_ERROR(
+collectPGOFuncNameStrings(
+FuncNames1, (DoCompression && compression::profile::isAvailable()),
+FuncNameStrings1),
+Succeeded());
 
 // Compressing:
 std::string FuncNameStrings2;
-EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames2, (DoCompression && zlib::isAvailable()),
-  FuncNameStrings2),
-  Succeeded());
+EXPECT_THAT_ERROR(
+collectPGOFuncNameStrings(
+FuncNames2, (DoCompression && compression::profile::isAvailable()),
+FuncNameStrings2),
+Succeeded());
 
 for (int Padding = 0; Padding < 2; Padding++) {
   // Join with paddings :
Index: llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
===
--- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -740,7 +740,7 @@
 .str()
 .c_str());
 }
-if (!zlib::isAvailable())
+if (!compression::zlib::isAvailable())
   return createStringError(
   errc::invalid_argument,
   "LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
@@ -999,7 +999,7 @@
 "--decompress-debug-sections");
   }
 
-  if (Config.DecompressDebugSections && !zlib::isAvailable())
+  if (Config.DecompressDebugSections && !compression::zlib::isAvailable())
 return createStringError(
 errc::invalid_argument,
 "LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress");
Index: llvm/tools/llvm-mc/llvm-mc.cpp
===
--- llvm/tools/llvm-mc/llvm-mc.cpp
+++ llvm/tools/llvm-mc/llvm-mc.cpp
@@ -403,7 +403,7 @@
   MAI->setRelaxELFRelocations(RelaxELFRel);
 
   if (CompressDebugSections != DebugCompressionType::None) {
-if (!zlib::isAvailable()) {
+if (!compression::zlib::isAvailable()) {
   WithColor::error(errs(), ProgName)
   << "build tools with zlib to enable -compress-debug-sections";
   return 1;
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -23,11 +23,14 @@
 
 using namespace llvm;
 
-#if LLVM_ENABLE_ZLIB
+using namespace compression;
+
 static Error createError(StringRef Err) {
   return make_error(Err, inconvertibleErrorCode());
 }
 
+#if LLVM_ENABLE_ZLIB
+
 static StringRef convertZlibCodeToStrin

[PATCH] D128465: Zstandard as a second compression method to LLVM

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 440747.
ckissane edited the summary of this revision.
ckissane added a comment.

make it the second stage


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/Compression.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -92,6 +92,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,6 +20,9 @@
 #if LLVM_ENABLE_ZLIB
 #include 
 #endif
+#if LLVM_ENABLE_ZSTD
+#include 
+#endif
 
 using namespace llvm;
 
@@ -102,3 +105,63 @@
   llvm_unreachable("zlib::uncompress is unavailable");
 }
 #endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(StringRef InputBuffer,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  unsigned long CompressedBufferSize = ::ZSTD_compressBound(InputBuffer.size());
+  CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+  unsigned long CompressedSize = ::ZSTD_compress(
+  (Bytef *)CompressedBuffer.data(), CompressedBufferSize,
+  (const Bytef *)InputBuffer.data(), InputBuffer.size(), Level);
+  if (ZSTD_isError(CompressedSize))
+report_bad_alloc_error("Allocation failed");
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(CompressedBuffer.data(), CompressedSize);
+  CompressedBuffer.truncate(CompressedSize);
+}
+
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+   size_t &UncompressedSize) {
+  unsigned long long const rSize = ZSTD_getFrameContentSize(
+  (const Bytef *)InputBuffer.data(), InputBuffer.size());
+  size_t const Res =
+  ::ZSTD_decompress((Bytef *)UncompressedBuffer, rSize,
+(const Bytef *)InputBuffer.data(), InputBuffer.size());
+  UncompressedSize = Res;
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(UncompressedBuffer, UncompressedSize);
+  return Res != rSize ? createError(ZSTD_getErrorName(Res)) : Error::success();
+}
+
+Error zstd::uncompress(StringRef InputBuffer,
+   SmallVectorImpl &UncompressedBuffer,
+   size_t UncompressedSize) {
+  UncompressedBuffer.resize_for_overwrite(UncompressedSize);
+  Error E = zstd::uncompress(InputBuffer, UncompressedBuffer.data(),
+ UncompressedSize);
+  UncompressedBuffer.truncate(UncompressedSize);
+  return E;
+}
+
+#else
+bool zstd::isAvailable() { return false; }
+void zstd::compress(StringRef InputBuffer,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  llvm_unreachable("zstd::compress is unavailable");
+}
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+   size_t &UncompressedSize) {
+  llvm_unreachable("zstd::uncompress is unavailable");
+}
+Error zstd::uncompress(StringRef InputBuffer,
+   SmallVectorImpl &UncompressedBuffer,
+   size_t UncompressedSize) {
+  llvm_unreachable("zstd::uncompress is unavailable");
+}
+#endif
\ No newline at end of file
Index: llvm/include/llvm/Support/Compression.h
===
--- llvm/include/llvm/Support/Compression.h
+++ llvm/include/llvm/Support/Compression.h
@@ -43,7 +43,32 @@
 
 } // End of namespace zlib
 
+namespace zstd {
+
+static constexpr int NoCompression = -5;
+static constexpr int BestSpeedCompression = 1;
+static constexpr int DefaultCompression = 5;
+static constexpr int BestSizeCompression = 12;
+
+bool isAvailable();
+
+voi

[PATCH] D128754: Refactor LLVM compression namespaces

[Cole Kissane via Phabricator via cfe-commits]

ckissane marked an inline comment as done.
ckissane added inline comments.



Comment at: llvm/unittests/Support/CompressionTest.cpp:68
-  0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
-}

leonardchan wrote:
> Should this be replaced with `llvm::crc32` rather than deleting this?
No Because llvm::crc32 has it's own tests already


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128754/new/

https://reviews.llvm.org/D128754

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128754: Refactor LLVM compression namespaces

[Cole Kissane via Phabricator via cfe-commits]

ckissane marked an inline comment as done.
ckissane added inline comments.



Comment at: clang-tools-extra/clangd/index/Serialization.cpp:242
+return error(
+"Compressed string table, but compression::serialize is unavailable");
 

leonardchan wrote:
> nit: Could we add some function that returns a string of whatever compression 
> is used? This way we have a more descriptive error message showing what 
> specifically is unavailable. Same comment elsewhere there's logging/error 
> reporting but the string is "compression::serialize".
sure, I can add a AlgorithmName property to the zlib namespace and future 
compression namespaces, then we can print out 
compression::serialize::AlgorithmName to see what is being used


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128754/new/

https://reviews.llvm.org/D128754

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128754: Refactor LLVM compression namespaces

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 441101.
ckissane added a comment.

add AlgorithmName to compression namespaces


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128754/new/

https://reviews.llvm.org/D128754

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/CMakeLists.txt
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/docs/ReleaseNotes.rst
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/ProfileData/SampleProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -19,6 +19,8 @@
 
 using namespace llvm;
 
+using namespace compression;
+
 namespace {
 
 #if LLVM_ENABLE_ZLIB
@@ -62,12 +64,6 @@
   TestZlibCompression(BinaryDataStr, zlib::DefaultCompression);
 }
 
-TEST(CompressionTest, ZlibCRC32) {
-  EXPECT_EQ(
-  0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
-}
-
 #endif
 
 }
Index: llvm/unittests/ProfileData/InstrProfTest.cpp
===
--- llvm/unittests/ProfileData/InstrProfTest.cpp
+++ llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -1146,17 +1146,19 @@
   for (bool DoCompression : {false, true}) {
 // Compressing:
 std::string FuncNameStrings1;
-EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames1, (DoCompression && zlib::isAvailable()),
-  FuncNameStrings1),
-  Succeeded());
+EXPECT_THAT_ERROR(
+collectPGOFuncNameStrings(
+FuncNames1, (DoCompression && compression::profile::isAvailable()),
+FuncNameStrings1),
+Succeeded());
 
 // Compressing:
 std::string FuncNameStrings2;
-EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames2, (DoCompression && zlib::isAvailable()),
-  FuncNameStrings2),
-  Succeeded());
+EXPECT_THAT_ERROR(
+collectPGOFuncNameStrings(
+FuncNames2, (DoCompression && compression::profile::isAvailable()),
+FuncNameStrings2),
+Succeeded());
 
 for (int Padding = 0; Padding < 2; Padding++) {
   // Join with paddings :
Index: llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
===
--- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -740,7 +740,7 @@
 .str()
 .c_str());
 }
-if (!zlib::isAvailable())
+if (!compression::zlib::isAvailable())
   return createStringError(
   errc::invalid_argument,
   "LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
@@ -999,7 +999,7 @@
 "--decompress-debug-sections");
   }
 
-  if (Config.DecompressDebugSections && !zlib::isAvailable())
+  if (Config.DecompressDebugSections && !compression::zlib::isAvailable())
 return createStringError(
 errc::invalid_argument,
 "LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress");
Index: llvm/tools/llvm-mc/llvm-mc.cpp
===
--- llvm/tools/llvm-mc/llvm-mc.cpp
+++ llvm/tools/llvm-mc/llvm-mc.cpp
@@ -403,7 +403,7 @@
   MAI->setRelaxELFRelocations(RelaxELFRel);
 
   if (CompressDebugSections != DebugCompressionType::None) {
-if (!zlib::isAvailable()) {
+if (!compression::zlib::isAvailable()) {
   WithColor::error(errs(), ProgName)
   << "build tools with zlib to enable -compress-debug-sections";
   return 1;
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -23,11 +23,14 @@
 
 using namespace llvm;
 
-#if LLVM_ENABLE_ZLIB
+using namespace compression;
+
 static Error createError(StringRef Err) {
   return make_error(Err, inconvertibleErrorCode());
 }
 
+#if LLVM_ENABLE_ZLIB
+
 static Strin

[PATCH] D128465: Zstandard as a second compression method to LLVM

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 441102.
ckissane added a comment.

add AlgorithmName to compression namespaces


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/Compression.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -92,6 +92,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,6 +20,9 @@
 #if LLVM_ENABLE_ZLIB
 #include 
 #endif
+#if LLVM_ENABLE_ZSTD
+#include 
+#endif
 
 using namespace llvm;
 
@@ -102,3 +105,63 @@
   llvm_unreachable("zlib::uncompress is unavailable");
 }
 #endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(StringRef InputBuffer,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  unsigned long CompressedBufferSize = ::ZSTD_compressBound(InputBuffer.size());
+  CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+  unsigned long CompressedSize = ::ZSTD_compress(
+  (Bytef *)CompressedBuffer.data(), CompressedBufferSize,
+  (const Bytef *)InputBuffer.data(), InputBuffer.size(), Level);
+  if (ZSTD_isError(CompressedSize))
+report_bad_alloc_error("Allocation failed");
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(CompressedBuffer.data(), CompressedSize);
+  CompressedBuffer.truncate(CompressedSize);
+}
+
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+   size_t &UncompressedSize) {
+  unsigned long long const rSize = ZSTD_getFrameContentSize(
+  (const Bytef *)InputBuffer.data(), InputBuffer.size());
+  size_t const Res =
+  ::ZSTD_decompress((Bytef *)UncompressedBuffer, rSize,
+(const Bytef *)InputBuffer.data(), InputBuffer.size());
+  UncompressedSize = Res;
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(UncompressedBuffer, UncompressedSize);
+  return Res != rSize ? createError(ZSTD_getErrorName(Res)) : Error::success();
+}
+
+Error zstd::uncompress(StringRef InputBuffer,
+   SmallVectorImpl &UncompressedBuffer,
+   size_t UncompressedSize) {
+  UncompressedBuffer.resize_for_overwrite(UncompressedSize);
+  Error E = zstd::uncompress(InputBuffer, UncompressedBuffer.data(),
+ UncompressedSize);
+  UncompressedBuffer.truncate(UncompressedSize);
+  return E;
+}
+
+#else
+bool zstd::isAvailable() { return false; }
+void zstd::compress(StringRef InputBuffer,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  llvm_unreachable("zstd::compress is unavailable");
+}
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+   size_t &UncompressedSize) {
+  llvm_unreachable("zstd::uncompress is unavailable");
+}
+Error zstd::uncompress(StringRef InputBuffer,
+   SmallVectorImpl &UncompressedBuffer,
+   size_t UncompressedSize) {
+  llvm_unreachable("zstd::uncompress is unavailable");
+}
+#endif
\ No newline at end of file
Index: llvm/include/llvm/Support/Compression.h
===
--- llvm/include/llvm/Support/Compression.h
+++ llvm/include/llvm/Support/Compression.h
@@ -44,7 +44,33 @@
 
 } // End of namespace zlib
 
+namespace zstd {
+
+static constexpr std::string AlgorithmName = "zstd";
+static constexpr int NoCompression = -5;
+static constexpr int BestSpeedCompression = 1;
+static constexpr int DefaultCompression = 5;
+static constexpr int BestSizeCompression = 12;
+

[PATCH] D128667: [WIP] Add Zstd ELF support

[Cole Kissane via Phabricator via cfe-commits]

ckissane added a comment.

In D128667#3612956 , @MaskRay wrote:

> The lld/ change should be separate. And it needs tests. We should have 
> several tests such as mixed zlib and zstd input, malformed zstd input, 
> compressed level, etc. `OutputSections` should not have a new member. The 
> parallel output part will be tricky, too (see D117853 
> ). I can handle the lld/ELF part and leave 
> other stuff to you.
>
> llvm-objcopy should probably be a separate patch as well.
>
> - clang (Driver, cc1as) and MC (including `llvm-mc 
> --compress-debug-sections`) in one patch
> - lld/ELF
> - llvm-objcopy
>
>> Context not available.
>
> See 
> https://llvm.org/docs/Phabricator.html#requesting-a-review-via-the-web-interface
>  "To make reviews easier, please always include as much context as possible 
> with your diff!"

@MaskRay I would gladly accept your help on lld/ELF and I'll break this patch 
up into 2 patches:

- clang (Driver, cc1as) and MC (including `llvm-mc --compress-debug-sections`) 
in one patch
- llvm-objcopy

as suggested.
Also, I believe you requested a git repo for the whole picture on adding zstd: 
https://github.com/ckissane/llvm-project/tree/ckissane.add-zstd-compression


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128667/new/

https://reviews.llvm.org/D128667

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128754: Refactor LLVM compression namespaces

[Cole Kissane via Phabricator via cfe-commits]

ckissane marked an inline comment as done.
ckissane added a comment.

Marked name of algorithm comments as done


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128754/new/

https://reviews.llvm.org/D128754

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128754: Refactor LLVM compression namespaces

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 441107.
ckissane added a comment.

- feat: refactor llvm compression namespaces
- chore: update rel notes to note zlib::crc32 remove
- feat: add AlgorithmName to compression namespaces


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128754/new/

https://reviews.llvm.org/D128754

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/CMakeLists.txt
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/docs/ReleaseNotes.rst
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/ProfileData/SampleProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -19,6 +19,8 @@
 
 using namespace llvm;
 
+using namespace compression;
+
 namespace {
 
 #if LLVM_ENABLE_ZLIB
@@ -62,12 +64,6 @@
   TestZlibCompression(BinaryDataStr, zlib::DefaultCompression);
 }
 
-TEST(CompressionTest, ZlibCRC32) {
-  EXPECT_EQ(
-  0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
-}
-
 #endif
 
 }
Index: llvm/unittests/ProfileData/InstrProfTest.cpp
===
--- llvm/unittests/ProfileData/InstrProfTest.cpp
+++ llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -1146,17 +1146,19 @@
   for (bool DoCompression : {false, true}) {
 // Compressing:
 std::string FuncNameStrings1;
-EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames1, (DoCompression && zlib::isAvailable()),
-  FuncNameStrings1),
-  Succeeded());
+EXPECT_THAT_ERROR(
+collectPGOFuncNameStrings(
+FuncNames1, (DoCompression && compression::profile::isAvailable()),
+FuncNameStrings1),
+Succeeded());
 
 // Compressing:
 std::string FuncNameStrings2;
-EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames2, (DoCompression && zlib::isAvailable()),
-  FuncNameStrings2),
-  Succeeded());
+EXPECT_THAT_ERROR(
+collectPGOFuncNameStrings(
+FuncNames2, (DoCompression && compression::profile::isAvailable()),
+FuncNameStrings2),
+Succeeded());
 
 for (int Padding = 0; Padding < 2; Padding++) {
   // Join with paddings :
Index: llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
===
--- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -740,7 +740,7 @@
 .str()
 .c_str());
 }
-if (!zlib::isAvailable())
+if (!compression::zlib::isAvailable())
   return createStringError(
   errc::invalid_argument,
   "LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
@@ -999,7 +999,7 @@
 "--decompress-debug-sections");
   }
 
-  if (Config.DecompressDebugSections && !zlib::isAvailable())
+  if (Config.DecompressDebugSections && !compression::zlib::isAvailable())
 return createStringError(
 errc::invalid_argument,
 "LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress");
Index: llvm/tools/llvm-mc/llvm-mc.cpp
===
--- llvm/tools/llvm-mc/llvm-mc.cpp
+++ llvm/tools/llvm-mc/llvm-mc.cpp
@@ -403,7 +403,7 @@
   MAI->setRelaxELFRelocations(RelaxELFRel);
 
   if (CompressDebugSections != DebugCompressionType::None) {
-if (!zlib::isAvailable()) {
+if (!compression::zlib::isAvailable()) {
   WithColor::error(errs(), ProgName)
   << "build tools with zlib to enable -compress-debug-sections";
   return 1;
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -23,11 +23,14 @@
 
 using namespace llvm;
 
-#if LLVM_ENABLE_ZLIB
+using namespace compress

[PATCH] D128754: Refactor LLVM compression namespaces

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 441113.
ckissane added a comment.

- Merge remote-tracking branch 'origin/main' into 
ckissane.refactor-compression-namespace


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128754/new/

https://reviews.llvm.org/D128754

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/CMakeLists.txt
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/docs/ReleaseNotes.rst
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/ProfileData/SampleProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -19,6 +19,8 @@
 
 using namespace llvm;
 
+using namespace compression;
+
 namespace {
 
 #if LLVM_ENABLE_ZLIB
@@ -62,12 +64,6 @@
   TestZlibCompression(BinaryDataStr, zlib::DefaultCompression);
 }
 
-TEST(CompressionTest, ZlibCRC32) {
-  EXPECT_EQ(
-  0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
-}
-
 #endif
 
 }
Index: llvm/unittests/ProfileData/InstrProfTest.cpp
===
--- llvm/unittests/ProfileData/InstrProfTest.cpp
+++ llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -1146,17 +1146,19 @@
   for (bool DoCompression : {false, true}) {
 // Compressing:
 std::string FuncNameStrings1;
-EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames1, (DoCompression && zlib::isAvailable()),
-  FuncNameStrings1),
-  Succeeded());
+EXPECT_THAT_ERROR(
+collectPGOFuncNameStrings(
+FuncNames1, (DoCompression && compression::profile::isAvailable()),
+FuncNameStrings1),
+Succeeded());
 
 // Compressing:
 std::string FuncNameStrings2;
-EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames2, (DoCompression && zlib::isAvailable()),
-  FuncNameStrings2),
-  Succeeded());
+EXPECT_THAT_ERROR(
+collectPGOFuncNameStrings(
+FuncNames2, (DoCompression && compression::profile::isAvailable()),
+FuncNameStrings2),
+Succeeded());
 
 for (int Padding = 0; Padding < 2; Padding++) {
   // Join with paddings :
Index: llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
===
--- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -739,7 +739,7 @@
 .str()
 .c_str());
 }
-if (!zlib::isAvailable())
+if (!compression::zlib::isAvailable())
   return createStringError(
   errc::invalid_argument,
   "LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
@@ -998,7 +998,7 @@
 "--decompress-debug-sections");
   }
 
-  if (Config.DecompressDebugSections && !zlib::isAvailable())
+  if (Config.DecompressDebugSections && !compression::zlib::isAvailable())
 return createStringError(
 errc::invalid_argument,
 "LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress");
Index: llvm/tools/llvm-mc/llvm-mc.cpp
===
--- llvm/tools/llvm-mc/llvm-mc.cpp
+++ llvm/tools/llvm-mc/llvm-mc.cpp
@@ -403,7 +403,7 @@
   MAI->setRelaxELFRelocations(RelaxELFRel);
 
   if (CompressDebugSections != DebugCompressionType::None) {
-if (!zlib::isAvailable()) {
+if (!compression::zlib::isAvailable()) {
   WithColor::error(errs(), ProgName)
   << "build tools with zlib to enable -compress-debug-sections";
   return 1;
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -23,11 +23,14 @@
 
 using namespace llvm;
 
-#if LLVM_ENABLE_ZLIB
+using namespace compression;
+
 static Error createError(StringRef Err) {
   return

[PATCH] D128465: Zstandard as a second compression method to LLVM

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 441114.
ckissane edited the summary of this revision.
ckissane added a comment.

- feat: add zstd compression
- Merge branch 'ckissane.refactor-compression-namespace' into 
ckissane.add-zstd-compression
- Merge branch 'ckissane.refactor-compression-namespace' into 
ckissane.add-zstd-compression


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/Compression.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -92,6 +92,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,6 +20,9 @@
 #if LLVM_ENABLE_ZLIB
 #include 
 #endif
+#if LLVM_ENABLE_ZSTD
+#include 
+#endif
 
 using namespace llvm;
 
@@ -102,3 +105,63 @@
   llvm_unreachable("zlib::uncompress is unavailable");
 }
 #endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(StringRef InputBuffer,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  unsigned long CompressedBufferSize = ::ZSTD_compressBound(InputBuffer.size());
+  CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+  unsigned long CompressedSize = ::ZSTD_compress(
+  (Bytef *)CompressedBuffer.data(), CompressedBufferSize,
+  (const Bytef *)InputBuffer.data(), InputBuffer.size(), Level);
+  if (ZSTD_isError(CompressedSize))
+report_bad_alloc_error("Allocation failed");
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(CompressedBuffer.data(), CompressedSize);
+  CompressedBuffer.truncate(CompressedSize);
+}
+
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+   size_t &UncompressedSize) {
+  unsigned long long const rSize = ZSTD_getFrameContentSize(
+  (const Bytef *)InputBuffer.data(), InputBuffer.size());
+  size_t const Res =
+  ::ZSTD_decompress((Bytef *)UncompressedBuffer, rSize,
+(const Bytef *)InputBuffer.data(), InputBuffer.size());
+  UncompressedSize = Res;
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(UncompressedBuffer, UncompressedSize);
+  return Res != rSize ? createError(ZSTD_getErrorName(Res)) : Error::success();
+}
+
+Error zstd::uncompress(StringRef InputBuffer,
+   SmallVectorImpl &UncompressedBuffer,
+   size_t UncompressedSize) {
+  UncompressedBuffer.resize_for_overwrite(UncompressedSize);
+  Error E = zstd::uncompress(InputBuffer, UncompressedBuffer.data(),
+ UncompressedSize);
+  UncompressedBuffer.truncate(UncompressedSize);
+  return E;
+}
+
+#else
+bool zstd::isAvailable() { return false; }
+void zstd::compress(StringRef InputBuffer,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  llvm_unreachable("zstd::compress is unavailable");
+}
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+   size_t &UncompressedSize) {
+  llvm_unreachable("zstd::uncompress is unavailable");
+}
+Error zstd::uncompress(StringRef InputBuffer,
+   SmallVectorImpl &UncompressedBuffer,
+   size_t UncompressedSize) {
+  llvm_unreachable("zstd::uncompress is unavailable");
+}
+#endif
\ No newline at end of file
Index: llvm/include/llvm/Support/Compression.h
===
--- llvm/include/llvm/Support/Compression.h
+++ llvm/include/llvm/Support/Compression.h
@@ -44,7 +44,33 @@
 
 } // End of namespace zlib
 
+nam

[PATCH] D128953: chore: refactor compression namespaces making way for a possible introduction of alternatives to zlib compression in the llvm toolchain. Changes are as follows: * Relocate the `ll

[Cole Kissane via Phabricator via cfe-commits]

ckissane created this revision.
Herald added subscribers: wenlei, usaxena95, kadircet, arphaman, hiraditya, 
arichardson, emaste.
Herald added a reviewer: alexander-shaposhnikov.
Herald added a reviewer: rupprecht.
Herald added a reviewer: jhenderson.
Herald added a reviewer: MaskRay.
Herald added a project: All.
ckissane requested review of this revision.
Herald added subscribers: cfe-commits, llvm-commits, StephenFan.
Herald added projects: clang, LLVM, clang-tools-extra.

...that explictly needs `zlib` compression (IE zlib elf debug sections) should 
use `llvm::compression::zlib`.   * Code interfacing with compressed profile 
data should use `llvm::compression::profile`.   * Code interfacing with 
compressed serialized data should use `llvm::compression::serialize`


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D128953

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/docs/ReleaseNotes.rst
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -19,6 +19,8 @@
 
 using namespace llvm;
 
+using namespace llvm::compression;
+
 namespace {
 
 #if LLVM_ENABLE_ZLIB
Index: llvm/unittests/ProfileData/InstrProfTest.cpp
===
--- llvm/unittests/ProfileData/InstrProfTest.cpp
+++ llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -1146,17 +1146,19 @@
   for (bool DoCompression : {false, true}) {
 // Compressing:
 std::string FuncNameStrings1;
-EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames1, (DoCompression && zlib::isAvailable()),
-  FuncNameStrings1),
-  Succeeded());
+EXPECT_THAT_ERROR(
+collectPGOFuncNameStrings(
+FuncNames1, (DoCompression && compression::profile::isAvailable()),
+FuncNameStrings1),
+Succeeded());
 
 // Compressing:
 std::string FuncNameStrings2;
-EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames2, (DoCompression && zlib::isAvailable()),
-  FuncNameStrings2),
-  Succeeded());
+EXPECT_THAT_ERROR(
+collectPGOFuncNameStrings(
+FuncNames2, (DoCompression && compression::profile::isAvailable()),
+FuncNameStrings2),
+Succeeded());
 
 for (int Padding = 0; Padding < 2; Padding++) {
   // Join with paddings :
Index: llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
===
--- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -739,7 +739,7 @@
 .str()
 .c_str());
 }
-if (!zlib::isAvailable())
+if (!compression::zlib::isAvailable())
   return createStringError(
   errc::invalid_argument,
   "LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
@@ -998,7 +998,7 @@
 "--decompress-debug-sections");
   }
 
-  if (Config.DecompressDebugSections && !zlib::isAvailable())
+  if (Config.DecompressDebugSections && !compression::zlib::isAvailable())
 return createStringError(
 errc::invalid_argument,
 "LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress");
Index: llvm/tools/llvm-mc/llvm-mc.cpp
===
--- llvm/tools/llvm-mc/llvm-mc.cpp
+++ llvm/tools/llvm-mc/llvm-mc.cpp
@@ -403,7 +403,7 @@
   MAI->setRelaxELFRelocations(RelaxELFRel);
 
   if (CompressDebugSections != DebugCompressionType::None) {
-if (!zlib::isAvailable()) {
+if (!compression::zlib::isAvailable()) {
   WithColor::error(errs(), ProgName)
   << "build tools with zlib to enable -compress-debug-sections";
   return 1;
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -23,10 +23,9 @@
 
 using namespace llvm

[PATCH] D128754: Refactor LLVM compression namespaces

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 441537.
ckissane added a comment.

make part 1 after part 0 of refactor


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128754/new/

https://reviews.llvm.org/D128754

Files:
  llvm/docs/ReleaseNotes.rst
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/Compression.cpp
  llvm/unittests/Support/CompressionTest.cpp


Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -64,12 +64,6 @@
   TestZlibCompression(BinaryDataStr, zlib::DefaultCompression);
 }
 
-TEST(CompressionTest, ZlibCRC32) {
-  EXPECT_EQ(
-  0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
-}
-
 #endif
 
 }
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -84,10 +84,6 @@
   return E;
 }
 
-uint32_t zlib::crc32(StringRef Buffer) {
-  return ::crc32(0, (const Bytef *)Buffer.data(), Buffer.size());
-}
-
 #else
 bool zlib::isAvailable() { return false; }
 void zlib::compress(StringRef InputBuffer,
@@ -103,7 +99,4 @@
size_t UncompressedSize) {
   llvm_unreachable("zlib::uncompress is unavailable");
 }
-uint32_t zlib::crc32(StringRef Buffer) {
-  llvm_unreachable("zlib::crc32 is unavailable");
-}
 #endif
Index: llvm/include/llvm/Support/Compression.h
===
--- llvm/include/llvm/Support/Compression.h
+++ llvm/include/llvm/Support/Compression.h
@@ -42,8 +42,6 @@
  SmallVectorImpl &UncompressedBuffer,
  size_t UncompressedSize);
 
-uint32_t crc32(StringRef Buffer);
-
 } // End of namespace zlib
 
 namespace profile = llvm::compression::zlib;
Index: llvm/docs/ReleaseNotes.rst
===
--- llvm/docs/ReleaseNotes.rst
+++ llvm/docs/ReleaseNotes.rst
@@ -187,6 +187,7 @@
   * Code that explictly needs ``zlib`` compression (IE zlib elf debug 
sections) should use ``llvm::compression::zlib``.
   * Code interfacing with compressed profile data should use 
``llvm::compression::profile``.
   * Code interfacing with compressed serialized data should use 
``llvm::compression::serialize``.
+  * Remove crc32 from zlib compression namespace, people should use the 
``llvm::crc32`` instead.
 
 Changes to the Go bindings
 --


Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -64,12 +64,6 @@
   TestZlibCompression(BinaryDataStr, zlib::DefaultCompression);
 }
 
-TEST(CompressionTest, ZlibCRC32) {
-  EXPECT_EQ(
-  0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
-}
-
 #endif
 
 }
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -84,10 +84,6 @@
   return E;
 }
 
-uint32_t zlib::crc32(StringRef Buffer) {
-  return ::crc32(0, (const Bytef *)Buffer.data(), Buffer.size());
-}
-
 #else
 bool zlib::isAvailable() { return false; }
 void zlib::compress(StringRef InputBuffer,
@@ -103,7 +99,4 @@
size_t UncompressedSize) {
   llvm_unreachable("zlib::uncompress is unavailable");
 }
-uint32_t zlib::crc32(StringRef Buffer) {
-  llvm_unreachable("zlib::crc32 is unavailable");
-}
 #endif
Index: llvm/include/llvm/Support/Compression.h
===
--- llvm/include/llvm/Support/Compression.h
+++ llvm/include/llvm/Support/Compression.h
@@ -42,8 +42,6 @@
  SmallVectorImpl &UncompressedBuffer,
  size_t UncompressedSize);
 
-uint32_t crc32(StringRef Buffer);
-
 } // End of namespace zlib
 
 namespace profile = llvm::compression::zlib;
Index: llvm/docs/ReleaseNotes.rst
===
--- llvm/docs/ReleaseNotes.rst
+++ llvm/docs/ReleaseNotes.rst
@@ -187,6 +187,7 @@
   * Code that explictly needs ``zlib`` compression (IE zlib elf debug sections) should use ``llvm::compression::zlib``.
   * Code interfacing with compressed profile data should use ``llvm::compression::profile``.
   * Code interfacing with compressed serialized data should use ``llvm::compression::serialize``.
+  * Remove crc32 from zlib compression namespace, people should use the ``llvm::crc32`` instead.
 
 Changes to the Go bindings
 --
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listi

[PATCH] D128953: [NFC] refactor compression namespaces making way for a possible introduction of alternatives to zlib compression in the llvm toolchain.

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 441540.
ckissane added a comment.

- Merge branch 'ckissane.refactor-compression.part-0' of 
github.com:ckissane/llvm-project into ckissane.refactor-compression.part-0
- Merge branch 'main' into ckissane.refactor-compression.part-0


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128953/new/

https://reviews.llvm.org/D128953

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/docs/ReleaseNotes.rst
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -19,6 +19,8 @@
 
 using namespace llvm;
 
+using namespace llvm::compression;
+
 namespace {
 
 #if LLVM_ENABLE_ZLIB
Index: llvm/unittests/ProfileData/InstrProfTest.cpp
===
--- llvm/unittests/ProfileData/InstrProfTest.cpp
+++ llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -1146,17 +1146,19 @@
   for (bool DoCompression : {false, true}) {
 // Compressing:
 std::string FuncNameStrings1;
-EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames1, (DoCompression && zlib::isAvailable()),
-  FuncNameStrings1),
-  Succeeded());
+EXPECT_THAT_ERROR(
+collectPGOFuncNameStrings(
+FuncNames1, (DoCompression && compression::profile::isAvailable()),
+FuncNameStrings1),
+Succeeded());
 
 // Compressing:
 std::string FuncNameStrings2;
-EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames2, (DoCompression && zlib::isAvailable()),
-  FuncNameStrings2),
-  Succeeded());
+EXPECT_THAT_ERROR(
+collectPGOFuncNameStrings(
+FuncNames2, (DoCompression && compression::profile::isAvailable()),
+FuncNameStrings2),
+Succeeded());
 
 for (int Padding = 0; Padding < 2; Padding++) {
   // Join with paddings :
Index: llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
===
--- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -739,7 +739,7 @@
 .str()
 .c_str());
 }
-if (!zlib::isAvailable())
+if (!compression::zlib::isAvailable())
   return createStringError(
   errc::invalid_argument,
   "LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
@@ -998,7 +998,7 @@
 "--decompress-debug-sections");
   }
 
-  if (Config.DecompressDebugSections && !zlib::isAvailable())
+  if (Config.DecompressDebugSections && !compression::zlib::isAvailable())
 return createStringError(
 errc::invalid_argument,
 "LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress");
Index: llvm/tools/llvm-mc/llvm-mc.cpp
===
--- llvm/tools/llvm-mc/llvm-mc.cpp
+++ llvm/tools/llvm-mc/llvm-mc.cpp
@@ -403,7 +403,7 @@
   MAI->setRelaxELFRelocations(RelaxELFRel);
 
   if (CompressDebugSections != DebugCompressionType::None) {
-if (!zlib::isAvailable()) {
+if (!compression::zlib::isAvailable()) {
   WithColor::error(errs(), ProgName)
   << "build tools with zlib to enable -compress-debug-sections";
   return 1;
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -23,10 +23,9 @@
 
 using namespace llvm;
 
+using namespace llvm::compression;
+
 #if LLVM_ENABLE_ZLIB
-static Error createError(StringRef Err) {
-  return make_error(Err, inconvertibleErrorCode());
-}
 
 static StringRef convertZlibCodeToString(int Code) {
   switch (Code) {
@@ -70,15 +69,17 @@
   // Tell MemorySanitizer that zlib output buffer is fully initialized.
   // This avoids a false report when running LLVM with uninstrumented ZLib.
   __msan_unpoison(UncompressedBu

[PATCH] D128754: Refactor LLVM compression namespaces (Part 1: remove crc32)

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 441541.
ckissane added a comment.

- Merge branch 'ckissane.refactor-compression.part-0' into 
ckissane.refactor-compression.part-1


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128754/new/

https://reviews.llvm.org/D128754

Files:
  llvm/docs/ReleaseNotes.rst
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/Compression.cpp
  llvm/unittests/Support/CompressionTest.cpp


Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -64,12 +64,6 @@
   TestZlibCompression(BinaryDataStr, zlib::DefaultCompression);
 }
 
-TEST(CompressionTest, ZlibCRC32) {
-  EXPECT_EQ(
-  0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
-}
-
 #endif
 
 }
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -84,10 +84,6 @@
   return E;
 }
 
-uint32_t zlib::crc32(StringRef Buffer) {
-  return ::crc32(0, (const Bytef *)Buffer.data(), Buffer.size());
-}
-
 #else
 bool zlib::isAvailable() { return false; }
 void zlib::compress(StringRef InputBuffer,
@@ -103,7 +99,4 @@
size_t UncompressedSize) {
   llvm_unreachable("zlib::uncompress is unavailable");
 }
-uint32_t zlib::crc32(StringRef Buffer) {
-  llvm_unreachable("zlib::crc32 is unavailable");
-}
 #endif
Index: llvm/include/llvm/Support/Compression.h
===
--- llvm/include/llvm/Support/Compression.h
+++ llvm/include/llvm/Support/Compression.h
@@ -42,8 +42,6 @@
  SmallVectorImpl &UncompressedBuffer,
  size_t UncompressedSize);
 
-uint32_t crc32(StringRef Buffer);
-
 } // End of namespace zlib
 
 namespace profile = llvm::compression::zlib;
Index: llvm/docs/ReleaseNotes.rst
===
--- llvm/docs/ReleaseNotes.rst
+++ llvm/docs/ReleaseNotes.rst
@@ -187,6 +187,7 @@
   * Code that explictly needs ``zlib`` compression (IE zlib elf debug 
sections) should use ``llvm::compression::zlib``.
   * Code interfacing with compressed profile data should use 
``llvm::compression::profile``.
   * Code interfacing with compressed serialized data should use 
``llvm::compression::serialize``.
+  * Remove crc32 from zlib compression namespace, people should use the 
``llvm::crc32`` instead.
 
 Changes to the Go bindings
 --


Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -64,12 +64,6 @@
   TestZlibCompression(BinaryDataStr, zlib::DefaultCompression);
 }
 
-TEST(CompressionTest, ZlibCRC32) {
-  EXPECT_EQ(
-  0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
-}
-
 #endif
 
 }
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -84,10 +84,6 @@
   return E;
 }
 
-uint32_t zlib::crc32(StringRef Buffer) {
-  return ::crc32(0, (const Bytef *)Buffer.data(), Buffer.size());
-}
-
 #else
 bool zlib::isAvailable() { return false; }
 void zlib::compress(StringRef InputBuffer,
@@ -103,7 +99,4 @@
size_t UncompressedSize) {
   llvm_unreachable("zlib::uncompress is unavailable");
 }
-uint32_t zlib::crc32(StringRef Buffer) {
-  llvm_unreachable("zlib::crc32 is unavailable");
-}
 #endif
Index: llvm/include/llvm/Support/Compression.h
===
--- llvm/include/llvm/Support/Compression.h
+++ llvm/include/llvm/Support/Compression.h
@@ -42,8 +42,6 @@
  SmallVectorImpl &UncompressedBuffer,
  size_t UncompressedSize);
 
-uint32_t crc32(StringRef Buffer);
-
 } // End of namespace zlib
 
 namespace profile = llvm::compression::zlib;
Index: llvm/docs/ReleaseNotes.rst
===
--- llvm/docs/ReleaseNotes.rst
+++ llvm/docs/ReleaseNotes.rst
@@ -187,6 +187,7 @@
   * Code that explictly needs ``zlib`` compression (IE zlib elf debug sections) should use ``llvm::compression::zlib``.
   * Code interfacing with compressed profile data should use ``llvm::compression::profile``.
   * Code interfacing with compressed serialized data should use ``llvm::compression::serialize``.
+  * Remove crc32 from zlib compression namespace, people should use the ``llvm::crc32`` instead.
 
 Changes to the Go bindings
 --
___
cfe-commits mailing list
cfe-commits

[PATCH] D128953: [NFC] Refactor llvm::zlib namespace

[Cole Kissane via Phabricator via cfe-commits]

ckissane added inline comments.



Comment at: llvm/include/llvm/Support/Compression.h:49-51
+namespace profile = llvm::compression::zlib;
+
+namespace serialize = llvm::compression::zlib;

phosek wrote:
> I think we will need to support dynamically selecting (de)compression 
> algorithm for both profile and serialization. For example, we should be able 
> to use read profiles generated by an older version of LLVM that only 
> supported zlib, even if the new one also supports zstd. Given that, I'd omit 
> these and instead use `compression::zlib` everywhere.
Correct, however I think that doing this in the meantime helps make it clear 
semantically for what purpose each compression call is for. It will make 
finding all instances of each time of use much easier in the future, and 
promotes semantically thought out usage.

Therefore I see no reason to not make these explicit aliases as it will only 
ease transition in the future.




Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128953/new/

https://reviews.llvm.org/D128953

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128953: [NFC] Refactor llvm::zlib namespace

[Cole Kissane via Phabricator via cfe-commits]

ckissane added inline comments.



Comment at: llvm/lib/ProfileData/InstrProf.cpp:154
+OS << ("profile uses " + compression::profile::AlgorithmName +
+   " compression but the profile reader was built " + "without " +
+   compression::profile::AlgorithmName + " support");

MaskRay wrote:
> leonardchan wrote:
> > `" compression but the profile reader was built " + "without "` -> `" 
> > compression but the profile reader was built without "`
> Keep the diagnostic unchanged in this patch.
And the other similar changes?
I will note these are NFC because if zlib is being used (as it currently always 
is), it has the exact same output.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128953/new/

https://reviews.llvm.org/D128953

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128953: [NFC] Refactor llvm::zlib namespace

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 441757.
ckissane added a comment.

discard debug string changes


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128953/new/

https://reviews.llvm.org/D128953

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/docs/ReleaseNotes.rst
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -19,6 +19,8 @@
 
 using namespace llvm;
 
+using namespace llvm::compression;
+
 namespace {
 
 #if LLVM_ENABLE_ZLIB
Index: llvm/unittests/ProfileData/InstrProfTest.cpp
===
--- llvm/unittests/ProfileData/InstrProfTest.cpp
+++ llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -1146,17 +1146,19 @@
   for (bool DoCompression : {false, true}) {
 // Compressing:
 std::string FuncNameStrings1;
-EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames1, (DoCompression && zlib::isAvailable()),
-  FuncNameStrings1),
-  Succeeded());
+EXPECT_THAT_ERROR(
+collectPGOFuncNameStrings(
+FuncNames1, (DoCompression && compression::profile::isAvailable()),
+FuncNameStrings1),
+Succeeded());
 
 // Compressing:
 std::string FuncNameStrings2;
-EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames2, (DoCompression && zlib::isAvailable()),
-  FuncNameStrings2),
-  Succeeded());
+EXPECT_THAT_ERROR(
+collectPGOFuncNameStrings(
+FuncNames2, (DoCompression && compression::profile::isAvailable()),
+FuncNameStrings2),
+Succeeded());
 
 for (int Padding = 0; Padding < 2; Padding++) {
   // Join with paddings :
Index: llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
===
--- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -739,7 +739,7 @@
 .str()
 .c_str());
 }
-if (!zlib::isAvailable())
+if (!compression::zlib::isAvailable())
   return createStringError(
   errc::invalid_argument,
   "LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
@@ -998,7 +998,7 @@
 "--decompress-debug-sections");
   }
 
-  if (Config.DecompressDebugSections && !zlib::isAvailable())
+  if (Config.DecompressDebugSections && !compression::zlib::isAvailable())
 return createStringError(
 errc::invalid_argument,
 "LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress");
Index: llvm/tools/llvm-mc/llvm-mc.cpp
===
--- llvm/tools/llvm-mc/llvm-mc.cpp
+++ llvm/tools/llvm-mc/llvm-mc.cpp
@@ -403,7 +403,7 @@
   MAI->setRelaxELFRelocations(RelaxELFRel);
 
   if (CompressDebugSections != DebugCompressionType::None) {
-if (!zlib::isAvailable()) {
+if (!compression::zlib::isAvailable()) {
   WithColor::error(errs(), ProgName)
   << "build tools with zlib to enable -compress-debug-sections";
   return 1;
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -23,10 +23,9 @@
 
 using namespace llvm;
 
+using namespace llvm::compression;
+
 #if LLVM_ENABLE_ZLIB
-static Error createError(StringRef Err) {
-  return make_error(Err, inconvertibleErrorCode());
-}
 
 static StringRef convertZlibCodeToString(int Code) {
   switch (Code) {
@@ -70,15 +69,17 @@
   // Tell MemorySanitizer that zlib output buffer is fully initialized.
   // This avoids a false report when running LLVM with uninstrumented ZLib.
   __msan_unpoison(UncompressedBuffer, UncompressedSize);
-  return Res ? createError(convertZlibCodeToString(Res)) : Error::success();
+  return Res ? make_error(convertZlibCodeToString(Res),
+   inconve

[PATCH] D128953: [NFC] Refactor llvm::zlib namespace

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 441763.
ckissane edited the summary of this revision.
ckissane added a comment.

remove compression namespace alias work


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128953/new/

https://reviews.llvm.org/D128953

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/docs/ReleaseNotes.rst
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -19,6 +19,8 @@
 
 using namespace llvm;
 
+using namespace llvm::compression;
+
 namespace {
 
 #if LLVM_ENABLE_ZLIB
Index: llvm/unittests/ProfileData/InstrProfTest.cpp
===
--- llvm/unittests/ProfileData/InstrProfTest.cpp
+++ llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -1147,14 +1147,16 @@
 // Compressing:
 std::string FuncNameStrings1;
 EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames1, (DoCompression && zlib::isAvailable()),
+  FuncNames1,
+  (DoCompression && compression::zlib::isAvailable()),
   FuncNameStrings1),
   Succeeded());
 
 // Compressing:
 std::string FuncNameStrings2;
 EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames2, (DoCompression && zlib::isAvailable()),
+  FuncNames2,
+  (DoCompression && compression::zlib::isAvailable()),
   FuncNameStrings2),
   Succeeded());
 
Index: llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
===
--- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -739,7 +739,7 @@
 .str()
 .c_str());
 }
-if (!zlib::isAvailable())
+if (!compression::zlib::isAvailable())
   return createStringError(
   errc::invalid_argument,
   "LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
@@ -998,7 +998,7 @@
 "--decompress-debug-sections");
   }
 
-  if (Config.DecompressDebugSections && !zlib::isAvailable())
+  if (Config.DecompressDebugSections && !compression::zlib::isAvailable())
 return createStringError(
 errc::invalid_argument,
 "LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress");
Index: llvm/tools/llvm-mc/llvm-mc.cpp
===
--- llvm/tools/llvm-mc/llvm-mc.cpp
+++ llvm/tools/llvm-mc/llvm-mc.cpp
@@ -403,7 +403,7 @@
   MAI->setRelaxELFRelocations(RelaxELFRel);
 
   if (CompressDebugSections != DebugCompressionType::None) {
-if (!zlib::isAvailable()) {
+if (!compression::zlib::isAvailable()) {
   WithColor::error(errs(), ProgName)
   << "build tools with zlib to enable -compress-debug-sections";
   return 1;
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -23,10 +23,9 @@
 
 using namespace llvm;
 
+using namespace llvm::compression;
+
 #if LLVM_ENABLE_ZLIB
-static Error createError(StringRef Err) {
-  return make_error(Err, inconvertibleErrorCode());
-}
 
 static StringRef convertZlibCodeToString(int Code) {
   switch (Code) {
@@ -70,15 +69,17 @@
   // Tell MemorySanitizer that zlib output buffer is fully initialized.
   // This avoids a false report when running LLVM with uninstrumented ZLib.
   __msan_unpoison(UncompressedBuffer, UncompressedSize);
-  return Res ? createError(convertZlibCodeToString(Res)) : Error::success();
+  return Res ? make_error(convertZlibCodeToString(Res),
+   inconvertibleErrorCode())
+ : Error::success();
 }
 
 Error zlib::uncompress(StringRef InputBuffer,
SmallVectorImpl &UncompressedBuffer,
size_t UncompressedSize) {
   Uncompr

[PATCH] D128953: [NFC] Refactor llvm::zlib namespace

[Cole Kissane via Phabricator via cfe-commits]

ckissane marked 2 inline comments as done.
ckissane added a comment.

comments addressed


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128953/new/

https://reviews.llvm.org/D128953

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128754: [llvm] Refactor LLVM compression namespaces (Part 1: remove crc32)

[Cole Kissane via Phabricator via cfe-commits]

ckissane marked 3 inline comments as done.
ckissane added a comment.

addressed comments marked done


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128754/new/

https://reviews.llvm.org/D128754

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128754: [llvm] Refactor LLVM compression namespaces (Part 1: remove crc32)

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 441770.
ckissane added a comment.

updated diff


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128754/new/

https://reviews.llvm.org/D128754

Files:
  llvm/docs/ReleaseNotes.rst
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/Compression.cpp
  llvm/unittests/Support/CompressionTest.cpp


Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -64,12 +64,6 @@
   TestZlibCompression(BinaryDataStr, zlib::DefaultCompression);
 }
 
-TEST(CompressionTest, ZlibCRC32) {
-  EXPECT_EQ(
-  0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
-}
-
 #endif
 
 }
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -84,10 +84,6 @@
   return E;
 }
 
-uint32_t zlib::crc32(StringRef Buffer) {
-  return ::crc32(0, (const Bytef *)Buffer.data(), Buffer.size());
-}
-
 #else
 bool zlib::isAvailable() { return false; }
 void zlib::compress(StringRef InputBuffer,
@@ -103,7 +99,4 @@
size_t UncompressedSize) {
   llvm_unreachable("zlib::uncompress is unavailable");
 }
-uint32_t zlib::crc32(StringRef Buffer) {
-  llvm_unreachable("zlib::crc32 is unavailable");
-}
 #endif
Index: llvm/include/llvm/Support/Compression.h
===
--- llvm/include/llvm/Support/Compression.h
+++ llvm/include/llvm/Support/Compression.h
@@ -42,8 +42,6 @@
  SmallVectorImpl &UncompressedBuffer,
  size_t UncompressedSize);
 
-uint32_t crc32(StringRef Buffer);
-
 } // End of namespace zlib
 
 } // End of namespace compression
Index: llvm/docs/ReleaseNotes.rst
===
--- llvm/docs/ReleaseNotes.rst
+++ llvm/docs/ReleaseNotes.rst
@@ -184,6 +184,7 @@
   introduction of alternatives to zlib compression in the llvm toolchain.
   Changes are as follows:
   * Relocate the ``llvm::zlib`` namespace to ``llvm::compression::zlib``.
+  * Remove crc32 from zlib compression namespace, people should use the 
``llvm::crc32`` instead.
 
 Changes to the Go bindings
 --


Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -64,12 +64,6 @@
   TestZlibCompression(BinaryDataStr, zlib::DefaultCompression);
 }
 
-TEST(CompressionTest, ZlibCRC32) {
-  EXPECT_EQ(
-  0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
-}
-
 #endif
 
 }
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -84,10 +84,6 @@
   return E;
 }
 
-uint32_t zlib::crc32(StringRef Buffer) {
-  return ::crc32(0, (const Bytef *)Buffer.data(), Buffer.size());
-}
-
 #else
 bool zlib::isAvailable() { return false; }
 void zlib::compress(StringRef InputBuffer,
@@ -103,7 +99,4 @@
size_t UncompressedSize) {
   llvm_unreachable("zlib::uncompress is unavailable");
 }
-uint32_t zlib::crc32(StringRef Buffer) {
-  llvm_unreachable("zlib::crc32 is unavailable");
-}
 #endif
Index: llvm/include/llvm/Support/Compression.h
===
--- llvm/include/llvm/Support/Compression.h
+++ llvm/include/llvm/Support/Compression.h
@@ -42,8 +42,6 @@
  SmallVectorImpl &UncompressedBuffer,
  size_t UncompressedSize);
 
-uint32_t crc32(StringRef Buffer);
-
 } // End of namespace zlib
 
 } // End of namespace compression
Index: llvm/docs/ReleaseNotes.rst
===
--- llvm/docs/ReleaseNotes.rst
+++ llvm/docs/ReleaseNotes.rst
@@ -184,6 +184,7 @@
   introduction of alternatives to zlib compression in the llvm toolchain.
   Changes are as follows:
   * Relocate the ``llvm::zlib`` namespace to ``llvm::compression::zlib``.
+  * Remove crc32 from zlib compression namespace, people should use the ``llvm::crc32`` instead.
 
 Changes to the Go bindings
 --
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128953: [NFC] Refactor llvm::zlib namespace

[Cole Kissane via Phabricator via cfe-commits]

ckissane added inline comments.



Comment at: llvm/include/llvm/Support/Compression.h:27
 
+static constexpr std::string AlgorithmName = "zlib";
 static constexpr int NoCompression = 0;

MaskRay wrote:
> Is it still used?
> 
> Prefer StringRef if the string is backed from some storage.
No it is not, I will remove it and reintroduce it in the first patch that adds 
zstd



Comment at: llvm/unittests/Support/CompressionTest.cpp:21
 using namespace llvm;
 
+using namespace llvm::compression;

MaskRay wrote:
> Delete blank line between two `using`
will do


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128953/new/

https://reviews.llvm.org/D128953

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128953: [NFC] Refactor llvm::zlib namespace

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 443005.
ckissane added a comment.

- Merge branch 'ckissane.refactor-compression.part-0' of 
github.com:ckissane/llvm-project into ckissane.refactor-compression.part-0
- Merge branch 'main' into ckissane.refactor-compression.part-0
- compression refactor: undo string subs
- Merge branch 'ckissane.refactor-compression.part-0' into 
ckissane.refactor-compression.part-0.no-string-sub
- remove compression namespaace aliases
- update release notes to reflect no compression aliases
- chore: fmt and remove AlgorithmName from zlib namespace
- Merge remote-tracking branch 'origin/main' into 
ckissane.refactor-compression.part-0.no-string-sub


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128953/new/

https://reviews.llvm.org/D128953

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/docs/ReleaseNotes.rst
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -18,6 +18,7 @@
 #include "gtest/gtest.h"
 
 using namespace llvm;
+using namespace llvm::compression;
 
 namespace {
 
Index: llvm/unittests/ProfileData/InstrProfTest.cpp
===
--- llvm/unittests/ProfileData/InstrProfTest.cpp
+++ llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -1147,14 +1147,16 @@
 // Compressing:
 std::string FuncNameStrings1;
 EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames1, (DoCompression && zlib::isAvailable()),
+  FuncNames1,
+  (DoCompression && compression::zlib::isAvailable()),
   FuncNameStrings1),
   Succeeded());
 
 // Compressing:
 std::string FuncNameStrings2;
 EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames2, (DoCompression && zlib::isAvailable()),
+  FuncNames2,
+  (DoCompression && compression::zlib::isAvailable()),
   FuncNameStrings2),
   Succeeded());
 
Index: llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
===
--- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -739,7 +739,7 @@
 .str()
 .c_str());
 }
-if (!zlib::isAvailable())
+if (!compression::zlib::isAvailable())
   return createStringError(
   errc::invalid_argument,
   "LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
@@ -998,7 +998,7 @@
 "--decompress-debug-sections");
   }
 
-  if (Config.DecompressDebugSections && !zlib::isAvailable())
+  if (Config.DecompressDebugSections && !compression::zlib::isAvailable())
 return createStringError(
 errc::invalid_argument,
 "LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress");
Index: llvm/tools/llvm-mc/llvm-mc.cpp
===
--- llvm/tools/llvm-mc/llvm-mc.cpp
+++ llvm/tools/llvm-mc/llvm-mc.cpp
@@ -403,7 +403,7 @@
   MAI->setRelaxELFRelocations(RelaxELFRel);
 
   if (CompressDebugSections != DebugCompressionType::None) {
-if (!zlib::isAvailable()) {
+if (!compression::zlib::isAvailable()) {
   WithColor::error(errs(), ProgName)
   << "build tools with zlib to enable -compress-debug-sections";
   return 1;
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -23,10 +23,9 @@
 
 using namespace llvm;
 
+using namespace llvm::compression;
+
 #if LLVM_ENABLE_ZLIB
-static Error createError(StringRef Err) {
-  return make_error(Err, inconvertibleErrorCode());
-}
 
 static StringRef convertZlibCodeToString(int Code) {
   switch (Code) {
@@ -70,15 +69,17 @@
   // Tell MemorySanitizer that zlib output buffer is fu

[PATCH] D128953: [NFC] Refactor llvm::zlib namespace

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 443006.
ckissane added a comment.

- chore: delete an excess blank line


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128953/new/

https://reviews.llvm.org/D128953

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/docs/ReleaseNotes.rst
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -18,6 +18,7 @@
 #include "gtest/gtest.h"
 
 using namespace llvm;
+using namespace llvm::compression;
 
 namespace {
 
Index: llvm/unittests/ProfileData/InstrProfTest.cpp
===
--- llvm/unittests/ProfileData/InstrProfTest.cpp
+++ llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -1147,14 +1147,16 @@
 // Compressing:
 std::string FuncNameStrings1;
 EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames1, (DoCompression && zlib::isAvailable()),
+  FuncNames1,
+  (DoCompression && compression::zlib::isAvailable()),
   FuncNameStrings1),
   Succeeded());
 
 // Compressing:
 std::string FuncNameStrings2;
 EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames2, (DoCompression && zlib::isAvailable()),
+  FuncNames2,
+  (DoCompression && compression::zlib::isAvailable()),
   FuncNameStrings2),
   Succeeded());
 
Index: llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
===
--- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -739,7 +739,7 @@
 .str()
 .c_str());
 }
-if (!zlib::isAvailable())
+if (!compression::zlib::isAvailable())
   return createStringError(
   errc::invalid_argument,
   "LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
@@ -998,7 +998,7 @@
 "--decompress-debug-sections");
   }
 
-  if (Config.DecompressDebugSections && !zlib::isAvailable())
+  if (Config.DecompressDebugSections && !compression::zlib::isAvailable())
 return createStringError(
 errc::invalid_argument,
 "LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress");
Index: llvm/tools/llvm-mc/llvm-mc.cpp
===
--- llvm/tools/llvm-mc/llvm-mc.cpp
+++ llvm/tools/llvm-mc/llvm-mc.cpp
@@ -403,7 +403,7 @@
   MAI->setRelaxELFRelocations(RelaxELFRel);
 
   if (CompressDebugSections != DebugCompressionType::None) {
-if (!zlib::isAvailable()) {
+if (!compression::zlib::isAvailable()) {
   WithColor::error(errs(), ProgName)
   << "build tools with zlib to enable -compress-debug-sections";
   return 1;
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -22,11 +22,9 @@
 #endif
 
 using namespace llvm;
+using namespace llvm::compression;
 
 #if LLVM_ENABLE_ZLIB
-static Error createError(StringRef Err) {
-  return make_error(Err, inconvertibleErrorCode());
-}
 
 static StringRef convertZlibCodeToString(int Code) {
   switch (Code) {
@@ -70,15 +68,17 @@
   // Tell MemorySanitizer that zlib output buffer is fully initialized.
   // This avoids a false report when running LLVM with uninstrumented ZLib.
   __msan_unpoison(UncompressedBuffer, UncompressedSize);
-  return Res ? createError(convertZlibCodeToString(Res)) : Error::success();
+  return Res ? make_error(convertZlibCodeToString(Res),
+   inconvertibleErrorCode())
+ : Error::success();
 }
 
 Error zlib::uncompress(StringRef InputBuffer,
SmallVectorImpl &UncompressedBuffer,
size_t UncompressedSize) {
   Uncompres

[PATCH] D128953: [NFC] Refactor llvm::zlib namespace

[Cole Kissane via Phabricator via cfe-commits]

ckissane marked 5 inline comments as done.
ckissane added a comment.

mark some handled comments done


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128953/new/

https://reviews.llvm.org/D128953

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128754: [llvm] Remove unused and redundant crc32 funcction from llvm::compression::zlib namespace

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 443009.
ckissane edited the summary of this revision.
ckissane added a comment.

- remove crc32 from zlib compression namespace
- Merge branch 'ckissane.refactor-compression.part-0' into 
ckissane.refactor-compression.part-1
- Merge branch 'ckissane.refactor-compression.part-0.no-string-sub' into 
ckissane.refactor-compression.part-1
- Merge remote-tracking branch 
'origin/ckissane.refactor-compression.part-0.no-string-sub' into 
ckissane.refactor-compression.part-1


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128754/new/

https://reviews.llvm.org/D128754

Files:
  llvm/docs/ReleaseNotes.rst
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/Compression.cpp
  llvm/unittests/Support/CompressionTest.cpp


Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -63,12 +63,6 @@
   TestZlibCompression(BinaryDataStr, zlib::DefaultCompression);
 }
 
-TEST(CompressionTest, ZlibCRC32) {
-  EXPECT_EQ(
-  0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
-}
-
 #endif
 
 }
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -83,10 +83,6 @@
   return E;
 }
 
-uint32_t zlib::crc32(StringRef Buffer) {
-  return ::crc32(0, (const Bytef *)Buffer.data(), Buffer.size());
-}
-
 #else
 bool zlib::isAvailable() { return false; }
 void zlib::compress(StringRef InputBuffer,
@@ -102,7 +98,4 @@
size_t UncompressedSize) {
   llvm_unreachable("zlib::uncompress is unavailable");
 }
-uint32_t zlib::crc32(StringRef Buffer) {
-  llvm_unreachable("zlib::crc32 is unavailable");
-}
 #endif
Index: llvm/include/llvm/Support/Compression.h
===
--- llvm/include/llvm/Support/Compression.h
+++ llvm/include/llvm/Support/Compression.h
@@ -41,8 +41,6 @@
  SmallVectorImpl &UncompressedBuffer,
  size_t UncompressedSize);
 
-uint32_t crc32(StringRef Buffer);
-
 } // End of namespace zlib
 
 } // End of namespace compression
Index: llvm/docs/ReleaseNotes.rst
===
--- llvm/docs/ReleaseNotes.rst
+++ llvm/docs/ReleaseNotes.rst
@@ -206,6 +206,7 @@
   introduction of alternatives to zlib compression in the llvm toolchain.
   Changes are as follows:
   * Relocate the ``llvm::zlib`` namespace to ``llvm::compression::zlib``.
+  * Remove crc32 from zlib compression namespace, people should use the 
``llvm::crc32`` instead.
 
 Changes to the Go bindings
 --


Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -63,12 +63,6 @@
   TestZlibCompression(BinaryDataStr, zlib::DefaultCompression);
 }
 
-TEST(CompressionTest, ZlibCRC32) {
-  EXPECT_EQ(
-  0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
-}
-
 #endif
 
 }
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -83,10 +83,6 @@
   return E;
 }
 
-uint32_t zlib::crc32(StringRef Buffer) {
-  return ::crc32(0, (const Bytef *)Buffer.data(), Buffer.size());
-}
-
 #else
 bool zlib::isAvailable() { return false; }
 void zlib::compress(StringRef InputBuffer,
@@ -102,7 +98,4 @@
size_t UncompressedSize) {
   llvm_unreachable("zlib::uncompress is unavailable");
 }
-uint32_t zlib::crc32(StringRef Buffer) {
-  llvm_unreachable("zlib::crc32 is unavailable");
-}
 #endif
Index: llvm/include/llvm/Support/Compression.h
===
--- llvm/include/llvm/Support/Compression.h
+++ llvm/include/llvm/Support/Compression.h
@@ -41,8 +41,6 @@
  SmallVectorImpl &UncompressedBuffer,
  size_t UncompressedSize);
 
-uint32_t crc32(StringRef Buffer);
-
 } // End of namespace zlib
 
 } // End of namespace compression
Index: llvm/docs/ReleaseNotes.rst
===
--- llvm/docs/ReleaseNotes.rst
+++ llvm/docs/ReleaseNotes.rst
@@ -206,6 +206,7 @@
   introduction of alternatives to zlib compression in the llvm toolchain.
   Changes are as follows:
   * Relocate the ``llvm::zlib`` namespace to ``llvm::compression::zlib``.
+  * Remove crc32 from zlib compression namespace, people should use the ``llvm::crc32`` instead.
 
 Changes to the Go bindings
 --

[PATCH] D128754: [llvm] Remove unused and redundant crc32 funcction from llvm::compression::zlib namespace

[Cole Kissane via Phabricator via cfe-commits]

ckissane marked 4 inline comments as done.
ckissane added a comment.

mark handled comments as done


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128754/new/

https://reviews.llvm.org/D128754

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128465: Zstandard as a second compression method to LLVM

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 443035.
ckissane added a comment.

start breaking this up further, this is now the add cmake config patch


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/include/llvm/Config/llvm-config.h.cmake
===
--- llvm/include/llvm/Config/llvm-config.h.cmake
+++ llvm/include/llvm/Config/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/cmake/modules/LLVMConfig.cmake.in
===
--- llvm/cmake/modules/LLVMConfig.cmake.in
+++ llvm/cmake/modules/LLVMConfig.cmake.in
@@ -73,6 +73,12 @@
   find_package(ZLIB)
 endif()
 
+set(LLVM_ENABLE_ZSTD @LLVM_ENABLE_ZSTD@)
+if(LLVM_ENABLE_ZSTD)
+  set(ZSTD_ROOT @ZSTD_ROOT@)
+  find_package(ZSTD)
+endif()
+
 set(LLVM_ENABLE_LIBXML2 @LLVM_ENABLE_LIBXML2@)
 if(LLVM_ENABLE_LIBXML2)
   find_package(LibXml2)
Index: llvm/cmake/modules/FindZSTD.cmake
===
--- /dev/null
+++ llvm/cmake/modules/FindZSTD.cmake
@@ -0,0 +1,31 @@
+#
+#//===--===//
+#//
+#// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+#// See https://llvm.org/LICENSE.txt for license information.
+#// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#//
+#//===--===//
+#
+
+find_path(ZSTD_INCLUDE_DIR
+  NAMES zstd.h
+  HINTS ${ZSTD_ROOT_DIR}/include)
+
+find_library(ZSTD_LIBRARY
+  NAMES zstd
+  HINTS ${ZSTD_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(
+ZSTD DEFAULT_MSG
+ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+
+if(ZSTD_FOUND)
+set(ZSTD_LIBRARIES ${ZSTD_LIBRARY})
+set(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR})
+endif()
+
+mark_as_advanced(
+  ZSTD_INCLUDE_DIR
+  ZSTD_LIBRARY)
Index: llvm/cmake/config-ix.cmake
===
--- llvm/cmake/config-ix.cmake
+++ llvm/cmake/config-ix.cmake
@@ -136,6 +136,27 @@
   set(LLVM_ENABLE_ZLIB "${HAVE_ZLIB}")
 endif()
 
+if(LLVM_ENABLE_ZSTD)
+  if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON)
+find_package(ZSTD REQUIRED)
+  elseif(NOT LLVM_USE_SANITIZER MATCHES "Memory.*")
+find_package(ZSTD)
+  endif()
+  if(ZSTD_FOUND)
+# Check if zstd we found is usable; for example, we may have found a 32-bit
+# library on a 64-bit system which would result in a link-time failure.
+cmake_push_check_state()
+list(APPEND CMAKE_REQUIRED_INCLUDES ${ZSTD_INCLUDE_DIR})
+list(APPEND CMAKE_REQUIRED_LIBRARIES ${ZSTD_LIBRARY})
+check_symbol_exists(ZSTD_compress zstd.h HAVE_ZSTD)
+cmake_pop_check_state()
+if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON AND NOT HAVE_ZSTD)
+  message(FATAL_ERROR "Failed to configure zstd")
+endif()
+  endif()
+  set(LLVM_ENABLE_ZSTD "${HAVE_ZSTD}")
+endif()
+
 if(LLVM_ENABLE_LIBXML2)
   if(LLVM_ENABLE_LIBXML2 STREQUAL FORCE_ON)
 find_package(LibXml2 REQUIRED)
Index: llvm/CMakeLists.txt
===
--- llvm/CMakeLists.txt
+++ llvm/CMakeLists.txt
@@ -438,6 +438,8 @@
 
 set(LLVM_ENABLE_ZLIB "ON" CACHE STRING "Use zlib for compression/decompression if available. Can be ON, OFF, or FORCE_ON")
 
+set(LLVM_ENABLE_ZSTD "ON" CACHE STRING "Use zstd for compression/decompression if available. Can be ON, OFF, or FORCE_ON")
+
 set(LLVM

[PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM

[Cole Kissane via Phabricator via cfe-commits]

ckissane marked 2 inline comments as done.
ckissane added a comment.

mark outdated comments done


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128953: [NFC] Refactor llvm::zlib namespace

[Cole Kissane via Phabricator via cfe-commits]

This revision was landed with ongoing or failed builds.
This revision was automatically updated to reflect the committed changes.
Closed by commit rGea61750c35a1: [NFC] Refactor llvm::zlib namespace (authored 
by ckissane).

Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128953/new/

https://reviews.llvm.org/D128953

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/docs/ReleaseNotes.rst
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -18,6 +18,7 @@
 #include "gtest/gtest.h"
 
 using namespace llvm;
+using namespace llvm::compression;
 
 namespace {
 
Index: llvm/unittests/ProfileData/InstrProfTest.cpp
===
--- llvm/unittests/ProfileData/InstrProfTest.cpp
+++ llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -1147,14 +1147,16 @@
 // Compressing:
 std::string FuncNameStrings1;
 EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames1, (DoCompression && zlib::isAvailable()),
+  FuncNames1,
+  (DoCompression && compression::zlib::isAvailable()),
   FuncNameStrings1),
   Succeeded());
 
 // Compressing:
 std::string FuncNameStrings2;
 EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames2, (DoCompression && zlib::isAvailable()),
+  FuncNames2,
+  (DoCompression && compression::zlib::isAvailable()),
   FuncNameStrings2),
   Succeeded());
 
Index: llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
===
--- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -739,7 +739,7 @@
 .str()
 .c_str());
 }
-if (!zlib::isAvailable())
+if (!compression::zlib::isAvailable())
   return createStringError(
   errc::invalid_argument,
   "LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
@@ -998,7 +998,7 @@
 "--decompress-debug-sections");
   }
 
-  if (Config.DecompressDebugSections && !zlib::isAvailable())
+  if (Config.DecompressDebugSections && !compression::zlib::isAvailable())
 return createStringError(
 errc::invalid_argument,
 "LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress");
Index: llvm/tools/llvm-mc/llvm-mc.cpp
===
--- llvm/tools/llvm-mc/llvm-mc.cpp
+++ llvm/tools/llvm-mc/llvm-mc.cpp
@@ -403,7 +403,7 @@
   MAI->setRelaxELFRelocations(RelaxELFRel);
 
   if (CompressDebugSections != DebugCompressionType::None) {
-if (!zlib::isAvailable()) {
+if (!compression::zlib::isAvailable()) {
   WithColor::error(errs(), ProgName)
   << "build tools with zlib to enable -compress-debug-sections";
   return 1;
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -22,11 +22,9 @@
 #endif
 
 using namespace llvm;
+using namespace llvm::compression;
 
 #if LLVM_ENABLE_ZLIB
-static Error createError(StringRef Err) {
-  return make_error(Err, inconvertibleErrorCode());
-}
 
 static StringRef convertZlibCodeToString(int Code) {
   switch (Code) {
@@ -70,15 +68,17 @@
   // Tell MemorySanitizer that zlib output buffer is fully initialized.
   // This avoids a false report when running LLVM with uninstrumented ZLib.
   __msan_unpoison(UncompressedBuffer, UncompressedSize);
-  return Res ? createError(convertZlibCodeToString(Res)) : Error::success();
+  return Res ? make_error(convertZlibCodeToString(Res),
+   inconvertibleErrorCode())
+ : Error::success();
 }
 
 Error zlib::uncompress(StringRef InputBuffer,

[PATCH] D128754: [llvm] Remove unused and redundant crc32 funcction from llvm::compression::zlib namespace

[Cole Kissane via Phabricator via cfe-commits]

This revision was automatically updated to reflect the committed changes.
Closed by commit rG96063bfa9088: [llvm] Remove unused and redundant crc32 
funcction from llvm::compression::zlib… (authored by ckissane).

Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128754/new/

https://reviews.llvm.org/D128754

Files:
  llvm/docs/ReleaseNotes.rst
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/Compression.cpp
  llvm/unittests/Support/CompressionTest.cpp


Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -63,12 +63,6 @@
   TestZlibCompression(BinaryDataStr, zlib::DefaultCompression);
 }
 
-TEST(CompressionTest, ZlibCRC32) {
-  EXPECT_EQ(
-  0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
-}
-
 #endif
 
 }
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -83,10 +83,6 @@
   return E;
 }
 
-uint32_t zlib::crc32(StringRef Buffer) {
-  return ::crc32(0, (const Bytef *)Buffer.data(), Buffer.size());
-}
-
 #else
 bool zlib::isAvailable() { return false; }
 void zlib::compress(StringRef InputBuffer,
@@ -102,7 +98,4 @@
size_t UncompressedSize) {
   llvm_unreachable("zlib::uncompress is unavailable");
 }
-uint32_t zlib::crc32(StringRef Buffer) {
-  llvm_unreachable("zlib::crc32 is unavailable");
-}
 #endif
Index: llvm/include/llvm/Support/Compression.h
===
--- llvm/include/llvm/Support/Compression.h
+++ llvm/include/llvm/Support/Compression.h
@@ -41,8 +41,6 @@
  SmallVectorImpl &UncompressedBuffer,
  size_t UncompressedSize);
 
-uint32_t crc32(StringRef Buffer);
-
 } // End of namespace zlib
 
 } // End of namespace compression
Index: llvm/docs/ReleaseNotes.rst
===
--- llvm/docs/ReleaseNotes.rst
+++ llvm/docs/ReleaseNotes.rst
@@ -206,6 +206,7 @@
   introduction of alternatives to zlib compression in the llvm toolchain.
   Changes are as follows:
   * Relocate the ``llvm::zlib`` namespace to ``llvm::compression::zlib``.
+  * Remove crc32 from zlib compression namespace, people should use the 
``llvm::crc32`` instead.
 
 Changes to the Go bindings
 --


Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -63,12 +63,6 @@
   TestZlibCompression(BinaryDataStr, zlib::DefaultCompression);
 }
 
-TEST(CompressionTest, ZlibCRC32) {
-  EXPECT_EQ(
-  0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
-}
-
 #endif
 
 }
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -83,10 +83,6 @@
   return E;
 }
 
-uint32_t zlib::crc32(StringRef Buffer) {
-  return ::crc32(0, (const Bytef *)Buffer.data(), Buffer.size());
-}
-
 #else
 bool zlib::isAvailable() { return false; }
 void zlib::compress(StringRef InputBuffer,
@@ -102,7 +98,4 @@
size_t UncompressedSize) {
   llvm_unreachable("zlib::uncompress is unavailable");
 }
-uint32_t zlib::crc32(StringRef Buffer) {
-  llvm_unreachable("zlib::crc32 is unavailable");
-}
 #endif
Index: llvm/include/llvm/Support/Compression.h
===
--- llvm/include/llvm/Support/Compression.h
+++ llvm/include/llvm/Support/Compression.h
@@ -41,8 +41,6 @@
  SmallVectorImpl &UncompressedBuffer,
  size_t UncompressedSize);
 
-uint32_t crc32(StringRef Buffer);
-
 } // End of namespace zlib
 
 } // End of namespace compression
Index: llvm/docs/ReleaseNotes.rst
===
--- llvm/docs/ReleaseNotes.rst
+++ llvm/docs/ReleaseNotes.rst
@@ -206,6 +206,7 @@
   introduction of alternatives to zlib compression in the llvm toolchain.
   Changes are as follows:
   * Relocate the ``llvm::zlib`` namespace to ``llvm::compression::zlib``.
+  * Remove crc32 from zlib compression namespace, people should use the ``llvm::crc32`` instead.
 
 Changes to the Go bindings
 --
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 443308.
ckissane added a comment.

- Merge remote-tracking branch 'origin/main' into ckissane.add-zstd.0-cmake
- added have_zstd to compiler-rt/test/lit.common.cfg.py, 
clang-tools-extra/clangd/test/lit.cfg.py and several lit.site.cfg.py.in files


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/test/lit.cfg.py
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.cfg.py
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  lld/test/lit.site.cfg.py.in
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  lldb/test/Shell/lit.site.cfg.py.in
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/docs/ReleaseNotes.rst
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/test/lit.site.cfg.py.in
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -18,6 +18,7 @@
 #include "gtest/gtest.h"
 
 using namespace llvm;
+using namespace llvm::compression;
 
 namespace {
 
@@ -62,12 +63,6 @@
   TestZlibCompression(BinaryDataStr, zlib::DefaultCompression);
 }
 
-TEST(CompressionTest, ZlibCRC32) {
-  EXPECT_EQ(
-  0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
-}
-
 #endif
 
 }
Index: llvm/unittests/ProfileData/InstrProfTest.cpp
===
--- llvm/unittests/ProfileData/InstrProfTest.cpp
+++ llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -1147,14 +1147,16 @@
 // Compressing:
 std::string FuncNameStrings1;
 EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames1, (DoCompression && zlib::isAvailable()),
+  FuncNames1,
+  (DoCompression && compression::zlib::isAvailable()),
   FuncNameStrings1),
   Succeeded());
 
 // Compressing:
 std::string FuncNameStrings2;
 EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames2, (DoCompression && zlib::isAvailable()),
+  FuncNames2,
+  (DoCompression && compression::zlib::isAvailable()),
   FuncNameStrings2),
   Succeeded());
 
Index: llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
===
--- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -739,7 +739,7 @@
 .str()
 .c_str());
 }
-if (!zlib::isAvailable())
+if (!compression::zlib::isAvailable())
   return createStringError(
   errc::invalid_argument,
   "LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
@@ -998,7 +998,7 @@
 "--decompress-debug-sections");
   }
 
-  if (Config.DecompressDebugSections && !zlib::isAvailable())
+  if (Config.DecompressDebugSections && !compression::zlib::isAvailable())
 return createStringError(
   

[PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM

[Cole Kissane via Phabricator via cfe-commits]

ckissane marked 2 inline comments as done.
ckissane added a comment.

marked fixed comments as done


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 443310.
ckissane added a comment.

- Merge remote-tracking branch 'origin/main' into ckissane.add-zstd.0-cmake


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/test/lit.cfg.py
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.cfg.py
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lld/test/lit.site.cfg.py.in
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  lldb/test/Shell/lit.site.cfg.py.in
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/test/lit.site.cfg.py.in
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/test/lit.site.cfg.py.in
===
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
Index: llvm/include/llvm/Config/llvm-config.h.cmake
===
--- llvm/include/llvm/Config/llvm-config.h.cmake
+++ llvm/include/llvm/Config/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/cmake/modules/LLVMConfig.cmake.in
===
--- llvm/cmake/modules/LLVMConfig.cmake.in
+++ llvm/cmake/modules/LLVMConfig.cmake.in
@@ -73,6 +73,12 @@
   find_package(ZLIB)
 endif()
 
+set(LLVM_ENABLE_ZSTD @LLVM_ENABLE_ZSTD@)
+if(LLVM_ENABLE_ZSTD)
+  set(ZSTD_ROOT @ZSTD_ROOT@)
+  find_package(ZSTD)
+endif()
+
 set(LLVM_ENABLE_LIBXML2 @LLVM_ENABLE_LIBXML2@)
 if(LLVM_ENABLE_LIBXML2)
   find_package(LibXml2)
Index: llvm/cmake/modules/FindZSTD.cmake
===
--- /dev/null
+++ llvm/cmake/modules/FindZSTD.cmake
@@ -0,0 +1,21 @@
+find_path(ZSTD_INCLUDE_DIR
+  NAMES zstd.h
+  HINTS ${ZSTD_ROOT_DIR}/include)
+
+find_library(ZSTD_LIBRARY
+  NAMES zstd
+  HINTS ${ZSTD_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(
+ZSTD DEFAULT_MSG
+ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+
+if(ZSTD_FOUND)
+set(ZSTD_LIBRARIES ${ZSTD_LIBRARY})
+set(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR})
+endif()
+
+mark_as_advanced(
+  ZSTD_INCLUDE_DIR
+  ZSTD_LIBRARY)
Index: llvm/cmake/config-ix.cmake
===
--- llvm/cmake/config-ix.cmake
+++ llvm/cmake/config-ix.cmake
@@ -136,6 +136,27 @@
   set(LLVM_ENABLE_ZLIB "${HAVE_ZLIB}")
 endif()
 
+if(LLVM_ENABLE_ZSTD)
+  if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON)
+find_package(ZSTD REQUIRED)
+  elseif(NOT LLVM_USE_SANITIZER MATCHES "Memory.*")
+find_package(ZSTD)
+  endif()
+  if(ZSTD_FOUND)
+# Check if zstd we found is usable; for example, we may have found a 32-bit
+# library on a 64-bit system which would result in a link-time failure.
+cmake_push_check_state()
+list(APPEND CMAKE_REQUIRED_INCLUDES ${ZSTD_INCLUDE_DIR})
+list(APPEND CMAKE_REQUIRED_LIBRARIES ${ZSTD_LIBRARY})
+check_symbol_exists(ZSTD_compress zstd.h HAVE_ZSTD)
+cmake_pop_check_state()
+if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON AND NOT HAVE_ZSTD)
+  message(FATAL_ERROR "Failed to configure zstd")
+endif()
+  endif()
+  set(LLVM_ENABLE_ZSTD "${HAVE_ZSTD}")
+endif()
+
 if(LLVM_ENABLE_LIBXML2)
   if(LLVM_ENABLE_LIBXML2 STREQUAL FORCE_ON)
 find_package(LibXml2 REQUIRED)
Index: llvm/CMakeLists.txt
===
--- llvm/CMakeLists.txt
+++ llvm/C

[PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM

[Cole Kissane via Phabricator via cfe-commits]

This revision was landed with ongoing or failed builds.
This revision was automatically updated to reflect the committed changes.
Closed by commit rGf07caf20b9d3: [llvm] cmake config groundwork to have ZSTD in 
LLVM (authored by ckissane).

Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/test/lit.cfg.py
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.cfg.py
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lld/test/lit.site.cfg.py.in
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  lldb/test/Shell/lit.site.cfg.py.in
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/test/lit.site.cfg.py.in
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/test/lit.site.cfg.py.in
===
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
Index: llvm/include/llvm/Config/llvm-config.h.cmake
===
--- llvm/include/llvm/Config/llvm-config.h.cmake
+++ llvm/include/llvm/Config/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/cmake/modules/LLVMConfig.cmake.in
===
--- llvm/cmake/modules/LLVMConfig.cmake.in
+++ llvm/cmake/modules/LLVMConfig.cmake.in
@@ -73,6 +73,12 @@
   find_package(ZLIB)
 endif()
 
+set(LLVM_ENABLE_ZSTD @LLVM_ENABLE_ZSTD@)
+if(LLVM_ENABLE_ZSTD)
+  set(ZSTD_ROOT @ZSTD_ROOT@)
+  find_package(ZSTD)
+endif()
+
 set(LLVM_ENABLE_LIBXML2 @LLVM_ENABLE_LIBXML2@)
 if(LLVM_ENABLE_LIBXML2)
   find_package(LibXml2)
Index: llvm/cmake/modules/FindZSTD.cmake
===
--- /dev/null
+++ llvm/cmake/modules/FindZSTD.cmake
@@ -0,0 +1,21 @@
+find_path(ZSTD_INCLUDE_DIR
+  NAMES zstd.h
+  HINTS ${ZSTD_ROOT_DIR}/include)
+
+find_library(ZSTD_LIBRARY
+  NAMES zstd
+  HINTS ${ZSTD_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(
+ZSTD DEFAULT_MSG
+ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+
+if(ZSTD_FOUND)
+set(ZSTD_LIBRARIES ${ZSTD_LIBRARY})
+set(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR})
+endif()
+
+mark_as_advanced(
+  ZSTD_INCLUDE_DIR
+  ZSTD_LIBRARY)
Index: llvm/cmake/config-ix.cmake
===
--- llvm/cmake/config-ix.cmake
+++ llvm/cmake/config-ix.cmake
@@ -136,6 +136,27 @@
   set(LLVM_ENABLE_ZLIB "${HAVE_ZLIB}")
 endif()
 
+if(LLVM_ENABLE_ZSTD)
+  if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON)
+find_package(ZSTD REQUIRED)
+  elseif(NOT LLVM_USE_SANITIZER MATCHES "Memory.*")
+find_package(ZSTD)
+  endif()
+  if(ZSTD_FOUND)
+# Check if zstd we found is usable; for example, we may have found a 32-bit
+# library on a 64-bit system which would result in a link-time failure.
+cmake_push_check_state()
+list(APPEND CMAKE_REQUIRED_INCLUDES ${ZSTD_INCLUDE_DIR})
+list(APPEND CMAKE_REQUIRED_LIBRARIES ${ZSTD_LIBRARY})
+check_symbol_exists(ZSTD_compress zstd.h HAVE_ZSTD)
+cmake_pop_check_state()
+if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON AND NOT HAVE_ZSTD)
+  message(FATAL_ERROR "Failed to configure zstd")
+endif()
+  endif()
+  set(LLVM_ENABLE_ZSTD "${HAVE_ZSTD}")
+endif()
+
 if(LLVM_ENABLE_LIBXML2)
   if(LLVM_ENABLE_LIBXML2 STREQUAL FORCE_ON)
 find_package(LibXml2 REQUIRED)
Index: llvm/CMakeLists.txt
=

[PATCH] D130516: [Support] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane added inline comments.



Comment at: clang/lib/Serialization/ASTWriter.cpp:2003-2004
   // consumers will not want its contents.
+  llvm::compression::CompressionAlgorithm CompressionScheme =
+  llvm::compression::ZlibCompressionAlgorithm();
+

dblaikie wrote:
> ckissane wrote:
> > dblaikie wrote:
> > > Doesn't this cause slicing & end up with the base implementation?
> > > 
> > > (also the base class `CompressionAlgorithm` has no virtual functions, so 
> > > I'm not sure how this is meant to work - does this code all work? Then I 
> > > must be missing some things - how does this work?)
> > You are correct to observe that this patch does not fully pass around 
> > pointers to instances of the classes, however, because I don't pass 
> > pointers and the currently repetitive nature of the compression classes, 
> > this still functions correctly.
> > In short, a follow-up patch (which I will shortly upload) will convert this 
> > to using class instances and passing those around.
> > Including reworking functions throughout llvm-project to take advantage of 
> > this.
> > I am aiming to take this 2 step process to cut down on making an already 
> > large pass larger.
> > Let me know if you have any concerns or ideas.
> But I'm not sure how this patch works correctly - wouldn't the line below 
> (`CompressionScheme.supported()`) call `CompressionAlgorithm::supported()` 
> which always returns false?
good catch


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D130516: compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane added a comment.

In D130516#3683384 , @dblaikie wrote:

> Any chance this could be split up to be more directly comparable to 
> https://reviews.llvm.org/D130458 ?

yes definitely! Doing so now!


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D130516: compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 448176.
ckissane added a comment.

- fix compression inheritence and some more compression class helpers


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/InstrProfCorrelator.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -22,31 +22,43 @@
 
 namespace {
 
-#if LLVM_ENABLE_ZLIB
-static void testZlibCompression(StringRef Input, int Level) {
+static void testCompressionAlgorithm(
+StringRef Input, int Level,
+compression::CompressionAlgorithm *CompressionScheme,
+std::string ExpectedDestinationBufferTooSmallErrorMessage) {
   SmallVector Compressed;
   SmallVector Uncompressed;
-  zlib::compress(arrayRefFromStringRef(Input), Compressed, Level);
+  CompressionScheme->compress(arrayRefFromStringRef(Input), Compressed, Level);
 
   // Check that uncompressed buffer is the same as original.
-  Error E = zlib::uncompress(Compressed, Uncompressed, Input.size());
+  Error E =
+  CompressionScheme->decompress(Compressed, Uncompressed, Input.size());
   consumeError(std::move(E));
 
   EXPECT_EQ(Input, toStringRef(Uncompressed));
   if (Input.size() > 0) {
 // Uncompression fails if expected length is too short.
-E = zlib::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("zlib error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+E = CompressionScheme->decompress(Compressed, Uncompressed,
+  Input.size() - 1);
+EXPECT_EQ(ExpectedDestinationBufferTooSmallErrorMessage,
+  llvm::toString(std::move(E)));
   }
 }
 
+#if LLVM_ENABLE_ZLIB
+static void testZlibCompression(StringRef Input, int Level) {
+  testCompressionAlgorithm(Input, Level, new ZlibCompressionAlgorithm(),
+   "zlib error: Z_BUF_ERROR");
+}
+
 TEST(CompressionTest, Zlib) {
-  testZlibCompression("", zlib::DefaultCompression);
+  compression::CompressionAlgorithm *CompressionScheme =
+  new compression::ZlibCompressionAlgorithm();
+  testZlibCompression("", CompressionScheme->getDefaultLevel());
 
-  testZlibCompression("hello, world!", zlib::NoCompression);
-  testZlibCompression("hello, world!", zlib::BestSizeCompression);
-  testZlibCompression("hello, world!", zlib::BestSpeedCompression);
-  testZlibCompression("hello, world!", zlib::DefaultCompression);
+  testZlibCompression("hello, world!", CompressionScheme->getBestSizeLevel());
+  testZlibCompression("hello, world!", CompressionScheme->getBestSpeedLevel());
+  testZlibCompression("hello, world!", CompressionScheme->getDefaultLevel());
 
   const size_t kSize = 1024;
   char BinaryData[kSize];
@@ -54,38 +66,27 @@
 BinaryData[i] = i & 255;
   StringRef BinaryDataStr(BinaryData, kSize);
 
-  testZlibCompression(BinaryDataStr, zlib::NoCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
-  testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+  testZlibCompression(BinaryDataStr, CompressionScheme->getBestSizeLevel());
+  testZlibCompression(BinaryDataStr, CompressionScheme->getBestSpeedLevel());
+  testZlibCompression(BinaryDataStr, CompressionScheme->getDefaultLevel());
 }
 #endif
 
 #if LLVM_ENABLE_ZSTD
-static void testZstdCompression(StringRef Input, int Level) {
-  SmallVector Compressed;
-  SmallVector Uncompressed;
-  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
 
-  // Check that uncompressed buffer is the same as original.
-  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
-  consumeError(std::move(E));
-
-  EXPECT_EQ(Input, toStringRef(Uncompressed));
-  if (Input.size() > 0) {
-// Uncompression fails if expected length is too short.
-E = zstd::unc

[PATCH] D130516: compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane added inline comments.



Comment at: llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp:54
+  CompressionScheme =
+  CompressionScheme->when(Compress && DoInstrProfNameCompression)
+  ->whenSupported();

note the helpers such as when(bool), whensupported() and notNone()


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D130516: compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 448181.
ckissane added a comment.

- use CompressionScheme->notNone() in InstrProf


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/InstrProfCorrelator.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -22,31 +22,43 @@
 
 namespace {
 
-#if LLVM_ENABLE_ZLIB
-static void testZlibCompression(StringRef Input, int Level) {
+static void testCompressionAlgorithm(
+StringRef Input, int Level,
+compression::CompressionAlgorithm *CompressionScheme,
+std::string ExpectedDestinationBufferTooSmallErrorMessage) {
   SmallVector Compressed;
   SmallVector Uncompressed;
-  zlib::compress(arrayRefFromStringRef(Input), Compressed, Level);
+  CompressionScheme->compress(arrayRefFromStringRef(Input), Compressed, Level);
 
   // Check that uncompressed buffer is the same as original.
-  Error E = zlib::uncompress(Compressed, Uncompressed, Input.size());
+  Error E =
+  CompressionScheme->decompress(Compressed, Uncompressed, Input.size());
   consumeError(std::move(E));
 
   EXPECT_EQ(Input, toStringRef(Uncompressed));
   if (Input.size() > 0) {
 // Uncompression fails if expected length is too short.
-E = zlib::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("zlib error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+E = CompressionScheme->decompress(Compressed, Uncompressed,
+  Input.size() - 1);
+EXPECT_EQ(ExpectedDestinationBufferTooSmallErrorMessage,
+  llvm::toString(std::move(E)));
   }
 }
 
+#if LLVM_ENABLE_ZLIB
+static void testZlibCompression(StringRef Input, int Level) {
+  testCompressionAlgorithm(Input, Level, new ZlibCompressionAlgorithm(),
+   "zlib error: Z_BUF_ERROR");
+}
+
 TEST(CompressionTest, Zlib) {
-  testZlibCompression("", zlib::DefaultCompression);
+  compression::CompressionAlgorithm *CompressionScheme =
+  new compression::ZlibCompressionAlgorithm();
+  testZlibCompression("", CompressionScheme->getDefaultLevel());
 
-  testZlibCompression("hello, world!", zlib::NoCompression);
-  testZlibCompression("hello, world!", zlib::BestSizeCompression);
-  testZlibCompression("hello, world!", zlib::BestSpeedCompression);
-  testZlibCompression("hello, world!", zlib::DefaultCompression);
+  testZlibCompression("hello, world!", CompressionScheme->getBestSizeLevel());
+  testZlibCompression("hello, world!", CompressionScheme->getBestSpeedLevel());
+  testZlibCompression("hello, world!", CompressionScheme->getDefaultLevel());
 
   const size_t kSize = 1024;
   char BinaryData[kSize];
@@ -54,38 +66,27 @@
 BinaryData[i] = i & 255;
   StringRef BinaryDataStr(BinaryData, kSize);
 
-  testZlibCompression(BinaryDataStr, zlib::NoCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
-  testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+  testZlibCompression(BinaryDataStr, CompressionScheme->getBestSizeLevel());
+  testZlibCompression(BinaryDataStr, CompressionScheme->getBestSpeedLevel());
+  testZlibCompression(BinaryDataStr, CompressionScheme->getDefaultLevel());
 }
 #endif
 
 #if LLVM_ENABLE_ZSTD
-static void testZstdCompression(StringRef Input, int Level) {
-  SmallVector Compressed;
-  SmallVector Uncompressed;
-  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
 
-  // Check that uncompressed buffer is the same as original.
-  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
-  consumeError(std::move(E));
-
-  EXPECT_EQ(Input, toStringRef(Uncompressed));
-  if (Input.size() > 0) {
-// Uncompression fails if expected length is too short.
-E = zstd::uncompress(Compressed, Un

[PATCH] D130516: compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane added inline comments.



Comment at: clang-tools-extra/clangd/index/Serialization.cpp:32
+llvm::compression::CompressionAlgorithm *StringTableCompressionScheme =
+new llvm::compression::ZlibCompressionAlgorithm();
+

dblaikie wrote:
> We're generally trying to avoid global ctors in LLVM. So at most this should 
> be a static local variable in a function that accesses the algorithm (though 
> perhaps these "compression algorithm" classes shouldn't be accessible 
> directly, and only through singleton accessors in the defined alongside - 
> like there's no reason for LLVM to contain more than one instance of 
> ZlibCompressionAlgorithm, I think?)
your idea seems correct to me, however
some algorithms, such as zstd, support running in multiple threads, so this 
might influence our decision.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D130667: feat: use compression class for: clang ast serial zstd option + prof data compression variants + zstd elf + ztsd objcopy support

[Cole Kissane via Phabricator via cfe-commits]

ckissane created this revision.
ckissane added reviewers: leonardchan, phosek, dblaikie.
Herald added subscribers: Enna1, wenlei, usaxena95, abrachet, kadircet, 
arphaman, hiraditya, arichardson, mgorny, emaste.
Herald added a reviewer: alexander-shaposhnikov.
Herald added a reviewer: rupprecht.
Herald added a reviewer: jhenderson.
Herald added a reviewer: MaskRay.
Herald added a project: All.
ckissane requested review of this revision.
Herald added subscribers: cfe-commits, llvm-commits, Sanitizers, StephenFan.
Herald added projects: clang, Sanitizers, LLVM, clang-tools-extra.

Merge remote-tracking branch 'origin/main' into ckissane.compression-class

[MC] fix merge issues with removal of GNU compression

Merge remote-tracking branch 'origin/main' into ckissane.compression-class

feat: compression class + clang ast serial zstd option + zstd elf + ztsd 
objcopy support

[MC] fix merge issues with removal of GNU compression

Merge branch 'ckissane.compression-class' of github.com:ckissane/llvm-project 
into ckissane.compression-class

fix compression class inheritence and passing + elf zstd support + ast writer 
and profiledata have configurable compression + command line opt compression 
support (see usage in InstrProf.cpp)

update usages of -enable-name-compression=false to be -name-compression=none

do not exit on parse of bad compression cl::opt

Merge remote-tracking branch 'origin/main' into ckissane.compression-class

fix compression class usage in some profile data tests

tiny cleanup of using NoneCompressionAlgorithm::AlgorithmId

Merge remote-tracking branch 'origin/ckissane.compression-class-simple' into 
ckissane.compression-class


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D130667

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/index/Serialization.h
  clang/include/clang/Serialization/ASTBitCodes.h
  clang/include/clang/Serialization/ASTWriter.h
  clang/lib/CodeGen/CoverageMappingGen.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Frontend/ASTUnit.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  clang/lib/Serialization/GeneratePCH.cpp
  clang/test/CodeGen/coverage-compilation-dir.c
  clang/test/CoverageMapping/abspath.cpp
  clang/test/CoverageMapping/ir.c
  clang/test/Profile/coverage-prefix-map.c
  compiler-rt/include/profile/InstrProfData.inc
  compiler-rt/test/profile/instrprof-darwin-dead-strip.c
  compiler-rt/test/profile/instrprof-gc-sections.c
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  lld/test/CMakeLists.txt
  lld/test/ELF/compressed-debug-input-zstd.s
  lld/test/ELF/compressed-input-err-zstd.s
  lld/test/lit.site.cfg.py.in
  llvm/include/llvm/BinaryFormat/ELF.h
  llvm/include/llvm/MC/MCTargetOptions.h
  llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
  llvm/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/ProfileData/InstrProfData.inc
  llvm/include/llvm/ProfileData/SampleProfWriter.h
  llvm/include/llvm/Support/CommandLine.h
  llvm/include/llvm/Support/Compression.h
  llvm/include/llvm/Transforms/Instrumentation.h
  llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/CommandLine.cpp
  llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
  llvm/test/tools/llvm-cov/coverage-prefix-map.test
  llvm/test/tools/llvm-cov/multiple-objects-not-all-instrumented.test
  llvm/test/tools/llvm-cov/multiple-objects.test
  llvm/test/tools/llvm-cov/multithreaded-report.test
  llvm/test/tools/llvm-cov/sources-specified.test
  llvm/test/tools/llvm-profdata/c-general.test
  llvm/test/tools/llvm-profdata/memprof-merge.test
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/tools/llvm-objcopy/ObjcopyOpts.td
  llvm/tools/llvm-profdata/llvm-profdata.cpp
  llvm/tools/llvm-profgen/ProfileGenerator.cpp
  llvm/unittests/ProfileData/CoverageMappingTest.cpp
  llvm/unittests/ProfileData/SampleProfTest.cpp

Index: llvm/unittests/ProfileData/SampleProfTest.cpp
===
--- llvm/unittests/ProfileData/SampleProfTest.cpp
+++ llvm/unittests/ProfileData/SampleProfTest.cpp
@@ -50,7 +50,8 @@
 std::error_code EC;
 std::unique_ptr OS(
 new raw_fd_ostream(Profile, EC, sys::fs::OF_None));
-auto WriterOrErr = SampleProfileWriter::create(OS, Format);
+auto WriterOrErr = SampleProfileWriter::create(
+OS, Format, new compression::ZlibCompressionAlgorithm());
 ASSERT_TRUE(NoError(WriterOrErr.getError()));
 Writer = std::move(WriterOrErr.get(

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane marked 2 inline comments as done.
ckissane added a comment.

marked outdated comments as done


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 448469.
ckissane added a comment.

- make compression singletons


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/InstrProfCorrelator.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -22,31 +22,43 @@
 
 namespace {
 
-#if LLVM_ENABLE_ZLIB
-static void testZlibCompression(StringRef Input, int Level) {
+static void testCompressionAlgorithm(
+StringRef Input, int Level,
+compression::CompressionAlgorithm *CompressionScheme,
+std::string ExpectedDestinationBufferTooSmallErrorMessage) {
   SmallVector Compressed;
   SmallVector Uncompressed;
-  zlib::compress(arrayRefFromStringRef(Input), Compressed, Level);
+  CompressionScheme->compress(arrayRefFromStringRef(Input), Compressed, Level);
 
   // Check that uncompressed buffer is the same as original.
-  Error E = zlib::uncompress(Compressed, Uncompressed, Input.size());
+  Error E =
+  CompressionScheme->decompress(Compressed, Uncompressed, Input.size());
   consumeError(std::move(E));
 
   EXPECT_EQ(Input, toStringRef(Uncompressed));
   if (Input.size() > 0) {
 // Uncompression fails if expected length is too short.
-E = zlib::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("zlib error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+E = CompressionScheme->decompress(Compressed, Uncompressed,
+  Input.size() - 1);
+EXPECT_EQ(ExpectedDestinationBufferTooSmallErrorMessage,
+  llvm::toString(std::move(E)));
   }
 }
 
+#if LLVM_ENABLE_ZLIB
+static void testZlibCompression(StringRef Input, int Level) {
+  testCompressionAlgorithm(Input, Level, ZlibCompression,
+   "zlib error: Z_BUF_ERROR");
+}
+
 TEST(CompressionTest, Zlib) {
-  testZlibCompression("", zlib::DefaultCompression);
+  compression::CompressionAlgorithm *CompressionScheme =
+  compression::ZlibCompression;
+  testZlibCompression("", CompressionScheme->getDefaultLevel());
 
-  testZlibCompression("hello, world!", zlib::NoCompression);
-  testZlibCompression("hello, world!", zlib::BestSizeCompression);
-  testZlibCompression("hello, world!", zlib::BestSpeedCompression);
-  testZlibCompression("hello, world!", zlib::DefaultCompression);
+  testZlibCompression("hello, world!", CompressionScheme->getBestSizeLevel());
+  testZlibCompression("hello, world!", CompressionScheme->getBestSpeedLevel());
+  testZlibCompression("hello, world!", CompressionScheme->getDefaultLevel());
 
   const size_t kSize = 1024;
   char BinaryData[kSize];
@@ -54,38 +66,27 @@
 BinaryData[i] = i & 255;
   StringRef BinaryDataStr(BinaryData, kSize);
 
-  testZlibCompression(BinaryDataStr, zlib::NoCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
-  testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+  testZlibCompression(BinaryDataStr, CompressionScheme->getBestSizeLevel());
+  testZlibCompression(BinaryDataStr, CompressionScheme->getBestSpeedLevel());
+  testZlibCompression(BinaryDataStr, CompressionScheme->getDefaultLevel());
 }
 #endif
 
 #if LLVM_ENABLE_ZSTD
-static void testZstdCompression(StringRef Input, int Level) {
-  SmallVector Compressed;
-  SmallVector Uncompressed;
-  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
 
-  // Check that uncompressed buffer is the same as original.
-  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
-  consumeError(std::move(E));
-
-  EXPECT_EQ(Input, toStringRef(Uncompressed));
-  if (Input.size() > 0) {
-// Uncompression fails if expected length is too short.
-E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("D

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 448473.
ckissane added a comment.

- fix usage of CompressionAlgorithmFromId


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/InstrProfCorrelator.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -22,31 +22,43 @@
 
 namespace {
 
-#if LLVM_ENABLE_ZLIB
-static void testZlibCompression(StringRef Input, int Level) {
+static void testCompressionAlgorithm(
+StringRef Input, int Level,
+compression::CompressionAlgorithm *CompressionScheme,
+std::string ExpectedDestinationBufferTooSmallErrorMessage) {
   SmallVector Compressed;
   SmallVector Uncompressed;
-  zlib::compress(arrayRefFromStringRef(Input), Compressed, Level);
+  CompressionScheme->compress(arrayRefFromStringRef(Input), Compressed, Level);
 
   // Check that uncompressed buffer is the same as original.
-  Error E = zlib::uncompress(Compressed, Uncompressed, Input.size());
+  Error E =
+  CompressionScheme->decompress(Compressed, Uncompressed, Input.size());
   consumeError(std::move(E));
 
   EXPECT_EQ(Input, toStringRef(Uncompressed));
   if (Input.size() > 0) {
 // Uncompression fails if expected length is too short.
-E = zlib::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("zlib error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+E = CompressionScheme->decompress(Compressed, Uncompressed,
+  Input.size() - 1);
+EXPECT_EQ(ExpectedDestinationBufferTooSmallErrorMessage,
+  llvm::toString(std::move(E)));
   }
 }
 
+#if LLVM_ENABLE_ZLIB
+static void testZlibCompression(StringRef Input, int Level) {
+  testCompressionAlgorithm(Input, Level, ZlibCompression,
+   "zlib error: Z_BUF_ERROR");
+}
+
 TEST(CompressionTest, Zlib) {
-  testZlibCompression("", zlib::DefaultCompression);
+  compression::CompressionAlgorithm *CompressionScheme =
+  compression::ZlibCompression;
+  testZlibCompression("", CompressionScheme->getDefaultLevel());
 
-  testZlibCompression("hello, world!", zlib::NoCompression);
-  testZlibCompression("hello, world!", zlib::BestSizeCompression);
-  testZlibCompression("hello, world!", zlib::BestSpeedCompression);
-  testZlibCompression("hello, world!", zlib::DefaultCompression);
+  testZlibCompression("hello, world!", CompressionScheme->getBestSizeLevel());
+  testZlibCompression("hello, world!", CompressionScheme->getBestSpeedLevel());
+  testZlibCompression("hello, world!", CompressionScheme->getDefaultLevel());
 
   const size_t kSize = 1024;
   char BinaryData[kSize];
@@ -54,38 +66,27 @@
 BinaryData[i] = i & 255;
   StringRef BinaryDataStr(BinaryData, kSize);
 
-  testZlibCompression(BinaryDataStr, zlib::NoCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
-  testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+  testZlibCompression(BinaryDataStr, CompressionScheme->getBestSizeLevel());
+  testZlibCompression(BinaryDataStr, CompressionScheme->getBestSpeedLevel());
+  testZlibCompression(BinaryDataStr, CompressionScheme->getDefaultLevel());
 }
 #endif
 
 #if LLVM_ENABLE_ZSTD
-static void testZstdCompression(StringRef Input, int Level) {
-  SmallVector Compressed;
-  SmallVector Uncompressed;
-  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
 
-  // Check that uncompressed buffer is the same as original.
-  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
-  consumeError(std::move(E));
-
-  EXPECT_EQ(Input, toStringRef(Uncompressed));
-  if (Input.size() > 0) {
-// Uncompression fails if expected length is too short.
-E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
-

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane added inline comments.



Comment at: llvm/lib/Support/Compression.cpp:30-32
+ZStdCompressionAlgorithm
+*llvm::compression::ZStdCompressionAlgorithm::Instance =
+new ZStdCompressionAlgorithm();

dblaikie wrote:
> leonardchan wrote:
> > Perhaps for each of these, you could instead have something like:
> > 
> > ```
> > ZStdCompressionAlgorithm *getZStdCompressionAlgorithm() {
> >   static ZStdCompressionAlgorithm* instance = new ZStdCompressionAlgorithm;
> >   return instance;
> > }
> > ```
> > 
> > This way the instances are only new'd when they're actually used.
> Yep, I'd mentioned/suggested that (so, seconding here) elsewhere encouraging 
> these to be singletons: https://reviews.llvm.org/D130516#3683384
> 
> And they don't even need to be 'new'd in that case, this would be fine:
> ```
> ZstdCompressionAlgorithm &getZstdCompressionAlgorithm() {
>   static ZstdCompressionAlgorithm C;
>   return C;
> }
> ```
> 
> Though I think maybe we don't need individual access to the algorithms, and 
> it'd be fine to have only a single entry point like this:
> ```
> CompressionAlgorithm *getCompressionAlgorithm(DebugCompressionType T) {
>   switch (T) {
>   case DebugCompressionType::ZStd: {
> static zstd::CompressionAlgorithm Zstd;
> if (zstd::isAvailable())
>   return &Zstd;
>   }
>   ...
>   }
>   return nullptr;
> }
> ```
> (or, possibly, we want to return non-null even if it isn't available, if we 
> include other things (like the configure macro name - so callers can use that 
> name to print helpful error messages - but then they have to explicitly check 
> if the algorithm is available after the call))
they currently already have singleton behavior i.e. 
`llvm::compression::ZStdCompressionAlgorithm::Instance` is the only place `new 
ZStdCompressionAlgorithm()` can be put into because the constructor is 
protected.

I'd rather not achieve "This way the instances are only new'd when they're 
actually used."
Because the rewards of that are relatively small, but it will make the code 
more verbose, I think the current pattern allows the best of both worlds of the 
namespace approach:
(`llvm::compression::zlib::compress` becomes 
`llvm::compression::ZlibCompression->compress`)
but they can be passed as class instances.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 449346.
ckissane added a comment.

- feat compression "enum" with methods


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/InstrProfCorrelator.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -22,31 +22,41 @@
 
 namespace {
 
-#if LLVM_ENABLE_ZLIB
-static void testZlibCompression(StringRef Input, int Level) {
+static void testCompressionAlgorithm(
+StringRef Input, int Level, compression::CompressionKind CompressionScheme,
+std::string ExpectedDestinationBufferTooSmallErrorMessage) {
   SmallVector Compressed;
   SmallVector Uncompressed;
-  zlib::compress(arrayRefFromStringRef(Input), Compressed, Level);
+  CompressionScheme->compress(arrayRefFromStringRef(Input), Compressed, Level);
 
   // Check that uncompressed buffer is the same as original.
-  Error E = zlib::uncompress(Compressed, Uncompressed, Input.size());
+  Error E =
+  CompressionScheme->decompress(Compressed, Uncompressed, Input.size());
   consumeError(std::move(E));
 
   EXPECT_EQ(Input, toStringRef(Uncompressed));
   if (Input.size() > 0) {
 // Uncompression fails if expected length is too short.
-E = zlib::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("zlib error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+E = CompressionScheme->decompress(Compressed, Uncompressed,
+  Input.size() - 1);
+EXPECT_EQ(ExpectedDestinationBufferTooSmallErrorMessage,
+  llvm::toString(std::move(E)));
   }
 }
 
+#if LLVM_ENABLE_ZLIB
+static void testZlibCompression(StringRef Input, int Level) {
+  testCompressionAlgorithm(Input, Level, CompressionKind::Zlib,
+   "zlib error: Z_BUF_ERROR");
+}
+
 TEST(CompressionTest, Zlib) {
-  testZlibCompression("", zlib::DefaultCompression);
+  compression::CompressionKind CompressionScheme = CompressionKind::Zlib;
+  testZlibCompression("", CompressionScheme->getDefaultLevel());
 
-  testZlibCompression("hello, world!", zlib::NoCompression);
-  testZlibCompression("hello, world!", zlib::BestSizeCompression);
-  testZlibCompression("hello, world!", zlib::BestSpeedCompression);
-  testZlibCompression("hello, world!", zlib::DefaultCompression);
+  testZlibCompression("hello, world!", CompressionScheme->getBestSizeLevel());
+  testZlibCompression("hello, world!", CompressionScheme->getBestSpeedLevel());
+  testZlibCompression("hello, world!", CompressionScheme->getDefaultLevel());
 
   const size_t kSize = 1024;
   char BinaryData[kSize];
@@ -54,38 +64,26 @@
 BinaryData[i] = i & 255;
   StringRef BinaryDataStr(BinaryData, kSize);
 
-  testZlibCompression(BinaryDataStr, zlib::NoCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
-  testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+  testZlibCompression(BinaryDataStr, CompressionScheme->getBestSizeLevel());
+  testZlibCompression(BinaryDataStr, CompressionScheme->getBestSpeedLevel());
+  testZlibCompression(BinaryDataStr, CompressionScheme->getDefaultLevel());
 }
 #endif
 
 #if LLVM_ENABLE_ZSTD
-static void testZstdCompression(StringRef Input, int Level) {
-  SmallVector Compressed;
-  SmallVector Uncompressed;
-  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
 
-  // Check that uncompressed buffer is the same as original.
-  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
-  consumeError(std::move(E));
-
-  EXPECT_EQ(Input, toStringRef(Uncompressed));
-  if (Input.size() > 0) {
-// Uncompression fails if expected length is too short.
-E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("Destination buffe

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 449355.
ckissane added a comment.

- make a zlib corruption check specific


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/InstrProfCorrelator.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -22,31 +22,41 @@
 
 namespace {
 
-#if LLVM_ENABLE_ZLIB
-static void testZlibCompression(StringRef Input, int Level) {
+static void testCompressionAlgorithm(
+StringRef Input, int Level, compression::CompressionKind CompressionScheme,
+std::string ExpectedDestinationBufferTooSmallErrorMessage) {
   SmallVector Compressed;
   SmallVector Uncompressed;
-  zlib::compress(arrayRefFromStringRef(Input), Compressed, Level);
+  CompressionScheme->compress(arrayRefFromStringRef(Input), Compressed, Level);
 
   // Check that uncompressed buffer is the same as original.
-  Error E = zlib::uncompress(Compressed, Uncompressed, Input.size());
+  Error E =
+  CompressionScheme->decompress(Compressed, Uncompressed, Input.size());
   consumeError(std::move(E));
 
   EXPECT_EQ(Input, toStringRef(Uncompressed));
   if (Input.size() > 0) {
 // Uncompression fails if expected length is too short.
-E = zlib::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("zlib error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+E = CompressionScheme->decompress(Compressed, Uncompressed,
+  Input.size() - 1);
+EXPECT_EQ(ExpectedDestinationBufferTooSmallErrorMessage,
+  llvm::toString(std::move(E)));
   }
 }
 
+#if LLVM_ENABLE_ZLIB
+static void testZlibCompression(StringRef Input, int Level) {
+  testCompressionAlgorithm(Input, Level, CompressionKind::Zlib,
+   "zlib error: Z_BUF_ERROR");
+}
+
 TEST(CompressionTest, Zlib) {
-  testZlibCompression("", zlib::DefaultCompression);
+  compression::CompressionKind CompressionScheme = CompressionKind::Zlib;
+  testZlibCompression("", CompressionScheme->getDefaultLevel());
 
-  testZlibCompression("hello, world!", zlib::NoCompression);
-  testZlibCompression("hello, world!", zlib::BestSizeCompression);
-  testZlibCompression("hello, world!", zlib::BestSpeedCompression);
-  testZlibCompression("hello, world!", zlib::DefaultCompression);
+  testZlibCompression("hello, world!", CompressionScheme->getBestSizeLevel());
+  testZlibCompression("hello, world!", CompressionScheme->getBestSpeedLevel());
+  testZlibCompression("hello, world!", CompressionScheme->getDefaultLevel());
 
   const size_t kSize = 1024;
   char BinaryData[kSize];
@@ -54,38 +64,26 @@
 BinaryData[i] = i & 255;
   StringRef BinaryDataStr(BinaryData, kSize);
 
-  testZlibCompression(BinaryDataStr, zlib::NoCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
-  testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+  testZlibCompression(BinaryDataStr, CompressionScheme->getBestSizeLevel());
+  testZlibCompression(BinaryDataStr, CompressionScheme->getBestSpeedLevel());
+  testZlibCompression(BinaryDataStr, CompressionScheme->getDefaultLevel());
 }
 #endif
 
 #if LLVM_ENABLE_ZSTD
-static void testZstdCompression(StringRef Input, int Level) {
-  SmallVector Compressed;
-  SmallVector Uncompressed;
-  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
 
-  // Check that uncompressed buffer is the same as original.
-  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
-  consumeError(std::move(E));
-
-  EXPECT_EQ(Input, toStringRef(Uncompressed));
-  if (Input.size() > 0) {
-// Uncompression fails if expected length is too short.
-E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("Destination buff

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 449365.
ckissane added a comment.

- trim down compression api: remove supported()


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/InstrProfCorrelator.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -22,31 +22,41 @@
 
 namespace {
 
-#if LLVM_ENABLE_ZLIB
-static void testZlibCompression(StringRef Input, int Level) {
+static void testCompressionAlgorithm(
+StringRef Input, int Level, compression::CompressionKind CompressionScheme,
+std::string ExpectedDestinationBufferTooSmallErrorMessage) {
   SmallVector Compressed;
   SmallVector Uncompressed;
-  zlib::compress(arrayRefFromStringRef(Input), Compressed, Level);
+  CompressionScheme->compress(arrayRefFromStringRef(Input), Compressed, Level);
 
   // Check that uncompressed buffer is the same as original.
-  Error E = zlib::uncompress(Compressed, Uncompressed, Input.size());
+  Error E =
+  CompressionScheme->decompress(Compressed, Uncompressed, Input.size());
   consumeError(std::move(E));
 
   EXPECT_EQ(Input, toStringRef(Uncompressed));
   if (Input.size() > 0) {
 // Uncompression fails if expected length is too short.
-E = zlib::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("zlib error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+E = CompressionScheme->decompress(Compressed, Uncompressed,
+  Input.size() - 1);
+EXPECT_EQ(ExpectedDestinationBufferTooSmallErrorMessage,
+  llvm::toString(std::move(E)));
   }
 }
 
+#if LLVM_ENABLE_ZLIB
+static void testZlibCompression(StringRef Input, int Level) {
+  testCompressionAlgorithm(Input, Level, CompressionKind::Zlib,
+   "zlib error: Z_BUF_ERROR");
+}
+
 TEST(CompressionTest, Zlib) {
-  testZlibCompression("", zlib::DefaultCompression);
+  compression::CompressionKind CompressionScheme = CompressionKind::Zlib;
+  testZlibCompression("", CompressionScheme->getDefaultLevel());
 
-  testZlibCompression("hello, world!", zlib::NoCompression);
-  testZlibCompression("hello, world!", zlib::BestSizeCompression);
-  testZlibCompression("hello, world!", zlib::BestSpeedCompression);
-  testZlibCompression("hello, world!", zlib::DefaultCompression);
+  testZlibCompression("hello, world!", CompressionScheme->getBestSizeLevel());
+  testZlibCompression("hello, world!", CompressionScheme->getBestSpeedLevel());
+  testZlibCompression("hello, world!", CompressionScheme->getDefaultLevel());
 
   const size_t kSize = 1024;
   char BinaryData[kSize];
@@ -54,38 +64,26 @@
 BinaryData[i] = i & 255;
   StringRef BinaryDataStr(BinaryData, kSize);
 
-  testZlibCompression(BinaryDataStr, zlib::NoCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
-  testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+  testZlibCompression(BinaryDataStr, CompressionScheme->getBestSizeLevel());
+  testZlibCompression(BinaryDataStr, CompressionScheme->getBestSpeedLevel());
+  testZlibCompression(BinaryDataStr, CompressionScheme->getDefaultLevel());
 }
 #endif
 
 #if LLVM_ENABLE_ZSTD
-static void testZstdCompression(StringRef Input, int Level) {
-  SmallVector Compressed;
-  SmallVector Uncompressed;
-  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
 
-  // Check that uncompressed buffer is the same as original.
-  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
-  consumeError(std::move(E));
-
-  EXPECT_EQ(Input, toStringRef(Uncompressed));
-  if (Input.size() > 0) {
-// Uncompression fails if expected length is too short.
-E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("Destinat

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 449375.
ckissane added a comment.

- CompressionKind: clean up param names to == op


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/InstrProfCorrelator.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -22,31 +22,41 @@
 
 namespace {
 
-#if LLVM_ENABLE_ZLIB
-static void testZlibCompression(StringRef Input, int Level) {
+static void testCompressionAlgorithm(
+StringRef Input, int Level, compression::CompressionKind CompressionScheme,
+std::string ExpectedDestinationBufferTooSmallErrorMessage) {
   SmallVector Compressed;
   SmallVector Uncompressed;
-  zlib::compress(arrayRefFromStringRef(Input), Compressed, Level);
+  CompressionScheme->compress(arrayRefFromStringRef(Input), Compressed, Level);
 
   // Check that uncompressed buffer is the same as original.
-  Error E = zlib::uncompress(Compressed, Uncompressed, Input.size());
+  Error E =
+  CompressionScheme->decompress(Compressed, Uncompressed, Input.size());
   consumeError(std::move(E));
 
   EXPECT_EQ(Input, toStringRef(Uncompressed));
   if (Input.size() > 0) {
 // Uncompression fails if expected length is too short.
-E = zlib::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("zlib error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+E = CompressionScheme->decompress(Compressed, Uncompressed,
+  Input.size() - 1);
+EXPECT_EQ(ExpectedDestinationBufferTooSmallErrorMessage,
+  llvm::toString(std::move(E)));
   }
 }
 
+#if LLVM_ENABLE_ZLIB
+static void testZlibCompression(StringRef Input, int Level) {
+  testCompressionAlgorithm(Input, Level, CompressionKind::Zlib,
+   "zlib error: Z_BUF_ERROR");
+}
+
 TEST(CompressionTest, Zlib) {
-  testZlibCompression("", zlib::DefaultCompression);
+  compression::CompressionKind CompressionScheme = CompressionKind::Zlib;
+  testZlibCompression("", CompressionScheme->getDefaultLevel());
 
-  testZlibCompression("hello, world!", zlib::NoCompression);
-  testZlibCompression("hello, world!", zlib::BestSizeCompression);
-  testZlibCompression("hello, world!", zlib::BestSpeedCompression);
-  testZlibCompression("hello, world!", zlib::DefaultCompression);
+  testZlibCompression("hello, world!", CompressionScheme->getBestSizeLevel());
+  testZlibCompression("hello, world!", CompressionScheme->getBestSpeedLevel());
+  testZlibCompression("hello, world!", CompressionScheme->getDefaultLevel());
 
   const size_t kSize = 1024;
   char BinaryData[kSize];
@@ -54,38 +64,26 @@
 BinaryData[i] = i & 255;
   StringRef BinaryDataStr(BinaryData, kSize);
 
-  testZlibCompression(BinaryDataStr, zlib::NoCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
-  testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+  testZlibCompression(BinaryDataStr, CompressionScheme->getBestSizeLevel());
+  testZlibCompression(BinaryDataStr, CompressionScheme->getBestSpeedLevel());
+  testZlibCompression(BinaryDataStr, CompressionScheme->getDefaultLevel());
 }
 #endif
 
 #if LLVM_ENABLE_ZSTD
-static void testZstdCompression(StringRef Input, int Level) {
-  SmallVector Compressed;
-  SmallVector Uncompressed;
-  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
 
-  // Check that uncompressed buffer is the same as original.
-  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
-  consumeError(std::move(E));
-
-  EXPECT_EQ(Input, toStringRef(Uncompressed));
-  if (Input.size() > 0) {
-// Uncompression fails if expected length is too short.
-E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("Destina

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane added a comment.

In D130516#3694366 , @dblaikie wrote:

> In D130516#3694151 , @MaskRay wrote:
>
>> In D130516#3688236 , @dblaikie 
>> wrote:
>>
>>> In D130516#3688123 , @MaskRay 
>>> wrote:
>>>
 I'd like to make a few arguments for the current namespace+free function 
 design, as opposed to the class+member function design as explored in this 
 patch (but thanks for the exploration!).
 Let's discuss several use cases.

 (a) if a use case just calls compress/uncompress. The class design has 
 slightly more boilerplate as it needs to get the algorithm class, a new 
 instance, or a singleton instance.
 For each new use, the number of lines may not differ, but the involvement 
 of a a static class member or an instance make the reader wonder whether 
 the object will be reused or thrown away.
 There is some slight cognitive burden.
 The class design has a non-trivial one-shot cost to have a function 
 returning the singleton instance.
>>>
>>> Though there must've been a condition that dominates this use somewhere - 
>>> I'd suggest that condition could be where the algorithm is retrieved, and 
>>> then passed to this code to use unconditionally.
>>>
>>> If the algorithm object is const and raw pointers/references are used, I 
>>> think it makes it clear to the reader that there's no ownership here, and 
>>> it's not stateful when compressing/decompressing.
>>
>> A pointer to a singleton compression class is isomorphic to an `enum class 
>> CompressionType` variable.
>
> I don't mean to suggest that either design is fundamentally more or less 
> functional - I'm totally OK with/agree that both design directions allow the 
> implementation of all the desired final/end-user-visible functionality.
>
> I'm trying to make a point about which, I think, achieves that goal in a 
> "better" way - that's the space of design discussions, I think - what kinds 
> of (developer, maintenance, etc) costs different designs incur.
>
>> Using an enum variable doesn't lose any usage pattern we can do with a 
>> pointer to a singleton compression class.
>
> I agree that either design doesn't change what's possible - I do, though, 
> think that the "usage patterns" are meaningfully different between the two 
> designs.
>
>> An enum variable allows more patterns, as the allowed values are enumerable 
>> (we don't need to worry about -Wswitch for the uses).
>>
>> Say, we do
>>
>>   auto *algo = !compression::ZlibCompression;
>>   if (!algo)
>> ...
>>   
>>   
>>   algo->compress(...);
>>
>> either together or apart, the result is similar to the following but with 
>> (IMO) slightly larger cognitive burden:
>>
>>   if (!compression::isAvailable(format))
>> ...
>>   
>>   compression::compress(format);
>
> Specifically two APIs that are related (it's important/necessary to check for 
> availability before calling compress or decompress) in their contracts but 
> unrelated in their API use makes it easier to misuse the APIs and have a 
> situation where the availability check doesn't cover the usage. That's what I 
> think is important/important to discuss here.
>
 (b) zlib compress/uncompress immediately following an availability check.

   // free function
   if (!compression::zlib::isAvailable())
 errs() << "cannot compress: " << 
 compression::zlib::buildConfigurationHint();
   
   // class
   auto *algo = !compression::ZlibCompression;
   if (!algo->isAvailable()) {
 errs() << "cannot compress: " << algo->buildConfigurationHint();
   }
>>>
>>> I think maybe this code might end up looking like:
>>>
>>>   Algo *algo = getAlgo(Zlib)
>>>   if (!algo)
>>> errs() ...
>>>
>>> It's possible that this function would return non-null even for a 
>>> non-available algorithm if we wanted to communicate other things (like the 
>>> cmake macro name to enable to add the functionality)
>>
>> I think this is similarly achieved with an enum variable.
>> With the class based approach, a pointer has a static type of the ancestor 
>> compression class and a dynamic type of any possible algorithm.
>> This is not different from that: the enum variable may have a value the enum 
>> class supports.
>
> I agree that the code is similar in either case, but with a small difference 
> that is important to me - that accessing the algorithm necessarily (to some 
> degree - you could still have code that doesn't test the 
> condition/dereferences null, the same way that code can dereference an empty 
> Optional without checking first - but at least the API I'm suggesting makes 
> clear there's a connection between availability and usage).
>
 (c) zlib/zstd compress/uncompress immediately following an availability 
 check.

   // free function
   if 

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane added a comment.

In D130516#3694422 , @dblaikie wrote:

> The current code here still seems more complicated than I'd prefer - looks 
> like currently the size/speed/default levels are currently unused, so maybe 
> we can omit those for now, knowing they will be added?
> And the CompressionKind with all its operator overloads seems like a lot of 
> surface area that is pretty non-obvious for usage - boolean testable, logical 
> operator overloads, etc.
> Could we have only one decompress/compress function each, for now?
> & maybe leave out the name/enum from the base class for now, add it in later 
> (& I think I mentionted in another comment those properties can be 
> non-virtual, maybe even direct const members - passed into the base through 
> the ctor from the derived class)

Yes, I can continue to trim down the implementation! I agree with your 
sentiment.

> Maybe it's easier if I either post a patch, or at least more explicitly flesh 
> out what I'm picturing/proposing/suggesting:
> Header:
>
>   struct CompressionAlgorithm {
> virtual void Compress(...);
> virtual void Decompress(...);
>   };
>   enum class CompressionType {
> Zlib, Zstd
>   };
>   CompressionAlgorithm *getCompressionAlgorithm(CompressionType);
>
> Implementation:
>
>   #if LLVM_ENABLE_ZLIB
>   struct ZlibCompressionAlgorthim : CompressionAlgorithm {
> void Compress(...) { ... }
> void Decompress(...) { ...}
>   }
>   #endif
>   ...
>   CompressionAlgorithm *getCompressionAlgorithm(CompressionType T) {
> switch (T) {
> case CompressionType::Zlib: {
>   #if LLVM_ENABLE_ZLIB
>   static ZlibCompressionAlgorithm A;
>   return &A;
>   #else
>   break;
>   #endif
> }
>   ...
> }
> return nullptr;
>   }

I agree with some of this, I have some strong thoughts I would like to work out 
about the whole nullptr being none or unsupported a little preemptively IMO.

>   Usage:
>
> if (CompressionAlgorithm *C = getCompressionAlgorithm(CompressionType::Zlib) {
>
>   C->compress(...);
>
> }
>
>   

currently, you can do

  if (CompressionKind C = CompressionKind::Zlib) {
C->compress(...);
  }



> And, yeah, I think it'd be suitable to eventually add name, type, 
> size/speed/default levels:
>
>   struct CompressionAlgorithm {
> const StringRef Name;
> const CompressionType Type;
> const int DefaultLevel;
> const int BestSizeLevel;
> const int BestSpeedLevel;
> virtual void Compress(...);
> virtual void Decompress(...);
>   protected:
> CompressionAlgorithm(StringRef Name, CompressionType Type, ...) : 
> Name(Name), Type(Type), ... {}
>   }
>
>   struct ZlibCompressionAlgorithm : CompressionAlgorithm {
> ZlibCompressionAlgorithm() : CompressionAlgorithm("zlib", 
> CompressionType::Zlib, 5, 10, 1) { }
> /* as before */
>   };
>   ...
>
> Though those can be added as needed - good to keep in mind that they're a 
> useful direction to go, but might simplify the review/discussion to omit them 
> for now.




Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 449421.
ckissane added a comment.

- compression api: greatly simplify implementation


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/InstrProfCorrelator.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -22,31 +22,41 @@
 
 namespace {
 
-#if LLVM_ENABLE_ZLIB
-static void testZlibCompression(StringRef Input, int Level) {
+static void testCompressionAlgorithm(
+StringRef Input, int Level, compression::CompressionKind CompressionScheme,
+std::string ExpectedDestinationBufferTooSmallErrorMessage) {
   SmallVector Compressed;
   SmallVector Uncompressed;
-  zlib::compress(arrayRefFromStringRef(Input), Compressed, Level);
+  CompressionScheme->compress(arrayRefFromStringRef(Input), Compressed, Level);
 
   // Check that uncompressed buffer is the same as original.
-  Error E = zlib::uncompress(Compressed, Uncompressed, Input.size());
+  Error E =
+  CompressionScheme->decompress(Compressed, Uncompressed, Input.size());
   consumeError(std::move(E));
 
   EXPECT_EQ(Input, toStringRef(Uncompressed));
   if (Input.size() > 0) {
 // Uncompression fails if expected length is too short.
-E = zlib::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("zlib error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+E = CompressionScheme->decompress(Compressed, Uncompressed,
+  Input.size() - 1);
+EXPECT_EQ(ExpectedDestinationBufferTooSmallErrorMessage,
+  llvm::toString(std::move(E)));
   }
 }
 
+#if LLVM_ENABLE_ZLIB
+static void testZlibCompression(StringRef Input, int Level) {
+  testCompressionAlgorithm(Input, Level, CompressionKind::Zlib,
+   "zlib error: Z_BUF_ERROR");
+}
+
 TEST(CompressionTest, Zlib) {
-  testZlibCompression("", zlib::DefaultCompression);
+  compression::CompressionKind CompressionScheme = CompressionKind::Zlib;
+  testZlibCompression("", CompressionScheme->DefaultLevel);
 
-  testZlibCompression("hello, world!", zlib::NoCompression);
-  testZlibCompression("hello, world!", zlib::BestSizeCompression);
-  testZlibCompression("hello, world!", zlib::BestSpeedCompression);
-  testZlibCompression("hello, world!", zlib::DefaultCompression);
+  testZlibCompression("hello, world!", CompressionScheme->BestSizeLevel);
+  testZlibCompression("hello, world!", CompressionScheme->BestSpeedLevel);
+  testZlibCompression("hello, world!", CompressionScheme->DefaultLevel);
 
   const size_t kSize = 1024;
   char BinaryData[kSize];
@@ -54,38 +64,26 @@
 BinaryData[i] = i & 255;
   StringRef BinaryDataStr(BinaryData, kSize);
 
-  testZlibCompression(BinaryDataStr, zlib::NoCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
-  testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSizeLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSpeedLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->DefaultLevel);
 }
 #endif
 
 #if LLVM_ENABLE_ZSTD
-static void testZstdCompression(StringRef Input, int Level) {
-  SmallVector Compressed;
-  SmallVector Uncompressed;
-  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
 
-  // Check that uncompressed buffer is the same as original.
-  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
-  consumeError(std::move(E));
-
-  EXPECT_EQ(Input, toStringRef(Uncompressed));
-  if (Input.size() > 0) {
-// Uncompression fails if expected length is too short.
-E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("Destination buffer is too small", llvm::

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane added inline comments.



Comment at: llvm/include/llvm/Support/Compression.h:48-51
+  Error decompress(ArrayRef Input, uint8_t *UncompressedBuffer,
+   size_t &UncompressedSize) {
+return Decompress(Input, UncompressedBuffer, UncompressedSize);
+  }

dblaikie wrote:
> Could we skip this wrapper & just have the underlying function (& also we 
> shouldn't be overloading by case like this anyway - please name all the 
> decompress/compress functions with the same case/spelling)
good point


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 449772.
ckissane added a comment.

- remove uppercase Compress, Decompress


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/InstrProfCorrelator.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -22,31 +22,41 @@
 
 namespace {
 
-#if LLVM_ENABLE_ZLIB
-static void testZlibCompression(StringRef Input, int Level) {
+static void testCompressionAlgorithm(
+StringRef Input, int Level, compression::CompressionKind CompressionScheme,
+std::string ExpectedDestinationBufferTooSmallErrorMessage) {
   SmallVector Compressed;
   SmallVector Uncompressed;
-  zlib::compress(arrayRefFromStringRef(Input), Compressed, Level);
+  CompressionScheme->compress(arrayRefFromStringRef(Input), Compressed, Level);
 
   // Check that uncompressed buffer is the same as original.
-  Error E = zlib::uncompress(Compressed, Uncompressed, Input.size());
+  Error E =
+  CompressionScheme->decompress(Compressed, Uncompressed, Input.size());
   consumeError(std::move(E));
 
   EXPECT_EQ(Input, toStringRef(Uncompressed));
   if (Input.size() > 0) {
 // Uncompression fails if expected length is too short.
-E = zlib::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("zlib error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+E = CompressionScheme->decompress(Compressed, Uncompressed,
+  Input.size() - 1);
+EXPECT_EQ(ExpectedDestinationBufferTooSmallErrorMessage,
+  llvm::toString(std::move(E)));
   }
 }
 
+#if LLVM_ENABLE_ZLIB
+static void testZlibCompression(StringRef Input, int Level) {
+  testCompressionAlgorithm(Input, Level, CompressionKind::Zlib,
+   "zlib error: Z_BUF_ERROR");
+}
+
 TEST(CompressionTest, Zlib) {
-  testZlibCompression("", zlib::DefaultCompression);
+  compression::CompressionKind CompressionScheme = CompressionKind::Zlib;
+  testZlibCompression("", CompressionScheme->DefaultLevel);
 
-  testZlibCompression("hello, world!", zlib::NoCompression);
-  testZlibCompression("hello, world!", zlib::BestSizeCompression);
-  testZlibCompression("hello, world!", zlib::BestSpeedCompression);
-  testZlibCompression("hello, world!", zlib::DefaultCompression);
+  testZlibCompression("hello, world!", CompressionScheme->BestSizeLevel);
+  testZlibCompression("hello, world!", CompressionScheme->BestSpeedLevel);
+  testZlibCompression("hello, world!", CompressionScheme->DefaultLevel);
 
   const size_t kSize = 1024;
   char BinaryData[kSize];
@@ -54,38 +64,26 @@
 BinaryData[i] = i & 255;
   StringRef BinaryDataStr(BinaryData, kSize);
 
-  testZlibCompression(BinaryDataStr, zlib::NoCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
-  testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSizeLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSpeedLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->DefaultLevel);
 }
 #endif
 
 #if LLVM_ENABLE_ZSTD
-static void testZstdCompression(StringRef Input, int Level) {
-  SmallVector Compressed;
-  SmallVector Uncompressed;
-  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
 
-  // Check that uncompressed buffer is the same as original.
-  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
-  consumeError(std::move(E));
-
-  EXPECT_EQ(Input, toStringRef(Uncompressed));
-  if (Input.size() > 0) {
-// Uncompression fails if expected length is too short.
-E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("Destination buffer is too small", llvm::toString(st

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 450093.
ckissane added a comment.

- remove compression kind || && overload


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/InstrProfCorrelator.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -22,31 +22,41 @@
 
 namespace {
 
-#if LLVM_ENABLE_ZLIB
-static void testZlibCompression(StringRef Input, int Level) {
+static void testCompressionAlgorithm(
+StringRef Input, int Level, compression::CompressionKind CompressionScheme,
+std::string ExpectedDestinationBufferTooSmallErrorMessage) {
   SmallVector Compressed;
   SmallVector Uncompressed;
-  zlib::compress(arrayRefFromStringRef(Input), Compressed, Level);
+  CompressionScheme->compress(arrayRefFromStringRef(Input), Compressed, Level);
 
   // Check that uncompressed buffer is the same as original.
-  Error E = zlib::uncompress(Compressed, Uncompressed, Input.size());
+  Error E =
+  CompressionScheme->decompress(Compressed, Uncompressed, Input.size());
   consumeError(std::move(E));
 
   EXPECT_EQ(Input, toStringRef(Uncompressed));
   if (Input.size() > 0) {
 // Uncompression fails if expected length is too short.
-E = zlib::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("zlib error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+E = CompressionScheme->decompress(Compressed, Uncompressed,
+  Input.size() - 1);
+EXPECT_EQ(ExpectedDestinationBufferTooSmallErrorMessage,
+  llvm::toString(std::move(E)));
   }
 }
 
+#if LLVM_ENABLE_ZLIB
+static void testZlibCompression(StringRef Input, int Level) {
+  testCompressionAlgorithm(Input, Level, CompressionKind::Zlib,
+   "zlib error: Z_BUF_ERROR");
+}
+
 TEST(CompressionTest, Zlib) {
-  testZlibCompression("", zlib::DefaultCompression);
+  compression::CompressionKind CompressionScheme = CompressionKind::Zlib;
+  testZlibCompression("", CompressionScheme->DefaultLevel);
 
-  testZlibCompression("hello, world!", zlib::NoCompression);
-  testZlibCompression("hello, world!", zlib::BestSizeCompression);
-  testZlibCompression("hello, world!", zlib::BestSpeedCompression);
-  testZlibCompression("hello, world!", zlib::DefaultCompression);
+  testZlibCompression("hello, world!", CompressionScheme->BestSizeLevel);
+  testZlibCompression("hello, world!", CompressionScheme->BestSpeedLevel);
+  testZlibCompression("hello, world!", CompressionScheme->DefaultLevel);
 
   const size_t kSize = 1024;
   char BinaryData[kSize];
@@ -54,38 +64,26 @@
 BinaryData[i] = i & 255;
   StringRef BinaryDataStr(BinaryData, kSize);
 
-  testZlibCompression(BinaryDataStr, zlib::NoCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
-  testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSizeLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSpeedLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->DefaultLevel);
 }
 #endif
 
 #if LLVM_ENABLE_ZSTD
-static void testZstdCompression(StringRef Input, int Level) {
-  SmallVector Compressed;
-  SmallVector Uncompressed;
-  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
 
-  // Check that uncompressed buffer is the same as original.
-  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
-  consumeError(std::move(E));
-
-  EXPECT_EQ(Input, toStringRef(Uncompressed));
-  if (Input.size() > 0) {
-// Uncompression fails if expected length is too short.
-E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("Destination buffer is too small", llvm::toString(s

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane added a comment.

@dblaikie @MaskRay I would like it if you could all take another look.

In response to @dblaikie 's comments about implementation weight I have greatly 
simplified the implementation, including removing extra capitalized function 
overloads (Compress, Decompress), and removing || and && operator overrides. 
Also adopting the recently suggested class impl in part.

I also believe similar semantics to the nullptr suggestion have been achieved, 
due to the bool cast returning supported status, and the Unknown type acting as 
a nullptr of sorts, being always unsupported and -1 (255) as a uint8. 
`Optional(Unknown)` is returned from `getOptionalCompressionKind(uint8_t)` when 
it is not 0 (NoneType()), 1 (`Optional(Zlib)`), or 2 (`Optional(ZStd)`).

In places where explict compression must be used `CompressionKind` can be 
passed around, and possibly optional (sometimes none) compression is 
represented as `llvm::Optional` which I have type aliased as 
`OptionalCompressionKind`


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 450169.
ckissane edited the summary of this revision.
ckissane added a comment.

- fix some nits in Compression.h


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/InstrProfCorrelator.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -22,31 +22,41 @@
 
 namespace {
 
-#if LLVM_ENABLE_ZLIB
-static void testZlibCompression(StringRef Input, int Level) {
+static void testCompressionAlgorithm(
+StringRef Input, int Level, compression::CompressionKind CompressionScheme,
+std::string ExpectedDestinationBufferTooSmallErrorMessage) {
   SmallVector Compressed;
   SmallVector Uncompressed;
-  zlib::compress(arrayRefFromStringRef(Input), Compressed, Level);
+  CompressionScheme->compress(arrayRefFromStringRef(Input), Compressed, Level);
 
   // Check that uncompressed buffer is the same as original.
-  Error E = zlib::uncompress(Compressed, Uncompressed, Input.size());
+  Error E =
+  CompressionScheme->decompress(Compressed, Uncompressed, Input.size());
   consumeError(std::move(E));
 
   EXPECT_EQ(Input, toStringRef(Uncompressed));
   if (Input.size() > 0) {
 // Uncompression fails if expected length is too short.
-E = zlib::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("zlib error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+E = CompressionScheme->decompress(Compressed, Uncompressed,
+  Input.size() - 1);
+EXPECT_EQ(ExpectedDestinationBufferTooSmallErrorMessage,
+  llvm::toString(std::move(E)));
   }
 }
 
+#if LLVM_ENABLE_ZLIB
+static void testZlibCompression(StringRef Input, int Level) {
+  testCompressionAlgorithm(Input, Level, CompressionKind::Zlib,
+   "zlib error: Z_BUF_ERROR");
+}
+
 TEST(CompressionTest, Zlib) {
-  testZlibCompression("", zlib::DefaultCompression);
+  compression::CompressionKind CompressionScheme = CompressionKind::Zlib;
+  testZlibCompression("", CompressionScheme->DefaultLevel);
 
-  testZlibCompression("hello, world!", zlib::NoCompression);
-  testZlibCompression("hello, world!", zlib::BestSizeCompression);
-  testZlibCompression("hello, world!", zlib::BestSpeedCompression);
-  testZlibCompression("hello, world!", zlib::DefaultCompression);
+  testZlibCompression("hello, world!", CompressionScheme->BestSizeLevel);
+  testZlibCompression("hello, world!", CompressionScheme->BestSpeedLevel);
+  testZlibCompression("hello, world!", CompressionScheme->DefaultLevel);
 
   const size_t kSize = 1024;
   char BinaryData[kSize];
@@ -54,38 +64,26 @@
 BinaryData[i] = i & 255;
   StringRef BinaryDataStr(BinaryData, kSize);
 
-  testZlibCompression(BinaryDataStr, zlib::NoCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
-  testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSizeLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSpeedLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->DefaultLevel);
 }
 #endif
 
 #if LLVM_ENABLE_ZSTD
-static void testZstdCompression(StringRef Input, int Level) {
-  SmallVector Compressed;
-  SmallVector Uncompressed;
-  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
 
-  // Check that uncompressed buffer is the same as original.
-  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
-  consumeError(std::move(E));
-
-  EXPECT_EQ(Input, toStringRef(Uncompressed));
-  if (Input.size() > 0) {
-// Uncompression fails if expected length is too short.
-E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("Destination 

[PATCH] D130667: feat: use compression class for: clang ast serial zstd option + prof data compression variants + zstd elf + ztsd objcopy support

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 450172.
ckissane added a comment.

- Merge branch 'ckissane.compression-class' of github.com:ckissane/llvm-project 
into ckissane.compression-class
- Merge remote-tracking branch 'origin/ckissane.compression-class-simple' into 
ckissane.compression-class
- Merge remote-tracking branch 'origin/ckissane.compression-class-simple' into 
ckissane.compression-class
- Merge remote-tracking branch 'origin/ckissane.compression-class-simple' into 
ckissane.compression-class


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130667/new/

https://reviews.llvm.org/D130667

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/index/Serialization.h
  clang/include/clang/Serialization/ASTBitCodes.h
  clang/include/clang/Serialization/ASTWriter.h
  clang/lib/CodeGen/CoverageMappingGen.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Frontend/ASTUnit.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  clang/lib/Serialization/GeneratePCH.cpp
  clang/test/CodeGen/coverage-compilation-dir.c
  clang/test/CoverageMapping/abspath.cpp
  clang/test/CoverageMapping/ir.c
  clang/test/Profile/coverage-prefix-map.c
  compiler-rt/include/profile/InstrProfData.inc
  compiler-rt/test/profile/instrprof-darwin-dead-strip.c
  compiler-rt/test/profile/instrprof-gc-sections.c
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  lld/test/CMakeLists.txt
  lld/test/ELF/compressed-debug-input-zstd.s
  lld/test/ELF/compressed-input-err-zstd.s
  lld/test/lit.site.cfg.py.in
  llvm/include/llvm/BinaryFormat/ELF.h
  llvm/include/llvm/MC/MCTargetOptions.h
  llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
  llvm/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/ProfileData/InstrProfData.inc
  llvm/include/llvm/ProfileData/SampleProfWriter.h
  llvm/include/llvm/Support/CommandLine.h
  llvm/include/llvm/Support/Compression.h
  llvm/include/llvm/Transforms/Instrumentation.h
  llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/CommandLine.cpp
  llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
  llvm/test/tools/llvm-cov/coverage-prefix-map.test
  llvm/test/tools/llvm-cov/multiple-objects-not-all-instrumented.test
  llvm/test/tools/llvm-cov/multiple-objects.test
  llvm/test/tools/llvm-cov/multithreaded-report.test
  llvm/test/tools/llvm-cov/sources-specified.test
  llvm/test/tools/llvm-profdata/c-general.test
  llvm/test/tools/llvm-profdata/memprof-merge.test
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/tools/llvm-objcopy/ObjcopyOpts.td
  llvm/tools/llvm-profdata/llvm-profdata.cpp
  llvm/tools/llvm-profgen/ProfileGenerator.cpp
  llvm/unittests/ProfileData/CoverageMappingTest.cpp
  llvm/unittests/ProfileData/SampleProfTest.cpp

Index: llvm/unittests/ProfileData/SampleProfTest.cpp
===
--- llvm/unittests/ProfileData/SampleProfTest.cpp
+++ llvm/unittests/ProfileData/SampleProfTest.cpp
@@ -50,7 +50,8 @@
 std::error_code EC;
 std::unique_ptr OS(
 new raw_fd_ostream(Profile, EC, sys::fs::OF_None));
-auto WriterOrErr = SampleProfileWriter::create(OS, Format);
+auto WriterOrErr = SampleProfileWriter::create(
+OS, Format, compression::CompressionKind::Zlib);
 ASSERT_TRUE(NoError(WriterOrErr.getError()));
 Writer = std::move(WriterOrErr.get());
   }
Index: llvm/unittests/ProfileData/CoverageMappingTest.cpp
===
--- llvm/unittests/ProfileData/CoverageMappingTest.cpp
+++ llvm/unittests/ProfileData/CoverageMappingTest.cpp
@@ -932,7 +932,9 @@
 {
   raw_string_ostream OS(EncodedFilenames);
   CoverageFilenamesSectionWriter Writer(Paths);
-  Writer.write(OS, Compress);
+  Writer.write(OS, Compress ? (compression::OptionalCompressionKind)
+  compression::CompressionKind::Zlib
+: NoneType());
 }
 
 std::vector ReadFilenames;
@@ -956,7 +958,9 @@
 {
   raw_string_ostream OS(EncodedFilenames);
   CoverageFilenamesSectionWriter Writer(Paths);
-  Writer.write(OS, Compress);
+  Writer.write(OS, Compress ? (compression::OptionalCompressionKind)
+  compression::CompressionKind::Zlib
+: NoneType());
 }
 
 StringRef CompilationDir = "out";
Index: llvm/tools/llvm-profgen/ProfileGenerator.cpp
=

[PATCH] D130667: feat: use compression class for: clang ast serial zstd option + prof data compression variants + zstd elf + ztsd objcopy support

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 450174.
ckissane added a comment.

- Merge remote-tracking branch 'origin/ckissane.compression-class-simple' into 
ckissane.compression-class
- remove erroneous declation of operator StringRef()


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130667/new/

https://reviews.llvm.org/D130667

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/index/Serialization.h
  clang/include/clang/Serialization/ASTBitCodes.h
  clang/include/clang/Serialization/ASTWriter.h
  clang/lib/CodeGen/CoverageMappingGen.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Frontend/ASTUnit.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  clang/lib/Serialization/GeneratePCH.cpp
  clang/test/CodeGen/coverage-compilation-dir.c
  clang/test/CoverageMapping/abspath.cpp
  clang/test/CoverageMapping/ir.c
  clang/test/Profile/coverage-prefix-map.c
  compiler-rt/include/profile/InstrProfData.inc
  compiler-rt/test/profile/instrprof-darwin-dead-strip.c
  compiler-rt/test/profile/instrprof-gc-sections.c
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  lld/test/CMakeLists.txt
  lld/test/ELF/compressed-debug-input-zstd.s
  lld/test/ELF/compressed-input-err-zstd.s
  lld/test/lit.site.cfg.py.in
  llvm/include/llvm/BinaryFormat/ELF.h
  llvm/include/llvm/MC/MCTargetOptions.h
  llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
  llvm/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/ProfileData/InstrProfData.inc
  llvm/include/llvm/ProfileData/SampleProfWriter.h
  llvm/include/llvm/Support/CommandLine.h
  llvm/include/llvm/Support/Compression.h
  llvm/include/llvm/Transforms/Instrumentation.h
  llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/CommandLine.cpp
  llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
  llvm/test/tools/llvm-cov/coverage-prefix-map.test
  llvm/test/tools/llvm-cov/multiple-objects-not-all-instrumented.test
  llvm/test/tools/llvm-cov/multiple-objects.test
  llvm/test/tools/llvm-cov/multithreaded-report.test
  llvm/test/tools/llvm-cov/sources-specified.test
  llvm/test/tools/llvm-profdata/c-general.test
  llvm/test/tools/llvm-profdata/memprof-merge.test
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/tools/llvm-objcopy/ObjcopyOpts.td
  llvm/tools/llvm-profdata/llvm-profdata.cpp
  llvm/tools/llvm-profgen/ProfileGenerator.cpp
  llvm/unittests/ProfileData/CoverageMappingTest.cpp
  llvm/unittests/ProfileData/SampleProfTest.cpp

Index: llvm/unittests/ProfileData/SampleProfTest.cpp
===
--- llvm/unittests/ProfileData/SampleProfTest.cpp
+++ llvm/unittests/ProfileData/SampleProfTest.cpp
@@ -50,7 +50,8 @@
 std::error_code EC;
 std::unique_ptr OS(
 new raw_fd_ostream(Profile, EC, sys::fs::OF_None));
-auto WriterOrErr = SampleProfileWriter::create(OS, Format);
+auto WriterOrErr = SampleProfileWriter::create(
+OS, Format, compression::CompressionKind::Zlib);
 ASSERT_TRUE(NoError(WriterOrErr.getError()));
 Writer = std::move(WriterOrErr.get());
   }
Index: llvm/unittests/ProfileData/CoverageMappingTest.cpp
===
--- llvm/unittests/ProfileData/CoverageMappingTest.cpp
+++ llvm/unittests/ProfileData/CoverageMappingTest.cpp
@@ -932,7 +932,9 @@
 {
   raw_string_ostream OS(EncodedFilenames);
   CoverageFilenamesSectionWriter Writer(Paths);
-  Writer.write(OS, Compress);
+  Writer.write(OS, Compress ? (compression::OptionalCompressionKind)
+  compression::CompressionKind::Zlib
+: NoneType());
 }
 
 std::vector ReadFilenames;
@@ -956,7 +958,9 @@
 {
   raw_string_ostream OS(EncodedFilenames);
   CoverageFilenamesSectionWriter Writer(Paths);
-  Writer.write(OS, Compress);
+  Writer.write(OS, Compress ? (compression::OptionalCompressionKind)
+  compression::CompressionKind::Zlib
+: NoneType());
 }
 
 StringRef CompilationDir = "out";
Index: llvm/tools/llvm-profgen/ProfileGenerator.cpp
===
--- llvm/tools/llvm-profgen/ProfileGenerator.cpp
+++ llvm/tools/llvm-profgen/ProfileGenerator.cpp
@@ -160,7 +160,8 @@
 }
 
 void ProfileGeneratorBase::write() {
-  auto WriterOrErr = SampleProfileWriter::cre

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 450176.
ckissane added a comment.

- move if into switch


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/InstrProfCorrelator.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -22,31 +22,41 @@
 
 namespace {
 
-#if LLVM_ENABLE_ZLIB
-static void testZlibCompression(StringRef Input, int Level) {
+static void testCompressionAlgorithm(
+StringRef Input, int Level, compression::CompressionKind CompressionScheme,
+std::string ExpectedDestinationBufferTooSmallErrorMessage) {
   SmallVector Compressed;
   SmallVector Uncompressed;
-  zlib::compress(arrayRefFromStringRef(Input), Compressed, Level);
+  CompressionScheme->compress(arrayRefFromStringRef(Input), Compressed, Level);
 
   // Check that uncompressed buffer is the same as original.
-  Error E = zlib::uncompress(Compressed, Uncompressed, Input.size());
+  Error E =
+  CompressionScheme->decompress(Compressed, Uncompressed, Input.size());
   consumeError(std::move(E));
 
   EXPECT_EQ(Input, toStringRef(Uncompressed));
   if (Input.size() > 0) {
 // Uncompression fails if expected length is too short.
-E = zlib::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("zlib error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+E = CompressionScheme->decompress(Compressed, Uncompressed,
+  Input.size() - 1);
+EXPECT_EQ(ExpectedDestinationBufferTooSmallErrorMessage,
+  llvm::toString(std::move(E)));
   }
 }
 
+#if LLVM_ENABLE_ZLIB
+static void testZlibCompression(StringRef Input, int Level) {
+  testCompressionAlgorithm(Input, Level, CompressionKind::Zlib,
+   "zlib error: Z_BUF_ERROR");
+}
+
 TEST(CompressionTest, Zlib) {
-  testZlibCompression("", zlib::DefaultCompression);
+  compression::CompressionKind CompressionScheme = CompressionKind::Zlib;
+  testZlibCompression("", CompressionScheme->DefaultLevel);
 
-  testZlibCompression("hello, world!", zlib::NoCompression);
-  testZlibCompression("hello, world!", zlib::BestSizeCompression);
-  testZlibCompression("hello, world!", zlib::BestSpeedCompression);
-  testZlibCompression("hello, world!", zlib::DefaultCompression);
+  testZlibCompression("hello, world!", CompressionScheme->BestSizeLevel);
+  testZlibCompression("hello, world!", CompressionScheme->BestSpeedLevel);
+  testZlibCompression("hello, world!", CompressionScheme->DefaultLevel);
 
   const size_t kSize = 1024;
   char BinaryData[kSize];
@@ -54,38 +64,26 @@
 BinaryData[i] = i & 255;
   StringRef BinaryDataStr(BinaryData, kSize);
 
-  testZlibCompression(BinaryDataStr, zlib::NoCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
-  testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSizeLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSpeedLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->DefaultLevel);
 }
 #endif
 
 #if LLVM_ENABLE_ZSTD
-static void testZstdCompression(StringRef Input, int Level) {
-  SmallVector Compressed;
-  SmallVector Uncompressed;
-  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
 
-  // Check that uncompressed buffer is the same as original.
-  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
-  consumeError(std::move(E));
-
-  EXPECT_EQ(Input, toStringRef(Uncompressed));
-  if (Input.size() > 0) {
-// Uncompression fails if expected length is too short.
-E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
-  }

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane added inline comments.



Comment at: llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp:54-60
+  OptionalCompressionScheme = compression::noneIfUnsupported(
+  (Compress && DoInstrProfNameCompression) ? OptionalCompressionScheme
+   : llvm::NoneType());
+
+  bool doCompression = bool(OptionalCompressionScheme);
+
+  if (doCompression) {

dblaikie wrote:
> This still seems like a lot of hoops to jump through - why 
> "noneIfUnsupported" rather than either having the compression scheme (I think 
> it could be the CompressionAlgorithm itself, rather than having the separate 
> OptionalCompressionKind abstraction) either be null itself, or expose an 
> "isAvailable" operation directly on the CompressionAlgorithm?
> 
> Even if the CompressionKind/OptionalCompressionKind/CompressionAlgorithm 
> abstractions are kept, I'm not sure why the above code is preferred over, say:
> 
> ```
> if (Compress && DoInstrProfNameCompression && OptionalCompressionScheme /* 
> .isAvailable(), if we want to be more explicit */) {
>   ...
> }
> ```
> 
> What's the benefit that `noneIfUnsupported` is providing? (& generally I'd 
> expect the `Compress && DoInstrProfNameCompression` to be tested/exit early 
> before even naming/constructing/querying/doing anything with the compression 
> scheme/algorithm/etc - so there'd be no need to combine the tests for 
> availability and the tests for whether compression was requested)
> 
> Perhaps this API is motivated by a desire to implement something much closer 
> to the original code than is necessary/suitable? Or some other use 
> case/benefit I'm not quite understanding yet?
I shall remove `noneIfUnsupported`. You express good points, we can simply 
check `if(OptionalCompressionScheme && *OptionalCompressionScheme)` where 
necessary.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane added inline comments.



Comment at: llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp:54-60
+  OptionalCompressionScheme = compression::noneIfUnsupported(
+  (Compress && DoInstrProfNameCompression) ? OptionalCompressionScheme
+   : llvm::NoneType());
+
+  bool doCompression = bool(OptionalCompressionScheme);
+
+  if (doCompression) {

ckissane wrote:
> dblaikie wrote:
> > This still seems like a lot of hoops to jump through - why 
> > "noneIfUnsupported" rather than either having the compression scheme (I 
> > think it could be the CompressionAlgorithm itself, rather than having the 
> > separate OptionalCompressionKind abstraction) either be null itself, or 
> > expose an "isAvailable" operation directly on the CompressionAlgorithm?
> > 
> > Even if the CompressionKind/OptionalCompressionKind/CompressionAlgorithm 
> > abstractions are kept, I'm not sure why the above code is preferred over, 
> > say:
> > 
> > ```
> > if (Compress && DoInstrProfNameCompression && OptionalCompressionScheme /* 
> > .isAvailable(), if we want to be more explicit */) {
> >   ...
> > }
> > ```
> > 
> > What's the benefit that `noneIfUnsupported` is providing? (& generally I'd 
> > expect the `Compress && DoInstrProfNameCompression` to be tested/exit early 
> > before even naming/constructing/querying/doing anything with the 
> > compression scheme/algorithm/etc - so there'd be no need to combine the 
> > tests for availability and the tests for whether compression was requested)
> > 
> > Perhaps this API is motivated by a desire to implement something much 
> > closer to the original code than is necessary/suitable? Or some other use 
> > case/benefit I'm not quite understanding yet?
> I shall remove `noneIfUnsupported`. You express good points, we can simply 
> check `if(OptionalCompressionScheme && *OptionalCompressionScheme)` where 
> necessary.
though it will make a lot of existing code patterns less clear, and more verbose


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane added inline comments.



Comment at: llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp:54-60
+  OptionalCompressionScheme = compression::noneIfUnsupported(
+  (Compress && DoInstrProfNameCompression) ? OptionalCompressionScheme
+   : llvm::NoneType());
+
+  bool doCompression = bool(OptionalCompressionScheme);
+
+  if (doCompression) {

ckissane wrote:
> ckissane wrote:
> > dblaikie wrote:
> > > This still seems like a lot of hoops to jump through - why 
> > > "noneIfUnsupported" rather than either having the compression scheme (I 
> > > think it could be the CompressionAlgorithm itself, rather than having the 
> > > separate OptionalCompressionKind abstraction) either be null itself, or 
> > > expose an "isAvailable" operation directly on the CompressionAlgorithm?
> > > 
> > > Even if the CompressionKind/OptionalCompressionKind/CompressionAlgorithm 
> > > abstractions are kept, I'm not sure why the above code is preferred over, 
> > > say:
> > > 
> > > ```
> > > if (Compress && DoInstrProfNameCompression && OptionalCompressionScheme 
> > > /* .isAvailable(), if we want to be more explicit */) {
> > >   ...
> > > }
> > > ```
> > > 
> > > What's the benefit that `noneIfUnsupported` is providing? (& generally 
> > > I'd expect the `Compress && DoInstrProfNameCompression` to be tested/exit 
> > > early before even naming/constructing/querying/doing anything with the 
> > > compression scheme/algorithm/etc - so there'd be no need to combine the 
> > > tests for availability and the tests for whether compression was 
> > > requested)
> > > 
> > > Perhaps this API is motivated by a desire to implement something much 
> > > closer to the original code than is necessary/suitable? Or some other use 
> > > case/benefit I'm not quite understanding yet?
> > I shall remove `noneIfUnsupported`. You express good points, we can simply 
> > check `if(OptionalCompressionScheme && *OptionalCompressionScheme)` where 
> > necessary.
> though it will make a lot of existing code patterns less clear, and more 
> verbose
and sometimes you really do need to re code the exact thing `noneIfUnsupported` 
encapsulates...


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 450338.
ckissane added a comment.

- cleanup some compression nits


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/InstrProfCorrelator.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -22,31 +22,41 @@
 
 namespace {
 
-#if LLVM_ENABLE_ZLIB
-static void testZlibCompression(StringRef Input, int Level) {
+static void testCompressionAlgorithm(
+StringRef Input, int Level, CompressionKind CompressionScheme,
+std::string ExpectedDestinationBufferTooSmallErrorMessage) {
   SmallVector Compressed;
   SmallVector Uncompressed;
-  zlib::compress(arrayRefFromStringRef(Input), Compressed, Level);
+  CompressionScheme->compress(arrayRefFromStringRef(Input), Compressed, Level);
 
   // Check that uncompressed buffer is the same as original.
-  Error E = zlib::uncompress(Compressed, Uncompressed, Input.size());
+  Error E =
+  CompressionScheme->decompress(Compressed, Uncompressed, Input.size());
   consumeError(std::move(E));
 
   EXPECT_EQ(Input, toStringRef(Uncompressed));
   if (Input.size() > 0) {
 // Uncompression fails if expected length is too short.
-E = zlib::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("zlib error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+E = CompressionScheme->decompress(Compressed, Uncompressed,
+  Input.size() - 1);
+EXPECT_EQ(ExpectedDestinationBufferTooSmallErrorMessage,
+  llvm::toString(std::move(E)));
   }
 }
 
+#if LLVM_ENABLE_ZLIB
+static void testZlibCompression(StringRef Input, int Level) {
+  testCompressionAlgorithm(Input, Level, CompressionKind::Zlib,
+   "zlib error: Z_BUF_ERROR");
+}
+
 TEST(CompressionTest, Zlib) {
-  testZlibCompression("", zlib::DefaultCompression);
+  CompressionKind CompressionScheme = CompressionKind::Zlib;
+  testZlibCompression("", CompressionScheme->DefaultLevel);
 
-  testZlibCompression("hello, world!", zlib::NoCompression);
-  testZlibCompression("hello, world!", zlib::BestSizeCompression);
-  testZlibCompression("hello, world!", zlib::BestSpeedCompression);
-  testZlibCompression("hello, world!", zlib::DefaultCompression);
+  testZlibCompression("hello, world!", CompressionScheme->BestSizeLevel);
+  testZlibCompression("hello, world!", CompressionScheme->BestSpeedLevel);
+  testZlibCompression("hello, world!", CompressionScheme->DefaultLevel);
 
   const size_t kSize = 1024;
   char BinaryData[kSize];
@@ -54,38 +64,26 @@
 BinaryData[i] = i & 255;
   StringRef BinaryDataStr(BinaryData, kSize);
 
-  testZlibCompression(BinaryDataStr, zlib::NoCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
-  testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSizeLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSpeedLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->DefaultLevel);
 }
 #endif
 
 #if LLVM_ENABLE_ZSTD
-static void testZstdCompression(StringRef Input, int Level) {
-  SmallVector Compressed;
-  SmallVector Uncompressed;
-  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
 
-  // Check that uncompressed buffer is the same as original.
-  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
-  consumeError(std::move(E));
-
-  EXPECT_EQ(Input, toStringRef(Uncompressed));
-  if (Input.size() > 0) {
-// Uncompression fails if expected length is too short.
-E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
-  }
+static void te

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane added inline comments.



Comment at: llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp:54-60
+  OptionalCompressionScheme = compression::noneIfUnsupported(
+  (Compress && DoInstrProfNameCompression) ? OptionalCompressionScheme
+   : llvm::NoneType());
+
+  bool doCompression = bool(OptionalCompressionScheme);
+
+  if (doCompression) {

dblaikie wrote:
> ckissane wrote:
> > ckissane wrote:
> > > ckissane wrote:
> > > > dblaikie wrote:
> > > > > This still seems like a lot of hoops to jump through - why 
> > > > > "noneIfUnsupported" rather than either having the compression scheme 
> > > > > (I think it could be the CompressionAlgorithm itself, rather than 
> > > > > having the separate OptionalCompressionKind abstraction) either be 
> > > > > null itself, or expose an "isAvailable" operation directly on the 
> > > > > CompressionAlgorithm?
> > > > > 
> > > > > Even if the 
> > > > > CompressionKind/OptionalCompressionKind/CompressionAlgorithm 
> > > > > abstractions are kept, I'm not sure why the above code is preferred 
> > > > > over, say:
> > > > > 
> > > > > ```
> > > > > if (Compress && DoInstrProfNameCompression && 
> > > > > OptionalCompressionScheme /* .isAvailable(), if we want to be more 
> > > > > explicit */) {
> > > > >   ...
> > > > > }
> > > > > ```
> > > > > 
> > > > > What's the benefit that `noneIfUnsupported` is providing? (& 
> > > > > generally I'd expect the `Compress && DoInstrProfNameCompression` to 
> > > > > be tested/exit early before even naming/constructing/querying/doing 
> > > > > anything with the compression scheme/algorithm/etc - so there'd be no 
> > > > > need to combine the tests for availability and the tests for whether 
> > > > > compression was requested)
> > > > > 
> > > > > Perhaps this API is motivated by a desire to implement something much 
> > > > > closer to the original code than is necessary/suitable? Or some other 
> > > > > use case/benefit I'm not quite understanding yet?
> > > > I shall remove `noneIfUnsupported`. You express good points, we can 
> > > > simply check `if(OptionalCompressionScheme && 
> > > > *OptionalCompressionScheme)` where necessary.
> > > though it will make a lot of existing code patterns less clear, and more 
> > > verbose
> > and sometimes you really do need to re code the exact thing 
> > `noneIfUnsupported` encapsulates...
> Are there examples within LLVM that you can show compare/contrast 
> `noneIfUnsupported` helps?
yes, I'll paste a couple here


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane added inline comments.



Comment at: llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp:54-60
+  OptionalCompressionScheme = compression::noneIfUnsupported(
+  (Compress && DoInstrProfNameCompression) ? OptionalCompressionScheme
+   : llvm::NoneType());
+
+  bool doCompression = bool(OptionalCompressionScheme);
+
+  if (doCompression) {

ckissane wrote:
> dblaikie wrote:
> > ckissane wrote:
> > > ckissane wrote:
> > > > ckissane wrote:
> > > > > dblaikie wrote:
> > > > > > This still seems like a lot of hoops to jump through - why 
> > > > > > "noneIfUnsupported" rather than either having the compression 
> > > > > > scheme (I think it could be the CompressionAlgorithm itself, rather 
> > > > > > than having the separate OptionalCompressionKind abstraction) 
> > > > > > either be null itself, or expose an "isAvailable" operation 
> > > > > > directly on the CompressionAlgorithm?
> > > > > > 
> > > > > > Even if the 
> > > > > > CompressionKind/OptionalCompressionKind/CompressionAlgorithm 
> > > > > > abstractions are kept, I'm not sure why the above code is preferred 
> > > > > > over, say:
> > > > > > 
> > > > > > ```
> > > > > > if (Compress && DoInstrProfNameCompression && 
> > > > > > OptionalCompressionScheme /* .isAvailable(), if we want to be more 
> > > > > > explicit */) {
> > > > > >   ...
> > > > > > }
> > > > > > ```
> > > > > > 
> > > > > > What's the benefit that `noneIfUnsupported` is providing? (& 
> > > > > > generally I'd expect the `Compress && DoInstrProfNameCompression` 
> > > > > > to be tested/exit early before even 
> > > > > > naming/constructing/querying/doing anything with the compression 
> > > > > > scheme/algorithm/etc - so there'd be no need to combine the tests 
> > > > > > for availability and the tests for whether compression was 
> > > > > > requested)
> > > > > > 
> > > > > > Perhaps this API is motivated by a desire to implement something 
> > > > > > much closer to the original code than is necessary/suitable? Or 
> > > > > > some other use case/benefit I'm not quite understanding yet?
> > > > > I shall remove `noneIfUnsupported`. You express good points, we can 
> > > > > simply check `if(OptionalCompressionScheme && 
> > > > > *OptionalCompressionScheme)` where necessary.
> > > > though it will make a lot of existing code patterns less clear, and 
> > > > more verbose
> > > and sometimes you really do need to re code the exact thing 
> > > `noneIfUnsupported` encapsulates...
> > Are there examples within LLVM that you can show compare/contrast 
> > `noneIfUnsupported` helps?
> yes, I'll paste a couple here
Ok, So I believe I was mistaken.
In older versions of this patch there was a none compression implementation 
that just did a memcpy,  this made a natural fall through state, which made 
this type of pattern advantageous.
However, this is no longer the case.
Hence I will remove this without further adue.
Thank you for your astute observation!


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 450343.
ckissane added a comment.

- remove OptionalCompressionKind noneIfUnsupported


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/InstrProfCorrelator.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -22,31 +22,41 @@
 
 namespace {
 
-#if LLVM_ENABLE_ZLIB
-static void testZlibCompression(StringRef Input, int Level) {
+static void testCompressionAlgorithm(
+StringRef Input, int Level, CompressionKind CompressionScheme,
+std::string ExpectedDestinationBufferTooSmallErrorMessage) {
   SmallVector Compressed;
   SmallVector Uncompressed;
-  zlib::compress(arrayRefFromStringRef(Input), Compressed, Level);
+  CompressionScheme->compress(arrayRefFromStringRef(Input), Compressed, Level);
 
   // Check that uncompressed buffer is the same as original.
-  Error E = zlib::uncompress(Compressed, Uncompressed, Input.size());
+  Error E =
+  CompressionScheme->decompress(Compressed, Uncompressed, Input.size());
   consumeError(std::move(E));
 
   EXPECT_EQ(Input, toStringRef(Uncompressed));
   if (Input.size() > 0) {
 // Uncompression fails if expected length is too short.
-E = zlib::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("zlib error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+E = CompressionScheme->decompress(Compressed, Uncompressed,
+  Input.size() - 1);
+EXPECT_EQ(ExpectedDestinationBufferTooSmallErrorMessage,
+  llvm::toString(std::move(E)));
   }
 }
 
+#if LLVM_ENABLE_ZLIB
+static void testZlibCompression(StringRef Input, int Level) {
+  testCompressionAlgorithm(Input, Level, CompressionKind::Zlib,
+   "zlib error: Z_BUF_ERROR");
+}
+
 TEST(CompressionTest, Zlib) {
-  testZlibCompression("", zlib::DefaultCompression);
+  CompressionKind CompressionScheme = CompressionKind::Zlib;
+  testZlibCompression("", CompressionScheme->DefaultLevel);
 
-  testZlibCompression("hello, world!", zlib::NoCompression);
-  testZlibCompression("hello, world!", zlib::BestSizeCompression);
-  testZlibCompression("hello, world!", zlib::BestSpeedCompression);
-  testZlibCompression("hello, world!", zlib::DefaultCompression);
+  testZlibCompression("hello, world!", CompressionScheme->BestSizeLevel);
+  testZlibCompression("hello, world!", CompressionScheme->BestSpeedLevel);
+  testZlibCompression("hello, world!", CompressionScheme->DefaultLevel);
 
   const size_t kSize = 1024;
   char BinaryData[kSize];
@@ -54,38 +64,26 @@
 BinaryData[i] = i & 255;
   StringRef BinaryDataStr(BinaryData, kSize);
 
-  testZlibCompression(BinaryDataStr, zlib::NoCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
-  testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSizeLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSpeedLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->DefaultLevel);
 }
 #endif
 
 #if LLVM_ENABLE_ZSTD
-static void testZstdCompression(StringRef Input, int Level) {
-  SmallVector Compressed;
-  SmallVector Uncompressed;
-  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
 
-  // Check that uncompressed buffer is the same as original.
-  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
-  consumeError(std::move(E));
-
-  EXPECT_EQ(Input, toStringRef(Uncompressed));
-  if (Input.size() > 0) {
-// Uncompression fails if expected length is too short.
-E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
-

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane added inline comments.



Comment at: clang/lib/Serialization/ASTReader.cpp:1474-1475
+  if (!CompressionScheme) {
+Error("compression class " +
+  (CompressionScheme->Name + " is not available").str());
 return nullptr;

leonardchan wrote:
> I think this `Error` takes a StringRef, so I think you could have:
> 
> ```
> return Error("compression class " + CompressionScheme->Name + " is not 
> available");
> ```
unfortunately that doesn't work (I tried)


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 450345.
ckissane added a comment.

- format error string


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/InstrProfCorrelator.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -22,31 +22,41 @@
 
 namespace {
 
-#if LLVM_ENABLE_ZLIB
-static void testZlibCompression(StringRef Input, int Level) {
+static void testCompressionAlgorithm(
+StringRef Input, int Level, CompressionKind CompressionScheme,
+std::string ExpectedDestinationBufferTooSmallErrorMessage) {
   SmallVector Compressed;
   SmallVector Uncompressed;
-  zlib::compress(arrayRefFromStringRef(Input), Compressed, Level);
+  CompressionScheme->compress(arrayRefFromStringRef(Input), Compressed, Level);
 
   // Check that uncompressed buffer is the same as original.
-  Error E = zlib::uncompress(Compressed, Uncompressed, Input.size());
+  Error E =
+  CompressionScheme->decompress(Compressed, Uncompressed, Input.size());
   consumeError(std::move(E));
 
   EXPECT_EQ(Input, toStringRef(Uncompressed));
   if (Input.size() > 0) {
 // Uncompression fails if expected length is too short.
-E = zlib::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("zlib error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+E = CompressionScheme->decompress(Compressed, Uncompressed,
+  Input.size() - 1);
+EXPECT_EQ(ExpectedDestinationBufferTooSmallErrorMessage,
+  llvm::toString(std::move(E)));
   }
 }
 
+#if LLVM_ENABLE_ZLIB
+static void testZlibCompression(StringRef Input, int Level) {
+  testCompressionAlgorithm(Input, Level, CompressionKind::Zlib,
+   "zlib error: Z_BUF_ERROR");
+}
+
 TEST(CompressionTest, Zlib) {
-  testZlibCompression("", zlib::DefaultCompression);
+  CompressionKind CompressionScheme = CompressionKind::Zlib;
+  testZlibCompression("", CompressionScheme->DefaultLevel);
 
-  testZlibCompression("hello, world!", zlib::NoCompression);
-  testZlibCompression("hello, world!", zlib::BestSizeCompression);
-  testZlibCompression("hello, world!", zlib::BestSpeedCompression);
-  testZlibCompression("hello, world!", zlib::DefaultCompression);
+  testZlibCompression("hello, world!", CompressionScheme->BestSizeLevel);
+  testZlibCompression("hello, world!", CompressionScheme->BestSpeedLevel);
+  testZlibCompression("hello, world!", CompressionScheme->DefaultLevel);
 
   const size_t kSize = 1024;
   char BinaryData[kSize];
@@ -54,38 +64,26 @@
 BinaryData[i] = i & 255;
   StringRef BinaryDataStr(BinaryData, kSize);
 
-  testZlibCompression(BinaryDataStr, zlib::NoCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
-  testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSizeLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSpeedLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->DefaultLevel);
 }
 #endif
 
 #if LLVM_ENABLE_ZSTD
-static void testZstdCompression(StringRef Input, int Level) {
-  SmallVector Compressed;
-  SmallVector Uncompressed;
-  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
 
-  // Check that uncompressed buffer is the same as original.
-  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
-  consumeError(std::move(E));
-
-  EXPECT_EQ(Input, toStringRef(Uncompressed));
-  if (Input.size() > 0) {
-// Uncompression fails if expected length is too short.
-E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
-  }
+static void testZStdComp

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 450392.
ckissane added a comment.

- fix InputSection decompress issue


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/InstrProfCorrelator.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -22,31 +22,41 @@
 
 namespace {
 
-#if LLVM_ENABLE_ZLIB
-static void testZlibCompression(StringRef Input, int Level) {
+static void testCompressionAlgorithm(
+StringRef Input, int Level, CompressionKind CompressionScheme,
+std::string ExpectedDestinationBufferTooSmallErrorMessage) {
   SmallVector Compressed;
   SmallVector Uncompressed;
-  zlib::compress(arrayRefFromStringRef(Input), Compressed, Level);
+  CompressionScheme->compress(arrayRefFromStringRef(Input), Compressed, Level);
 
   // Check that uncompressed buffer is the same as original.
-  Error E = zlib::uncompress(Compressed, Uncompressed, Input.size());
+  Error E =
+  CompressionScheme->decompress(Compressed, Uncompressed, Input.size());
   consumeError(std::move(E));
 
   EXPECT_EQ(Input, toStringRef(Uncompressed));
   if (Input.size() > 0) {
 // Uncompression fails if expected length is too short.
-E = zlib::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("zlib error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+E = CompressionScheme->decompress(Compressed, Uncompressed,
+  Input.size() - 1);
+EXPECT_EQ(ExpectedDestinationBufferTooSmallErrorMessage,
+  llvm::toString(std::move(E)));
   }
 }
 
+#if LLVM_ENABLE_ZLIB
+static void testZlibCompression(StringRef Input, int Level) {
+  testCompressionAlgorithm(Input, Level, CompressionKind::Zlib,
+   "zlib error: Z_BUF_ERROR");
+}
+
 TEST(CompressionTest, Zlib) {
-  testZlibCompression("", zlib::DefaultCompression);
+  CompressionKind CompressionScheme = CompressionKind::Zlib;
+  testZlibCompression("", CompressionScheme->DefaultLevel);
 
-  testZlibCompression("hello, world!", zlib::NoCompression);
-  testZlibCompression("hello, world!", zlib::BestSizeCompression);
-  testZlibCompression("hello, world!", zlib::BestSpeedCompression);
-  testZlibCompression("hello, world!", zlib::DefaultCompression);
+  testZlibCompression("hello, world!", CompressionScheme->BestSizeLevel);
+  testZlibCompression("hello, world!", CompressionScheme->BestSpeedLevel);
+  testZlibCompression("hello, world!", CompressionScheme->DefaultLevel);
 
   const size_t kSize = 1024;
   char BinaryData[kSize];
@@ -54,38 +64,26 @@
 BinaryData[i] = i & 255;
   StringRef BinaryDataStr(BinaryData, kSize);
 
-  testZlibCompression(BinaryDataStr, zlib::NoCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
-  testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSizeLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSpeedLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->DefaultLevel);
 }
 #endif
 
 #if LLVM_ENABLE_ZSTD
-static void testZstdCompression(StringRef Input, int Level) {
-  SmallVector Compressed;
-  SmallVector Uncompressed;
-  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
 
-  // Check that uncompressed buffer is the same as original.
-  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
-  consumeError(std::move(E));
-
-  EXPECT_EQ(Input, toStringRef(Uncompressed));
-  if (Input.size() > 0) {
-// Uncompression fails if expected length is too short.
-E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
-  }
+static voi

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 450401.
ckissane added a comment.

- Merge remote-tracking branch 'origin/main' into 
ckissane.compression-class-simple


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/InstrProfCorrelator.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -22,31 +22,41 @@
 
 namespace {
 
-#if LLVM_ENABLE_ZLIB
-static void testZlibCompression(StringRef Input, int Level) {
+static void testCompressionAlgorithm(
+StringRef Input, int Level, CompressionKind CompressionScheme,
+std::string ExpectedDestinationBufferTooSmallErrorMessage) {
   SmallVector Compressed;
   SmallVector Uncompressed;
-  zlib::compress(arrayRefFromStringRef(Input), Compressed, Level);
+  CompressionScheme->compress(arrayRefFromStringRef(Input), Compressed, Level);
 
   // Check that uncompressed buffer is the same as original.
-  Error E = zlib::uncompress(Compressed, Uncompressed, Input.size());
+  Error E =
+  CompressionScheme->decompress(Compressed, Uncompressed, Input.size());
   consumeError(std::move(E));
 
   EXPECT_EQ(Input, toStringRef(Uncompressed));
   if (Input.size() > 0) {
 // Uncompression fails if expected length is too short.
-E = zlib::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("zlib error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+E = CompressionScheme->decompress(Compressed, Uncompressed,
+  Input.size() - 1);
+EXPECT_EQ(ExpectedDestinationBufferTooSmallErrorMessage,
+  llvm::toString(std::move(E)));
   }
 }
 
+#if LLVM_ENABLE_ZLIB
+static void testZlibCompression(StringRef Input, int Level) {
+  testCompressionAlgorithm(Input, Level, CompressionKind::Zlib,
+   "zlib error: Z_BUF_ERROR");
+}
+
 TEST(CompressionTest, Zlib) {
-  testZlibCompression("", zlib::DefaultCompression);
+  CompressionKind CompressionScheme = CompressionKind::Zlib;
+  testZlibCompression("", CompressionScheme->DefaultLevel);
 
-  testZlibCompression("hello, world!", zlib::NoCompression);
-  testZlibCompression("hello, world!", zlib::BestSizeCompression);
-  testZlibCompression("hello, world!", zlib::BestSpeedCompression);
-  testZlibCompression("hello, world!", zlib::DefaultCompression);
+  testZlibCompression("hello, world!", CompressionScheme->BestSizeLevel);
+  testZlibCompression("hello, world!", CompressionScheme->BestSpeedLevel);
+  testZlibCompression("hello, world!", CompressionScheme->DefaultLevel);
 
   const size_t kSize = 1024;
   char BinaryData[kSize];
@@ -54,38 +64,26 @@
 BinaryData[i] = i & 255;
   StringRef BinaryDataStr(BinaryData, kSize);
 
-  testZlibCompression(BinaryDataStr, zlib::NoCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
-  testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSizeLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSpeedLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->DefaultLevel);
 }
 #endif
 
 #if LLVM_ENABLE_ZSTD
-static void testZstdCompression(StringRef Input, int Level) {
-  SmallVector Compressed;
-  SmallVector Uncompressed;
-  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
 
-  // Check that uncompressed buffer is the same as original.
-  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
-  consumeError(std::move(E));
-
-  EXPECT_EQ(Input, toStringRef(Uncompressed));
-  if (Input.size() > 0) {
-// Uncompression fails if expected length is too short.
-E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("Destination buffer is too small"

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 451247.
ckissane added a comment.

- address review comments


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

Files:
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/ProfileData/InstrProf.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/InstrProfCorrelator.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp

Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -22,31 +22,41 @@
 
 namespace {
 
-#if LLVM_ENABLE_ZLIB
-static void testZlibCompression(StringRef Input, int Level) {
+static void testCompressionAlgorithm(
+StringRef Input, int Level, CompressionKind CompressionScheme,
+std::string ExpectedDestinationBufferTooSmallErrorMessage) {
   SmallVector Compressed;
   SmallVector Uncompressed;
-  zlib::compress(arrayRefFromStringRef(Input), Compressed, Level);
+  CompressionScheme->compress(arrayRefFromStringRef(Input), Compressed, Level);
 
   // Check that uncompressed buffer is the same as original.
-  Error E = zlib::uncompress(Compressed, Uncompressed, Input.size());
+  Error E =
+  CompressionScheme->decompress(Compressed, Uncompressed, Input.size());
   consumeError(std::move(E));
 
   EXPECT_EQ(Input, toStringRef(Uncompressed));
   if (Input.size() > 0) {
 // Uncompression fails if expected length is too short.
-E = zlib::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("zlib error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+E = CompressionScheme->decompress(Compressed, Uncompressed,
+  Input.size() - 1);
+EXPECT_EQ(ExpectedDestinationBufferTooSmallErrorMessage,
+  llvm::toString(std::move(E)));
   }
 }
 
+#if LLVM_ENABLE_ZLIB
+static void testZlibCompression(StringRef Input, int Level) {
+  testCompressionAlgorithm(Input, Level, CompressionKind::Zlib,
+   "zlib error: Z_BUF_ERROR");
+}
+
 TEST(CompressionTest, Zlib) {
-  testZlibCompression("", zlib::DefaultCompression);
+  CompressionKind CompressionScheme = CompressionKind::Zlib;
+  testZlibCompression("", CompressionScheme->DefaultLevel);
 
-  testZlibCompression("hello, world!", zlib::NoCompression);
-  testZlibCompression("hello, world!", zlib::BestSizeCompression);
-  testZlibCompression("hello, world!", zlib::BestSpeedCompression);
-  testZlibCompression("hello, world!", zlib::DefaultCompression);
+  testZlibCompression("hello, world!", CompressionScheme->BestSizeLevel);
+  testZlibCompression("hello, world!", CompressionScheme->BestSpeedLevel);
+  testZlibCompression("hello, world!", CompressionScheme->DefaultLevel);
 
   const size_t kSize = 1024;
   char BinaryData[kSize];
@@ -54,38 +64,26 @@
 BinaryData[i] = i & 255;
   StringRef BinaryDataStr(BinaryData, kSize);
 
-  testZlibCompression(BinaryDataStr, zlib::NoCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
-  testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
-  testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSizeLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->BestSpeedLevel);
+  testZlibCompression(BinaryDataStr, CompressionScheme->DefaultLevel);
 }
 #endif
 
 #if LLVM_ENABLE_ZSTD
-static void testZstdCompression(StringRef Input, int Level) {
-  SmallVector Compressed;
-  SmallVector Uncompressed;
-  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
 
-  // Check that uncompressed buffer is the same as original.
-  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
-  consumeError(std::move(E));
-
-  EXPECT_EQ(Input, toStringRef(Uncompressed));
-  if (Input.size() > 0) {
-// Uncompression fails if expected length is too short.
-E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
-EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
-  }
+static void testZStd

[PATCH] D130516: [llvm] compression classes

[Cole Kissane via Phabricator via cfe-commits]

ckissane added a comment.

In D130516#3703691 , @dblaikie wrote:

> This is looking pretty close to what I've been picturing - the only thing 
> remaining is that I think we could get rid of `CompressionKind` and access 
> the `CompressionAlgorithm` directly - basically the contents of 
> `CompressionKind::operator->` could be a free/public function `const 
> CompressionAlgorithm &getCompressionAlgorithm(enum class CompressionKind);` - 
> and have it return a reference to the local static implementation, with a 
> none implementation (for those profile cases where you want to pass in an 
> algorithm if it's available, or none) and one implementation for each of 
> zlib/zstd?

I can see what you are asking for, however since its behavior is essentially 
the same, and still uses both enum values and class implementations, I don't 
see any practical advantages (though if there is something I am failing to 
observe let me know).
Additionally, I can see some disadvantages in largely increasing code verbosity 
across the codebase. 
A snippet of an example from elf: `if(CompressionKind::Zlib) ...`
would turn into `if(getCompressionAlgorithm(CompressionKind::Zlib)) ...`
Because of this I am leaning away from such an implementation change.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D130516/new/

https://reviews.llvm.org/D130516

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 443687.
ckissane added a comment.

clean a space and make accurate


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/test/lit.cfg.py
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.cfg.py
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lld/test/lit.site.cfg.py.in
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  lldb/test/Shell/lit.site.cfg.py.in
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/lib/Support/CMakeLists.txt
  llvm/test/lit.site.cfg.py.in
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/test/lit.site.cfg.py.in
===
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
Index: llvm/lib/Support/CMakeLists.txt
===
--- llvm/lib/Support/CMakeLists.txt
+++ llvm/lib/Support/CMakeLists.txt
@@ -25,6 +25,10 @@
   set(imported_libs ZLIB::ZLIB)
 endif()
 
+if(LLVM_ENABLE_ZSTD)
+  list(APPEND imported_libs zstd)
+endif()
+
 if( MSVC OR MINGW )
   # libuuid required for FOLDERID_Profile usage in lib/Support/Windows/Path.inc.
   # advapi32 required for CryptAcquireContextW in lib/Support/Windows/Path.inc.
Index: llvm/include/llvm/Config/llvm-config.h.cmake
===
--- llvm/include/llvm/Config/llvm-config.h.cmake
+++ llvm/include/llvm/Config/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/cmake/modules/LLVMConfig.cmake.in
===
--- llvm/cmake/modules/LLVMConfig.cmake.in
+++ llvm/cmake/modules/LLVMConfig.cmake.in
@@ -73,6 +73,12 @@
   find_package(ZLIB)
 endif()
 
+set(LLVM_ENABLE_ZSTD @LLVM_ENABLE_ZSTD@)
+if(LLVM_ENABLE_ZSTD)
+  set(ZSTD_ROOT @ZSTD_ROOT@)
+  find_package(ZSTD)
+endif()
+
 set(LLVM_ENABLE_LIBXML2 @LLVM_ENABLE_LIBXML2@)
 if(LLVM_ENABLE_LIBXML2)
   find_package(LibXml2)
Index: llvm/cmake/modules/FindZSTD.cmake
===
--- /dev/null
+++ llvm/cmake/modules/FindZSTD.cmake
@@ -0,0 +1,21 @@
+find_path(ZSTD_INCLUDE_DIR
+  NAMES zstd.h
+  HINTS ${ZSTD_ROOT_DIR}/include)
+
+find_library(ZSTD_LIBRARY
+  NAMES zstd
+  HINTS ${ZSTD_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(
+ZSTD DEFAULT_MSG
+ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+
+if(ZSTD_FOUND)
+set(ZSTD_LIBRARIES ${ZSTD_LIBRARY})
+set(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR})
+endif()
+
+mark_as_advanced(
+  ZSTD_INCLUDE_DIR
+  ZSTD_LIBRARY)
Index: llvm/cmake/config-ix.cmake
===
--- llvm/cmake/config-ix.cmake
+++ llvm/cmake/config-ix.cmake
@@ -136,6 +136,27 @@
   set(LLVM_ENABLE_ZLIB "${HAVE_ZLIB}")
 endif()
 
+if(LLVM_ENABLE_ZSTD)
+  if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON)
+find_package(ZSTD REQUIRED)
+  elseif(NOT LLVM_USE_SANITIZER MATCHES "Memory.*")
+find_package(ZSTD)
+  endif()
+  if(ZSTD_FOUND)
+# Check if zstd we found is usable; for example, we may have found a 32-bit
+# library on a 64-bit system which would result in a link-time failure.
+cmake_push_check_state()
+list(APPEND CMAKE_REQUIRED_INCLUDES ${ZSTD_INCLUDE_DIR})
+list(APPEND CMAKE_REQUIRED_LIBRARIES ${ZSTD_LIBRARY})
+check_symbol_exists(ZSTD

[PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM

[Cole Kissane via Phabricator via cfe-commits]

ckissane added inline comments.



Comment at: lld/test/lit.site.cfg.py.in:21
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@

MaskRay wrote:
> MaskRay wrote:
> > This needs a change in lld/test/CMakeLists.txt
> > 
> > Looks like you haven't run `ninja check-lld` as I mentioned.
> Actually I am preparing a lld patch and you probably should drop have_zstd 
> from lld/ for this patch.
I get this when I try to run that:
`ninja: error: unknown target 'check-lld', did you mean 'check-lit'?`



Comment at: lld/test/lit.site.cfg.py.in:21
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@

ckissane wrote:
> MaskRay wrote:
> > MaskRay wrote:
> > > This needs a change in lld/test/CMakeLists.txt
> > > 
> > > Looks like you haven't run `ninja check-lld` as I mentioned.
> > Actually I am preparing a lld patch and you probably should drop have_zstd 
> > from lld/ for this patch.
> I get this when I try to run that:
> `ninja: error: unknown target 'check-lld', did you mean 'check-lit'?`
Got it


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 444044.
ckissane added a comment.

- make sure LLVM_ENABLE_ZSTD OFF turns into 0
- add zstd compression namespace


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/test/lit.cfg.py
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.cfg.py
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lld/test/lit.site.cfg.py.in
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  lldb/test/Shell/lit.site.cfg.py.in
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/test/lit.site.cfg.py.in
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/test/lit.site.cfg.py.in
===
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,6 +20,9 @@
 #if LLVM_ENABLE_ZLIB
 #include 
 #endif
+#if LLVM_ENABLE_ZSTD
+#include 
+#endif
 
 using namespace llvm;
 using namespace llvm::compression;
@@ -99,3 +102,66 @@
   llvm_unreachable("zlib::uncompress is unavailable");
 }
 #endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(StringRef InputBuffer,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  unsigned long CompressedBufferSize = ::ZSTD_compressBound(InputBuffer.size());
+  CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+  unsigned long CompressedSize = ::ZSTD_compress(
+  (char *)CompressedBuffer.data(), CompressedBufferSize,
+  (const char *)InputBuffer.data(), InputBuffer.size(), Level);
+  if (ZSTD_isError(CompressedSize))
+report_bad_alloc_error("Allocation failed");
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(CompressedBuffer.data(), CompressedSize);
+  CompressedBuffer.truncate(CompressedSize);
+}
+
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+   size_t &UncompressedSize) {
+  unsigned long long const FrameContentSize = ZSTD_getFrameContentSize(
+  (const char *)InputBuffer.data(), InputBuffer.size());
+  size_t const Res =
+  ::ZSTD_decompress((char *)UncompressedBuffer, FrameContentSize,
+(const char *)InputBuffer.data(), InputBuffer.size());
+  UncompressedSize = Res;
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(UncompressedBuffer, UncompressedSize);
+  return Res != FrameContentSize
+ ? make_error(ZSTD_getErrorName(Res),
+   inconvertibleErrorCode())
+ : Error::success();
+}
+
+Error zstd::uncompress(StringRef InputBuffer,
+   SmallVectorImpl &UncompressedBuffer,
+   size_t UncompressedSize) {
+  UncompressedBuffer.resize_for_overwrite(UncompressedSize);
+  Error E = zstd::uncompress(InputBuffer, UncompressedBuffer.data(),
+ UncompressedSize);
+  UncompressedBuffer.truncate(UncompressedSize);
+  return E;
+}
+
+#else
+bool zstd::isAvailable() { return false; }
+void zstd::compress(StringRef InputBuffer,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  llvm_unreachable("zstd::compress is unavailable");
+}
+Error 

[PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM and add ZSTD compression namespace

[Cole Kissane via Phabricator via cfe-commits]

ckissane added a comment.

In D128465#3642997 , @MaskRay wrote:

> As I mentioned, the proper approach is to add zstd functionality along with 
> the CMake change, instead of adding CMake to all llvm-project components 
> without a way to test them.

@MaskRay, I have now done this and ran the ldd tests as requested:

  With LLVM_ENABLE_ZSTD=ON
  $ ninja check-lld
  Testing Time: 8.98s
Unsupported  :   17
Passed   : 2638
Expectedly Failed:1
  With LLVM_ENABLE_ZSTD=OFF
  $ ninja check-lld
  Testing Time: 8.95s
Unsupported  :   17
Passed   : 2638
Expectedly Failed:1


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM and add ZSTD compression namespace

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 444078.
ckissane added a comment.

- add zstd tests to CompressionTest.cpp


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/test/lit.cfg.py
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.cfg.py
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lld/test/lit.site.cfg.py.in
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  lldb/test/Shell/lit.site.cfg.py.in
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/test/lit.site.cfg.py.in
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -65,4 +65,46 @@
 
 #endif
 
+#if LLVM_ENABLE_ZSTD
+
+void TestZstdCompression(StringRef Input, int Level) {
+  SmallString<32> Compressed;
+  SmallString<32> Uncompressed;
+
+  zstd::compress(Input, Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, Uncompressed);
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  TestZstdCompression("", zstd::DefaultCompression);
+
+  TestZstdCompression("hello, world!", zstd::NoCompression);
+  TestZstdCompression("hello, world!", zstd::BestSizeCompression);
+  TestZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  TestZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i) {
+BinaryData[i] = i & 255;
+  }
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  TestZstdCompression(BinaryDataStr, zstd::NoCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  TestZstdCompression(BinaryDataStr, zstd::DefaultCompression);
+}
+
+#endif
 }
Index: llvm/test/lit.site.cfg.py.in
===
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,6 +20,9 @@
 #if LLVM_ENABLE_ZLIB
 #include 
 #endif
+#if LLVM_ENABLE_ZSTD
+#include 
+#endif
 
 using namespace llvm;
 using namespace llvm::compression;
@@ -57,7 +60,9 @@
   // Tell MemorySanitizer that zlib output buffer is fully initialized.
   // This avoids a false report when running LLVM with uninstrumented ZLib.
   __msan_unpoison(CompressedBuffer.data(), CompressedSize);
-  CompressedBuffer.truncate(CompressedSize);
+  if (CompressedSize < CompressedBuffer.size()) {
+CompressedBuffer.truncate(CompressedSize);
+  }
 }
 
 Error zlib::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
@@ -79,7 +84,9 @@
   UncompressedBuffer.resize_for_overwrite(UncompressedSize);
   Error E = zlib::uncompress(InputBuffer, UncompressedBuffer.data(),
  UncompressedSize);
-  UncompressedBuffer.truncate(UncompressedSize);
+  if (UncompressedSize < Uncompr

[PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM and add ZSTD compression namespace

[Cole Kissane via Phabricator via cfe-commits]

ckissane added a comment.

In D128465#3646258 , @MaskRay wrote:

> In D128465#3646203 , @ckissane 
> wrote:
>
>> In D128465#3642997 , @MaskRay 
>> wrote:
>>
>>> As I mentioned, the proper approach is to add zstd functionality along with 
>>> the CMake change, instead of adding CMake to all llvm-project components 
>>> without a way to test them.
>>
>> @MaskRay, I have now done this and ran the ldd tests as requested:
>>
>>   With LLVM_ENABLE_ZSTD=ON
>>   $ ninja check-lld
>>   Testing Time: 8.98s
>> Unsupported  :   17
>> Passed   : 2638
>> Expectedly Failed:1
>>   With LLVM_ENABLE_ZSTD=OFF
>>   $ ninja check-lld
>>   Testing Time: 8.95s
>> Unsupported  :   17
>> Passed   : 2638
>> Expectedly Failed:1
>
> I request that you abandon this patch and incorporate the llvm cmake part 
> into the llvm patch which you actually use zstd.
> It is not appropriate to add zstd cmake to llvm-project components which 
> don't use zstd.

Let me see if I understand this correctly:
Are you saying that I should abandon this patch, and create a new patch that:

- adds findZSTD.cmake
- adds zstd to compression namespace
- adds tests to CompressionTest.cpp
- and does the **minimal** amount of cmake work to have the flags and test work
  - thus differing per component cmake config to patches like how you are doing 
in https://reviews.llvm.org/D129406

Is this correct?

I realize though that then https://reviews.llvm.org/D129406 or similar would be 
"the [first] llvm patch which actually use[s] zstd"


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM and add ZSTD compression namespace

[Cole Kissane via Phabricator via cfe-commits]

ckissane added a comment.

In D128465#3646601 , @MaskRay wrote:

> In D128465#3646561 , @ckissane 
> wrote:
>
>> In D128465#3646258 , @MaskRay 
>> wrote:
>>
>>> In D128465#3646203 , @ckissane 
>>> wrote:
>>>
 In D128465#3642997 , @MaskRay 
 wrote:

> As I mentioned, the proper approach is to add zstd functionality along 
> with the CMake change, instead of adding CMake to all llvm-project 
> components without a way to test them.

 @MaskRay, I have now done this and ran the ldd tests as requested:

   With LLVM_ENABLE_ZSTD=ON
   $ ninja check-lld
   Testing Time: 8.98s
 Unsupported  :   17
 Passed   : 2638
 Expectedly Failed:1
   With LLVM_ENABLE_ZSTD=OFF
   $ ninja check-lld
   Testing Time: 8.95s
 Unsupported  :   17
 Passed   : 2638
 Expectedly Failed:1
>>>
>>> I request that you abandon this patch and incorporate the llvm cmake part 
>>> into the llvm patch which you actually use zstd.
>>> It is not appropriate to add zstd cmake to llvm-project components which 
>>> don't use zstd.
>>
>> Let me see if I understand this correctly:
>> Are you saying that I should abandon this patch, and create a new patch that:
>>
>> - adds findZSTD.cmake
>> - adds zstd to compression namespace
>> - adds tests to CompressionTest.cpp
>> - and does the **minimal** amount of cmake work to have the flags and test 
>> work
>>   - thus differing per component cmake config to patches like how you are 
>> doing in https://reviews.llvm.org/D129406
>>
>> Is this correct?
>>
>> I realize though that then https://reviews.llvm.org/D129406 or similar would 
>> be "the [first] llvm patch which actually use[s] zstd"
>
> You can find a component in llvm which uses zstd (e.g. lib/Support). Add the 
> functionality with tests (e.g. a CompressionTest.cpp unittest) along with the 
> CMake/(optional Bazel, gn) changes.
> Keep clang, clang-tools-extra, lld, etc unchanged. These components don't yet 
> use zstd and the CMake changes in this patch will be untestable.

Got it!, this patch already has such tests (CompressionTest.cpp unittest) in 
lib/Support, so I should just remove the untestable cmake config.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM and add ZSTD compression namespace

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 444374.
ckissane added a comment.

- Trim CMAKE modifications to minimum needed


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/test/lit.site.cfg.py.in
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -65,4 +65,46 @@
 
 #endif
 
+#if LLVM_ENABLE_ZSTD
+
+void TestZstdCompression(StringRef Input, int Level) {
+  SmallString<32> Compressed;
+  SmallString<32> Uncompressed;
+
+  zstd::compress(Input, Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, Uncompressed);
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  TestZstdCompression("", zstd::DefaultCompression);
+
+  TestZstdCompression("hello, world!", zstd::NoCompression);
+  TestZstdCompression("hello, world!", zstd::BestSizeCompression);
+  TestZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  TestZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i) {
+BinaryData[i] = i & 255;
+  }
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  TestZstdCompression(BinaryDataStr, zstd::NoCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  TestZstdCompression(BinaryDataStr, zstd::DefaultCompression);
+}
+
+#endif
 }
Index: llvm/test/lit.site.cfg.py.in
===
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,6 +20,9 @@
 #if LLVM_ENABLE_ZLIB
 #include 
 #endif
+#if LLVM_ENABLE_ZSTD
+#include 
+#endif
 
 using namespace llvm;
 using namespace llvm::compression;
@@ -57,7 +60,9 @@
   // Tell MemorySanitizer that zlib output buffer is fully initialized.
   // This avoids a false report when running LLVM with uninstrumented ZLib.
   __msan_unpoison(CompressedBuffer.data(), CompressedSize);
-  CompressedBuffer.truncate(CompressedSize);
+  if (CompressedSize < CompressedBuffer.size()) {
+CompressedBuffer.truncate(CompressedSize);
+  }
 }
 
 Error zlib::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
@@ -79,7 +84,9 @@
   UncompressedBuffer.resize_for_overwrite(UncompressedSize);
   Error E = zlib::uncompress(InputBuffer, UncompressedBuffer.data(),
  UncompressedSize);
-  UncompressedBuffer.truncate(UncompressedSize);
+  if (UncompressedSize < UncompressedBuffer.size()) {
+UncompressedBuffer.truncate(UncompressedSize);
+  }
   return E;
 }
 
@@ -99,3 +106,67 @@
   llvm_unreachable("zlib::uncompress is unavailable");
 }
 #endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(StringRef InputBuffer,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  unsigned long CompressedBufferSize = ::ZSTD_compressBound(InputBuffer.size());
+  CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+  

[PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM and add ZSTD compression namespace

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 00.
ckissane added a comment.

tidy some code


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/test/lit.site.cfg.py.in
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -23,11 +23,9 @@
 namespace {
 
 #if LLVM_ENABLE_ZLIB
-
-void TestZlibCompression(StringRef Input, int Level) {
+static void testZlibCompression(StringRef Input, int Level) {
   SmallString<32> Compressed;
   SmallString<32> Uncompressed;
-
   zlib::compress(Input, Compressed, Level);
 
   // Check that uncompressed buffer is the same as original.
@@ -43,26 +41,63 @@
 }
 
 TEST(CompressionTest, Zlib) {
-  TestZlibCompression("", zlib::DefaultCompression);
+  testZlibCompression("", zlib::DefaultCompression);
 
-  TestZlibCompression("hello, world!", zlib::NoCompression);
-  TestZlibCompression("hello, world!", zlib::BestSizeCompression);
-  TestZlibCompression("hello, world!", zlib::BestSpeedCompression);
-  TestZlibCompression("hello, world!", zlib::DefaultCompression);
+  testZlibCompression("hello, world!", zlib::NoCompression);
+  testZlibCompression("hello, world!", zlib::BestSizeCompression);
+  testZlibCompression("hello, world!", zlib::BestSpeedCompression);
+  testZlibCompression("hello, world!", zlib::DefaultCompression);
 
   const size_t kSize = 1024;
   char BinaryData[kSize];
-  for (size_t i = 0; i < kSize; ++i) {
+  for (size_t i = 0; i < kSize; ++i)
 BinaryData[i] = i & 255;
-  }
   StringRef BinaryDataStr(BinaryData, kSize);
 
-  TestZlibCompression(BinaryDataStr, zlib::NoCompression);
-  TestZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
-  TestZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
-  TestZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+  testZlibCompression(BinaryDataStr, zlib::NoCompression);
+  testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
+  testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
+  testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+}
+#endif
+
+#if LLVM_ENABLE_ZSTD
+static void testZstdCompression(StringRef Input, int Level) {
+  SmallString<32> Compressed;
+  SmallString<32> Uncompressed;
+  zstd::compress(Input, Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, Uncompressed);
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
+  }
 }
 
+TEST(CompressionTest, Zstd) {
+  testZstdCompression("", zstd::DefaultCompression);
+
+  testZstdCompression("hello, world!", zstd::NoCompression);
+  testZstdCompression("hello, world!", zstd::BestSizeCompression);
+  testZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  testZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i)
+BinaryData[i] = i & 255;
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  testZstdCompression(BinaryDataStr, zstd::NoCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  testZstdCompression(BinaryDataStr, zstd::DefaultCompression);
+}
 #endif
 
 }
Index: llvm/test/lit.site.cfg.py.in
===
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_EN

[PATCH] D128465: [llvm] add zstd to `llvm::compression` namespace

[Cole Kissane via Phabricator via cfe-commits]

ckissane marked 6 inline comments as done.
ckissane added a comment.

Mark handled comments from @MaskRay as done


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128465: [llvm] add zstd to `llvm::compression` namespace

[Cole Kissane via Phabricator via cfe-commits]

ckissane added inline comments.



Comment at: llvm/lib/Support/Compression.cpp:63
   __msan_unpoison(CompressedBuffer.data(), CompressedSize);
-  CompressedBuffer.truncate(CompressedSize);
+  if (CompressedSize < CompressedBuffer.size())
+CompressedBuffer.truncate(CompressedSize);

phosek wrote:
> I'd make changes to zlib separately.
Here https://reviews.llvm.org/D129698


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D128465: [llvm] add zstd to `llvm::compression` namespace

[Cole Kissane via Phabricator via cfe-commits]

ckissane updated this revision to Diff 32.
ckissane added a comment.

- Merge branch main


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/test/lit.site.cfg.py.in
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -61,4 +61,43 @@
 }
 #endif
 
+#if LLVM_ENABLE_ZSTD
+static void testZstdCompression(StringRef Input, int Level) {
+  SmallString<32> Compressed;
+  SmallString<32> Uncompressed;
+  zstd::compress(Input, Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, Uncompressed);
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  testZstdCompression("", zstd::DefaultCompression);
+
+  testZstdCompression("hello, world!", zstd::NoCompression);
+  testZstdCompression("hello, world!", zstd::BestSizeCompression);
+  testZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  testZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i)
+BinaryData[i] = i & 255;
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  testZstdCompression(BinaryDataStr, zstd::NoCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  testZstdCompression(BinaryDataStr, zstd::DefaultCompression);
+}
+#endif
+
 }
Index: llvm/test/lit.site.cfg.py.in
===
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,6 +20,9 @@
 #if LLVM_ENABLE_ZLIB
 #include 
 #endif
+#if LLVM_ENABLE_ZSTD
+#include 
+#endif
 
 using namespace llvm;
 using namespace llvm::compression;
@@ -101,3 +104,65 @@
   llvm_unreachable("zlib::uncompress is unavailable");
 }
 #endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(StringRef InputBuffer,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  unsigned long CompressedBufferSize = ::ZSTD_compressBound(InputBuffer.size());
+  CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+  unsigned long CompressedSize = ::ZSTD_compress(
+  (char *)CompressedBuffer.data(), CompressedBufferSize,
+  (const char *)InputBuffer.data(), InputBuffer.size(), Level);
+  if (ZSTD_isError(CompressedSize))
+report_bad_alloc_error("Allocation failed");
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(CompressedBuffer.data(), CompressedSize);
+  if (CompressedSize < CompressedBuffer.size())
+CompressedBuffer.truncate(CompressedSize);
+}
+
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+   size_t &UncompressedSize) {
+  const size_t Res =
+  ::ZSTD_decompress((char *)UncompressedBuffer, UncompressedSize,
+(const char *)InputBuffer.data(), InputBu