This is an automated email from the ASF dual-hosted git repository.
twice pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/kvrocks.git
The following commit(s) were added to refs/heads/unstable by this push:
new ec523ccec feat: Add support for dictionary compression settings (#3425)
ec523ccec is described below
commit ec523ccec1fe77bde23b942bba1bdef1542448e9
Author: nagisa-kunhah <[email protected]>
AuthorDate: Thu Apr 9 13:04:31 2026 +0800
feat: Add support for dictionary compression settings (#3425)
issue: #3409
---------
Co-authored-by: 纪华裕 <[email protected]>
Co-authored-by: Twice <[email protected]>
---
kvrocks.conf | 13 +++++++++++++
src/config/config.cc | 4 ++++
src/config/config.h | 2 ++
src/storage/storage.cc | 2 ++
tests/cppunit/config_test.cc | 28 ++++++++++++++++++++++++++++
tests/cppunit/storage_test.cc | 26 ++++++++++++++++++++++++++
tests/gocase/unit/config/config_test.go | 6 +++++-
7 files changed, 80 insertions(+), 1 deletion(-)
diff --git a/kvrocks.conf b/kvrocks.conf
index 12add2d19..12b019ca1 100644
--- a/kvrocks.conf
+++ b/kvrocks.conf
@@ -923,6 +923,19 @@ rocksdb.compression snappy
# compression library as mentioned above)
rocksdb.compression_level 32767
+# Maximum per-SST dictionary size for block compression. Set to 0 to disable
+# dictionary compression.
+#
+# Default: 0
+rocksdb.compression_max_dict_bytes 0
+
+# Maximum amount of training data used by the ZSTD dictionary trainer. This is
+# most useful when rocksdb.compression is set to zstd and
rocksdb.compression_max_dict_bytes
+# is non-zero.
+#
+# Default: 0
+rocksdb.compression_zstd_max_train_bytes 0
+
# If non-zero, we perform bigger reads when doing compaction. If you're
# running RocksDB on spinning disks, you should set this to at least 2MB.
# That way RocksDB's compaction is doing sequential instead of random reads.
diff --git a/src/config/config.cc b/src/config/config.cc
index e7bf980ce..416eabd9a 100644
--- a/src/config/config.cc
+++ b/src/config/config.cc
@@ -254,6 +254,10 @@ Config::Config() {
new EnumField<rocksdb::CompressionType>(&rocks_db.compression,
compression_types,
rocksdb::CompressionType::kNoCompression)},
{"rocksdb.compression_level", true, new
IntField(&rocks_db.compression_level, 32767, INT_MIN, INT_MAX)},
+ {"rocksdb.compression_max_dict_bytes", true,
+ new UInt32Field(&rocks_db.compression_max_dict_bytes, 0, 0,
UINT32_MAX)},
+ {"rocksdb.compression_zstd_max_train_bytes", true,
+ new UInt32Field(&rocks_db.compression_zstd_max_train_bytes, 0, 0,
UINT32_MAX)},
{"rocksdb.compression_start_level", false,
new IntField(&rocks_db.compression_start_level, 2, 0,
KVROCKS_MAX_LSM_LEVEL - 1)},
{"rocksdb.block_size", true, new IntField(&rocks_db.block_size, 16384,
0, INT_MAX)},
diff --git a/src/config/config.h b/src/config/config.h
index 675bbd34c..159fb83b5 100644
--- a/src/config/config.h
+++ b/src/config/config.h
@@ -236,6 +236,8 @@ struct Config {
rocksdb::CompressionType compression;
int compression_start_level;
int compression_level;
+ uint32_t compression_max_dict_bytes;
+ uint32_t compression_zstd_max_train_bytes;
bool disable_auto_compactions;
bool enable_blob_files;
int min_blob_size;
diff --git a/src/storage/storage.cc b/src/storage/storage.cc
index ca0f17daa..288524eb0 100644
--- a/src/storage/storage.cc
+++ b/src/storage/storage.cc
@@ -178,6 +178,8 @@ rocksdb::Options Storage::InitRocksDBOptions() {
options.write_buffer_size = config_->rocks_db.write_buffer_size * MiB;
options.num_levels = KVROCKS_MAX_LSM_LEVEL;
options.compression_opts.level = config_->rocks_db.compression_level;
+ options.compression_opts.max_dict_bytes =
config_->rocks_db.compression_max_dict_bytes;
+ options.compression_opts.zstd_max_train_bytes =
config_->rocks_db.compression_zstd_max_train_bytes;
options.compression_per_level.resize(options.num_levels);
options.wal_compression = config_->rocks_db.wal_compression;
for (int i = 0; i < options.num_levels; ++i) {
diff --git a/tests/cppunit/config_test.cc b/tests/cppunit/config_test.cc
index b7d931be4..3c78a48f0 100644
--- a/tests/cppunit/config_test.cc
+++ b/tests/cppunit/config_test.cc
@@ -134,6 +134,8 @@ TEST(Config, GetAndSet) {
{"rocksdb.row_cache_size", "100"},
{"rocksdb.rate_limiter_auto_tuned", "yes"},
{"rocksdb.compression_level", "32767"},
+ {"rocksdb.compression_max_dict_bytes", "16384"},
+ {"rocksdb.compression_zstd_max_train_bytes", "262144"},
{"rocksdb.wal_compression", "no"},
{"histogram-bucket-boundaries", "10,100,1000,10000"},
@@ -189,6 +191,32 @@ TEST(Config, Rewrite) {
unlink(path);
}
+TEST(Config, LoadRocksDBDictionaryCompressionOptions) {
+ const char *path = "test.conf";
+ unlink(path);
+
+ std::ofstream output_file(path, std::ios::out);
+ output_file << "rocksdb.compression_max_dict_bytes 16384"
+ << "\n";
+ output_file << "rocksdb.compression_zstd_max_train_bytes 262144"
+ << "\n";
+ output_file.close();
+
+ Config config;
+ ASSERT_TRUE(config.Load(CLIOptions(path)).IsOK());
+ ASSERT_EQ(config.rocks_db.compression_max_dict_bytes, 16384U);
+ ASSERT_EQ(config.rocks_db.compression_zstd_max_train_bytes, 262144U);
+
+ ASSERT_TRUE(config.Rewrite({}).IsOK());
+
+ Config rewritten_config;
+ ASSERT_TRUE(rewritten_config.Load(CLIOptions(path)).IsOK());
+ ASSERT_EQ(rewritten_config.rocks_db.compression_max_dict_bytes, 16384U);
+ ASSERT_EQ(rewritten_config.rocks_db.compression_zstd_max_train_bytes,
262144U);
+
+ unlink(path);
+}
+
TEST(Config, ParseConfigLine) {
ASSERT_EQ(*ParseConfigLine(""), ConfigKV{});
ASSERT_EQ(*ParseConfigLine("# hello"), ConfigKV{});
diff --git a/tests/cppunit/storage_test.cc b/tests/cppunit/storage_test.cc
index 96fffd335..3b403e376 100644
--- a/tests/cppunit/storage_test.cc
+++ b/tests/cppunit/storage_test.cc
@@ -22,8 +22,10 @@
#include <gtest/gtest.h>
#include <status.h>
#include <storage/storage.h>
+#include <unistd.h>
#include <filesystem>
+#include <fstream>
TEST(Storage, CreateBackup) {
std::error_code ec;
@@ -103,3 +105,27 @@ TEST(Storage, ReadOnlyTransactions) {
std::filesystem::remove_all(config.db_dir, ec);
ASSERT_TRUE(!ec);
}
+
+TEST(Storage, RocksDBDictionaryCompressionOptions) {
+ const char *path = "test_storage_options.conf";
+ unlink(path);
+
+ std::ofstream output_file(path, std::ios::out);
+ output_file << "rocksdb.compression_max_dict_bytes 16384\n";
+ output_file << "rocksdb.compression_zstd_max_train_bytes 262144\n";
+ output_file.close();
+
+ Config config;
+ ASSERT_TRUE(config.Load(CLIOptions(path)).IsOK());
+ config.db_dir = "test_storage_options_dir";
+
+ auto storage = std::make_unique<engine::Storage>(&config);
+ auto s = storage->Open();
+ ASSERT_TRUE(s.IsOK());
+
+ const auto options = storage->GetDB()->GetOptions();
+ EXPECT_EQ(options.compression_opts.max_dict_bytes, 16384U);
+ EXPECT_EQ(options.compression_opts.zstd_max_train_bytes, 262144U);
+
+ unlink(path);
+}
diff --git a/tests/gocase/unit/config/config_test.go
b/tests/gocase/unit/config/config_test.go
index 1dcb4f8a5..b9e087e9b 100644
--- a/tests/gocase/unit/config/config_test.go
+++ b/tests/gocase/unit/config/config_test.go
@@ -123,10 +123,14 @@ func TestConfigSetCompression(t *testing.T) {
require.NoError(t, rdb.Do(ctx, "SET", "foo", "bar").Err())
configKey := "rocksdb.compression"
+ vals, err := rdb.ConfigGet(ctx, configKey).Result()
+ require.NoError(t, err)
+ require.EqualValues(t, "no", vals[configKey])
+
supportedCompressions := []string{"no", "snappy", "zlib", "lz4", "zstd"}
for _, compression := range supportedCompressions {
require.NoError(t, rdb.ConfigSet(ctx, configKey,
compression).Err())
- vals, err := rdb.ConfigGet(ctx, configKey).Result()
+ vals, err = rdb.ConfigGet(ctx, configKey).Result()
require.NoError(t, err)
require.EqualValues(t, compression, vals[configKey])
}
