This is an automated email from the ASF dual-hosted git repository.
twice pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/kvrocks.git
The following commit(s) were added to refs/heads/unstable by this push:
new e9591f993 feat(logs): enable saving the slow logs into the file (#2903)
e9591f993 is described below
commit e9591f9936d30680cdd08f56f1664bb2aab1d934
Author: Raphael <[email protected]>
AuthorDate: Thu May 8 10:40:53 2025 +0800
feat(logs): enable saving the slow logs into the file (#2903)
Co-authored-by: Twice <[email protected]>
Co-authored-by: hulk <[email protected]>
---
kvrocks.conf | 5 +++++
src/common/logging.h | 5 +++++
src/config/config.cc | 9 +++++++++
src/config/config.h | 7 +++++++
src/server/server.cc | 1 +
src/stats/log_collector.cc | 25 +++++++++++++++++++++++++
src/stats/log_collector.h | 6 ++++++
7 files changed, 58 insertions(+)
diff --git a/kvrocks.conf b/kvrocks.conf
index 1c4bcaf34..3efd1ce61 100644
--- a/kvrocks.conf
+++ b/kvrocks.conf
@@ -498,6 +498,11 @@ slowlog-log-slower-than 100000
# You can reclaim memory used by the slow log with SLOWLOG RESET.
slowlog-max-len 128
+# Dump slow logs to logfiles with this level, off means don't dump.
+# Possible values: info, warning, off
+# Default: off
+slowlog-dump-logfile-level off
+
# If you run kvrocks from upstart or systemd, kvrocks can interact with your
# supervision tree. Options:
# supervised no - no supervision interaction
diff --git a/src/common/logging.h b/src/common/logging.h
index e145fa5a0..6ddda6d93 100644
--- a/src/common/logging.h
+++ b/src/common/logging.h
@@ -45,6 +45,11 @@ struct FormatMessageWithLoc {
spdlog::source_loc current_loc;
};
+template <typename... Args>
+inline void log(spdlog::level::level_enum lvl, FormatMessageWithLoc fmt, Args
&&...args) { // NOLINT
+ spdlog::default_logger_raw()->log(fmt.current_loc, lvl, fmt.fmt,
std::forward<Args>(args)...);
+}
+
template <typename... Args>
inline void debug(FormatMessageWithLoc fmt, Args &&...args) { // NOLINT
spdlog::default_logger_raw()->log(fmt.current_loc, spdlog::level::debug,
fmt.fmt, std::forward<Args>(args)...);
diff --git a/src/config/config.cc b/src/config/config.cc
index 2a6dac050..1668dd026 100644
--- a/src/config/config.cc
+++ b/src/config/config.cc
@@ -208,6 +208,9 @@ Config::Config() {
{"slowlog-log-slower-than", false, new
IntField(&slowlog_log_slower_than, 200000, -1, INT_MAX)},
{"profiling-sample-commands", false, new
StringField(&profiling_sample_commands_str_, "")},
{"slowlog-max-len", false, new IntField(&slowlog_max_len, 128, 0,
INT_MAX)},
+ {"slowlog-dump-logfile-level", false,
+ new EnumField<spdlog::level::level_enum>(&slowlog_dump_logfile_level,
slowlog_dump_logfile_levels,
+ spdlog::level::off)},
{"purge-backup-on-fullsync", false, new
YesNoField(&purge_backup_on_fullsync, false)},
{"rename-command", true, new MultiStringField(&rename_command_,
std::vector<std::string>{})},
{"auto-resize-block-and-sst", false, new
YesNoField(&auto_resize_block_and_sst, true)},
@@ -539,6 +542,12 @@ void Config::initFieldCallback() {
srv->GetSlowLog()->SetMaxEntries(slowlog_max_len);
return Status::OK();
}},
+ {"slowlog-dump-logfile-level",
+ [this](Server *srv, [[maybe_unused]] const std::string &k,
[[maybe_unused]] const std::string &v) -> Status {
+ if (!srv) return Status::OK();
+ srv->GetSlowLog()->SetDumpToLogfileLevel(slowlog_dump_logfile_level);
+ return Status::OK();
+ }},
{"max-db-size",
[](Server *srv, [[maybe_unused]] const std::string &k, [[maybe_unused]]
const std::string &v) -> Status {
if (!srv) return Status::OK();
diff --git a/src/config/config.h b/src/config/config.h
index 615dea649..c43bf7b29 100644
--- a/src/config/config.h
+++ b/src/config/config.h
@@ -62,6 +62,12 @@ const std::vector<ConfigEnum<spdlog::level::level_enum>>
log_levels{
{"error", spdlog::level::err}, {"fatal", spdlog::level::critical},
};
+const std::vector<ConfigEnum<spdlog::level::level_enum>>
slowlog_dump_logfile_levels{
+ {"info", spdlog::level::info},
+ {"warning", spdlog::level::warn},
+ {"off", spdlog::level::off},
+};
+
enum class BlockCacheType { kCacheTypeLRU = 0, kCacheTypeHCC };
struct CLIOptions {
@@ -104,6 +110,7 @@ struct Config {
int max_backup_keep_hours = 24;
int slowlog_log_slower_than = 100000;
int slowlog_max_len = 128;
+ spdlog::level::level_enum slowlog_dump_logfile_level = spdlog::level::off;
uint64_t proto_max_bulk_len = 512 * 1024 * 1024;
bool daemonize = false;
SupervisedMode supervised_mode = kSupervisedNone;
diff --git a/src/server/server.cc b/src/server/server.cc
index 09aef5147..71ed03fff 100644
--- a/src/server/server.cc
+++ b/src/server/server.cc
@@ -115,6 +115,7 @@ Server::Server(engine::Storage *storage, Config *config)
AdjustOpenFilesLimit();
slow_log_.SetMaxEntries(config->slowlog_max_len);
+ slow_log_.SetDumpToLogfileLevel(config->slowlog_dump_logfile_level);
perf_log_.SetMaxEntries(config->profiling_sample_record_max_len);
}
diff --git a/src/stats/log_collector.cc b/src/stats/log_collector.cc
index ae4415492..74be9b118 100644
--- a/src/stats/log_collector.cc
+++ b/src/stats/log_collector.cc
@@ -38,6 +38,22 @@ std::string SlowEntry::ToRedisString() const {
return output;
}
+void SlowEntry::DumpToLogFile(spdlog::level::level_enum level) const {
+ if (level == spdlog::level::off) {
+ return;
+ }
+
+ std::string cmd;
+ if (args.size() > 0) {
+ for (const auto &arg : args) {
+ cmd.append(arg).append(" ");
+ }
+ cmd.pop_back();
+ }
+ log(level, "[slowlog] id: {}, timestamp: {}, duration: {}, cmd: {}, ip: {},
port: {}, client_name: {}", id, time,
+ duration, cmd, ip, port, client_name);
+}
+
std::string PerfEntry::ToRedisString() const {
std::string output;
output.append(redis::MultiLen(6));
@@ -78,6 +94,12 @@ void LogCollector<T>::SetMaxEntries(int64_t max_entries) {
max_entries_ = max_entries;
}
+template <class T>
+void LogCollector<T>::SetDumpToLogfileLevel(spdlog::level::level_enum level) {
+ std::lock_guard<std::mutex> guard(mu_);
+ dump_to_logfile_level_ = level;
+}
+
template <class T>
void LogCollector<T>::PushEntry(std::unique_ptr<T> &&entry) {
std::lock_guard<std::mutex> guard(mu_);
@@ -86,6 +108,9 @@ void LogCollector<T>::PushEntry(std::unique_ptr<T> &&entry) {
if (max_entries_ > 0 && !entries_.empty() && entries_.size() >=
static_cast<size_t>(max_entries_)) {
entries_.pop_back();
}
+ if (dump_to_logfile_level_ != spdlog::level::off) {
+ entry->DumpToLogFile(dump_to_logfile_level_);
+ }
entries_.push_front(std::move(entry));
}
diff --git a/src/stats/log_collector.h b/src/stats/log_collector.h
index d67893d3f..e6efb0816 100644
--- a/src/stats/log_collector.h
+++ b/src/stats/log_collector.h
@@ -31,6 +31,8 @@
#include <string>
#include <vector>
+#include "spdlog/common.h"
+
class SlowEntry {
public:
uint64_t id;
@@ -41,6 +43,7 @@ class SlowEntry {
std::string ip;
uint32_t port;
std::string ToRedisString() const;
+ void DumpToLogFile(spdlog::level::level_enum) const;
};
class PerfEntry {
@@ -53,6 +56,7 @@ class PerfEntry {
std::string iostats_context;
std::string ToRedisString() const;
+ void DumpToLogFile(spdlog::level::level_enum) const {};
};
template <class T>
@@ -67,10 +71,12 @@ class LogCollector {
void SetMaxEntries(int64_t max_entries);
void PushEntry(std::unique_ptr<T> &&entry);
std::string GetLatestEntries(int64_t cnt);
+ void SetDumpToLogfileLevel(spdlog::level::level_enum level);
private:
std::mutex mu_;
uint64_t id_ = 0;
int64_t max_entries_ = 128;
std::deque<std::unique_ptr<T>> entries_;
+ spdlog::level::level_enum dump_to_logfile_level_ = spdlog::level::off;
};
