Changeset: 6f9c7107b8f4 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/6f9c7107b8f4 Modified Files: gdk/ChangeLog.Dec2023 gdk/gdk_logger.c gdk/gdk_logger_internals.h Branch: Dec2023 Log Message:
Lucas' implementation of time-based WAL rotation. diffs (145 lines): diff --git a/gdk/ChangeLog.Dec2023 b/gdk/ChangeLog.Dec2023 --- a/gdk/ChangeLog.Dec2023 +++ b/gdk/ChangeLog.Dec2023 @@ -1,6 +1,10 @@ # ChangeLog file for GDK # This file is updated with Maddlog +* Tue Dec 12 2023 Lucas Pereira <[email protected]> +- Introduced options wal_max_dropped, wal_max_file_age and + wal_max_file_size that control the write-ahead log file rotation. + * Mon Nov 20 2023 Sjoerd Mullender <[email protected]> - Removed function BATroles to set column names on BATs. diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c --- a/gdk/gdk_logger.c +++ b/gdk/gdk_logger.c @@ -1091,6 +1091,7 @@ log_open_output(logger *lg) assert(current && current->next == NULL); new_range->cnt = current->cnt; current->next = new_range; + lg->file_age = GDKusec(); return GDK_SUCCEED; } @@ -2250,6 +2251,17 @@ log_new(int debug, const char *fn, const logger *lg; char filename[FILENAME_MAX]; + lng max_dropped = GDKgetenv_int("wal_max_dropped", 100000); + lng max_file_age = GDKgetenv_int("wal_max_file_age", 600); + lng max_file_size = 0; + + if (GDKdebug & FORCEMITOMASK) { + max_file_size = 2048; /* 2 KiB */ + } else { + const char *max_file_size_str = GDKgetenv("wal_max_file_size"); + max_file_size = max_file_size_str ? strtoul(max_file_size_str, NULL, 10) : 2147483648; + } + if (!GDKinmemory(0) && MT_path_absolute(logdir)) { TRC_CRITICAL(GDK, "logdir must be relative path\n"); return NULL; @@ -2269,6 +2281,11 @@ log_new(int debug, const char *fn, const .postfuncp = postfuncp, .funcdata = funcdata, + .max_dropped = max_dropped >= 0 ? max_dropped : 100000, + .file_age = 0, + .max_file_age = max_file_age >= 0 ? max_file_age * 1000000 : 600000000, + .max_file_size = max_file_size >= 0 ? max_file_size : 2147483648, + .id = 0, .saved_id = getBBPlogno(), /* get saved log numer from bbp */ .saved_tid = (int) getBBPtransid(), /* get saved transaction id from bbp */ @@ -2494,10 +2511,19 @@ log_activate(logger *lg) { bool flush_cleanup = false; gdk_return res = GDK_SUCCEED; + + const lng current_file_size = LOG_DISABLED(lg) ? 0 : (lng) getfilepos(getFile(lg->current->output_log)); + + if (current_file_size == -1) + return GDK_FAIL; + rotation_lock(lg); if (!lg->flushnow && !lg->current->next && - ATOMIC_GET(&lg->current->drops) > 100000 && + current_file_size > 2 && + (ATOMIC_GET(&lg->current->drops) > (ulng)lg->max_dropped || + current_file_size > lg->max_file_size || + (GDKusec() - lg->file_age) > lg->max_file_age) && (ulng) ATOMIC_GET(&lg->current->last_ts) > 0 && lg->saved_id + 1 == lg->id && ATOMIC_GET(&lg->current->refcount) == 1 /* no pending work on this file */ ) { @@ -3047,12 +3073,6 @@ log_delta(logger *lg, BAT *uid, BAT *uva return ok; } -#define DBLKSZ 8192 -#define SEGSZ (64*DBLKSZ) - -#define LOG_MINI (LL_CONSTANT(2)*1024) -#define LOG_LARGE (LL_CONSTANT(2)*1024*1024*1024) - static inline bool check_rotation_conditions(logger *lg) { @@ -3063,10 +3083,20 @@ check_rotation_conditions(logger *lg) return false; /* do not rotate if there is already a prepared next current */ if (mnstr_errnr(lg->current->output_log) != MNSTR_NO__ERROR) return true; - const lng p = (lng) getfilepos(getFile(lg->current->output_log)); - - const lng log_large = (ATOMIC_GET(&GDKdebug) & FORCEMITOMASK) ? LOG_MINI : LOG_LARGE; - bool res = (p > log_large) || (lg->saved_id + 1 >= lg->id && ATOMIC_GET(&lg->current->drops) > 100000); + const lng current_file_size = (lng) getfilepos(getFile(lg->current->output_log)); + + if (current_file_size == -1) + return false; + + assert(current_file_size >= 0); + + if (current_file_size == 2) + return false; + + bool res = (lg->saved_id + 1 >= lg->id && ATOMIC_GET(&lg->current->drops) > (ulng)lg->max_dropped) || + current_file_size > lg->max_file_size || + (GDKusec() - lg->file_age) > lg->max_file_age; + return res; } diff --git a/gdk/gdk_logger_internals.h b/gdk/gdk_logger_internals.h --- a/gdk/gdk_logger_internals.h +++ b/gdk/gdk_logger_internals.h @@ -40,17 +40,25 @@ struct logger { int8_t type_id[128]; /* mapping from GDK type nr to logger type id */ // CHECK writer only - lng total_cnt; /* When logging the content of a bats in multiple runs, total_cnt is used the very first to signal this and keep track in the logging*/ + lng total_cnt; /* When logging the content of a bats in + * multiple runs, total_cnt is used the + * very first to signal this and keep + * track in the logging*/ void *rbuf; size_t rbufsize; void *wbuf; size_t wbufsize; + lng max_dropped; /* default 100000 */ + lng file_age; /* log file age */ + lng max_file_age; /* default 10 mins */ + lng max_file_size; /* default 2 GiB */ // synchronized by combination of store->flush and rotation_lock ulng id; /* current log output file id */ ulng saved_id; /* id of last fully handled log file */ int tid; /* current transaction id */ - int saved_tid; /* id of transaction which was flushed out (into BBP storage) */ + int saved_tid; /* id of transaction which was flushed + * out (into BBP storage) */ // synchronized by rotation_lock logged_range *current; _______________________________________________ checkin-list mailing list -- [email protected] To unsubscribe send an email to [email protected]
