Changeset: 8f3ba20b071e for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8f3ba20b071e
Modified Files:
sql/backends/monet5/sql_scenario.c
sql/server/sql_mvc.c
sql/server/sql_mvc.h
sql/storage/bat/bat_storage.c
sql/storage/bat/bat_table.c
sql/storage/sql_storage.h
sql/storage/store.c
Branch: Dec2016
Log Message:
vacuum catalog tables when db is idle.
diffs (truncated from 310 to 300 lines):
diff --git a/sql/backends/monet5/sql_scenario.c
b/sql/backends/monet5/sql_scenario.c
--- a/sql/backends/monet5/sql_scenario.c
+++ b/sql/backends/monet5/sql_scenario.c
@@ -203,7 +203,7 @@ SQLepilogue(void *ret)
return res;
}
-MT_Id sqllogthread, minmaxthread;
+MT_Id sqllogthread, idlethread;
static str
SQLinit(void)
@@ -251,12 +251,10 @@ SQLinit(void)
throw(SQL, "SQLinit", "Starting log manager failed");
}
GDKregister(sqllogthread);
-#if 0
- if (MT_create_thread(&minmaxthread, (void (*)(void *))
mvc_minmaxmanager, NULL, MT_THR_JOINABLE) != 0) {
- throw(SQL, "SQLinit", "Starting minmax manager failed");
+ if (MT_create_thread(&idlethread, (void (*)(void *)) mvc_idlemanager,
NULL, MT_THR_JOINABLE) != 0) {
+ throw(SQL, "SQLinit", "Starting idle manager failed");
}
- GDKregister(minmaxthread);
-#endif
+ GDKregister(idlethread);
return MAL_SUCCEED;
}
diff --git a/sql/server/sql_mvc.c b/sql/server/sql_mvc.c
--- a/sql/server/sql_mvc.c
+++ b/sql/server/sql_mvc.c
@@ -179,11 +179,11 @@ mvc_logmanager(void)
}
void
-mvc_minmaxmanager(void)
+mvc_idlemanager(void)
{
- Thread thr = THRnew("minmaxmanager");
+ Thread thr = THRnew("idlemanager");
- minmax_manager();
+ idle_manager();
THRdel(thr);
}
diff --git a/sql/server/sql_mvc.h b/sql/server/sql_mvc.h
--- a/sql/server/sql_mvc.h
+++ b/sql/server/sql_mvc.h
@@ -131,7 +131,7 @@ typedef struct mvc {
extern int mvc_init(int debug, store_type store, int ro, int su, backend_stack
stk);
extern void mvc_exit(void);
extern void mvc_logmanager(void);
-extern void mvc_minmaxmanager(void);
+extern void mvc_idlemanager(void);
extern mvc *mvc_create(int clientid, backend_stack stk, int debug, bstream
*rs, stream *ws);
extern void mvc_reset(mvc *m, bstream *rs, stream *ws, int debug, int
globalvars);
diff --git a/sql/storage/bat/bat_storage.c b/sql/storage/bat/bat_storage.c
--- a/sql/storage/bat/bat_storage.c
+++ b/sql/storage/bat/bat_storage.c
@@ -902,6 +902,62 @@ count_del(sql_trans *tr, sql_table *t)
return d->cnt;
}
+static size_t
+count_col_upd(sql_trans *tr, sql_column *c)
+{
+ sql_delta *b;
+
+ assert (isTable(c->t)) ;
+ if (!c->data) {
+ sql_column *oc = tr_find_column(tr->parent, c);
+ c->data = timestamp_delta(oc->data, tr->stime);
+ }
+ b = c->data;
+ if (!b)
+ return 1;
+ return b->ucnt;
+}
+
+static size_t
+count_idx_upd(sql_trans *tr, sql_idx *i)
+{
+ sql_delta *b;
+
+ assert (isTable(i->t)) ;
+ if (!i->data) {
+ sql_idx *oi = tr_find_idx(tr->parent, i);
+ i->data = timestamp_delta(oi->data, tr->stime);
+ }
+ b = i->data;
+ if (!b)
+ return 0;
+ return b->ucnt;
+}
+
+static size_t
+count_upd(sql_trans *tr, sql_table *t)
+{
+ node *n;
+
+ if (!isTable(t))
+ return 0;
+
+ for( n = t->columns.set->h; n; n = n->next) {
+ sql_column *c = n->data;
+
+ if (count_col_upd(tr, c))
+ return 1;
+ }
+ if (t->idxs.set)
+ for( n = t->idxs.set->h; n; n = n->next) {
+ sql_idx *i = n->data;
+
+ if (count_idx_upd(tr, i))
+ return 1;
+ }
+ return 0;
+}
+
static int
sorted_col(sql_trans *tr, sql_column *col)
{
@@ -2466,6 +2522,7 @@ bat_storage_init( store_functions *sf)
sf->delete_tab = (delete_tab_fptr)&delete_tab;
sf->count_del = (count_del_fptr)&count_del;
+ sf->count_upd = (count_upd_fptr)&count_upd;
sf->count_col = (count_col_fptr)&count_col;
sf->count_idx = (count_idx_fptr)&count_idx;
sf->dcount_col = (dcount_col_fptr)&dcount_col;
diff --git a/sql/storage/bat/bat_table.c b/sql/storage/bat/bat_table.c
--- a/sql/storage/bat/bat_table.c
+++ b/sql/storage/bat/bat_table.c
@@ -470,6 +470,32 @@ rids_diff(sql_trans *tr, rids *l, sql_co
return l;
}
+static int
+table_vacuum(sql_trans *tr, sql_table *t)
+{
+ BAT *tids = delta_cands(tr, t);
+ BAT **cols;
+ node *n;
+
+ cols = NEW_ARRAY(BAT*, cs_size(&t->columns));
+ for (n = t->columns.set->h; n; n = n->next) {
+ sql_column *c = n->data;
+ BAT *v = store_funcs.bind_col(tr, c, RDONLY);
+
+ cols[c->colnr] = BATproject(tids, v);
+ BBPunfix(v->batCacheid);
+ }
+ sql_trans_clear_table(tr, t);
+ for (n = t->columns.set->h; n; n = n->next) {
+ sql_column *c = n->data;
+
+ store_funcs.append_col(tr, c, cols[c->colnr], TYPE_bat);
+ BBPunfix(cols[c->colnr]->batCacheid);
+ }
+ _DELETE(cols);
+ return SQL_OK;
+}
+
int
bat_table_init( table_functions *tf )
{
@@ -479,6 +505,7 @@ bat_table_init( table_functions *tf )
tf->column_update_value = column_update_value;
tf->table_insert = table_insert;
tf->table_delete = table_delete;
+ tf->table_vacuum = table_vacuum;
tf->rids_select = rids_select;
tf->rids_orderby = rids_orderby;
diff --git a/sql/storage/sql_storage.h b/sql/storage/sql_storage.h
--- a/sql/storage/sql_storage.h
+++ b/sql/storage/sql_storage.h
@@ -47,6 +47,7 @@ typedef void *(*column_find_value_fptr)(
typedef int (*column_update_value_fptr)(sql_trans *tr, sql_column *c, oid rid,
void *value);
typedef int (*table_insert_fptr)(sql_trans *tr, sql_table *t, ...);
typedef int (*table_delete_fptr)(sql_trans *tr, sql_table *t, oid rid);
+typedef int (*table_vacuum_fptr)(sql_trans *tr, sql_table *t);
typedef struct rids {
BUN cur;
@@ -92,6 +93,7 @@ typedef struct table_functions {
column_update_value_fptr column_update_value;
table_insert_fptr table_insert;
table_delete_fptr table_delete;
+ table_vacuum_fptr table_vacuum;
rids_select_fptr rids_select;
rids_orderby_fptr rids_orderby;
@@ -130,6 +132,7 @@ typedef int (*delete_tab_fptr) (sql_tran
-- check for sortedness
*/
typedef size_t (*count_del_fptr) (sql_trans *tr, sql_table *t);
+typedef size_t (*count_upd_fptr) (sql_trans *tr, sql_table *t);
typedef size_t (*count_col_fptr) (sql_trans *tr, sql_column *c, int all /* all
or new only */);
typedef size_t (*count_idx_fptr) (sql_trans *tr, sql_idx *i, int all /* all or
new only */);
typedef size_t (*dcount_col_fptr) (sql_trans *tr, sql_column *c);
@@ -207,6 +210,7 @@ typedef struct store_functions {
delete_tab_fptr delete_tab;
count_del_fptr count_del;
+ count_upd_fptr count_upd;
count_col_fptr count_col;
count_idx_fptr count_idx;
dcount_col_fptr dcount_col;
@@ -323,7 +327,7 @@ extern void store_exit(void);
extern void store_apply_deltas(void);
extern void store_flush_log(void);
extern void store_manager(void);
-extern void minmax_manager(void);
+extern void idle_manager(void);
extern void store_lock(void);
extern void store_unlock(void);
diff --git a/sql/storage/store.c b/sql/storage/store.c
--- a/sql/storage/store.c
+++ b/sql/storage/store.c
@@ -1700,6 +1700,46 @@ store_flush_log(void)
need_flush = 1;
}
+static int
+store_needs_vacuum( sql_trans *tr )
+{
+ sql_schema *s = find_sql_schema(tr, "sys");
+ node *n;
+
+ for( n = s->tables.set->h; n; n = n->next) {
+ sql_table *t = n->data;
+ sql_column *c = t->columns.set->h->data;
+
+ /* no inserts, updates and enough deletes ? */
+ if (!store_funcs.count_col(tr, c, 0) &&
+ !store_funcs.count_upd(tr, t) &&
+ store_funcs.count_del(tr, t) > 128)
+ return 1;
+ }
+ return 0;
+}
+
+static void
+store_vacuum( sql_trans *tr )
+{
+ /* tables */
+ sql_schema *s = find_sql_schema(tr, "sys");
+ node *n;
+
+ for( n = s->tables.set->h; n; n = n->next) {
+ sql_table *t = n->data;
+ sql_column *c = t->columns.set->h->data;
+ int cnt = 0;
+
+ if (!store_funcs.count_col(tr, c, 0) &&
+ !store_funcs.count_upd(tr, t) &&
+ (cnt = store_funcs.count_del(tr, t)) > 128) {
+ /*printf("vacuum (%d) %s\n", cnt, t->base.name);*/
+ table_funcs.table_vacuum(tr, t);
+ }
+ }
+}
+
void
store_manager(void)
{
@@ -1791,23 +1831,32 @@ store_manager(void)
}
void
-minmax_manager(void)
-{
+idle_manager(void)
+{
+ const int timeout = GDKdebug & FORCEMITOMASK ? 10 : 50;
+
while (!GDKexiting()) {
+ sql_session *s;
int t;
- for (t = 30000; t > 0; t -= 50) {
- MT_sleep_ms(50);
+ for (t = 5000; t > 0; t -= timeout) {
+ MT_sleep_ms(timeout);
if (GDKexiting())
return;
}
MT_lock_set(&bs_lock);
- if (store_nr_active || GDKexiting()) {
+ if (store_nr_active || GDKexiting() ||
!store_needs_vacuum(gtrans)) {
MT_lock_unset(&bs_lock);
continue;
}
- if (store_funcs.gtrans_minmax)
- store_funcs.gtrans_minmax(gtrans);
+
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
