Changeset: b6eff70df7f0 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/b6eff70df7f0
Modified Files:
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_cat.c
sql/backends/monet5/sql_execute.c
sql/backends/monet5/sql_gencode.c
sql/backends/monet5/sql_statement.c
sql/backends/monet5/sql_upgrades.c
sql/server/rel_optimizer.c
sql/server/rel_optimizer.h
sql/server/sql_mvc.c
sql/server/sql_mvc.h
sql/server/sql_partition.c
Branch: sqloptimizer
Log Message:
Adding optimizer benchmark information. Cleanup
diffs (truncated from 364 to 300 lines):
diff --git a/sql/backends/monet5/rel_bin.c b/sql/backends/monet5/rel_bin.c
--- a/sql/backends/monet5/rel_bin.c
+++ b/sql/backends/monet5/rel_bin.c
@@ -3904,7 +3904,7 @@ sql_parse(backend *be, sql_schema *s, co
sql_rel *rel = rel_parse(be->mvc, s, query, mode);
stmt *sq = NULL;
- if (rel && (rel = sql_processrelation(be->mvc, rel, 1, 1, 1)))
+ if (rel && (rel = sql_processrelation(be->mvc, rel, 0, 1, 1, 1)))
sq = rel_bin(be, rel);
return sq;
}
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -123,11 +123,12 @@ sql_symbol2relation(backend *be, symbol
sql_query *query = query_create(be->mvc);
lng Tbegin;
int extra_opts = be->mvc->emode != m_prepare;
+ int profile = be->mvc->emode == m_plan;
rel = rel_semantic(query, sym);
Tbegin = GDKusec();
if (rel)
- rel = sql_processrelation(be->mvc, rel, 1, extra_opts,
extra_opts);
+ rel = sql_processrelation(be->mvc, rel, profile, 1, extra_opts,
extra_opts);
if (rel)
rel = rel_partition(be->mvc, rel);
if (rel && (rel_no_mitosis(be->mvc, rel) ||
rel_need_distinct_query(rel)))
@@ -450,7 +451,7 @@ create_table_or_view(mvc *sql, char *sna
r = rel_parse(sql, s, nt->query, m_deps);
if (r)
- r = sql_processrelation(sql, r, 0, 0, 0);
+ r = sql_processrelation(sql, r, 0, 0, 0, 0);
if (r) {
list *blist = rel_dependencies(sql, r);
if (mvc_create_dependencies(sql, blist, nt->base.id,
VIEW_DEPENDENCY)) {
diff --git a/sql/backends/monet5/sql_cat.c b/sql/backends/monet5/sql_cat.c
--- a/sql/backends/monet5/sql_cat.c
+++ b/sql/backends/monet5/sql_cat.c
@@ -571,7 +571,7 @@ create_trigger(mvc *sql, char *sname, ch
}
r = rel_parse(sql, s, buf, m_deps);
if (r)
- r = sql_processrelation(sql, r, 0, 0, 0);
+ r = sql_processrelation(sql, r, 0, 0, 0, 0);
if (r) {
list *blist = rel_dependencies(sql, r);
if (mvc_create_dependencies(sql, blist,
tri->base.id, TRIGGER_DEPENDENCY)) {
@@ -1060,7 +1060,7 @@ create_func(mvc *sql, char *sname, char
}
r = rel_parse(sql, s, buf, m_deps);
if (r)
- r = sql_processrelation(sql, r, 0, 0, 0);
+ r = sql_processrelation(sql, r, 0, 0, 0, 0);
if (r) {
node *n;
list *blist = rel_dependencies(sql, r);
diff --git a/sql/backends/monet5/sql_execute.c
b/sql/backends/monet5/sql_execute.c
--- a/sql/backends/monet5/sql_execute.c
+++ b/sql/backends/monet5/sql_execute.c
@@ -775,7 +775,7 @@ RAstatement(Client c, MalBlkPtr mb, MalS
refs = sa_list(m->sa);
rel = rel_read(m, *expr, &pos, refs);
if (*opt && rel)
- rel = sql_processrelation(m, rel, 0, 0, 0);
+ rel = sql_processrelation(m, rel, 0, 0, 0, 0);
if (!rel) {
if (strlen(m->errstr) > 6 && m->errstr[5] == '!')
msg = createException(SQL, "RAstatement", "%s",
m->errstr);
@@ -972,7 +972,7 @@ RAstatement2(Client cntxt, MalBlkPtr mb,
refs = sa_list(m->sa);
rel = rel_read(m, expr, &pos, refs);
if (rel)
- rel = sql_processrelation(m, rel, 0, 0, 0);
+ rel = sql_processrelation(m, rel, 0, 0, 0, 0);
if (!rel) {
if (strlen(m->errstr) > 6 && m->errstr[5] == '!')
msg = createException(SQL, "RAstatement2", "%s",
m->errstr);
diff --git a/sql/backends/monet5/sql_gencode.c
b/sql/backends/monet5/sql_gencode.c
--- a/sql/backends/monet5/sql_gencode.c
+++ b/sql/backends/monet5/sql_gencode.c
@@ -1206,7 +1206,7 @@ backend_create_sql_func(backend *be, sql
r = rel_parse(m, f->s, f->query, m_instantiate);
if (r)
- r = sql_processrelation(m, r, 1, 1, 0);
+ r = sql_processrelation(m, r, 0, 1, 1, 0);
if (!r)
return -1;
diff --git a/sql/backends/monet5/sql_statement.c
b/sql/backends/monet5/sql_statement.c
--- a/sql/backends/monet5/sql_statement.c
+++ b/sql/backends/monet5/sql_statement.c
@@ -3548,7 +3548,7 @@ stmt_func(backend *be, stmt *ops, const
rel->p = prop_remove(rel->p, p);
/* sql_processrelation may split projections, so make sure the topmost
relation only contains references */
rel = rel_project(be->mvc->sa, rel, rel_projections(be->mvc, rel, NULL,
1, 1));
- if (!(rel = sql_processrelation(be->mvc, rel, 0, 1, 1)))
+ if (!(rel = sql_processrelation(be->mvc, rel, 0, 0, 1, 1)))
return NULL;
if (p) {
p->p = rel->p;
diff --git a/sql/backends/monet5/sql_upgrades.c
b/sql/backends/monet5/sql_upgrades.c
--- a/sql/backends/monet5/sql_upgrades.c
+++ b/sql/backends/monet5/sql_upgrades.c
@@ -753,7 +753,7 @@ sql_update_nov2019_missing_dependencies(
r = rel_parse(sql, s, relt, m_deps);
if (r)
- r = sql_processrelation(sql, r, 0, 0,
0);
+ r = sql_processrelation(sql, r, 0, 0,
0, 0);
if (r) {
list *id_l = rel_dependencies(sql, r);
@@ -789,7 +789,7 @@ sql_update_nov2019_missing_dependencies(
r = rel_parse(sql, s, relt, m_deps);
if (r)
- r = sql_processrelation(sql, r,
0, 0, 0);
+ r = sql_processrelation(sql, r,
0, 0, 0, 0);
if (r) {
list *id_l =
rel_dependencies(sql, r);
@@ -817,7 +817,7 @@ sql_update_nov2019_missing_dependencies(
r = rel_parse(sql, s, relt,
m_deps);
if (r)
- r =
sql_processrelation(sql, r, 0, 0, 0);
+ r =
sql_processrelation(sql, r, 0, 0, 0, 0);
if (r) {
list *id_l =
rel_dependencies(sql, r);
diff --git a/sql/server/rel_optimizer.c b/sql/server/rel_optimizer.c
--- a/sql/server/rel_optimizer.c
+++ b/sql/server/rel_optimizer.c
@@ -635,6 +635,7 @@ rel_setjoins_2_joingroupby(visitor *v, g
/* the definition of a single SQL optimizer */
typedef struct sql_optimizer {
+ const int index;
const char *name;
bool (*can_run_optimizer)(visitor *v, global_props *gp);
sql_rel *(*run_optimizer)(visitor *v, global_props *gp, sql_rel *rel);
@@ -642,38 +643,40 @@ typedef struct sql_optimizer {
/* these optimizers/rewriters run in a cycle loop */
const sql_optimizer pre_sql_optimizers[] = {
- {"split_select", can_split_select, rel_split_select},
- {"push_project_down", can_push_project_down, rel_push_project_down},
- {"merge_projects", can_merge_projects, rel_merge_projects},
- {"push_project_up", can_push_project_up, rel_push_project_up},
- {"split_project", can_split_project, rel_split_project},
- {"remove_redundant_join", can_remove_redundant_join,
rel_remove_redundant_join},
- {"simplify_math", can_simplify_math, rel_simplify_math},
- {"optimize_exps", can_optimize_exps, rel_optimize_exps},
- {"optimize_select_and_joins_bottomup",
can_optimize_select_and_joins_bottomup, rel_optimize_select_and_joins_bottomup},
- {"project_reduce_casts", can_project_reduce_casts,
rel_project_reduce_casts},
- {"optimize_unions_bottomup", can_optimize_unions_bottomup,
rel_optimize_unions_bottomup},
- {"optimize_projections", can_optimize_projections,
rel_optimize_projections},
- {"optimize_joins", can_optimize_joins, rel_optimize_joins},
- {"join_order", can_join_order, rel_join_order},
- {"optimize_semi_and_anti", can_optimize_semi_and_anti,
rel_optimize_semi_and_anti},
- {"optimize_select_and_joins_topdown",
can_optimize_select_and_joins_topdown, rel_optimize_select_and_joins_topdown},
- {"optimize_unions_topdown", can_optimize_unions_topdown,
rel_optimize_unions_topdown},
- {"dce", can_dce, rel_dce},
- {"push_func_and_select_down", can_push_func_and_select_down,
rel_push_func_and_select_down},
- {"push_topn_and_sample_down", can_push_topn_and_sample_down,
rel_push_topn_and_sample_down},
- {"distinct_project2groupby", can_distinct_project2groupby,
rel_distinct_project2groupby},
- {"merge_table_rewrite", can_merge_table_rewrite,
rel_merge_table_rewrite},
- {NULL, NULL, NULL}
+ { 0, "split_select", can_split_select, rel_split_select},
+ { 1, "push_project_down", can_push_project_down, rel_push_project_down},
+ { 2, "merge_projects", can_merge_projects, rel_merge_projects},
+ { 3, "push_project_up", can_push_project_up, rel_push_project_up},
+ { 4, "split_project", can_split_project, rel_split_project},
+ { 5, "remove_redundant_join", can_remove_redundant_join,
rel_remove_redundant_join},
+ { 6, "simplify_math", can_simplify_math, rel_simplify_math},
+ { 7, "optimize_exps", can_optimize_exps, rel_optimize_exps},
+ { 8, "optimize_select_and_joins_bottomup",
can_optimize_select_and_joins_bottomup, rel_optimize_select_and_joins_bottomup},
+ { 9, "project_reduce_casts", can_project_reduce_casts,
rel_project_reduce_casts},
+ {10, "optimize_unions_bottomup", can_optimize_unions_bottomup,
rel_optimize_unions_bottomup},
+ {11, "optimize_projections", can_optimize_projections,
rel_optimize_projections},
+ {12, "optimize_joins", can_optimize_joins, rel_optimize_joins},
+ {13, "join_order", can_join_order, rel_join_order},
+ {14, "optimize_semi_and_anti", can_optimize_semi_and_anti,
rel_optimize_semi_and_anti},
+ {15, "optimize_select_and_joins_topdown",
can_optimize_select_and_joins_topdown, rel_optimize_select_and_joins_topdown},
+ {16, "optimize_unions_topdown", can_optimize_unions_topdown,
rel_optimize_unions_topdown},
+ {17, "dce", can_dce, rel_dce},
+ {18, "push_func_and_select_down", can_push_func_and_select_down,
rel_push_func_and_select_down},
+ {19, "push_topn_and_sample_down", can_push_topn_and_sample_down,
rel_push_topn_and_sample_down},
+ {20, "distinct_project2groupby", can_distinct_project2groupby,
rel_distinct_project2groupby},
+ {21, "merge_table_rewrite", can_merge_table_rewrite,
rel_merge_table_rewrite},
+ { 0, NULL, NULL, NULL}
};
/* these optimizers/rewriters only run once after the cycle loop */
const sql_optimizer post_sql_optimizers[] = {
- {"push_select_up", can_push_select_up, rel_push_select_up}, /* run
rel_push_select_up only once at the end to avoid an infinite optimization loop
*/
- {"setjoins_2_joingroupby", can_setjoins_2_joingroupby,
rel_setjoins_2_joingroupby},
- {NULL, NULL, NULL}
+ {22, "push_select_up", can_push_select_up, rel_push_select_up}, /* run
rel_push_select_up only once at the end to avoid an infinite optimization loop
*/
+ {23, "setjoins_2_joingroupby", can_setjoins_2_joingroupby,
rel_setjoins_2_joingroupby},
+ { 0, NULL, NULL, NULL}
};
+#define NOPTIMIZERS 24
+
/* make sure the outer project (without order by or distinct) has all the
aliases */
static sql_rel *
rel_keep_renames(mvc *sql, sql_rel *rel)
@@ -723,19 +726,32 @@ calculate_opt_level(mvc *sql, sql_rel *r
return 1;
}
-static sql_rel *
-run_optimizer_set(visitor *v, sql_rel *rel, global_props *gp, const
sql_optimizer *set)
+static inline sql_rel *
+run_optimizer_set(visitor *v, sql_optimizer_run *runs, sql_rel *rel,
global_props *gp, const sql_optimizer *set)
{
+ /* if 'runs' is set, it means profiling is intended */
for (int i = 0 ; set[i].name ; i++) {
- if (set[i].can_run_optimizer(v, gp))
- rel = set[i].run_optimizer(v, gp, rel);
+ if (set[i].can_run_optimizer(v, gp)) {
+ if (runs) {
+ sql_optimizer_run *run = &(runs[set[i].index]);
+ run->name = set[i].name;
+ int changes = v->changes;
+ lng clk = GDKusec();
+ rel = set[i].run_optimizer(v, gp, rel);
+ run->time += (GDKusec() - clk);
+ run->nchanges += (v->changes - changes);
+ } else {
+ rel = set[i].run_optimizer(v, gp, rel);
+ }
+ }
}
return rel;
}
+/* 'profile' means to benchmark each individual optimizer run */
/* 'instantiate' means to rewrite logical tables: (merge, remote, replica
tables) */
sql_rel *
-rel_optimizer(mvc *sql, sql_rel *rel, int instantiate, int value_based_opt,
int storage_based_opt)
+rel_optimizer(mvc *sql, sql_rel *rel, int profile, int instantiate, int
value_based_opt, int storage_based_opt)
{
global_props gp = (global_props) {.cnt = {0}, .instantiate =
(uint8_t)instantiate, .opt_cycle = 0};
visitor v = { .sql = sql, .value_based_opt = value_based_opt,
.storage_based_opt = storage_based_opt, .changes = 1, .data = &gp };
@@ -743,6 +759,7 @@ rel_optimizer(mvc *sql, sql_rel *rel, in
if (!(rel = rel_keep_renames(sql, rel)))
return rel;
+ sql_optimizer_run *runs = profile ? sa_zalloc(sql->ta, NOPTIMIZERS *
sizeof(sql_optimizer_run)) : NULL;
for ( ;rel && gp.opt_cycle < 20 && v.changes; gp.opt_cycle++) {
v.changes = 0;
gp = (global_props) {.cnt = {0}, .instantiate =
(uint8_t)instantiate, .opt_cycle = gp.opt_cycle};
@@ -750,14 +767,14 @@ rel_optimizer(mvc *sql, sql_rel *rel, in
gp.opt_level = calculate_opt_level(sql, rel);
if (gp.opt_level == 0 && !gp.needs_mergetable_rewrite)
break;
- rel = run_optimizer_set(&v, rel, &gp, pre_sql_optimizers);
+ rel = run_optimizer_set(&v, runs, rel, &gp, pre_sql_optimizers);
}
#ifndef NDEBUG
assert(gp.opt_cycle < 20);
#endif
/* these optimizers run statistics gathered by the last optimization
cycle */
- rel = run_optimizer_set(&v, rel, &gp, post_sql_optimizers);
+ rel = run_optimizer_set(&v, runs, rel, &gp, post_sql_optimizers);
/* merge table rewrites may introduce remote or replica tables */
/* at the moment, make sure the remote table rewriters always run last
*/
diff --git a/sql/server/rel_optimizer.h b/sql/server/rel_optimizer.h
--- a/sql/server/rel_optimizer.h
+++ b/sql/server/rel_optimizer.h
@@ -12,19 +12,6 @@
#include "sql_mvc.h"
#include "sql_relation.h"
-/* a single SQL optimizer run */
-typedef struct {
- const char *name;
- int nchanges;
- lng time;
-} sql_optimizer_run;
-
-/* an optimized SQL query */
-typedef struct {
- sql_optimizer_run *runs;
- sql_rel *rel;
-} sql_optimized_query;
-
-extern sql_rel *rel_optimizer(mvc *sql, sql_rel *rel, int instantiate, int
value_based_opt, int storage_based_opt);
+extern sql_rel *rel_optimizer(mvc *sql, sql_rel *rel, int profile, int
instantiate, int value_based_opt, int storage_based_opt);
#endif /*_REL_OPTIMIZER_H_*/
diff --git a/sql/server/sql_mvc.c b/sql/server/sql_mvc.c
_______________________________________________
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org
