Changeset: 4e66b3f863d1 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4e66b3f863d1
Added Files:
monetdb5/optimizer/opt_matpack.c
monetdb5/optimizer/opt_matpack.h
Modified Files:
monetdb5/modules/mal/mat.c
monetdb5/modules/mal/mat.h
monetdb5/modules/mal/mat.mal
monetdb5/optimizer/Makefile.ag
monetdb5/optimizer/opt_pipes.c
monetdb5/optimizer/opt_support.c
monetdb5/optimizer/opt_wrapper.c
monetdb5/optimizer/optimizer.mal
Branch: Feb2013
Log Message:
Incremental MAT pack optimizer
mat.pack behaves like a blocking operator, which it isn't.
The semantics simply requires to append the arguments to form
a new BAT.
Before dataflow optimizer steps in, we should break compound
mat.pack operations into an incremental one. This will reduce
the resource footprint and improve use of parallelism.
(transplanted from 62ff93eca5093b7d99994c790bd7839907bfd64e)
diffs (281 lines):
diff --git a/monetdb5/modules/mal/mat.c b/monetdb5/modules/mal/mat.c
--- a/monetdb5/modules/mal/mat.c
+++ b/monetdb5/modules/mal/mat.c
@@ -133,6 +133,55 @@ MATpackInternal(MalStkPtr stk, InstrPtr
return MAL_SUCCEED;
}
+/*
+ * Enable incremental packing. The SQL front-end requires
+ * fixed oid sequences.
+ */
+str
+MATpackIncrement(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
+{
+ int bid, *ret = (int*) getArgReference(stk,p,0);
+ BAT *b, *bb, *bn;
+
+ (void) cntxt;
+ b = BATdescriptor( bid = stk->stk[getArg(p,1)].val.ival);
+ if ( b == NULL)
+ throw(MAL, "mat.pack", RUNTIME_OBJECT_MISSING);
+ if ( bid < 0 )
+ b = BATmirror(b);
+ assert(BAThdense(b));
+
+ if ( getArgType(mb,p,2) == TYPE_int){
+ /* first step */
+ bn = BATnew(TYPE_void, b->ttype, BATcount(b) *
stk->stk[getArg(p,2)].val.ival);
+ if (bn == NULL)
+ throw(MAL, "mat.pack", MAL_MALLOC_FAIL);
+ BATsettrivprop(bn);
+ BATseqbase(bn, b->H->seq);
+ BATseqbase(BATmirror(bn), b->T->seq);
+ BATappend(bn,b,FALSE);
+ BBPreleaseref(b->batCacheid);
+ assert(!bn->H->nil || !bn->H->nonil);
+ assert(!bn->T->nil || !bn->T->nonil);
+ BBPkeepref(*ret = bn->batCacheid);
+ } else {
+ /* remaining steps */
+ bb = BATdescriptor(stk->stk[getArg(p,2)].val.ival);
+ if ( bb ){
+ if (BATcount(b) == 0)
+ BATseqbase(b, bb->H->seq);
+ if (BATcount(b) == 0)
+ BATseqbase(BATmirror(b), bb->T->seq);
+ BATappend(b,bb,FALSE);
+ }
+ assert(!b->H->nil || !b->H->nonil);
+ assert(!b->T->nil || !b->T->nonil);
+ BBPkeepref(*ret = b->batCacheid);
+ BBPreleaseref(bb->batCacheid);
+ }
+ return MAL_SUCCEED;
+}
+
static str
MATpackSliceInternal(MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
diff --git a/monetdb5/modules/mal/mat.h b/monetdb5/modules/mal/mat.h
--- a/monetdb5/modules/mal/mat.h
+++ b/monetdb5/modules/mal/mat.h
@@ -36,6 +36,7 @@
mat_export str MATpack(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
mat_export str MATpack2(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
mat_export str MATpack3(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
+mat_export str MATpackIncrement(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr p);
mat_export str MATpackValues(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr p);
mat_export str MATpackSlice(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr p);
mat_export str MATnewIterator(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr p);
diff --git a/monetdb5/modules/mal/mat.mal b/monetdb5/modules/mal/mat.mal
--- a/monetdb5/modules/mal/mat.mal
+++ b/monetdb5/modules/mal/mat.mal
@@ -21,6 +21,14 @@ pattern pack3(b:bat[:oid,:any_2]...):bat
address MATpack3
comment "Materialize the MAT into a BAT by considering the heads as void.
(used in centipede)";
+pattern packIncrement(b:bat[:oid,:any_2],pieces:int):bat[:oid,:any_2]
+address MATpackIncrement
+comment "Prepare incremental mat pack";
+
+pattern packIncrement(b:bat[:oid,:any_2],c:bat[:oid,:any_2]):bat[:oid,:any_2]
+address MATpackIncrement
+comment "Prepare incremental mat pack";
+
pattern slice(first:wrd, last:wrd, b:bat[:oid,:any_2]...):bat[:oid,:any_2]
address MATpackSlice
comment "Materialize a sliced MAT into a BAT";
diff --git a/monetdb5/optimizer/Makefile.ag b/monetdb5/optimizer/Makefile.ag
--- a/monetdb5/optimizer/Makefile.ag
+++ b/monetdb5/optimizer/Makefile.ag
@@ -50,6 +50,7 @@ lib_optimizer = {
opt_joinpath.c opt_joinpath.h \
opt_macro.c opt_macro.h \
opt_mapreduce.c opt_mapreduce.h \
+ opt_matpack.c opt_matpack.h \
opt_mergetable.c opt_mergetable.h \
opt_mitosis.c opt_mitosis.h \
opt_multiplex.c opt_multiplex.h \
diff --git a/monetdb5/optimizer/opt_matpack.c b/monetdb5/optimizer/opt_matpack.c
new file mode 100644
--- /dev/null
+++ b/monetdb5/optimizer/opt_matpack.c
@@ -0,0 +1,69 @@
+/*
+ * The contents of this file are subject to the MonetDB Public License
+ * Version 1.1 (the "License"); you may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ * http://www.monetdb.org/Legal/MonetDBLicense
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
+ * License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * The Original Code is the MonetDB Database System.
+ *
+ * The Initial Developer of the Original Code is CWI.
+ * Portions created by CWI are Copyright (C) 1997-July 2008 CWI.
+ * Copyright August 2008-2012 MonetDB B.V.
+ * All Rights Reserved.
+ *
+*/
+/*
+ * This simple module unrolls the mat.pack into an incremental sequence.
+ * This could speedup parallel processing and releases resources faster.
+ */
+#include "monetdb_config.h"
+#include "opt_matpack.h"
+
+int
+OPTmatpackImplementation(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci)
+{
+ int v, i, j, limit;
+ InstrPtr p,q;
+ int actions = 0;
+ InstrPtr *old;
+ char *packIncrementRef = putName("packIncrement", 13);
+
+ (void) pci;
+ (void) cntxt;
+ (void) stk; /* to fool compilers */
+ old= mb->stmt;
+ limit= mb->stop;
+ if ( newMalBlkStmt(mb,mb->stop) < 0)
+ return 0;
+ for (i = 0; i < limit; i++) {
+ p = old[i];
+ if( getModuleId(p) == matRef && getFunctionId(p) == packRef &&
isaBatType(getArgType(mb,p,1))) {
+ q = newStmt(mb, matRef, packIncrementRef);
+ v = getArg(q,0);
+ setVarType(mb,v,getArgType(mb,p,1));
+ q = pushArgument(mb, q, getArg(p,1));
+ q = pushInt(mb,q, p->argc - p->retc);
+
+ for ( j = 2; j < p->argc; j++) {
+ q = newStmt(mb,matRef, packIncrementRef);
+ q = pushArgument(mb, q, v);
+ q = pushArgument(mb, q, getArg(p,j));
+ setVarType(mb,getArg(q,0),getVarType(mb,v));
+ v = getArg(q,0);
+ }
+ getArg(q,0) = getArg(p,0);
+ freeInstruction(p);
+ continue;
+ }
+ pushInstruction(mb,p);
+ }
+ GDKfree(old);
+ DEBUGoptimizers
+ mnstr_printf(cntxt->fdout,"#opt_matpack: %d statements
marked\n", actions);
+ return actions;
+}
diff --git a/monetdb5/optimizer/opt_matpack.h b/monetdb5/optimizer/opt_matpack.h
new file mode 100644
--- /dev/null
+++ b/monetdb5/optimizer/opt_matpack.h
@@ -0,0 +1,34 @@
+/*
+ * The contents of this file are subject to the MonetDB Public License
+ * Version 1.1 (the "License"); you may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ * http://www.monetdb.org/Legal/MonetDBLicense
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
+ * License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * The Original Code is the MonetDB Database System.
+ *
+ * The Initial Developer of the Original Code is CWI.
+ * Portions created by CWI are Copyright (C) 1997-July 2008 CWI.
+ * Copyright August 2008-2012 MonetDB B.V.
+ * All Rights Reserved.
+*/
+#ifndef _OPT_MATPACK_
+#define _OPT_MATPACK_
+#include "opt_prelude.h"
+#include "opt_support.h"
+#include "mal_interpreter.h"
+#include "mal_instruction.h"
+#include "mal_function.h"
+
+opt_export int OPTmatpackImplementation(Client cntxt, MalBlkPtr mb, MalStkPtr
stk, InstrPtr pci);
+
+/* #define _DEBUG_OPT_MATPACK_ */
+#undef DEBUG_OPT_MATPACK
+#define DEBUG_OPT_MATPACK 1
+#define OPTDEBUGmatpack if ( optDebug & (1 <<DEBUG_OPT_MATPACK) )
+
+#endif
diff --git a/monetdb5/optimizer/opt_pipes.c b/monetdb5/optimizer/opt_pipes.c
--- a/monetdb5/optimizer/opt_pipes.c
+++ b/monetdb5/optimizer/opt_pipes.c
@@ -70,8 +70,8 @@ struct PIPELINES {
"optimizer.emptySet();"
"optimizer.aliases();"
"optimizer.pushselect();"
- "optimizer.mitosis();"
- "optimizer.mergetable();"
+ "optimizer.mitosis();"
+ "optimizer.mergetable();"
"optimizer.deadcode();"
"optimizer.commonTerms();"
//"optimizer.groups();"
@@ -79,6 +79,7 @@ struct PIPELINES {
"optimizer.reorder();"
"optimizer.deadcode();"
"optimizer.reduce();"
+ //"optimizer.matpack();"
"optimizer.dataflow();"
"optimizer.history();"
"optimizer.multiplex();"
diff --git a/monetdb5/optimizer/opt_support.c b/monetdb5/optimizer/opt_support.c
--- a/monetdb5/optimizer/opt_support.c
+++ b/monetdb5/optimizer/opt_support.c
@@ -164,6 +164,7 @@ struct OPTcatalog {
{"joinPath", 0, 0, 0, DEBUG_OPT_JOINPATH},
{"macro", 0, 0, 0, DEBUG_OPT_MACRO},
{"mapreduce", 0, 0, 0, DEBUG_OPT_MAPREDUCE},
+{"matpack", 0, 0, 0, DEBUG_OPT_MATPACK},
{"mergetable", 0, 0, 0, DEBUG_OPT_MERGETABLE},
{"mitosis", 0, 0, 0, DEBUG_OPT_MITOSIS},
{"multiplex", 0, 0, 0, DEBUG_OPT_MULTIPLEX},
diff --git a/monetdb5/optimizer/opt_wrapper.c b/monetdb5/optimizer/opt_wrapper.c
--- a/monetdb5/optimizer/opt_wrapper.c
+++ b/monetdb5/optimizer/opt_wrapper.c
@@ -58,6 +58,7 @@ All Rights Reserved.
#include "opt_inline.h"
#include "opt_joinpath.h"
#include "opt_mapreduce.h"
+#include "opt_matpack.h"
#include "opt_mergetable.h"
#include "opt_mitosis.h"
#include "opt_multiplex.h"
@@ -99,6 +100,7 @@ struct{
{"inline", &OPTinlineImplementation},
{"joinPath", &OPTjoinPathImplementation},
{"mapreduce", &OPTmapreduceImplementation},
+ {"matpack", &OPTmatpackImplementation},
{"mergetable", &OPTmergetableImplementation},
{"mitosis", &OPTmitosisImplementation},
{"multiplex", &OPTmultiplexImplementation},
diff --git a/monetdb5/optimizer/optimizer.mal b/monetdb5/optimizer/optimizer.mal
--- a/monetdb5/optimizer/optimizer.mal
+++ b/monetdb5/optimizer/optimizer.mal
@@ -358,6 +358,13 @@ pattern optimizer.multiplex(mod:str,fcn:
address OPTwrapper
comment "Compiler for multiplexed instructions.";
+#opt_matpack
+pattern optimizer.matpack():str
+address OPTwrapper;
+pattern optimizer.matpack(mod:str, fcn:str):str
+address OPTwrapper
+comment "Unroll the mat.pack operation";
+
#opt_octopus.mal
module octopus;
pattern optimizer.octopus():str
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
http://mail.monetdb.org/mailman/listinfo/checkin-list
