On 2018/04/05 12:14, Amit Langote wrote: > I will post comments on your v19 later today.
I looked at it and here are my thoughts on it after having for the first time looked very closely at it. * Regarding PartitionPruneInfo: I think the names of the fields could be improved -- like pruning_steps instead prunesteps, unpruned_parts instead of allpartindexs. The latter is even a bit misleading because it doesn't in fact contain *all* partition indexes, only those that are unpruned, because each either has a subpath or it appears in (unpruned) partitioned_rels list. Also, I didn't like the name subnodeindex and subpartindex very much. subplan_indexes and parent_indexes would sound more informative to me. * make_partition_pruneinfo has a parameter resultRelations that's not used anywhere * In make_partition_pruneinfo() allsubnodeindex = palloc(sizeof(int) * root->simple_rel_array_size); allsubpartindex = palloc(sizeof(int) * root->simple_rel_array_size); I think these arrays need to have root->simple_rel_array_size + 1 elements, because they're subscripted using RT indexes which start at 1. * Also in make_partition_pruneinfo() /* Initialize to -1 to indicate the rel was not found */ for (i = 0; i < root->simple_rel_array_size; i++) { allsubnodeindex[i] = -1; allsubpartindex[i] = -1; } Maybe, allocate the arrays above mentioned using palloc0 and don't do this initialization. Instead make the indexes that are stored in these start with 1 and consider 0 as invalid entries. * Regarding the code added in execPartition.c and execPartition.h: I wondered why PartitionedRelPruning is named the way it is. I saw many parallels with PartitionDispatchData both in terms of the main thing it consists of, that is, the map that translates partition indexes as in partition descriptor to that of subplans or of some other executor structure. Also, I imagine you tried to mimic PartitionTupleRouting with PartitionPruning but not throughout. For example, tuple routing struct pointer variables are throughout called proute, whereas PartitionPruning ones are called partprune instead of, say, pprune. Consistency would help, imho. * Instead of nesting PartitionedRelPruning inside another, just store them in a global flat array in the PartitionPruning, like PartitionTupleRouting does for PartitionDispatch of individual partitioned tables in the tree. typedef struct PartitionedRelPruning { int nparts; int *subnodeindex; struct PartitionedRelPruning **subpartprune; * I don't see why there needs to be nparts in the above, because it already has a PartitionPruneContext member which has that information. In fact, I made most of changes myself while going through the code. Please see attached the delta patch. It also tweaks quite a few other things including various comments. I think parts of it apply to 0001, 0003, and 0004 patches. See if this looks good to you. Thanks, Amit
diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index 17da8cdbd3..1041871e51 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -39,11 +39,11 @@ static char *ExecBuildSlotPartitionKeyDescription(Relation rel, bool *isnull, int maxfieldlen); static List *adjust_partition_tlist(List *tlist, TupleConversionMap *map); -static void find_subplans_for_extparams_recurse( - PartitionedRelPruning *partrelprune, +static void find_subplans_for_extparams_recurse(PartitionPruningDispatch *all_ppd, + int dispatch_offset, Bitmapset **validsubplans); -static void find_subplans_for_allparams_recurse( - PartitionedRelPruning *partrelprune, +static void find_subplans_for_allparams_recurse(PartitionPruningDispatch *all_ppd, + int dispatch_offset, Bitmapset **validsubplans); @@ -1343,27 +1343,27 @@ adjust_partition_tlist(List *tlist, TupleConversionMap *map) PartitionPruning * ExecSetupPartitionPruning(PlanState *planstate, List *partitionpruneinfo) { - PartitionedRelPruning *partrelprunes; - PartitionPruning *partprune; + PartitionPruning *pprune; + PartitionPruningDispatch *all_ppd; ListCell *lc; int i; Assert(partitionpruneinfo != NIL); - partprune = (PartitionPruning *) palloc(sizeof(PartitionPruning)); - partrelprunes = (PartitionedRelPruning *) - palloc(sizeof(PartitionedRelPruning) * - list_length(partitionpruneinfo)); + pprune = (PartitionPruning *) palloc(sizeof(PartitionPruning)); + all_ppd = pprune->partition_dispatch_info = + (PartitionPruningDispatchData **) + palloc(sizeof(PartitionPruningDispatchData *) * + list_length(partitionpruneinfo)); /* * The first item in the array contains the details for the query's target * partition, so record that as the root of the partition hierarchy. */ - partprune->partrelpruning = partrelprunes; - partprune->npartrelpruning = list_length(partitionpruneinfo); - partprune->extparams = NULL; - partprune->execparams = NULL; - partprune->allparams = NULL; + pprune->num_dispatch = list_length(partitionpruneinfo); + pprune->extparams = NULL; + pprune->execparams = NULL; + pprune->allparams = NULL; /* * Create a sub memory context which we'll use when making calls to the @@ -1372,79 +1372,70 @@ ExecSetupPartitionPruning(PlanState *planstate, List *partitionpruneinfo) * call the function in this context to avoid any memory leaking in the * executor's memory context. */ - partprune->prune_context = AllocSetContextCreate(CurrentMemoryContext, - "Partition Prune", - ALLOCSET_DEFAULT_SIZES); + pprune->prune_context = AllocSetContextCreate(CurrentMemoryContext, + "Partition Prune", + ALLOCSET_DEFAULT_SIZES); i = 0; foreach(lc, partitionpruneinfo) { PartitionPruneInfo *pinfo = (PartitionPruneInfo *) lfirst(lc); - PartitionedRelPruning *partrelprune = &partrelprunes[i]; - PartitionPruneContext *context = &partrelprune->context; + PartitionPruningDispatch *ppd = &all_ppd[i]; + PartitionPruneContext *context; PartitionDesc partdesc; Relation rel; PartitionKey partkey; int partnatts; - int j; - partrelprune->allpartindexes = bms_copy(pinfo->allpartindexes); - partrelprune->nparts = pinfo->nparts; - partrelprune->subnodeindex = palloc(sizeof(int) * pinfo->nparts); - partrelprune->subpartprune = palloc(sizeof(PartitionedRelPruning *) * - pinfo->nparts); + *ppd = (PartitionPruningDispatch) + palloc(sizeof(PartitionPruningDispatchData)); + context = &((*ppd)->context); + (*ppd)->pruning_steps = pinfo->pruning_steps; + (*ppd)->extparams = bms_copy(pinfo->extparams); + (*ppd)->allparams = bms_union(pinfo->extparams, pinfo->execparams); + (*ppd)->unpruned_parts = bms_copy(pinfo->unpruned_parts); + + /* Initialize PartitionPruneContext struct. */ /* - * We must make a copy of this rather than pointing directly to the - * plan's version as we may end up making modifications to it later. + * Note that the relation must've been already locked in + * ExecLockNonLeafAppendTables() */ - memcpy(partrelprune->subnodeindex, pinfo->subnodeindex, - sizeof(int) * pinfo->nparts); - - for (j = 0; j < pinfo->nparts; j++) - { - int subpartidx = pinfo->subpartindex[j]; - - Assert(subpartidx < list_length(partitionpruneinfo)); - - if (subpartidx >= 0) - partrelprune->subpartprune[j] = &partrelprunes[subpartidx]; - else - partrelprune->subpartprune[j] = NULL; - } - rel = relation_open(pinfo->reloid, NoLock); - partkey = RelationGetPartitionKey(rel); partdesc = RelationGetPartitionDesc(rel); context->strategy = partkey->strategy; context->partnatts = partnatts = partkey->partnatts; - context->partopcintype = partkey->partopcintype; context->partopfamily = partkey->partopfamily; context->partcollation = partkey->partcollation; context->partsupfunc = partkey->partsupfunc; - context->nparts = pinfo->nparts; + context->nparts = partdesc->nparts; context->boundinfo = partition_bounds_copy(partdesc->boundinfo, partkey); context->planstate = planstate; context->safeparams = NULL; /* empty for now */ - partrelprune->prunesteps = pinfo->prunesteps; + (*ppd)->subplan_indexes = palloc(sizeof(int) * context->nparts); + (*ppd)->parent_indexes = palloc(sizeof(int) * context->nparts); - partrelprune->extparams = bms_copy(pinfo->extparams); - partrelprune->allparams = bms_union(pinfo->extparams, - pinfo->execparams); - - partprune->extparams = bms_add_members(partprune->extparams, - pinfo->extparams); - - partprune->execparams = bms_add_members(partprune->execparams, - pinfo->execparams); + /* + * Copy from the planner's arrays as is, for now. This might change + * after performing pruning with external params that's done at the + * plan startup. + */ + memcpy((*ppd)->subplan_indexes, pinfo->subplan_indexes, + sizeof(int) * context->nparts); + memcpy((*ppd)->parent_indexes, pinfo->parent_indexes, + sizeof(int) * context->nparts); + /* Add the params of this pinfo to the global bitmapset in pprune. */ + pprune->extparams = bms_add_members(pprune->extparams, + pinfo->extparams); + pprune->execparams = bms_add_members(pprune->execparams, + pinfo->execparams); relation_close(rel, NoLock); - i++; } @@ -1452,16 +1443,15 @@ ExecSetupPartitionPruning(PlanState *planstate, List *partitionpruneinfo) * Cache the union of the Param ids of both types. This saves having to * recalculate it everytime we need to know what they are. */ - partprune->allparams = bms_union(partprune->extparams, - partprune->execparams); + pprune->allparams = bms_union(pprune->extparams, pprune->execparams); - return partprune; + return pprune; } /* * ExecFindInitialMatchingSubPlans * Determine which subset of subplan nodes we need to initialize based - * on the details stored in 'partprune'. Here we only determine the + * on the details stored in 'pprune'. Here we only determine the * matching partitions using values known during plan startup, which is * only external Params. Exec Params will be unknown at this time. We * must delay pruning using exec Params until the actual executor run. @@ -1473,14 +1463,13 @@ ExecSetupPartitionPruning(PlanState *planstate, List *partitionpruneinfo) * return its matching subnode indexes assuming that the caller discarded * the original non-matching subnodes. * - * This function must only be called if 'partprune' has any extparams. + * This function must only be called if 'pprune' has any extparams. * * 'nsubplans' must be passed as the total number of unpruned subplans. */ Bitmapset * -ExecFindInitialMatchingSubPlans(PartitionPruning *partprune, int nsubplans) +ExecFindInitialMatchingSubPlans(PartitionPruning *pprune, int nsubplans) { - PartitionedRelPruning *partrelprune; MemoryContext oldcontext; Bitmapset *result = NULL; @@ -1488,25 +1477,27 @@ ExecFindInitialMatchingSubPlans(PartitionPruning *partprune, int nsubplans) * Ensure there's actually external params, or we've not been called * already. */ - Assert(!bms_is_empty(partprune->extparams)); - - partrelprune = partprune->partrelpruning; + Assert(!bms_is_empty(pprune->extparams)); /* * Switch to a temp context to avoid leaking memory in the * executor's memory context. */ - oldcontext = MemoryContextSwitchTo(partprune->prune_context); + oldcontext = MemoryContextSwitchTo(pprune->prune_context); - /* Determine which subplans match these external params */ - find_subplans_for_extparams_recurse(partrelprune, &result); + /* + * Determine which subplans match using external params. We ask it to + * start pruning with 0th partitioned table, that is, the root. + */ + find_subplans_for_extparams_recurse(pprune->partition_dispatch_info, 0, + &result); MemoryContextSwitchTo(oldcontext); /* Move to the correct memory context */ result = bms_copy(result); - MemoryContextReset(partprune->prune_context); + MemoryContextReset(pprune->prune_context); /* * Record that partition pruning has been performed for external params. @@ -1514,8 +1505,8 @@ ExecFindInitialMatchingSubPlans(PartitionPruning *partprune, int nsubplans) * with the same input and also so that ExecFindMatchingSubPlans is aware * that pruning has already been done for external Params. */ - bms_free(partprune->extparams); - partprune->extparams = NULL; + bms_free(pprune->extparams); + pprune->extparams = NULL; /* * If any subplans were pruned, we must re-sequence the subplan indexes so @@ -1532,7 +1523,7 @@ ExecFindInitialMatchingSubPlans(PartitionPruning *partprune, int nsubplans) * First we must build a map which allows us to map the old subplan * index into the new one. */ - subplanidxmap = (int *) palloc(sizeof(int) * nsubplans); + subplanidxmap = (int *) palloc0(sizeof(int) * nsubplans); newidx = 0; for (i = 0; i < nsubplans; i++) { @@ -1543,28 +1534,28 @@ ExecFindInitialMatchingSubPlans(PartitionPruning *partprune, int nsubplans) } /* - * Now we can re-sequence each PartitionPruneInfo's subnodeindex - * so that they point to the new index of the subnode. + * Now we can re-sequence each PartitionPruningDispatch's indexes + * so that they point to the new indexes of subplans. */ - for (i = 0; i < partprune->npartrelpruning; i++) + for (i = 0; i < pprune->num_dispatch; i++) { - PartitionedRelPruning *partrelprune; + PartitionPruningDispatch ppd; int j; - partrelprune = &partprune->partrelpruning[i]; + ppd = pprune->partition_dispatch_info[i]; /* - * We also need to reset the allpartindexes field so that it + * We also need to reset the unpruned_parts field so that it * only contains partition indexes that we actually still have - * subnodeindexes for. It seems easier to build a fresh one, - * rather than trying to update the existing one. + * a valid value in subplan_indexes for. It seems easier to build + * a fresh one, rather than trying to update the existing one. */ - bms_free(partrelprune->allpartindexes); - partrelprune->allpartindexes = NULL; + bms_free(ppd->unpruned_parts); + ppd->unpruned_parts = NULL; - for (j = 0; j < partrelprune->nparts; j++) + for (j = 0; j < ppd->context.nparts; j++) { - int oldidx = partrelprune->subnodeindex[j]; + int oldidx = ppd->subplan_indexes[j]; /* * If this partition existed as a subplan then change the old @@ -1575,12 +1566,11 @@ ExecFindInitialMatchingSubPlans(PartitionPruning *partprune, int nsubplans) */ if (oldidx >= 0) { - partrelprune->subnodeindex[j] = subplanidxmap[oldidx]; + ppd->subplan_indexes[j] = subplanidxmap[oldidx]; - if (subplanidxmap[oldidx] >= 0) - partrelprune->allpartindexes = - bms_add_member(partrelprune->allpartindexes, - j); + if (ppd->subplan_indexes[j] >= 0) + ppd->unpruned_parts = + bms_add_member(ppd->unpruned_parts, j); } } } @@ -1597,10 +1587,12 @@ ExecFindInitialMatchingSubPlans(PartitionPruning *partprune, int nsubplans) * Recursive worker function for ExecFindInitialMatchingSubPlans. */ static void -find_subplans_for_extparams_recurse(PartitionedRelPruning *partrelprune, +find_subplans_for_extparams_recurse(PartitionPruningDispatch *all_ppd, + int dispatch_offset, Bitmapset **validsubplans) { - PartitionPruneContext *context = &partrelprune->context; + PartitionPruningDispatch ppd = all_ppd[dispatch_offset]; + PartitionPruneContext *context = &ppd->context; Bitmapset *partset; int i; @@ -1616,23 +1608,29 @@ find_subplans_for_extparams_recurse(PartitionedRelPruning *partrelprune, * any subpartitioned tables as we may find their partition keys match * some Params at their level. */ - if (!bms_is_empty(partrelprune->extparams)) + if (!bms_is_empty(ppd->extparams)) { - context->safeparams = partrelprune->extparams; - partset = get_matching_partitions(context, partrelprune->prunesteps); + context->safeparams = ppd->extparams; + partset = get_matching_partitions(context, ppd->pruning_steps); } else - partset = partrelprune->allpartindexes; + partset = ppd->unpruned_parts; /* Translate partset into subnode indexes */ i = -1; while ((i = bms_next_member(partset, i)) >= 0) { - if (partrelprune->subnodeindex[i] >= 0) + /* If the chosen partition is a leaf partition, add its subplan. */ + if (ppd->subplan_indexes[i] >= 0) *validsubplans = bms_add_member(*validsubplans, - partrelprune->subnodeindex[i]); - else if (partrelprune->subpartprune[i] != NULL) - find_subplans_for_extparams_recurse(partrelprune->subpartprune[i], + ppd->subplan_indexes[i]); + /* + * Else if it's a partitioned table, recurse to perform pruning + * for its own partitions. + */ + else if (ppd->parent_indexes[i]) + find_subplans_for_extparams_recurse(all_ppd, + ppd->parent_indexes[i], validsubplans); else { @@ -1650,31 +1648,34 @@ find_subplans_for_extparams_recurse(PartitionedRelPruning *partrelprune, /* * ExecFindMatchingSubPlans * Determine which subplans match the the pruning steps detailed in - * 'partprune' for the current Param values. + * 'pprune' for the current Param values. */ Bitmapset * -ExecFindMatchingSubPlans(PartitionPruning *partprune) +ExecFindMatchingSubPlans(PartitionPruning *pprune) { - PartitionedRelPruning *partrelprune; MemoryContext oldcontext; Bitmapset *result = NULL; - partrelprune = partprune->partrelpruning; - /* * Switch to a temp context to avoid leaking memory in the * executor's memory context. */ - oldcontext = MemoryContextSwitchTo(partprune->prune_context); + oldcontext = MemoryContextSwitchTo(pprune->prune_context); - find_subplans_for_allparams_recurse(partrelprune, &result); + /* + * Determine which subplans match using all the params, including both + * the external and executor params. We ask it to start pruning with + * 0th partitioned table, that is, the root. + */ + find_subplans_for_allparams_recurse(pprune->partition_dispatch_info, 0, + &result); MemoryContextSwitchTo(oldcontext); /* Move to the correct memory context */ result = bms_copy(result); - MemoryContextReset(partprune->prune_context); + MemoryContextReset(pprune->prune_context); return result; } @@ -1684,10 +1685,12 @@ ExecFindMatchingSubPlans(PartitionPruning *partprune) * Recursive worker function for ExecFindMatchingSubPlans. */ static void -find_subplans_for_allparams_recurse(PartitionedRelPruning *partrelprune, +find_subplans_for_allparams_recurse(PartitionPruningDispatch *all_ppd, + int dispatch_offset, Bitmapset **validsubplans) { - PartitionPruneContext *context = &partrelprune->context; + PartitionPruningDispatch ppd = all_ppd[dispatch_offset]; + PartitionPruneContext *context = &ppd->context; Bitmapset *partset; int i; @@ -1703,23 +1706,30 @@ find_subplans_for_allparams_recurse(PartitionedRelPruning *partrelprune, * any subpartitioned tables as we may find their partition keys match * some Params at their level. */ - if (!bms_is_empty(partrelprune->allparams)) + if (!bms_is_empty(ppd->allparams)) { - context->safeparams = partrelprune->allparams; - partset = get_matching_partitions(context, partrelprune->prunesteps); + context->safeparams = ppd->allparams; + partset = get_matching_partitions(context, ppd->pruning_steps); } else - partset = partrelprune->allpartindexes; + partset = ppd->unpruned_parts; /* Translate partset into subnode indexes */ i = -1; while ((i = bms_next_member(partset, i)) >= 0) { - if (partrelprune->subnodeindex[i] >= 0) - *validsubplans = bms_add_member(*validsubplans, - partrelprune->subnodeindex[i]); - else if (partrelprune->subpartprune[i] != NULL) - find_subplans_for_allparams_recurse(partrelprune->subpartprune[i], + /* If the chosen partition is a leaf partition, add its subplan. */ + if (ppd->subplan_indexes[i] >= 0) + *validsubplans = + bms_add_member(*validsubplans, + ppd->subplan_indexes[i]); + /* + * Else if it's a partitioned table, recurse to perform pruning + * for its own partitions. + */ + else if (ppd->parent_indexes[i] >= 0) + find_subplans_for_allparams_recurse(all_ppd, + ppd->parent_indexes[i], validsubplans); else { diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index 5286ada835..c589be4563 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -126,22 +126,22 @@ ExecInitAppend(Append *node, EState *estate, int eflags) /* If run-time partition pruning is enabled, then setup that up now */ if (node->part_prune_infos != NIL) { - PartitionPruning *partprune; + PartitionPruning *pprune; ExecAssignExprContext(estate, &appendstate->ps); - partprune = ExecSetupPartitionPruning(&appendstate->ps, - node->part_prune_infos); + pprune = ExecSetupPartitionPruning(&appendstate->ps, + node->part_prune_infos); /* * When there are external params matching the partition key we may be * able to prune away Append subplans now. */ - if (!bms_is_empty(partprune->extparams)) + if (!bms_is_empty(pprune->extparams)) { /* Determine which subplans match the external params */ - validsubplans = ExecFindInitialMatchingSubPlans(partprune, - list_length(node->appendplans)); + nplans = list_length(node->appendplans); + validsubplans = ExecFindInitialMatchingSubPlans(pprune, nplans); /* * If no subplans match the given parameters then we must handle @@ -173,10 +173,10 @@ ExecInitAppend(Append *node, EState *estate, int eflags) * If there's no exec params then no further pruning can be done, we * can just set the valid subplans to all remaining subplans. */ - if (bms_is_empty(partprune->execparams)) + if (bms_is_empty(pprune->execparams)) appendstate->as_valid_subplans = bms_add_range(NULL, 0, nplans - 1); - appendstate->partition_pruning = partprune; + appendstate->partition_pruning = pprune; } else diff --git a/src/backend/executor/nodeMergeAppend.c b/src/backend/executor/nodeMergeAppend.c index d9cf911f4f..cf2c1ec6dc 100644 --- a/src/backend/executor/nodeMergeAppend.c +++ b/src/backend/executor/nodeMergeAppend.c @@ -91,22 +91,22 @@ ExecInitMergeAppend(MergeAppend *node, EState *estate, int eflags) /* If run-time partition pruning is enabled, then setup that up now */ if (node->part_prune_infos != NIL) { - PartitionPruning *partprune; + PartitionPruning *pprune; ExecAssignExprContext(estate, &mergestate->ps); - partprune = ExecSetupPartitionPruning(&mergestate->ps, - node->part_prune_infos); + pprune = ExecSetupPartitionPruning(&mergestate->ps, + node->part_prune_infos); /* * When there are external params matching the partition key we may be * able to prune away MergeAppend subplans now. */ - if (!bms_is_empty(partprune->extparams)) + if (!bms_is_empty(pprune->extparams)) { /* Determine which subplans match the external params */ - validsubplans = ExecFindInitialMatchingSubPlans(partprune, - list_length(node->mergeplans)); + nplans = list_length(node->mergeplans); + validsubplans = ExecFindInitialMatchingSubPlans(pprune, nplans); /* * If no subplans match the given parameters then we must handle @@ -140,13 +140,13 @@ ExecInitMergeAppend(MergeAppend *node, EState *estate, int eflags) * Otherwise we set the valid subplans to NULL so that they can be * determined during actual execution. */ - if (bms_is_empty(partprune->execparams)) + if (bms_is_empty(pprune->execparams)) mergestate->ms_valid_subplans = bms_add_range(NULL, 0, nplans - 1); else mergestate->ms_valid_subplans = NULL; - mergestate->partition_pruning = partprune; + mergestate->partition_pruning = pprune; } else diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 739a023965..722e79be5b 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -2175,11 +2175,11 @@ _copyPartitionPruneInfo(const PartitionPruneInfo *from) PartitionPruneInfo *newnode = makeNode(PartitionPruneInfo); COPY_SCALAR_FIELD(reloid); - COPY_NODE_FIELD(prunesteps); - COPY_BITMAPSET_FIELD(allpartindexes); + COPY_NODE_FIELD(pruning_steps); + COPY_BITMAPSET_FIELD(unpruned_parts); COPY_SCALAR_FIELD(nparts); - COPY_POINTER_FIELD(subnodeindex, from->nparts * sizeof(int)); - COPY_POINTER_FIELD(subpartindex, from->nparts * sizeof(int)); + COPY_POINTER_FIELD(subplan_indexes, from->nparts * sizeof(int)); + COPY_POINTER_FIELD(parent_indexes, from->nparts * sizeof(int)); COPY_BITMAPSET_FIELD(extparams); COPY_BITMAPSET_FIELD(execparams); diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index e31b6a9c33..88e7f08551 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -1757,17 +1757,17 @@ _outPartitionPruneInfo(StringInfo str, const PartitionPruneInfo *node) WRITE_NODE_TYPE("PARTITIONPRUNEINFO"); WRITE_OID_FIELD(reloid); - WRITE_NODE_FIELD(prunesteps); - WRITE_BITMAPSET_FIELD(allpartindexes); + WRITE_NODE_FIELD(pruning_steps); + WRITE_BITMAPSET_FIELD(unpruned_parts); WRITE_INT_FIELD(nparts); - appendStringInfoString(str, " :subnodeindex"); + appendStringInfoString(str, " :subplan_indexes"); for (i = 0; i < node->nparts; i++) - appendStringInfo(str, " %d", node->subnodeindex[i]); + appendStringInfo(str, " %d", node->subplan_indexes[i]); - appendStringInfoString(str, " :subpartindex"); + appendStringInfoString(str, " :parent_indexes"); for (i = 0; i < node->nparts; i++) - appendStringInfo(str, " %d", node->subpartindex[i]); + appendStringInfo(str, " %d", node->parent_indexes[i]); WRITE_BITMAPSET_FIELD(extparams); WRITE_BITMAPSET_FIELD(execparams); diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 5bf3d28c51..6e059ec568 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -1363,11 +1363,11 @@ _readPartitionPruneInfo(void) READ_LOCALS(PartitionPruneInfo); READ_OID_FIELD(reloid); - READ_NODE_FIELD(prunesteps); - READ_BITMAPSET_FIELD(allpartindexes); + READ_NODE_FIELD(pruning_steps); + READ_BITMAPSET_FIELD(unpruned_parts); READ_INT_FIELD(nparts); - READ_INT_ARRAY(subnodeindex, local_node->nparts); - READ_INT_ARRAY(subpartindex, local_node->nparts); + READ_INT_ARRAY(subplan_indexes, local_node->nparts); + READ_INT_ARRAY(parent_indexes, local_node->nparts); READ_BITMAPSET_FIELD(extparams); READ_BITMAPSET_FIELD(execparams); diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index d6c94846d3..c33e0ab206 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -1107,9 +1107,10 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path) * and allow translation of partition indexes into subpath indexes. */ if (prunequal != NIL) - partpruneinfos = make_partition_pruneinfo(root, - best_path->partitioned_rels, NIL, - best_path->subpaths, prunequal); + partpruneinfos = + make_partition_pruneinfo(root, + best_path->partitioned_rels, + best_path->subpaths, prunequal); } /* @@ -1257,9 +1258,10 @@ create_merge_append_plan(PlannerInfo *root, MergeAppendPath *best_path) * and allow translation of partition indexes into subpath indexes. */ if (prunequal != NIL) - partpruneinfos = make_partition_pruneinfo(root, - best_path->partitioned_rels, NIL, - best_path->subpaths, prunequal); + partpruneinfos = + make_partition_pruneinfo(root, + best_path->partitioned_rels, + best_path->subpaths, prunequal); } node->partitioned_rels = best_path->partitioned_rels; diff --git a/src/backend/optimizer/util/partprune.c b/src/backend/optimizer/util/partprune.c index e8815cb697..5483023dc2 100644 --- a/src/backend/optimizer/util/partprune.c +++ b/src/backend/optimizer/util/partprune.c @@ -238,131 +238,113 @@ generate_partition_pruning_steps(RelOptInfo *rel, List *clauses, /* * make_partition_pruneinfo - * Return a List of PartitionPruneInfos, one for each 'partitioned_rel', - * or NIL if no Params were found matching the partition key, in which - * case run-time partition pruning is useless. + * Return a List of PartitionPruneInfos, one for each relation in + * 'partitioned_rel' or NIL if no Params matching the partition key + * were found, in which case, run-time partition pruning is useless. * - * Here we index the subpaths by partition index so that we're able to - * translate the output of get_matching_partitions into subpath indexes to - * possibly allow for further partition pruning to be performed during - * execution. + * Each PartitionPruneInfo node consists of a map to translate the partition + * indexes as output by get_matching_partitions to the indexes of their + * corresponding subplans in the array of subplans that contains entries + * corresponding to all the partitions in the tree that are selected by the + * planner. */ List * -make_partition_pruneinfo(PlannerInfo *root, List *partition_rels, - List *resultRelations, List *subpaths, - List *prunequal) +make_partition_pruneinfo(PlannerInfo *root, List *partitioned_rels, + List *subpaths, List *prunequal) { - RangeTblEntry *rte; - RelOptInfo *parentpart; ListCell *lc; List *pinfolist = NIL; - int *allsubnodeindex; - int *allsubpartindex; + int *all_subpath_indexes; + int *all_parent_indexes; int i; bool gotparam = false; /* - * Allocate two arrays, one to allow quick lookups of the 'subpaths' index - * of a relation by relid and another to lookup the 'partitioned_rel' - * index by relid. + * Create a mapping from RT indexes of the subpaths' relations to their + * ordinal position in the list. Same for the RT indexes appearing in + * partitioned_rels list. Note that RT indexes start with 1, so must + * allocate space for root->simple_rel_array_size + 1 integers. */ - allsubnodeindex = palloc(sizeof(int) * root->simple_rel_array_size); - allsubpartindex = palloc(sizeof(int) * root->simple_rel_array_size); - - /* Initialize to -1 to indicate the rel was not found */ - for (i = 0; i < root->simple_rel_array_size; i++) - { - allsubnodeindex[i] = -1; - allsubpartindex[i] = -1; - } + all_subpath_indexes = palloc0(sizeof(int) * + root->simple_rel_array_size + 1); + all_parent_indexes = palloc0(sizeof(int) * + root->simple_rel_array_size + 1); /* * Now loop over each subpath and fill in the index of the subpath for the - * subpath's relid. + * subpath's relid. Make the indexes start at 1, so all 0 entries in the + * array would mean a relation that is not in this partition tree or is + * pruned. */ - if (resultRelations != NIL) + i = 1; + foreach(lc, subpaths) { - i = 0; - foreach(lc, resultRelations) - { - int resultrel = lfirst_int(lc); - Assert(resultrel < root->simple_rel_array_size); - allsubnodeindex[resultrel] = i; - i++; - } - } - else - { - i = 0; - foreach(lc, subpaths) - { - Path *path = (Path *) lfirst(lc); - RelOptInfo *pathrel = path->parent; + Path *path = (Path *) lfirst(lc); + RelOptInfo *pathrel = path->parent; - Assert(IS_SIMPLE_REL(pathrel)); - Assert(pathrel->relid < root->simple_rel_array_size); + Assert(IS_SIMPLE_REL(pathrel)); + Assert(pathrel->relid < root->simple_rel_array_size); - allsubnodeindex[pathrel->relid] = i; - i++; - } + all_subpath_indexes[pathrel->relid] = i; + i++; } - /* Likewise for the partition_rels */ - i = 0; - foreach(lc, partition_rels) + /* Likewise for the partitioned_rels */ + i = 1; + foreach(lc, partitioned_rels) { Index rti = lfirst_int(lc); Assert(rti < root->simple_rel_array_size); - - allsubpartindex[rti] = i; + all_parent_indexes[rti] = i; i++; } - /* We now build a PartitionPruneInfo for each partition_rels */ + /* We now build a PartitionPruneInfo for each partitioned_rels */ i = 0; - foreach(lc, partition_rels) + foreach(lc, partitioned_rels) { + RangeTblEntry *rte; + RelOptInfo *parent; Index rti = lfirst_int(lc); - RelOptInfo *subpart = find_base_rel(root, rti); + RelOptInfo *rel = find_base_rel(root, rti); PartitionPruneInfo *pinfo; - int nparts = subpart->nparts; - int *subnodeindex; - int *subpartindex; - List *partprunequal; + int nparts = rel->nparts; + int *subplan_indexes; + int *parent_indexes; + List *translated_prunequal; bool constfalse; - rte = root->simple_rte_array[subpart->relid]; - + rte = root->simple_rte_array[rel->relid]; pinfo = makeNode(PartitionPruneInfo); pinfo->reloid = rte->relid; /* * The first item in the list is the target partitioned relation. The - * quals belong to this relation, so require no translation. + * quals belong to this relation, so require no translation. Also, + * save the first rel as the root parent for all subsequent rels in + * partitioned_rels. */ if (i == 0) { - parentpart = subpart; - partprunequal = prunequal; + parent = rel; + translated_prunequal = prunequal; } else - { /* * For sub-partitioned tables the columns may not be in the same * order as the parent, so we must translate the prunequal to make * it compatible with this relation. */ - partprunequal = (List *) + translated_prunequal = (List *) adjust_appendrel_attrs_multilevel(root, (Node *) prunequal, - subpart->relids, - parentpart->relids); - } - - pinfo->prunesteps = generate_partition_pruning_steps(subpart, - partprunequal, - &constfalse); + rel->relids, + parent->relids); + pinfo->pruning_steps = + generate_partition_pruning_steps(rel, + translated_prunequal, + &constfalse); if (constfalse) { @@ -378,10 +360,11 @@ make_partition_pruneinfo(PlannerInfo *root, List *partition_rels, return NIL; } - pinfo->allpartindexes = NULL; + pinfo->unpruned_parts = NULL; pinfo->nparts = nparts; - pinfo->subnodeindex = subnodeindex = palloc(nparts * sizeof(int)); - pinfo->subpartindex = subpartindex = palloc(nparts * sizeof(int)); + pinfo->subplan_indexes = subplan_indexes = + palloc(nparts * sizeof(int)); + pinfo->parent_indexes = parent_indexes = palloc(nparts * sizeof(int)); pinfo->extparams = NULL; pinfo->execparams = NULL; @@ -390,7 +373,7 @@ make_partition_pruneinfo(PlannerInfo *root, List *partition_rels, * We'll not bother enabling run-time pruning if no params matched * the partition key at any level of partitioning. */ - gotparam |= pull_partkey_params(pinfo, pinfo->prunesteps); + gotparam |= pull_partkey_params(pinfo, pinfo->pruning_steps); /* * Loop over each partition of the partitioned rel and record the @@ -400,21 +383,28 @@ make_partition_pruneinfo(PlannerInfo *root, List *partition_rels, */ for (i = 0; i < nparts; i++) { - RelOptInfo *partrel = subpart->part_rels[i]; - int subnodeidx = allsubnodeindex[partrel->relid]; - int subpartidx = allsubpartindex[partrel->relid]; - - subnodeindex[i] = subnodeidx; - subpartindex[i] = subpartidx; + RelOptInfo *partrel = rel->part_rels[i]; + int subpath_index = all_subpath_indexes[partrel->relid]; + int parent_index = all_parent_indexes[partrel->relid]; /* - * Record the indexes of all the partition indexes that we have - * subnodes or subparts for. This allows an optimization to skip - * attempting any run-time pruning when no Params are found - * matching the partition key at this level. + * If this partition's path is in subpaths, add its offset to + * subplan_indexes. If it's not, -1 will be stored. */ - if (subnodeidx >= 0 || subpartidx >= 0) - pinfo->allpartindexes = bms_add_member(pinfo->allpartindexes, + subplan_indexes[i] = subpath_index - 1; + + /* + * If this partition is itself a partitioned table, add its offset + * to parent_indexes. If it's not, -1 will be stored. + */ + parent_indexes[i] = parent_index - 1; + + /* + * Record the indexes of all the partitions that each either has + * a subpath for or appears in partitioned_rels list. + */ + if (subpath_index > 0 || parent_index > 0) + pinfo->unpruned_parts = bms_add_member(pinfo->unpruned_parts, i); } @@ -422,8 +412,8 @@ make_partition_pruneinfo(PlannerInfo *root, List *partition_rels, i++; } - pfree(allsubnodeindex); - pfree(allsubpartindex); + pfree(all_subpath_indexes); + pfree(all_parent_indexes); if (gotparam) return pinfolist; diff --git a/src/include/executor/execPartition.h b/src/include/executor/execPartition.h index 140e0bdf1e..6afd513ee2 100644 --- a/src/include/executor/execPartition.h +++ b/src/include/executor/execPartition.h @@ -110,70 +110,72 @@ typedef struct PartitionTupleRouting } PartitionTupleRouting; /*----------------------- - * PartitionedRelPruning - Encapsulates all information required to support - * elimination of partitions in node types which support arbitrary Lists of - * subplans. Information stored here allows partprune.c's partition pruning - * functions to be called and the return value of partition indexes translated - * into the subpath indexes of node types such as Append, thus allowing us to - * bypass certain subnodes when we have proofs that indicate that no tuple - * matching the 'prunesteps' will be found within. + * PartitionPruningDispatch - Encapsulates all information required to perform + * partition pruning using the information provided in one of the node types + * used to scan partitioned tables, viz. Append and MergeAppend. Information + * stored here allows us to call partprune.c's partition pruning functions and + * translate the returned partition indexes into those of the subplans + * subplans contained in the Append or MergeAppend or the index of another + * PartitionPruningDispatch if we need to perform pruning one level down. * - * nparts The number of partitions which belong to this - * partitioned relation. Also defines the size of - * the 'subnodeindex' and 'subpartprune' arrays. - * subnodeindex An array of nparts containing the subnode - * index which matches this partition index, or - * -1 if there is no match. - * subpartprune An array of nparts containing the - * PartitionedRelPruning details this partition - * index for sub-partitioned tables. - * allpartindexes A Bitmapset of the partition index that we have - * subnodes mapped for. - * belong to this partition. - * context Contains the context details required to call - * the partition pruning code. - * prunesteps Contains list of PartitionPruneStep used to - * perform the actual pruning. + * context Contains the context details required to call + * the partition pruning code. + * pruning_steps Contains list of PartitionPruneStep used to + * perform the actual pruning. + * extparams IDs of external Params + * allparams IDs of both exeternal and executor Params + * unpruned_parts Indexes of partitions selected after executing + * pruning_steps + * subplan_indexes An array containing one value for each partition, + * which, if it's >= 0, is the index of its subplan + * in the correponding Append or MergeAppend node + * or -1 if the partition has been pruned or is not + * a leaf partition + * parent_indexes An array containing one value for each partition, + * which, if it's >= 0, is the index of its + * PartitionPruningDispatchData to perform further + * pruning with (that is, pruning of the partitions + * of the next level) or -1 if the partition has + * been pruned or is a leaf partitions. *----------------------- */ -typedef struct PartitionedRelPruning +typedef struct PartitionPruningDispatchData { - int nparts; - int *subnodeindex; - struct PartitionedRelPruning **subpartprune; - Bitmapset *allpartindexes; PartitionPruneContext context; - List *prunesteps; + List *pruning_steps; Bitmapset *extparams; Bitmapset *allparams; -} PartitionedRelPruning; + Bitmapset *unpruned_parts; + int *subplan_indexes; + int *parent_indexes; +} PartitionPruningDispatchData; + +typedef struct PartitionPruningDispatchData *PartitionPruningDispatch; /*----------------------- - * PartitionPruning - Encapsulates a hierarchy of PartitionedRelPruning - * structs and also stores all Param IDs which were found to match the - * partition keys of each partition. This struct can be attached to node - * types which support arbitrary Lists of subnodes containing partitions to - * allow subnodes to be eliminated due to the clauses being unable to match - * to any tuple that the subnode could possibly produce. + * PartitionPruning - Analogous to PartitionTupleRouting, this encapsulates + * the information needed to perform partition pruning for the partitioned + * tables in the tree * - * partrelpruning Array of PartitionedRelPruning for the node's target - * partitioned relation. First element contains the - * details for the target partitioned table. - * npartrelpruning Number of items in partrelpruning array. - * prune_context A memory context which can be used to call the query - * planner's partition prune functions. - * extparams All PARAM_EXTERN Param IDs which were found to match a - * partition key in each of the contained - * PartitionedRelPruning structs. - * execparams As above but for PARAM_EXEC. - * allparams Union of extparams and execparams, saved to avoid - * recalculation. + * partition_dispatch_info Array of PartitionPruningDispatch objects with + * one entry for each partitioned table in the + * partition tree. + * num_dispatch number of partitioned tables in the partition + * tree (= length of partition_dispatch_info[]) + * prune_context A memory context which can be used to call the + * planner's partition prune functions. + * extparams All PARAM_EXTERN Param IDs which were found to + * match a partition key in each of the contained + * PartitionedRelPruning structs. + * execparams As above but for PARAM_EXEC. + * allparams Union of extparams and execparams, saved to avoid + * recalculation. *----------------------- */ typedef struct PartitionPruning { - PartitionedRelPruning *partrelpruning; - int npartrelpruning; + PartitionPruningDispatch *partition_dispatch_info; + int num_dispatch; MemoryContext prune_context; Bitmapset *extparams; Bitmapset *execparams; @@ -201,8 +203,8 @@ extern HeapTuple ConvertPartitionTupleSlot(TupleConversionMap *map, extern void ExecCleanupTupleRouting(PartitionTupleRouting *proute); extern PartitionPruning *ExecSetupPartitionPruning(PlanState *planstate, List *partitionpruneinfo); -extern Bitmapset *ExecFindMatchingSubPlans(PartitionPruning *partprune); -extern Bitmapset *ExecFindInitialMatchingSubPlans(PartitionPruning *partprune, +extern Bitmapset *ExecFindMatchingSubPlans(PartitionPruning *pprune); +extern Bitmapset *ExecFindInitialMatchingSubPlans(PartitionPruning *pprune, int nsubplans); #endif /* EXECPARTITION_H */ diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h index f357473c6a..750784bbd8 100644 --- a/src/include/nodes/primnodes.h +++ b/src/include/nodes/primnodes.h @@ -1580,26 +1580,44 @@ typedef struct PartitionPruneStepCombine } PartitionPruneStepCombine; /*---------- - * PartitionPruneInfo - Details required to allow the executor to prune - * partitions. + * PartitionPruneInfo - Details about partition pruning that planner passes + * to the executor through the Plan node types that are used for partitioned + * tables * - * Here we store mapping details to allow translation of a partitioned table's - * index into subnode indexes for node types which support arbitrary numbers - * of sub nodes, such as Append. + * Here we store three pieces of information: + * + * 1. Partition pruning steps which contain references to Params that can + * only be evaluated during execution + * + * 2. Mapping details to allow translation of partition indexes (as stored in + * the table's partition descriptor to the indexes of the corresponding + * partition's subplan or if the partition is itself a partitioned table, + * the index of a struct that's in turn can be used to perform partition + * pruning for its own partition. + * + * 3. Param IDs for both both Param types. + * + * This closely resembles the PartitionDispatchData in execPartition.h, but + * since this is stored in a plan it's defined as a Node type. *---------- */ typedef struct PartitionPruneInfo { NodeTag type; - Oid reloid; /* Oid of partition rel */ - List *prunesteps; /* List of PartitionPruneStep */ - Bitmapset *allpartindexes; /* All part index we have subnodes for at this - * level */ - int nparts; /* length of the following arrays */ - int *subnodeindex; /* subnode index indexed by partition id */ - int *subpartindex; /* subpart index indexed by partition id */ - Bitmapset *extparams; /* All external ParamIDs seen in prunesteps */ - Bitmapset *execparams; /* All exec ParamIDs seen in prunesteps */ + Oid reloid; /* OID of the table this node is for */ + List *pruning_steps; /* List of PartitionPruneStep */ + Bitmapset *unpruned_parts; /* Indexes of all unpruned partitions */ + + int nparts; /* Total number of table's partitions */ + int *subplan_indexes; /* Map from partition indexes to + * subplan indexes, for leaf partitions */ + int *parent_indexes; /* Map from partition indexes to + * index of the pruning info struct, for + * partitioned partitions */ + Bitmapset *extparams; /* All external ParamIDs seen in + * pruning_steps */ + Bitmapset *execparams; /* All executor ParamIDs seen in + * pruning_steps */ } PartitionPruneInfo; #endif /* PRIMNODES_H */ diff --git a/src/include/optimizer/partprune.h b/src/include/optimizer/partprune.h index b7352d150c..65c0c23560 100644 --- a/src/include/optimizer/partprune.h +++ b/src/include/optimizer/partprune.h @@ -20,8 +20,8 @@ extern Relids prune_append_rel_partitions(RelOptInfo *rel); extern List *generate_partition_pruning_steps(RelOptInfo *rel, List *clauses, bool *constfalse); -extern List *make_partition_pruneinfo(PlannerInfo *root, List *partition_rels, - List *resultRelations, List *subpaths, +extern List *make_partition_pruneinfo(PlannerInfo *root, + List *partitioned_rels, List *subpaths, List *prunequal); #endif /* PARTPRUNE_H */