From d19997bb7587076cc1e3fc0c81a1a06655f93bb5 Mon Sep 17 00:00:00 2001 From: Maxime Schoemans Date: Mon, 13 Apr 2026 16:04:23 +0200 Subject: [PATCH v8 1/3] Improve range join selectivity estimation for <<, >>, && Teach rangejoinsel to estimate join selectivity for range operators using bound histogram statistics instead of falling back to fixed defaults. The estimation is based on a trapezoidal approximation of P(X < Y) by parallel-scanning the bound histograms of both sides. This improves planner row estimates especially when the two range columns have clearly separated or strongly overlapping distributions. Regression tests cover plan changes for representative range join cases. Based on: Repas, Luo, Schoemans, Sakr (2022) "Selectivity Estimation of Inequality Joins In Databases" https://doi.org/10.48550/arXiv.2206.07396 --- src/backend/utils/adt/rangetypes_selfuncs.c | 300 ++++++++++++++++++++ src/include/catalog/pg_operator.dat | 6 +- src/include/catalog/pg_proc.dat | 4 + src/test/regress/expected/rangetypes.out | 114 ++++++++ src/test/regress/sql/rangetypes.sql | 53 ++++ 5 files changed, 474 insertions(+), 3 deletions(-) diff --git a/src/backend/utils/adt/rangetypes_selfuncs.c b/src/backend/utils/adt/rangetypes_selfuncs.c index 75f1e7567d5..cc702f28610 100644 --- a/src/backend/utils/adt/rangetypes_selfuncs.c +++ b/src/backend/utils/adt/rangetypes_selfuncs.c @@ -1221,3 +1221,303 @@ calc_hist_selectivity_contains(TypeCacheEntry *typcache, return sum_frac; } + +/* + * Estimate join selectivity P(X < Y) using rangebound histograms. + * + * Based on: Diogo Repas, Zhicheng Luo, Maxime Schoemans, Mahmoud Sakr, 2022 + * "Selectivity Estimation of Inequality Joins In Databases" + * https://doi.org/10.48550/arXiv.2206.07396 + * + * hist1 and hist2 are arrays of RangeBound entries from the bounds histograms + * of two range-typed attributes X and Y, respectively. Each array has at + * least 2 entries (one histogram bin). The entries carry full bound metadata + * (lower/upper flag, inclusive/exclusive), and all comparisons use + * range_cmp_bounds() so that bound semantics are preserved. + * + * The algorithm models each attribute's distribution as a piecewise function + * derived from its histogram, then computes: + * P(X < Y) = 0.5 * sum( (F_X(prev) + F_X(cur)) * (F_Y(cur) - F_Y(prev)) ) + * by parallel-scanning both histograms. + * + * The initial fast-forward loops skip histogram entries that fall entirely + * before the other histogram's range, so the main loop only processes the + * overlapping region. Bounds checks are required because the histograms may + * be completely disjoint (e.g., all of X is below all of Y). + */ +static double +calc_hist_join_selectivity(TypeCacheEntry *typcache, + const RangeBound *hist1, int nhist1, + const RangeBound *hist2, int nhist2) +{ + int i, + j; + double selectivity = 0.0; + double prev_sel1 = -1.0; /* negative sentinel skips first iter */ + double prev_sel2 = 0.0; + + Assert(nhist1 > 1); + Assert(nhist2 > 1); + + /* + * Fast-forward past hist1 entries that are entirely below hist2[0], and + * vice versa. Bounds checks prevent out-of-bounds access when the + * histograms are fully disjoint. + */ + for (i = 0; i < nhist1 && + range_cmp_bounds(typcache, &hist1[i], &hist2[0]) < 0; i++) + ; + for (j = 0; j < nhist2 && + range_cmp_bounds(typcache, &hist2[j], &hist1[0]) < 0; j++) + ; + + /* + * Handle fully-separated histograms. When all bounds in hist1 are below + * all bounds in hist2, P(X < Y) is ~1.0. When all of hist2 is below + * hist1, P(X < Y) is ~0.0. We return immediately rather than falling + * into the overlap walk with invalid indices. + */ + if (i >= nhist1) + return 1.0; + if (j >= nhist2) + return 0.0; + + /* Walk the overlapping region of both histograms */ + while (i < nhist1 && j < nhist2) + { + double cur_sel1, + cur_sel2; + RangeBound cur_sync; + int cmp; + + cmp = range_cmp_bounds(typcache, &hist1[i], &hist2[j]); + if (cmp < 0) + cur_sync = hist1[i++]; + else if (cmp > 0) + cur_sync = hist2[j++]; + else + { + /* Equal bounds: advance both */ + cur_sync = hist1[i]; + i++; + j++; + } + cur_sel1 = calc_hist_selectivity_scalar(typcache, &cur_sync, + hist1, nhist1, false); + cur_sel2 = calc_hist_selectivity_scalar(typcache, &cur_sync, + hist2, nhist2, false); + + /* Skip the first iteration (no previous point yet) */ + if (prev_sel1 >= 0) + selectivity += (prev_sel1 + cur_sel1) * (cur_sel2 - prev_sel2); + + prev_sel1 = cur_sel1; + prev_sel2 = cur_sel2; + } + + /* P(X < Y) = 0.5 * Sum(...) */ + selectivity /= 2; + + /* Include remainder of hist2 if hist1 was exhausted first */ + if (j < nhist2) + selectivity += 1 - prev_sel2; + + return selectivity; +} + +/* + * rangejoinsel -- join selectivity for range-vs-range operators + * + * Supports: <<, >>, && + * These operators map directly to strict bound comparisons P(X < Y), + * which calc_hist_join_selectivity() estimates from bound histograms. + * Other range operators are left to their existing generic estimators. + */ +Datum +rangejoinsel(PG_FUNCTION_ARGS) +{ + PlannerInfo *root = (PlannerInfo *) PG_GETARG_POINTER(0); + Oid operator = PG_GETARG_OID(1); + List *args = (List *) PG_GETARG_POINTER(2); + SpecialJoinInfo *sjinfo = (SpecialJoinInfo *) PG_GETARG_POINTER(4); + VariableStatData vardata1; + VariableStatData vardata2; + Selectivity selec; + AttStatsSlot hist1; + AttStatsSlot hist2; + AttStatsSlot sslot; + bool have_hist1 = false; + bool have_hist2 = false; + TypeCacheEntry *typcache; + Form_pg_statistic stats1; + Form_pg_statistic stats2; + double empty_frac1; + double empty_frac2; + double null_frac1; + double null_frac2; + int nhist1; + int nhist2; + RangeBound *hist1_lower; + RangeBound *hist1_upper; + RangeBound *hist2_lower; + RangeBound *hist2_upper; + bool join_is_reversed; + bool empty; + int i; + + get_join_variables(root, args, sjinfo, &vardata1, &vardata2, + &join_is_reversed); + + selec = default_range_selectivity(operator); + + /* + * Acquire histogram stats for both sides. Each slot is tracked + * independently so we can release exactly what was acquired on any + * failure path. + */ + if (!HeapTupleIsValid(vardata1.statsTuple) || + !HeapTupleIsValid(vardata2.statsTuple)) + goto cleanup; + + if (vardata1.vartype != vardata2.vartype) + goto cleanup; + + memset(&hist1, 0, sizeof(hist1)); + memset(&hist2, 0, sizeof(hist2)); + + if (!get_attstatsslot(&hist1, vardata1.statsTuple, + STATISTIC_KIND_BOUNDS_HISTOGRAM, InvalidOid, + ATTSTATSSLOT_VALUES)) + goto cleanup; + have_hist1 = true; + + if (!get_attstatsslot(&hist2, vardata2.statsTuple, + STATISTIC_KIND_BOUNDS_HISTOGRAM, InvalidOid, + ATTSTATSSLOT_VALUES)) + goto cleanup; + have_hist2 = true; + + /* Initialize type cache */ + typcache = range_get_typcache(fcinfo, vardata1.vartype); + + /* Look up NULL and empty-range fractions */ + stats1 = (Form_pg_statistic) GETSTRUCT(vardata1.statsTuple); + stats2 = (Form_pg_statistic) GETSTRUCT(vardata2.statsTuple); + + null_frac1 = stats1->stanullfrac; + null_frac2 = stats2->stanullfrac; + + /* Try to get fraction of empty ranges for the first variable */ + if (get_attstatsslot(&sslot, vardata1.statsTuple, + STATISTIC_KIND_RANGE_LENGTH_HISTOGRAM, + InvalidOid, ATTSTATSSLOT_NUMBERS)) + { + if (sslot.nnumbers != 1) + elog(ERROR, "invalid empty fraction statistic"); + empty_frac1 = sslot.numbers[0]; + free_attstatsslot(&sslot); + } + else + { + empty_frac1 = 0.0; + } + + /* Try to get fraction of empty ranges for the second variable */ + if (get_attstatsslot(&sslot, vardata2.statsTuple, + STATISTIC_KIND_RANGE_LENGTH_HISTOGRAM, + InvalidOid, ATTSTATSSLOT_NUMBERS)) + { + if (sslot.nnumbers != 1) + elog(ERROR, "invalid empty fraction statistic"); + empty_frac2 = sslot.numbers[0]; + free_attstatsslot(&sslot); + } + else + { + empty_frac2 = 0.0; + } + + /* Convert range histograms to separate lower/upper bound arrays */ + nhist1 = hist1.nvalues; + hist1_lower = (RangeBound *) palloc(sizeof(RangeBound) * nhist1); + hist1_upper = (RangeBound *) palloc(sizeof(RangeBound) * nhist1); + for (i = 0; i < nhist1; i++) + { + range_deserialize(typcache, DatumGetRangeTypeP(hist1.values[i]), + &hist1_lower[i], &hist1_upper[i], &empty); + if (empty) + elog(ERROR, "bounds histogram contains an empty range"); + } + + nhist2 = hist2.nvalues; + hist2_lower = (RangeBound *) palloc(sizeof(RangeBound) * nhist2); + hist2_upper = (RangeBound *) palloc(sizeof(RangeBound) * nhist2); + for (i = 0; i < nhist2; i++) + { + range_deserialize(typcache, DatumGetRangeTypeP(hist2.values[i]), + &hist2_lower[i], &hist2_upper[i], &empty); + if (empty) + elog(ERROR, "bounds histogram contains an empty range"); + } + + /* Estimate selectivity based on the operator */ + switch (operator) + { + case OID_RANGE_OVERLAP_OP: + + /* + * A && B iff NOT(A << B) AND NOT(A >> B) = 1 - P(A.upper < + * B.lower) - P(B.upper < A.lower) + */ + selec = 1; + selec -= calc_hist_join_selectivity(typcache, + hist1_upper, nhist1, + hist2_lower, nhist2); + selec -= calc_hist_join_selectivity(typcache, + hist2_upper, nhist2, + hist1_lower, nhist1); + break; + + case OID_RANGE_LEFT_OP: + /* A << B iff upper(A) < lower(B) */ + selec = calc_hist_join_selectivity(typcache, + hist1_upper, nhist1, + hist2_lower, nhist2); + break; + + case OID_RANGE_RIGHT_OP: + /* A >> B iff upper(B) < lower(A) */ + selec = calc_hist_join_selectivity(typcache, + hist2_upper, nhist2, + hist1_lower, nhist1); + break; + + default: + /* Unsupported operator; keep the default selectivity */ + goto cleanup; + } + + /* The histogram-based selectivity applies to non-empty ranges only */ + selec *= (1 - empty_frac1) * (1 - empty_frac2); + + /* + * For the supported operators (<<, >>, &&), empty ranges always produce + * false, so no empty-fraction adjustment is needed. + */ + + /* All range operators are strict */ + selec *= (1 - null_frac1) * (1 - null_frac2); + +cleanup: + if (have_hist2) + free_attstatsslot(&hist2); + if (have_hist1) + free_attstatsslot(&hist1); + + ReleaseVariableStats(vardata1); + ReleaseVariableStats(vardata2); + + CLAMP_PROBABILITY(selec); + + PG_RETURN_FLOAT8((float8) selec); +} diff --git a/src/include/catalog/pg_operator.dat b/src/include/catalog/pg_operator.dat index 1465f13120a..5ea4434f9fa 100644 --- a/src/include/catalog/pg_operator.dat +++ b/src/include/catalog/pg_operator.dat @@ -3094,7 +3094,7 @@ oprname => '&&', oprleft => 'anyrange', oprright => 'anyrange', oprresult => 'bool', oprcom => '&&(anyrange,anyrange)', oprcode => 'range_overlaps', oprrest => 'rangesel', - oprjoin => 'areajoinsel' }, + oprjoin => 'rangejoinsel' }, { oid => '3889', oid_symbol => 'OID_RANGE_CONTAINS_ELEM_OP', descr => 'contains', oprname => '@>', oprleft => 'anyrange', oprright => 'anyelement', @@ -3122,12 +3122,12 @@ oprname => '<<', oprleft => 'anyrange', oprright => 'anyrange', oprresult => 'bool', oprcom => '>>(anyrange,anyrange)', oprcode => 'range_before', oprrest => 'rangesel', - oprjoin => 'scalarltjoinsel' }, + oprjoin => 'rangejoinsel' }, { oid => '3894', oid_symbol => 'OID_RANGE_RIGHT_OP', descr => 'is right of', oprname => '>>', oprleft => 'anyrange', oprright => 'anyrange', oprresult => 'bool', oprcom => '<<(anyrange,anyrange)', oprcode => 'range_after', oprrest => 'rangesel', - oprjoin => 'scalargtjoinsel' }, + oprjoin => 'rangejoinsel' }, { oid => '3895', oid_symbol => 'OID_RANGE_OVERLAPS_LEFT_OP', descr => 'overlaps or is left of', oprname => '&<', oprleft => 'anyrange', oprright => 'anyrange', diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 99fa9a6ede2..c6a707acae4 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -12919,4 +12919,8 @@ proname => 'hashoid8extended', prorettype => 'int8', proargtypes => 'oid8 int8', prosrc => 'hashoid8extended' }, +{ oid => '8355', descr => 'join selectivity for range operators', + proname => 'rangejoinsel', provolatile => 's', prorettype => 'float8', + proargtypes => 'internal oid internal int2 internal', + prosrc => 'rangejoinsel' }, ] diff --git a/src/test/regress/expected/rangetypes.out b/src/test/regress/expected/rangetypes.out index e062a4e5c2c..2fc5b770f90 100644 --- a/src/test/regress/expected/rangetypes.out +++ b/src/test/regress/expected/rangetypes.out @@ -2033,3 +2033,117 @@ select * from text_support_test where t <@ textrange_supp('a', 'd'); drop table text_support_test; drop type textrange_supp; +-- +-- test selectivity of range join operators +-- +create table test_range_join_1 (ir1 int4range); +create table test_range_join_2 (ir2 int4range); +create table test_range_join_3 (ir3 int4range); +insert into test_range_join_1 select int4range(g, g+10) from generate_series(1, 1000) g; +insert into test_range_join_1 select int4range(g, g+100) from generate_series(1, 1000, 10) g; +insert into test_range_join_2 select int4range(g, g+10) from generate_series(1, 500) g; +insert into test_range_join_2 select int4range(g, g+100) from generate_series(1, 500, 10) g; +insert into test_range_join_3 select int4range(g, g+10) from generate_series(501, 1000) g; +insert into test_range_join_3 select int4range(g, g+100) from generate_series(501, 1000, 10) g; +analyze test_range_join_1; +analyze test_range_join_2; +analyze test_range_join_3; +-- reorder joins based on computed selectivity +explain (costs off) select count(*) from test_range_join_1, test_range_join_2, test_range_join_3 where ir1 && ir2 and ir2 && ir3; + QUERY PLAN +----------------------------------------------------------------------------------- + Aggregate + -> Nested Loop + Join Filter: (test_range_join_1.ir1 && test_range_join_2.ir2) + -> Seq Scan on test_range_join_1 + -> Materialize + -> Nested Loop + Join Filter: (test_range_join_2.ir2 && test_range_join_3.ir3) + -> Seq Scan on test_range_join_2 + -> Materialize + -> Seq Scan on test_range_join_3 +(10 rows) + +explain (costs off) select count(*) from test_range_join_1, test_range_join_2, test_range_join_3 where ir1 << ir2 and ir2 << ir3; + QUERY PLAN +----------------------------------------------------------------------------- + Aggregate + -> Nested Loop + Join Filter: (test_range_join_2.ir2 << test_range_join_3.ir3) + -> Nested Loop + Join Filter: (test_range_join_1.ir1 << test_range_join_2.ir2) + -> Seq Scan on test_range_join_1 + -> Materialize + -> Seq Scan on test_range_join_2 + -> Materialize + -> Seq Scan on test_range_join_3 +(10 rows) + +explain (costs off) select count(*) from test_range_join_1, test_range_join_2, test_range_join_3 where ir1 >> ir2 and ir2 >> ir3; + QUERY PLAN +----------------------------------------------------------------------------- + Aggregate + -> Nested Loop + Join Filter: (test_range_join_1.ir1 >> test_range_join_2.ir2) + -> Nested Loop + Join Filter: (test_range_join_2.ir2 >> test_range_join_3.ir3) + -> Seq Scan on test_range_join_2 + -> Materialize + -> Seq Scan on test_range_join_3 + -> Seq Scan on test_range_join_1 +(9 rows) + +drop table test_range_join_1; +drop table test_range_join_2; +drop table test_range_join_3; +-- +-- test range join selectivity with fully disjoint histograms +-- (exercises the bounds-check logic when histograms do not overlap) +-- +create table test_range_join_lo (r int4range); +create table test_range_join_hi (r int4range); +-- low ranges: [1,11), [2,12), ... [500,510) +insert into test_range_join_lo select int4range(g, g+10) from generate_series(1, 500) g; +-- high ranges: [10001,10011), [10002,10012), ... [10500,10510) +insert into test_range_join_hi select int4range(g, g+10) from generate_series(10001, 10500) g; +analyze test_range_join_lo; +analyze test_range_join_hi; +-- lo << hi should produce a large selectivity (most pairs match) +-- lo >> hi should produce a near-zero selectivity +-- lo && hi should produce a near-zero selectivity (no overlap) +-- These should not crash and should produce stable plans. +explain (costs off) select count(*) from test_range_join_lo a, test_range_join_hi b where a.r << b.r; + QUERY PLAN +---------------------------------------------------- + Aggregate + -> Nested Loop + Join Filter: (a.r << b.r) + -> Seq Scan on test_range_join_lo a + -> Materialize + -> Seq Scan on test_range_join_hi b +(6 rows) + +explain (costs off) select count(*) from test_range_join_lo a, test_range_join_hi b where a.r >> b.r; + QUERY PLAN +---------------------------------------------------- + Aggregate + -> Nested Loop + Join Filter: (a.r >> b.r) + -> Seq Scan on test_range_join_lo a + -> Materialize + -> Seq Scan on test_range_join_hi b +(6 rows) + +explain (costs off) select count(*) from test_range_join_lo a, test_range_join_hi b where a.r && b.r; + QUERY PLAN +---------------------------------------------------- + Aggregate + -> Nested Loop + Join Filter: (a.r && b.r) + -> Seq Scan on test_range_join_lo a + -> Materialize + -> Seq Scan on test_range_join_hi b +(6 rows) + +drop table test_range_join_lo; +drop table test_range_join_hi; diff --git a/src/test/regress/sql/rangetypes.sql b/src/test/regress/sql/rangetypes.sql index 5c4b0337b7a..f69109da334 100644 --- a/src/test/regress/sql/rangetypes.sql +++ b/src/test/regress/sql/rangetypes.sql @@ -708,3 +708,56 @@ select * from text_support_test where t <@ textrange_supp('a', 'd'); drop table text_support_test; drop type textrange_supp; + +-- +-- test selectivity of range join operators +-- +create table test_range_join_1 (ir1 int4range); +create table test_range_join_2 (ir2 int4range); +create table test_range_join_3 (ir3 int4range); + +insert into test_range_join_1 select int4range(g, g+10) from generate_series(1, 1000) g; +insert into test_range_join_1 select int4range(g, g+100) from generate_series(1, 1000, 10) g; +insert into test_range_join_2 select int4range(g, g+10) from generate_series(1, 500) g; +insert into test_range_join_2 select int4range(g, g+100) from generate_series(1, 500, 10) g; +insert into test_range_join_3 select int4range(g, g+10) from generate_series(501, 1000) g; +insert into test_range_join_3 select int4range(g, g+100) from generate_series(501, 1000, 10) g; + +analyze test_range_join_1; +analyze test_range_join_2; +analyze test_range_join_3; + +-- reorder joins based on computed selectivity +explain (costs off) select count(*) from test_range_join_1, test_range_join_2, test_range_join_3 where ir1 && ir2 and ir2 && ir3; +explain (costs off) select count(*) from test_range_join_1, test_range_join_2, test_range_join_3 where ir1 << ir2 and ir2 << ir3; +explain (costs off) select count(*) from test_range_join_1, test_range_join_2, test_range_join_3 where ir1 >> ir2 and ir2 >> ir3; + +drop table test_range_join_1; +drop table test_range_join_2; +drop table test_range_join_3; + +-- +-- test range join selectivity with fully disjoint histograms +-- (exercises the bounds-check logic when histograms do not overlap) +-- +create table test_range_join_lo (r int4range); +create table test_range_join_hi (r int4range); + +-- low ranges: [1,11), [2,12), ... [500,510) +insert into test_range_join_lo select int4range(g, g+10) from generate_series(1, 500) g; +-- high ranges: [10001,10011), [10002,10012), ... [10500,10510) +insert into test_range_join_hi select int4range(g, g+10) from generate_series(10001, 10500) g; + +analyze test_range_join_lo; +analyze test_range_join_hi; + +-- lo << hi should produce a large selectivity (most pairs match) +-- lo >> hi should produce a near-zero selectivity +-- lo && hi should produce a near-zero selectivity (no overlap) +-- These should not crash and should produce stable plans. +explain (costs off) select count(*) from test_range_join_lo a, test_range_join_hi b where a.r << b.r; +explain (costs off) select count(*) from test_range_join_lo a, test_range_join_hi b where a.r >> b.r; +explain (costs off) select count(*) from test_range_join_lo a, test_range_join_hi b where a.r && b.r; + +drop table test_range_join_lo; +drop table test_range_join_hi; -- 2.50.1 (Apple Git-155)