From ac1a0f23f7deb3a9a17bb8e150601ee9aaaafa46 Mon Sep 17 00:00:00 2001 From: Haibo Yan Date: Mon, 6 Apr 2026 09:30:10 -0700 Subject: [PATCH v5] Improve range/range join selectivity estimation Teach rangejoinsel to estimate selected range/range join operators using range histogram statistics instead of falling back to fixed defaults. This improves planner row estimates for operators such as <<, >>, &&, especially when the two range columns have clearly separated or strongly overlapping distributions. Regression tests cover plan changes for representative range join cases. --- src/backend/utils/adt/rangetypes_selfuncs.c | 301 ++++++++++++++++++++ src/include/catalog/pg_operator.dat | 6 +- src/include/catalog/pg_proc.dat | 5 + src/test/regress/expected/rangetypes.out | 114 ++++++++ src/test/regress/sql/rangetypes.sql | 53 ++++ 5 files changed, 476 insertions(+), 3 deletions(-) diff --git a/src/backend/utils/adt/rangetypes_selfuncs.c b/src/backend/utils/adt/rangetypes_selfuncs.c index 75f1e7567d5..97ae19fbcd2 100644 --- a/src/backend/utils/adt/rangetypes_selfuncs.c +++ b/src/backend/utils/adt/rangetypes_selfuncs.c @@ -1221,3 +1221,304 @@ calc_hist_selectivity_contains(TypeCacheEntry *typcache, return sum_frac; } + +/* + * Estimate join selectivity P(X < Y) using rangebound histograms. + * + * Based on: Diogo Repas, Zhicheng Luo, Maxime Schoemans, Mahmoud Sakr, 2022 + * "Selectivity Estimation of Inequality Joins In Databases" + * https://doi.org/10.48550/arXiv.2206.07396 + * + * hist1 and hist2 are arrays of RangeBound entries from the bounds histograms + * of two range-typed attributes X and Y, respectively. Each array has at + * least 2 entries (one histogram bin). The entries carry full bound metadata + * (lower/upper flag, inclusive/exclusive), and all comparisons use + * range_cmp_bounds() so that bound semantics are preserved. + * + * The algorithm models each attribute's distribution as a piecewise function + * derived from its histogram, then computes: + * P(X < Y) = 0.5 * sum( (F_X(prev) + F_X(cur)) * (F_Y(cur) - F_Y(prev)) ) + * by parallel-scanning both histograms. + * + * The initial fast-forward loops skip histogram entries that fall entirely + * before the other histogram's range, so the main loop only processes the + * overlapping region. Bounds checks are required because the histograms may + * be completely disjoint (e.g., all of X is below all of Y). + */ +static double +calc_hist_join_selectivity(TypeCacheEntry *typcache, + const RangeBound *hist1, int nhist1, + const RangeBound *hist2, int nhist2) +{ + int i, + j; + double selectivity = 0.0; + double prev_sel1 = -1.0; /* negative sentinel skips first iter */ + double prev_sel2 = 0.0; + + Assert(nhist1 > 1); + Assert(nhist2 > 1); + + /* + * Fast-forward past hist1 entries that are entirely below hist2[0], and + * vice versa. Bounds checks prevent out-of-bounds access when the + * histograms are fully disjoint. + */ + for (i = 0; i < nhist1 && + range_cmp_bounds(typcache, &hist1[i], &hist2[0]) < 0; i++) + ; + for (j = 0; j < nhist2 && + range_cmp_bounds(typcache, &hist2[j], &hist1[0]) < 0; j++) + ; + + /* + * Handle fully-separated histograms. When all bounds in hist1 are below + * all bounds in hist2, P(X < Y) is ~1.0. When all of hist2 is below + * hist1, P(X < Y) is ~0.0. We return immediately rather than falling + * into the overlap walk with invalid indices. + */ + if (i >= nhist1) + return 1.0; + if (j >= nhist2) + return 0.0; + + /* Walk the overlapping region of both histograms */ + while (i < nhist1 && j < nhist2) + { + double cur_sel1, + cur_sel2; + RangeBound cur_sync; + + if (range_cmp_bounds(typcache, &hist1[i], &hist2[j]) < 0) + cur_sync = hist1[i++]; + else if (range_cmp_bounds(typcache, &hist1[i], &hist2[j]) > 0) + cur_sync = hist2[j++]; + else + { + /* Equal bounds: advance both */ + cur_sync = hist1[i]; + i++; + j++; + } + cur_sel1 = calc_hist_selectivity_scalar(typcache, &cur_sync, + hist1, nhist1, false); + cur_sel2 = calc_hist_selectivity_scalar(typcache, &cur_sync, + hist2, nhist2, false); + + /* Skip the first iteration (no previous point yet) */ + if (prev_sel1 >= 0) + selectivity += (prev_sel1 + cur_sel1) * (cur_sel2 - prev_sel2); + + prev_sel1 = cur_sel1; + prev_sel2 = cur_sel2; + } + + /* P(X < Y) = 0.5 * Sum(...) */ + selectivity /= 2; + + /* Include remainder of hist2 if hist1 was exhausted first */ + if (j < nhist2) + selectivity += 1 - prev_sel2; + + return selectivity; +} + +/* + * rangejoinsel -- join selectivity for range-vs-range operators + * + * Supports: <<, >>, && + * These operators map directly to strict bound comparisons P(X < Y), + * which calc_hist_join_selectivity() estimates from bound histograms. + * Other range operators are left to their existing generic estimators. + */ +Datum +rangejoinsel(PG_FUNCTION_ARGS) +{ + PlannerInfo *root = (PlannerInfo *) PG_GETARG_POINTER(0); + Oid operator = PG_GETARG_OID(1); + List *args = (List *) PG_GETARG_POINTER(2); + SpecialJoinInfo *sjinfo = (SpecialJoinInfo *) PG_GETARG_POINTER(4); + VariableStatData vardata1; + VariableStatData vardata2; + Selectivity selec; + AttStatsSlot hist1; + AttStatsSlot hist2; + AttStatsSlot sslot; + bool have_hist1 = false; + bool have_hist2 = false; + TypeCacheEntry *typcache; + Form_pg_statistic stats1; + Form_pg_statistic stats2; + double empty_frac1; + double empty_frac2; + double null_frac1; + double null_frac2; + int nhist1; + int nhist2; + RangeBound *hist1_lower; + RangeBound *hist1_upper; + RangeBound *hist2_lower; + RangeBound *hist2_upper; + bool empty; + int i; + + { + bool join_is_reversed; + + get_join_variables(root, args, sjinfo, &vardata1, &vardata2, + &join_is_reversed); + } + + selec = default_range_selectivity(operator); + + /* + * Acquire histogram stats for both sides. Each slot is tracked + * independently so we can release exactly what was acquired on any + * failure path. + */ + if (!HeapTupleIsValid(vardata1.statsTuple) || + !HeapTupleIsValid(vardata2.statsTuple)) + goto cleanup; + + if (vardata1.vartype != vardata2.vartype) + goto cleanup; + + memset(&hist1, 0, sizeof(hist1)); + memset(&hist2, 0, sizeof(hist2)); + + if (!get_attstatsslot(&hist1, vardata1.statsTuple, + STATISTIC_KIND_BOUNDS_HISTOGRAM, InvalidOid, + ATTSTATSSLOT_VALUES)) + goto cleanup; + have_hist1 = true; + + if (!get_attstatsslot(&hist2, vardata2.statsTuple, + STATISTIC_KIND_BOUNDS_HISTOGRAM, InvalidOid, + ATTSTATSSLOT_VALUES)) + goto cleanup; + have_hist2 = true; + + /* Initialize type cache */ + typcache = range_get_typcache(fcinfo, vardata1.vartype); + + /* Look up NULL and empty-range fractions */ + stats1 = (Form_pg_statistic) GETSTRUCT(vardata1.statsTuple); + stats2 = (Form_pg_statistic) GETSTRUCT(vardata2.statsTuple); + + null_frac1 = stats1->stanullfrac; + null_frac2 = stats2->stanullfrac; + + /* Try to get fraction of empty ranges for the first variable */ + if (get_attstatsslot(&sslot, vardata1.statsTuple, + STATISTIC_KIND_RANGE_LENGTH_HISTOGRAM, + InvalidOid, ATTSTATSSLOT_NUMBERS)) + { + if (sslot.nnumbers != 1) + elog(ERROR, "invalid empty fraction statistic"); + empty_frac1 = sslot.numbers[0]; + free_attstatsslot(&sslot); + } + else + { + empty_frac1 = 0.0; + } + + /* Try to get fraction of empty ranges for the second variable */ + if (get_attstatsslot(&sslot, vardata2.statsTuple, + STATISTIC_KIND_RANGE_LENGTH_HISTOGRAM, + InvalidOid, ATTSTATSSLOT_NUMBERS)) + { + if (sslot.nnumbers != 1) + elog(ERROR, "invalid empty fraction statistic"); + empty_frac2 = sslot.numbers[0]; + free_attstatsslot(&sslot); + } + else + { + empty_frac2 = 0.0; + } + + /* Convert range histograms to separate lower/upper bound arrays */ + nhist1 = hist1.nvalues; + hist1_lower = (RangeBound *) palloc(sizeof(RangeBound) * nhist1); + hist1_upper = (RangeBound *) palloc(sizeof(RangeBound) * nhist1); + for (i = 0; i < nhist1; i++) + { + range_deserialize(typcache, DatumGetRangeTypeP(hist1.values[i]), + &hist1_lower[i], &hist1_upper[i], &empty); + if (empty) + elog(ERROR, "bounds histogram contains an empty range"); + } + + nhist2 = hist2.nvalues; + hist2_lower = (RangeBound *) palloc(sizeof(RangeBound) * nhist2); + hist2_upper = (RangeBound *) palloc(sizeof(RangeBound) * nhist2); + for (i = 0; i < nhist2; i++) + { + range_deserialize(typcache, DatumGetRangeTypeP(hist2.values[i]), + &hist2_lower[i], &hist2_upper[i], &empty); + if (empty) + elog(ERROR, "bounds histogram contains an empty range"); + } + + /* Estimate selectivity based on the operator */ + switch (operator) + { + case OID_RANGE_OVERLAP_OP: + + /* + * A && B iff NOT(A << B) AND NOT(A >> B) + * = 1 - P(A.upper < B.lower) - P(B.upper < A.lower) + */ + selec = 1; + selec -= calc_hist_join_selectivity(typcache, + hist1_upper, nhist1, + hist2_lower, nhist2); + selec -= calc_hist_join_selectivity(typcache, + hist2_upper, nhist2, + hist1_lower, nhist1); + break; + + case OID_RANGE_LEFT_OP: + /* A << B iff upper(A) < lower(B) */ + selec = calc_hist_join_selectivity(typcache, + hist1_upper, nhist1, + hist2_lower, nhist2); + break; + + case OID_RANGE_RIGHT_OP: + /* A >> B iff upper(B) < lower(A) */ + selec = calc_hist_join_selectivity(typcache, + hist2_upper, nhist2, + hist1_lower, nhist1); + break; + + default: + /* Unsupported operator; keep the default selectivity */ + goto cleanup; + } + + /* The histogram-based selectivity applies to non-empty ranges only */ + selec *= (1 - empty_frac1) * (1 - empty_frac2); + + /* + * For the supported operators (<<, >>, &&), empty ranges always produce + * false, so no empty-fraction adjustment is needed. + */ + + /* All range operators are strict */ + selec *= (1 - null_frac1) * (1 - null_frac2); + +cleanup: + if (have_hist2) + free_attstatsslot(&hist2); + if (have_hist1) + free_attstatsslot(&hist1); + + ReleaseVariableStats(vardata1); + ReleaseVariableStats(vardata2); + + CLAMP_PROBABILITY(selec); + + PG_RETURN_FLOAT8((float8) selec); +} diff --git a/src/include/catalog/pg_operator.dat b/src/include/catalog/pg_operator.dat index 1465f13120a..5ea4434f9fa 100644 --- a/src/include/catalog/pg_operator.dat +++ b/src/include/catalog/pg_operator.dat @@ -3094,7 +3094,7 @@ oprname => '&&', oprleft => 'anyrange', oprright => 'anyrange', oprresult => 'bool', oprcom => '&&(anyrange,anyrange)', oprcode => 'range_overlaps', oprrest => 'rangesel', - oprjoin => 'areajoinsel' }, + oprjoin => 'rangejoinsel' }, { oid => '3889', oid_symbol => 'OID_RANGE_CONTAINS_ELEM_OP', descr => 'contains', oprname => '@>', oprleft => 'anyrange', oprright => 'anyelement', @@ -3122,12 +3122,12 @@ oprname => '<<', oprleft => 'anyrange', oprright => 'anyrange', oprresult => 'bool', oprcom => '>>(anyrange,anyrange)', oprcode => 'range_before', oprrest => 'rangesel', - oprjoin => 'scalarltjoinsel' }, + oprjoin => 'rangejoinsel' }, { oid => '3894', oid_symbol => 'OID_RANGE_RIGHT_OP', descr => 'is right of', oprname => '>>', oprleft => 'anyrange', oprright => 'anyrange', oprresult => 'bool', oprcom => '<<(anyrange,anyrange)', oprcode => 'range_after', oprrest => 'rangesel', - oprjoin => 'scalargtjoinsel' }, + oprjoin => 'rangejoinsel' }, { oid => '3895', oid_symbol => 'OID_RANGE_OVERLAPS_LEFT_OP', descr => 'overlaps or is left of', oprname => '&<', oprleft => 'anyrange', oprright => 'anyrange', diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 3ea17fc5629..c16aa8cec84 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -12787,6 +12787,11 @@ proname => 'error_on_null', proisstrict => 'f', prorettype => 'anyelement', proargtypes => 'anyelement', prosrc => 'pg_error_on_null' }, +{ oid => '8355', descr => 'join selectivity for range operators', + proname => 'rangejoinsel', provolatile => 's', prorettype => 'float8', + proargtypes => 'internal oid internal int2 internal', + prosrc => 'rangejoinsel' }, + { oid => '6321', descr => 'list of available WAL summary files', proname => 'pg_available_wal_summaries', prorows => '100', proretset => 't', provolatile => 'v', prorettype => 'record', proargtypes => '', diff --git a/src/test/regress/expected/rangetypes.out b/src/test/regress/expected/rangetypes.out index e062a4e5c2c..2fc5b770f90 100644 --- a/src/test/regress/expected/rangetypes.out +++ b/src/test/regress/expected/rangetypes.out @@ -2033,3 +2033,117 @@ select * from text_support_test where t <@ textrange_supp('a', 'd'); drop table text_support_test; drop type textrange_supp; +-- +-- test selectivity of range join operators +-- +create table test_range_join_1 (ir1 int4range); +create table test_range_join_2 (ir2 int4range); +create table test_range_join_3 (ir3 int4range); +insert into test_range_join_1 select int4range(g, g+10) from generate_series(1, 1000) g; +insert into test_range_join_1 select int4range(g, g+100) from generate_series(1, 1000, 10) g; +insert into test_range_join_2 select int4range(g, g+10) from generate_series(1, 500) g; +insert into test_range_join_2 select int4range(g, g+100) from generate_series(1, 500, 10) g; +insert into test_range_join_3 select int4range(g, g+10) from generate_series(501, 1000) g; +insert into test_range_join_3 select int4range(g, g+100) from generate_series(501, 1000, 10) g; +analyze test_range_join_1; +analyze test_range_join_2; +analyze test_range_join_3; +-- reorder joins based on computed selectivity +explain (costs off) select count(*) from test_range_join_1, test_range_join_2, test_range_join_3 where ir1 && ir2 and ir2 && ir3; + QUERY PLAN +----------------------------------------------------------------------------------- + Aggregate + -> Nested Loop + Join Filter: (test_range_join_1.ir1 && test_range_join_2.ir2) + -> Seq Scan on test_range_join_1 + -> Materialize + -> Nested Loop + Join Filter: (test_range_join_2.ir2 && test_range_join_3.ir3) + -> Seq Scan on test_range_join_2 + -> Materialize + -> Seq Scan on test_range_join_3 +(10 rows) + +explain (costs off) select count(*) from test_range_join_1, test_range_join_2, test_range_join_3 where ir1 << ir2 and ir2 << ir3; + QUERY PLAN +----------------------------------------------------------------------------- + Aggregate + -> Nested Loop + Join Filter: (test_range_join_2.ir2 << test_range_join_3.ir3) + -> Nested Loop + Join Filter: (test_range_join_1.ir1 << test_range_join_2.ir2) + -> Seq Scan on test_range_join_1 + -> Materialize + -> Seq Scan on test_range_join_2 + -> Materialize + -> Seq Scan on test_range_join_3 +(10 rows) + +explain (costs off) select count(*) from test_range_join_1, test_range_join_2, test_range_join_3 where ir1 >> ir2 and ir2 >> ir3; + QUERY PLAN +----------------------------------------------------------------------------- + Aggregate + -> Nested Loop + Join Filter: (test_range_join_1.ir1 >> test_range_join_2.ir2) + -> Nested Loop + Join Filter: (test_range_join_2.ir2 >> test_range_join_3.ir3) + -> Seq Scan on test_range_join_2 + -> Materialize + -> Seq Scan on test_range_join_3 + -> Seq Scan on test_range_join_1 +(9 rows) + +drop table test_range_join_1; +drop table test_range_join_2; +drop table test_range_join_3; +-- +-- test range join selectivity with fully disjoint histograms +-- (exercises the bounds-check logic when histograms do not overlap) +-- +create table test_range_join_lo (r int4range); +create table test_range_join_hi (r int4range); +-- low ranges: [1,11), [2,12), ... [500,510) +insert into test_range_join_lo select int4range(g, g+10) from generate_series(1, 500) g; +-- high ranges: [10001,10011), [10002,10012), ... [10500,10510) +insert into test_range_join_hi select int4range(g, g+10) from generate_series(10001, 10500) g; +analyze test_range_join_lo; +analyze test_range_join_hi; +-- lo << hi should produce a large selectivity (most pairs match) +-- lo >> hi should produce a near-zero selectivity +-- lo && hi should produce a near-zero selectivity (no overlap) +-- These should not crash and should produce stable plans. +explain (costs off) select count(*) from test_range_join_lo a, test_range_join_hi b where a.r << b.r; + QUERY PLAN +---------------------------------------------------- + Aggregate + -> Nested Loop + Join Filter: (a.r << b.r) + -> Seq Scan on test_range_join_lo a + -> Materialize + -> Seq Scan on test_range_join_hi b +(6 rows) + +explain (costs off) select count(*) from test_range_join_lo a, test_range_join_hi b where a.r >> b.r; + QUERY PLAN +---------------------------------------------------- + Aggregate + -> Nested Loop + Join Filter: (a.r >> b.r) + -> Seq Scan on test_range_join_lo a + -> Materialize + -> Seq Scan on test_range_join_hi b +(6 rows) + +explain (costs off) select count(*) from test_range_join_lo a, test_range_join_hi b where a.r && b.r; + QUERY PLAN +---------------------------------------------------- + Aggregate + -> Nested Loop + Join Filter: (a.r && b.r) + -> Seq Scan on test_range_join_lo a + -> Materialize + -> Seq Scan on test_range_join_hi b +(6 rows) + +drop table test_range_join_lo; +drop table test_range_join_hi; diff --git a/src/test/regress/sql/rangetypes.sql b/src/test/regress/sql/rangetypes.sql index 5c4b0337b7a..f69109da334 100644 --- a/src/test/regress/sql/rangetypes.sql +++ b/src/test/regress/sql/rangetypes.sql @@ -708,3 +708,56 @@ select * from text_support_test where t <@ textrange_supp('a', 'd'); drop table text_support_test; drop type textrange_supp; + +-- +-- test selectivity of range join operators +-- +create table test_range_join_1 (ir1 int4range); +create table test_range_join_2 (ir2 int4range); +create table test_range_join_3 (ir3 int4range); + +insert into test_range_join_1 select int4range(g, g+10) from generate_series(1, 1000) g; +insert into test_range_join_1 select int4range(g, g+100) from generate_series(1, 1000, 10) g; +insert into test_range_join_2 select int4range(g, g+10) from generate_series(1, 500) g; +insert into test_range_join_2 select int4range(g, g+100) from generate_series(1, 500, 10) g; +insert into test_range_join_3 select int4range(g, g+10) from generate_series(501, 1000) g; +insert into test_range_join_3 select int4range(g, g+100) from generate_series(501, 1000, 10) g; + +analyze test_range_join_1; +analyze test_range_join_2; +analyze test_range_join_3; + +-- reorder joins based on computed selectivity +explain (costs off) select count(*) from test_range_join_1, test_range_join_2, test_range_join_3 where ir1 && ir2 and ir2 && ir3; +explain (costs off) select count(*) from test_range_join_1, test_range_join_2, test_range_join_3 where ir1 << ir2 and ir2 << ir3; +explain (costs off) select count(*) from test_range_join_1, test_range_join_2, test_range_join_3 where ir1 >> ir2 and ir2 >> ir3; + +drop table test_range_join_1; +drop table test_range_join_2; +drop table test_range_join_3; + +-- +-- test range join selectivity with fully disjoint histograms +-- (exercises the bounds-check logic when histograms do not overlap) +-- +create table test_range_join_lo (r int4range); +create table test_range_join_hi (r int4range); + +-- low ranges: [1,11), [2,12), ... [500,510) +insert into test_range_join_lo select int4range(g, g+10) from generate_series(1, 500) g; +-- high ranges: [10001,10011), [10002,10012), ... [10500,10510) +insert into test_range_join_hi select int4range(g, g+10) from generate_series(10001, 10500) g; + +analyze test_range_join_lo; +analyze test_range_join_hi; + +-- lo << hi should produce a large selectivity (most pairs match) +-- lo >> hi should produce a near-zero selectivity +-- lo && hi should produce a near-zero selectivity (no overlap) +-- These should not crash and should produce stable plans. +explain (costs off) select count(*) from test_range_join_lo a, test_range_join_hi b where a.r << b.r; +explain (costs off) select count(*) from test_range_join_lo a, test_range_join_hi b where a.r >> b.r; +explain (costs off) select count(*) from test_range_join_lo a, test_range_join_hi b where a.r && b.r; + +drop table test_range_join_lo; +drop table test_range_join_hi; -- 2.52.0