This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new c200303022d fix: modify ExpressionFilter.canVectorizeMatcher and
AggregatorUtil.canVectorize to use ExpressionPlanner.plan (#19245)
c200303022d is described below
commit c200303022da0d3bf16422447d2bfc429a2a75a2
Author: Clint Wylie <[email protected]>
AuthorDate: Wed Apr 1 08:58:45 2026 -0700
fix: modify ExpressionFilter.canVectorizeMatcher and
AggregatorUtil.canVectorize to use ExpressionPlanner.plan (#19245)
changes:
* `ExpressionFilter.canVectorizeMatcher` and `AggregatorUtil.canVectorize`
now use `ExpressionPlanner.plan` to check for `Trait.VECTORIZABLE` instead of
calling `Expr.canVectorize` directly
* updated javadoc for `Expr.canVectorize` to indicate that it alone isn't
sufficient to decide if vector processing should be used
* update `BaseFilterTest.assertFilterMatchesSkipVectorize` to now assert
that the cursor factory is not vectorizable (or not a
`ColumnarFrameCursorFactory`) so that we do not accidentally skip vectorization
for things that are vectorizable, which would have caught the bug for
`ExpressionFilter`, fixed up tests that were incorrectly skipping vector
coverage
* update `AggregatorUtilTest` to cover `canVectorize`
---
.../druid/query/filter/BloomDimFilterTest.java | 12 +--
.../main/java/org/apache/druid/math/expr/Expr.java | 11 ++-
.../druid/query/aggregation/AggregatorUtil.java | 5 +-
.../druid/segment/filter/ExpressionFilter.java | 5 +-
.../query/aggregation/AggregatorUtilTest.java | 94 ++++++++++++++++++++++
.../filter/ArrayContainsElementFilterTests.java | 4 +-
.../druid/segment/filter/BaseFilterTest.java | 78 ++++++++++++------
.../druid/segment/filter/EqualityFilterTests.java | 4 +-
.../druid/segment/filter/ExpressionFilterTest.java | 42 +++++-----
.../filter/FloatAndDoubleFilteringTest.java | 6 +-
.../druid/segment/filter/JavaScriptFilterTest.java | 84 +++++++++----------
.../druid/segment/filter/LongFilteringTest.java | 4 +-
.../druid/segment/filter/TimeFilteringTest.java | 6 +-
13 files changed, 246 insertions(+), 109 deletions(-)
diff --git
a/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/filter/BloomDimFilterTest.java
b/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/filter/BloomDimFilterTest.java
index 82f4fcb621a..07e7d15931f 100644
---
a/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/filter/BloomDimFilterTest.java
+++
b/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/filter/BloomDimFilterTest.java
@@ -229,24 +229,24 @@ public class BloomDimFilterTest extends BaseFilterTest
@Test
public void testExpressionVirtualColumn() throws IOException
{
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
new BloomDimFilter("expr", bloomKFilter(1000, 1.1F), null),
List.of("0", "1", "2", "3", "4", "5")
);
- assertFilterMatchesSkipVectorize(new BloomDimFilter("expr",
bloomKFilter(1000, 1.2F), null), List.of());
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(new BloomDimFilter("expr", bloomKFilter(1000, 1.2F),
null), List.of());
+ assertFilterMatches(
new BloomDimFilter("exprDouble", bloomKFilter(1000, 2.1D), null),
List.of("0", "1", "2", "3", "4", "5")
);
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
new BloomDimFilter("exprDouble", bloomKFilter(1000, 2.2D), null),
List.of()
);
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
new BloomDimFilter("exprLong", bloomKFilter(1000, 3L), null),
List.of("0", "1", "2", "3", "4", "5")
);
- assertFilterMatchesSkipVectorize(new BloomDimFilter("exprLong",
bloomKFilter(1000, 4L), null), List.of());
+ assertFilterMatches(new BloomDimFilter("exprLong", bloomKFilter(1000, 4L),
null), List.of());
}
@Test
diff --git a/processing/src/main/java/org/apache/druid/math/expr/Expr.java
b/processing/src/main/java/org/apache/druid/math/expr/Expr.java
index 08e05c06579..53de7eabbc2 100644
--- a/processing/src/main/java/org/apache/druid/math/expr/Expr.java
+++ b/processing/src/main/java/org/apache/druid/math/expr/Expr.java
@@ -29,6 +29,7 @@ import org.apache.druid.java.util.common.ISE;
import org.apache.druid.math.expr.vector.ExprVectorProcessor;
import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.query.filter.ColumnIndexSelector;
+import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.ColumnType;
@@ -181,8 +182,14 @@ public interface Expr extends Cacheable
* Check if an expression can be 'vectorized', for a given set of inputs. If
this method returns true,
* {@link #asVectorProcessor} is expected to produce a {@link
ExprVectorProcessor} which can evaluate values in batches
* to use with vectorized query engines.
- *
- * @param inspector
+ * <p>
+ * Note that this method is insufficient by itself for determining to use
vector processors, as it only checks whether
+ * the expression tree itself supports vectorization, but ignores
schema-level constraints that
+ * {@link org.apache.druid.segment.virtual.ExpressionPlanner} enforces.
+ * <p>
+ * Most callers should instead prefer to use
+ * {@link
org.apache.druid.segment.virtual.ExpressionPlanner#plan(ColumnInspector, Expr)}
and check for
+ * {@link
org.apache.druid.segment.virtual.ExpressionPlan.Trait#VECTORIZABLE} on the
resulting plan.
*/
default boolean canVectorize(InputBindingInspector inspector)
{
diff --git
a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java
b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java
index 91d39402e77..6866052887a 100755
---
a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java
+++
b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java
@@ -41,6 +41,8 @@ import org.apache.druid.segment.column.Types;
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import org.apache.druid.segment.vector.VectorValueSelector;
+import org.apache.druid.segment.virtual.ExpressionPlan;
+import org.apache.druid.segment.virtual.ExpressionPlanner;
import org.apache.druid.segment.virtual.ExpressionSelectors;
import org.apache.druid.segment.virtual.ExpressionVectorSelectors;
@@ -392,7 +394,8 @@ public class AggregatorUtil
return capabilities == null || capabilities.isNumeric();
}
if (expression != null) {
- return fieldExpression.get().canVectorize(columnInspector);
+ ExpressionPlan plan = ExpressionPlanner.plan(columnInspector,
fieldExpression.get());
+ return plan.is(ExpressionPlan.Trait.VECTORIZABLE);
}
return false;
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java
b/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java
index 5b477d38a34..090acd3d1bf 100644
---
a/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java
+++
b/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java
@@ -48,6 +48,8 @@ import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.index.BitmapColumnIndex;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
+import org.apache.druid.segment.virtual.ExpressionPlan;
+import org.apache.druid.segment.virtual.ExpressionPlanner;
import org.apache.druid.segment.virtual.ExpressionSelectors;
import org.apache.druid.segment.virtual.ExpressionVectorSelectors;
@@ -72,7 +74,8 @@ public class ExpressionFilter implements Filter
@Override
public boolean canVectorizeMatcher(ColumnInspector inspector)
{
- return expr.get().canVectorize(inspector);
+ ExpressionPlan plan = ExpressionPlanner.plan(inspector, expr.get());
+ return plan.is(ExpressionPlan.Trait.VECTORIZABLE);
}
@Override
diff --git
a/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorUtilTest.java
b/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorUtilTest.java
index be97e65abd0..75aeb08d8a8 100644
---
a/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorUtilTest.java
+++
b/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorUtilTest.java
@@ -19,13 +19,21 @@
package org.apache.druid.query.aggregation;
+import com.google.common.base.Suppliers;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import org.apache.druid.java.util.common.Pair;
+import org.apache.druid.math.expr.Expr;
+import org.apache.druid.math.expr.Parser;
import org.apache.druid.query.QueryRunnerTestHelper;
import org.apache.druid.query.aggregation.post.ArithmeticPostAggregator;
import org.apache.druid.query.aggregation.post.ConstantPostAggregator;
import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator;
+import org.apache.druid.query.expression.TestExprMacroTable;
+import org.apache.druid.segment.ColumnInspector;
+import org.apache.druid.segment.column.ColumnCapabilities;
+import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
+import org.apache.druid.segment.column.ColumnType;
import org.junit.Assert;
import org.junit.Test;
@@ -33,6 +41,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
public class AggregatorUtilTest
{
@@ -199,4 +208,89 @@ public class AggregatorUtilTest
);
}
+ @Test
+ public void testCanVectorizeFieldNameWithNumericCapabilities()
+ {
+ ColumnInspector inspector = makeInspector(
+ Map.of("col",
ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.LONG))
+ );
+ Assert.assertTrue(AggregatorUtil.canVectorize(inspector, "col", null,
Suppliers.ofInstance(null)));
+ }
+
+ @Test
+ public void testCanVectorizeFieldNameWithNullCapabilities()
+ {
+ // null capabilities (unknown column) is treated as vectorizable
+ ColumnInspector inspector = makeInspector(Map.of());
+ Assert.assertTrue(AggregatorUtil.canVectorize(inspector, "unknown_col",
null, Suppliers.ofInstance(null)));
+ }
+
+ @Test
+ public void testCanVectorizeFieldNameWithStringCapabilities()
+ {
+ ColumnInspector inspector = makeInspector(
+ Map.of("col",
ColumnCapabilitiesImpl.createSimpleSingleValueStringColumnCapabilities())
+ );
+ Assert.assertFalse(AggregatorUtil.canVectorize(inspector, "col", null,
Suppliers.ofInstance(null)));
+ }
+
+ @Test
+ public void testCanVectorizeExpressionOnNumericColumns()
+ {
+ ColumnInspector inspector = makeInspector(
+ Map.of(
+ "long1",
ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.LONG),
+ "long2",
ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.LONG)
+ )
+ );
+ String expression = "long1 + long2";
+ Expr expr = Parser.parse(expression, TestExprMacroTable.INSTANCE);
+ Assert.assertTrue(AggregatorUtil.canVectorize(inspector, null, expression,
Suppliers.ofInstance(expr)));
+ }
+
+ @Test
+ public void testCanVectorizeExpressionWithIncompleteInputs()
+ {
+ // string column with unknown multi-valuedness results in
INCOMPLETE_INPUTS which blocks vectorization,
+ // even though the expression itself supports vectorization
+ ColumnInspector inspector = makeInspector(
+ Map.of(
+ "string_unknown", new
ColumnCapabilitiesImpl().setType(ColumnType.STRING)
+ )
+ );
+ String expression = "concat(string_unknown, 'x')";
+ Expr expr = Parser.parse(expression, TestExprMacroTable.INSTANCE);
+ Assert.assertFalse(AggregatorUtil.canVectorize(inspector, null,
expression, Suppliers.ofInstance(expr)));
+ }
+
+ @Test
+ public void testCanVectorizeExpressionWithNeedsApplied()
+ {
+ // multi-valued string column used in scalar context results in
NEEDS_APPLIED which blocks vectorization
+ ColumnInspector inspector = makeInspector(
+ Map.of(
+ "multi_string", new
ColumnCapabilitiesImpl().setType(ColumnType.STRING)
+
.setDictionaryEncoded(true)
+
.setHasBitmapIndexes(true)
+
.setDictionaryValuesUnique(true)
+
.setDictionaryValuesSorted(true)
+
.setHasMultipleValues(true)
+ )
+ );
+ String expression = "concat(multi_string, 'x')";
+ Expr expr = Parser.parse(expression, TestExprMacroTable.INSTANCE);
+ Assert.assertFalse(AggregatorUtil.canVectorize(inspector, null,
expression, Suppliers.ofInstance(expr)));
+ }
+
+ @Test
+ public void testCanVectorizeNeitherFieldNameNorExpression()
+ {
+ ColumnInspector inspector = makeInspector(Map.of());
+ Assert.assertFalse(AggregatorUtil.canVectorize(inspector, null, null,
Suppliers.ofInstance(null)));
+ }
+
+ private static ColumnInspector makeInspector(Map<String, ColumnCapabilities>
capabilitiesMap)
+ {
+ return capabilitiesMap::get;
+ }
}
diff --git
a/processing/src/test/java/org/apache/druid/segment/filter/ArrayContainsElementFilterTests.java
b/processing/src/test/java/org/apache/druid/segment/filter/ArrayContainsElementFilterTests.java
index 811fc646886..3e45aacd578 100644
---
a/processing/src/test/java/org/apache/druid/segment/filter/ArrayContainsElementFilterTests.java
+++
b/processing/src/test/java/org/apache/druid/segment/filter/ArrayContainsElementFilterTests.java
@@ -446,12 +446,12 @@ public class ArrayContainsElementFilterTests
{
// only auto schema supports array columns... skip other segment types
Assume.assumeFalse(testName.contains("frame (columnar)") ||
testName.contains("rowBasedWithoutTypeSignature"));
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
new ArrayContainsElementFilter("nestedArrayLong",
ColumnType.LONG_ARRAY, new Object[]{1L, 2L, 3L}, null),
ImmutableList.of("0", "2")
);
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
new ArrayContainsElementFilter("nestedArrayLong",
ColumnType.LONG_ARRAY, new Object[]{1L, 2L}, null),
ImmutableList.of()
);
diff --git
a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
index 70cd2fe4a1f..1644cf4720b 100644
---
a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
@@ -80,6 +80,7 @@ import org.apache.druid.segment.RowAdapters;
import org.apache.druid.segment.RowBasedColumnSelectorFactory;
import org.apache.druid.segment.RowBasedCursorFactory;
import org.apache.druid.segment.TestHelper;
+import org.apache.druid.segment.VirtualColumn;
import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature;
@@ -112,9 +113,11 @@ import org.junit.Rule;
import org.junit.rules.TemporaryFolder;
import org.junit.runners.Parameterized;
+import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.Closeable;
import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -122,6 +125,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Queue;
import java.util.Set;
import java.util.stream.Collectors;
@@ -984,11 +988,39 @@ public abstract class BaseFilterTest extends
InitializedNullHandlingTest
final DimFilter filter,
final String selectColumn
)
+ {
+ final Filter postFilteringFilter = makePostFilter(filter);
+
+ try (final CursorHolder cursorHolder =
cursorFactory.makeCursorHolder(makeVectorCursorBuildSpec(postFilteringFilter)))
{
+ final VectorCursor cursor = cursorHolder.asVectorCursor();
+ final SingleValueDimensionVectorSelector selector = cursor
+ .getColumnSelectorFactory()
+ .makeSingleValueDimensionSelector(new
DefaultDimensionSpec(selectColumn, selectColumn));
+
+ final List<String> values = new ArrayList<>();
+
+ while (!cursor.isDone()) {
+ cursor.advance();
+ }
+ cursor.reset();
+ while (!cursor.isDone()) {
+ final int[] rowVector = selector.getRowVector();
+ for (int i = 0; i < cursor.getCurrentVectorSize(); i++) {
+ values.add(selector.lookupName(rowVector[i]));
+ }
+ cursor.advance();
+ }
+
+ return values;
+ }
+ }
+
+ @Nonnull
+ private Filter makePostFilter(DimFilter filter)
{
final Filter theFilter = makeFilter(filter);
- final Filter postFilteringFilter = new Filter()
+ return new Filter()
{
-
@Override
public ValueMatcher makeMatcher(ColumnSelectorFactory factory)
{
@@ -1020,29 +1052,6 @@ public abstract class BaseFilterTest extends
InitializedNullHandlingTest
return null;
}
};
-
- try (final CursorHolder cursorHolder =
cursorFactory.makeCursorHolder(makeVectorCursorBuildSpec(postFilteringFilter)))
{
- final VectorCursor cursor = cursorHolder.asVectorCursor();
- final SingleValueDimensionVectorSelector selector = cursor
- .getColumnSelectorFactory()
- .makeSingleValueDimensionSelector(new
DefaultDimensionSpec(selectColumn, selectColumn));
-
- final List<String> values = new ArrayList<>();
-
- while (!cursor.isDone()) {
- cursor.advance();
- }
- cursor.reset();
- while (!cursor.isDone()) {
- final int[] rowVector = selector.getRowVector();
- for (int i = 0; i < cursor.getCurrentVectorSize(); i++) {
- values.add(selector.lookupName(rowVector[i]));
- }
- cursor.advance();
- }
-
- return values;
- }
}
private List<String> selectColumnValuesMatchingFilterUsingVectorCursor(
@@ -1291,6 +1300,25 @@ public abstract class BaseFilterTest extends
InitializedNullHandlingTest
expectedRows.size(),
selectCountUsingVectorizedFilteredAggregator(filter)
);
+ } else if (!(cursorFactory instanceof ColumnarFrameCursorFactory)) {
+ final List<VirtualColumn> relevant = new ArrayList<>();
+ Queue<String> toResolve = new ArrayDeque<>(filter.getRequiredColumns());
+ while (!toResolve.isEmpty()) {
+ VirtualColumn vc = VIRTUAL_COLUMNS.getVirtualColumn(toResolve.poll());
+ if (vc != null) {
+ relevant.add(vc);
+ toResolve.addAll(vc.requiredColumns());
+ }
+ }
+ final CursorBuildSpec spec = CursorBuildSpec.builder()
+
.setVirtualColumns(VirtualColumns.create(relevant))
+ // use makePostFilter since
some things with non-vectorizable
+ // matchers can still use
indexes
+
.setFilter(makePostFilter(filter))
+ .build();
+ try (CursorHolder holder = cursorFactory.makeCursorHolder(spec)) {
+ Assert.assertFalse(holder.canVectorize());
+ }
}
}
diff --git
a/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java
b/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java
index 4946bc96258..aa35112b67c 100644
---
a/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java
+++
b/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java
@@ -403,11 +403,11 @@ public class EqualityFilterTests
}
// array_offset behaves same on both mvds and arrays
- assertFilterMatchesSkipVectorize(
+ assertFilterMatchesSkipVectorizeUnlessFallback(
new EqualityFilter("vdim2-offset", ColumnType.STRING, "b", null),
ImmutableList.of("0")
);
- assertFilterMatchesSkipVectorize(
+ assertFilterMatchesSkipVectorizeUnlessFallback(
NotDimFilter.of(new EqualityFilter("vdim2-offset",
ColumnType.STRING, "b", null)),
ImmutableList.of()
);
diff --git
a/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java
b/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java
index b5791c04309..65c0f6f44d2 100644
---
a/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java
@@ -142,8 +142,8 @@ public class ExpressionFilterTest extends BaseFilterTest
// Empty String and "a" will not match
assertFilterMatches(edf("dim3 < 2"), ImmutableList.of("3", "4", "6", "9"));
assertFilterMatches(edf("dim3 < 2.0"), ImmutableList.of("3", "4", "6",
"9"));
- assertFilterMatchesSkipVectorize(edf("like(dim3, '1%')"),
ImmutableList.of("1", "3", "4", "6", "9"));
- assertFilterMatchesSkipVectorize(edf("array_contains(dim3, '1')"),
ImmutableList.of("3", "4", "6"));
+ assertFilterMatchesSkipVectorizeUnlessFallback(edf("like(dim3, '1%')"),
ImmutableList.of("1", "3", "4", "6", "9"));
+ assertFilterMatchesSkipVectorizeUnlessFallback(edf("array_contains(dim3,
'1')"), ImmutableList.of("3", "4", "6"));
}
@Test
@@ -169,6 +169,7 @@ public class ExpressionFilterTest extends BaseFilterTest
if (isAutoSchema()) {
return;
}
+
assertFilterMatchesSkipVectorize(edf("dim4 == ''"), ImmutableList.of("2"));
// AS per SQL standard null == null returns false.
assertFilterMatchesSkipVectorize(edf("dim4 == null"), ImmutableList.of());
@@ -194,25 +195,26 @@ public class ExpressionFilterTest extends BaseFilterTest
assertFilterMatchesSkipVectorize(edf("dim4 == '4'"), ImmutableList.of("4",
"5"));
assertFilterMatchesSkipVectorize(edf("concat(dim4, dim4) == '33'"),
ImmutableList.of("3"));
assertFilterMatchesSkipVectorize(edf("like(dim4, '4%')"),
ImmutableList.of("4", "5"));
- assertFilterMatchesSkipVectorize(edf("array_contains(dim4, '5')"),
ImmutableList.of("4", "5"));
- assertFilterMatchesSkipVectorize(edf("array_to_string(dim4, ':') ==
'4:5'"), ImmutableList.of("4", "5"));
+ // array functions on mvds can vectorize with fallback
+ assertFilterMatchesSkipVectorizeUnlessFallback(edf("array_contains(dim4,
'5')"), ImmutableList.of("4", "5"));
+ assertFilterMatchesSkipVectorizeUnlessFallback(edf("array_to_string(dim4,
':') == '4:5'"), ImmutableList.of("4", "5"));
}
@Test
public void testSingleAndMultiValuedStringColumn()
{
- assertFilterMatchesSkipVectorize(edf("array_contains(dim4, dim3)"),
ImmutableList.of("5", "9"));
+ assertFilterMatchesSkipVectorizeUnlessFallback(edf("array_contains(dim4,
dim3)"), ImmutableList.of("5", "9"));
}
@Test
public void testMvOverlap()
{
- assertFilterMatchesSkipVectorize(edf("mv_overlap(dim4, '1')"),
List.of("0"));
- assertFilterMatchesSkipVectorize(edf("mv_overlap(dim4, '4')"),
List.of("4", "5"));
- assertFilterMatchesSkipVectorize(edf("mv_overlap(dim4, array(1, 2, 3,
4))"), List.of("0", "3", "4", "5"));
- assertFilterMatchesSkipVectorize(edf("mv_overlap(dim4, dim3)"),
List.of("5", "9"));
- assertFilterMatchesSkipVectorize(edf("mv_overlap(dim4, null)"),
List.of("1", "6", "7", "8"));
- assertFilterMatchesSkipVectorize(edf("mv_overlap(dim4, [])"), List.of());
+ assertFilterMatchesSkipVectorizeUnlessFallback(edf("mv_overlap(dim4,
'1')"), List.of("0"));
+ assertFilterMatchesSkipVectorizeUnlessFallback(edf("mv_overlap(dim4,
'4')"), List.of("4", "5"));
+ assertFilterMatchesSkipVectorizeUnlessFallback(edf("mv_overlap(dim4,
array(1, 2, 3, 4))"), List.of("0", "3", "4", "5"));
+ assertFilterMatchesSkipVectorizeUnlessFallback(edf("mv_overlap(dim4,
dim3)"), List.of("5", "9"));
+ assertFilterMatchesSkipVectorizeUnlessFallback(edf("mv_overlap(dim4,
null)"), List.of("1", "6", "7", "8"));
+ assertFilterMatchesSkipVectorizeUnlessFallback(edf("mv_overlap(dim4,
[])"), List.of());
}
@Test
@@ -225,7 +227,7 @@ public class ExpressionFilterTest extends BaseFilterTest
assertFilterMatches(edf("dim1 < '2'"), ImmutableList.of("0", "1"));
assertFilterMatches(edf("dim1 < 2"), ImmutableList.of("0", "1"));
assertFilterMatches(edf("dim1 < 2.0"), ImmutableList.of("0", "1"));
- assertFilterMatchesSkipVectorize(edf("like(dim1, '1%')"),
ImmutableList.of("1"));
+ assertFilterMatchesSkipVectorizeUnlessFallback(edf("like(dim1, '1%')"),
ImmutableList.of("1"));
}
@Test
@@ -238,7 +240,7 @@ public class ExpressionFilterTest extends BaseFilterTest
assertFilterMatches(edf("dim2 < '2'"), ImmutableList.of("0", "1"));
assertFilterMatches(edf("dim2 < 2"), ImmutableList.of("0", "1"));
assertFilterMatches(edf("dim2 < 2.0"), ImmutableList.of("0", "1"));
- assertFilterMatchesSkipVectorize(edf("like(dim2, '1%')"),
ImmutableList.of("1"));
+ assertFilterMatchesSkipVectorizeUnlessFallback(edf("like(dim2, '1%')"),
ImmutableList.of("1"));
}
@Test
@@ -274,27 +276,27 @@ public class ExpressionFilterTest extends BaseFilterTest
@Test
public void testNullNotUnknown()
{
- assertFilterMatchesSkipVectorize(
+ assertFilterMatchesSkipVectorizeUnlessFallback(
edf("isfalse(dim5)"),
ImmutableList.of("0", "1", "3", "4", "5", "6", "7", "8")
);
- assertFilterMatchesSkipVectorize(
+ assertFilterMatchesSkipVectorizeUnlessFallback(
edf("!isfalse(dim5)"),
ImmutableList.of("2", "9")
);
- assertFilterMatchesSkipVectorize(
+ assertFilterMatchesSkipVectorizeUnlessFallback(
NotDimFilter.of(edf("isfalse(dim5)")),
ImmutableList.of("2", "9")
);
- assertFilterMatchesSkipVectorize(
+ assertFilterMatchesSkipVectorizeUnlessFallback(
edf("isfalse(notexist)"),
ImmutableList.of()
);
- assertFilterMatchesSkipVectorize(
+ assertFilterMatchesSkipVectorizeUnlessFallback(
edf("!isfalse(notexist)"), ImmutableList.of("0", "1", "2", "3", "4",
"5", "6", "7", "8", "9")
);
- assertFilterMatchesSkipVectorize(
+ assertFilterMatchesSkipVectorizeUnlessFallback(
NotDimFilter.of(edf("isfalse(notexist)")), ImmutableList.of("0", "1",
"2", "3", "4", "5", "6", "7", "8", "9")
);
}
@@ -325,7 +327,7 @@ public class ExpressionFilterTest extends BaseFilterTest
assertFilterMatches(edf("missing > '2'"), ImmutableList.of());
assertFilterMatches(edf("missing > 2"), ImmutableList.of());
assertFilterMatches(edf("missing > 2.0"), ImmutableList.of());
- assertFilterMatchesSkipVectorize(edf("like(missing, '1%')"),
ImmutableList.of());
+ assertFilterMatchesSkipVectorizeUnlessFallback(edf("like(missing, '1%')"),
ImmutableList.of());
}
@Test
diff --git
a/processing/src/test/java/org/apache/druid/segment/filter/FloatAndDoubleFilteringTest.java
b/processing/src/test/java/org/apache/druid/segment/filter/FloatAndDoubleFilteringTest.java
index 0ecbe9d0f04..251af2692bb 100644
---
a/processing/src/test/java/org/apache/druid/segment/filter/FloatAndDoubleFilteringTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/filter/FloatAndDoubleFilteringTest.java
@@ -189,13 +189,13 @@ public class FloatAndDoubleFilteringTest extends
BaseFilterTest
String jsFn = "function(x) { return(x === 3 || x === 5) }";
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
new JavaScriptDimFilter(columnName, jsFn, null,
JavaScriptConfig.getEnabledInstance()),
List.of("3", "5")
);
String jsFn2 = "function(x) { return(x === 3.0 || x === 5.0) }";
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
new JavaScriptDimFilter(columnName, jsFn2, null,
JavaScriptConfig.getEnabledInstance()),
List.of("3", "5")
);
@@ -316,7 +316,7 @@ public class FloatAndDoubleFilteringTest extends
BaseFilterTest
);
String jsFn = "function(x) { return(x === 'Wednesday' || x === 'Thursday')
}";
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
new JavaScriptDimFilter(columnName, jsFn, exfn,
JavaScriptConfig.getEnabledInstance()),
List.of("3", "4")
);
diff --git
a/processing/src/test/java/org/apache/druid/segment/filter/JavaScriptFilterTest.java
b/processing/src/test/java/org/apache/druid/segment/filter/JavaScriptFilterTest.java
index 0959479ab24..d9cdb39452b 100644
---
a/processing/src/test/java/org/apache/druid/segment/filter/JavaScriptFilterTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/filter/JavaScriptFilterTest.java
@@ -84,23 +84,23 @@ public class JavaScriptFilterTest extends BaseFilterTest
@Test
public void testSingleValueStringColumnWithoutNulls()
{
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim0",
jsNullFilter, null), ImmutableList.of());
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim0",
jsValueFilter(""), null), ImmutableList.of());
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim0",
jsValueFilter("0"), null), ImmutableList.of("0"));
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim0",
jsValueFilter("1"), null), ImmutableList.of("1"));
+ assertFilterMatches(newJavaScriptDimFilter("dim0", jsNullFilter, null),
ImmutableList.of());
+ assertFilterMatches(newJavaScriptDimFilter("dim0", jsValueFilter(""),
null), ImmutableList.of());
+ assertFilterMatches(newJavaScriptDimFilter("dim0", jsValueFilter("0"),
null), ImmutableList.of("0"));
+ assertFilterMatches(newJavaScriptDimFilter("dim0", jsValueFilter("1"),
null), ImmutableList.of("1"));
}
@Test
public void testSingleValueStringColumnWithNulls()
{
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim1",
jsNullFilter, null), ImmutableList.of());
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim1",
jsValueFilter(""), null), ImmutableList.of("0"));
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim1",
jsValueFilter("10"), null), ImmutableList.of("1"));
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim1",
jsValueFilter("2"), null), ImmutableList.of("2"));
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim1",
jsValueFilter("1"), null), ImmutableList.of("3"));
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim1",
jsValueFilter("abdef"), null), ImmutableList.of("4"));
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim1",
jsValueFilter("abc"), null), ImmutableList.of("5"));
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim1",
jsValueFilter("ab"), null), ImmutableList.of());
+ assertFilterMatches(newJavaScriptDimFilter("dim1", jsNullFilter, null),
ImmutableList.of());
+ assertFilterMatches(newJavaScriptDimFilter("dim1", jsValueFilter(""),
null), ImmutableList.of("0"));
+ assertFilterMatches(newJavaScriptDimFilter("dim1", jsValueFilter("10"),
null), ImmutableList.of("1"));
+ assertFilterMatches(newJavaScriptDimFilter("dim1", jsValueFilter("2"),
null), ImmutableList.of("2"));
+ assertFilterMatches(newJavaScriptDimFilter("dim1", jsValueFilter("1"),
null), ImmutableList.of("3"));
+ assertFilterMatches(newJavaScriptDimFilter("dim1", jsValueFilter("abdef"),
null), ImmutableList.of("4"));
+ assertFilterMatches(newJavaScriptDimFilter("dim1", jsValueFilter("abc"),
null), ImmutableList.of("5"));
+ assertFilterMatches(newJavaScriptDimFilter("dim1", jsValueFilter("ab"),
null), ImmutableList.of());
}
@Test
@@ -110,39 +110,39 @@ public class JavaScriptFilterTest extends BaseFilterTest
return;
}
// multi-val null......
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim2",
jsNullFilter, null), ImmutableList.of("1", "5"));
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim2",
jsValueFilter(""), null), ImmutableList.of("2"));
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(newJavaScriptDimFilter("dim2", jsNullFilter, null),
ImmutableList.of("1", "5"));
+ assertFilterMatches(newJavaScriptDimFilter("dim2", jsValueFilter(""),
null), ImmutableList.of("2"));
+ assertFilterMatches(
newJavaScriptDimFilter("dim2", jsValueFilter("a"), null),
ImmutableList.of("0", "3")
);
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim2",
jsValueFilter("b"), null), ImmutableList.of("0"));
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim2",
jsValueFilter("c"), null), ImmutableList.of("4"));
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim2",
jsValueFilter("d"), null), ImmutableList.of());
+ assertFilterMatches(newJavaScriptDimFilter("dim2", jsValueFilter("b"),
null), ImmutableList.of("0"));
+ assertFilterMatches(newJavaScriptDimFilter("dim2", jsValueFilter("c"),
null), ImmutableList.of("4"));
+ assertFilterMatches(newJavaScriptDimFilter("dim2", jsValueFilter("d"),
null), ImmutableList.of());
}
@Test
public void testMissingColumnSpecifiedInDimensionList()
{
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
newJavaScriptDimFilter("dim3", jsNullFilter, null),
ImmutableList.of("0", "1", "2", "3", "4", "5")
);
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim3",
jsValueFilter("a"), null), ImmutableList.of());
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim3",
jsValueFilter("b"), null), ImmutableList.of());
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim3",
jsValueFilter("c"), null), ImmutableList.of());
+ assertFilterMatches(newJavaScriptDimFilter("dim3", jsValueFilter("a"),
null), ImmutableList.of());
+ assertFilterMatches(newJavaScriptDimFilter("dim3", jsValueFilter("b"),
null), ImmutableList.of());
+ assertFilterMatches(newJavaScriptDimFilter("dim3", jsValueFilter("c"),
null), ImmutableList.of());
}
@Test
public void testMissingColumnNotSpecifiedInDimensionList()
{
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
newJavaScriptDimFilter("dim4", jsNullFilter, null),
ImmutableList.of("0", "1", "2", "3", "4", "5")
);
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim4",
jsValueFilter("a"), null), ImmutableList.of());
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim4",
jsValueFilter("b"), null), ImmutableList.of());
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim4",
jsValueFilter("c"), null), ImmutableList.of());
+ assertFilterMatches(newJavaScriptDimFilter("dim4", jsValueFilter("a"),
null), ImmutableList.of());
+ assertFilterMatches(newJavaScriptDimFilter("dim4", jsValueFilter("b"),
null), ImmutableList.of());
+ assertFilterMatches(newJavaScriptDimFilter("dim4", jsValueFilter("c"),
null), ImmutableList.of());
}
@Test
@@ -157,49 +157,49 @@ public class JavaScriptFilterTest extends BaseFilterTest
LookupExtractor mapExtractor = new MapLookupExtractor(stringMap, false);
LookupExtractionFn lookupFn = new LookupExtractionFn(mapExtractor, false,
"UNKNOWN", false, true);
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
newJavaScriptDimFilter("dim0", jsValueFilter("HELLO"), lookupFn),
ImmutableList.of("1")
);
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
newJavaScriptDimFilter("dim0", jsValueFilter("UNKNOWN"), lookupFn),
ImmutableList.of("0", "2", "3", "4", "5")
);
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
newJavaScriptDimFilter("dim1", jsValueFilter("HELLO"), lookupFn),
ImmutableList.of("3", "4")
);
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
newJavaScriptDimFilter("dim1", jsValueFilter("UNKNOWN"), lookupFn),
ImmutableList.of("0", "1", "2", "5")
);
if (!isAutoSchema()) {
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
newJavaScriptDimFilter("dim2", jsValueFilter("HELLO"), lookupFn),
ImmutableList.of("0", "3")
);
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
newJavaScriptDimFilter("dim2", jsValueFilter("UNKNOWN"), lookupFn),
ImmutableList.of("0", "1", "2", "4", "5")
);
}
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
newJavaScriptDimFilter("dim3", jsValueFilter("HELLO"), lookupFn),
ImmutableList.of()
);
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
newJavaScriptDimFilter("dim3", jsValueFilter("UNKNOWN"), lookupFn),
ImmutableList.of("0", "1", "2", "3", "4", "5")
);
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
newJavaScriptDimFilter("dim4", jsValueFilter("HELLO"), lookupFn),
ImmutableList.of()
);
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
newJavaScriptDimFilter("dim4", jsValueFilter("UNKNOWN"), lookupFn),
ImmutableList.of("0", "1", "2", "3", "4", "5")
);
@@ -208,12 +208,12 @@ public class JavaScriptFilterTest extends BaseFilterTest
@Test
public void testNumericNull()
{
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("f0",
jsNullFilter, null), ImmutableList.of("4"));
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("d0",
jsNullFilter, null), ImmutableList.of("2"));
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("l0",
jsNullFilter, null), ImmutableList.of("3"));
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("f0",
jsNumericValueFilter("5.5"), null), ImmutableList.of("2"));
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("d0",
jsNumericValueFilter("120.0245"), null), ImmutableList.of("3"));
- assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("l0",
jsNumericValueFilter("9001"), null), ImmutableList.of("4"));
+ assertFilterMatches(newJavaScriptDimFilter("f0", jsNullFilter, null),
ImmutableList.of("4"));
+ assertFilterMatches(newJavaScriptDimFilter("d0", jsNullFilter, null),
ImmutableList.of("2"));
+ assertFilterMatches(newJavaScriptDimFilter("l0", jsNullFilter, null),
ImmutableList.of("3"));
+ assertFilterMatches(newJavaScriptDimFilter("f0",
jsNumericValueFilter("5.5"), null), ImmutableList.of("2"));
+ assertFilterMatches(newJavaScriptDimFilter("d0",
jsNumericValueFilter("120.0245"), null), ImmutableList.of("3"));
+ assertFilterMatches(newJavaScriptDimFilter("l0",
jsNumericValueFilter("9001"), null), ImmutableList.of("4"));
}
@Test
diff --git
a/processing/src/test/java/org/apache/druid/segment/filter/LongFilteringTest.java
b/processing/src/test/java/org/apache/druid/segment/filter/LongFilteringTest.java
index a360b891692..91d78247ccc 100644
---
a/processing/src/test/java/org/apache/druid/segment/filter/LongFilteringTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/filter/LongFilteringTest.java
@@ -238,7 +238,7 @@ public class LongFilteringTest extends BaseFilterTest
);
String jsFn = "function(x) { return(x === 3 || x === 5) }";
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
new JavaScriptDimFilter(LONG_COLUMN, jsFn, null,
JavaScriptConfig.getEnabledInstance()),
List.of("3", "5")
);
@@ -346,7 +346,7 @@ public class LongFilteringTest extends BaseFilterTest
);
String jsFn = "function(x) { return(x === 'Wednesday' || x === 'Thursday')
}";
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
new JavaScriptDimFilter(LONG_COLUMN, jsFn, exfn,
JavaScriptConfig.getEnabledInstance()),
List.of("3", "4")
);
diff --git
a/processing/src/test/java/org/apache/druid/segment/filter/TimeFilteringTest.java
b/processing/src/test/java/org/apache/druid/segment/filter/TimeFilteringTest.java
index 6d7a074f068..8ead0e5f40f 100644
---
a/processing/src/test/java/org/apache/druid/segment/filter/TimeFilteringTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/filter/TimeFilteringTest.java
@@ -135,7 +135,7 @@ public class TimeFilteringTest extends BaseFilterTest
);
String jsFn = "function(x) { return(x === 3 || x === 5) }";
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
new JavaScriptDimFilter(ColumnHolder.TIME_COLUMN_NAME, jsFn, null,
JavaScriptConfig.getEnabledInstance()),
List.of("3", "5")
);
@@ -199,7 +199,7 @@ public class TimeFilteringTest extends BaseFilterTest
);
String jsFn = "function(x) { return(x === 'Wednesday' || x === 'Thursday')
}";
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
new JavaScriptDimFilter(ColumnHolder.TIME_COLUMN_NAME, jsFn, exfn,
JavaScriptConfig.getEnabledInstance()),
List.of("2", "3")
);
@@ -330,7 +330,7 @@ public class TimeFilteringTest extends BaseFilterTest
// increment timestamp by 2 hours
String timeBoosterJsFn = "function(x) { return(Number(x) + 7200000) }";
ExtractionFn exFn = new JavaScriptExtractionFn(timeBoosterJsFn, true,
JavaScriptConfig.getEnabledInstance());
- assertFilterMatchesSkipVectorize(
+ assertFilterMatches(
new IntervalDimFilter(
"dim0",
Collections.singletonList(Intervals.of("1970-01-01T02:00:00.001Z/1970-01-01T02:00:00.005Z")),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
