This is an automated email from the ASF dual-hosted git repository.
saurabhd336 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 083ab4e671 Fix issue with startree index metadata loading for columns
with '__' in name (#12554)
083ab4e671 is described below
commit 083ab4e671d1a8ef273956a0d01a343f9c1c4e54
Author: Saurabh Dubey <[email protected]>
AuthorDate: Wed Mar 6 13:09:15 2024 +0530
Fix issue with startree index metadata loading for columns with '__' in
name (#12554)
* Fix issue with startree metadata loading
* Fix array out of bounds
* Modify column names
* Move to optional
* Review comments
---------
Co-authored-by: Saurabh Dubey
<[email protected]>
Co-authored-by: Saurabh Dubey <[email protected]>
---
.../pinot/core/startree/v2/BaseStarTreeV2Test.java | 35 ++++++++++++----------
.../startree/v2/store/StarTreeIndexMapUtils.java | 8 +++--
2 files changed, 26 insertions(+), 17 deletions(-)
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java
b/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java
index 6737fc877c..14c148bbb2 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java
@@ -91,32 +91,37 @@ abstract class BaseStarTreeV2Test<R, A> {
private static final int NUM_SEGMENT_RECORDS = 100_000;
private static final int MAX_LEAF_RECORDS = RANDOM.nextInt(100) + 1;
- private static final String DIMENSION_D1 = "d1";
- private static final String DIMENSION_D2 = "d2";
+ // Using column names with '__' to make sure regular table columns with '__'
in the name aren't wrongly interpreted
+ // as AggregationFunctionColumnPair
+ private static final String DIMENSION_D1 = "d1__COLUMN_NAME";
+ private static final String DIMENSION_D2 = "__d2";
private static final int DIMENSION_CARDINALITY = 100;
private static final String METRIC = "m";
// Supported filters
- private static final String QUERY_FILTER_AND = " WHERE d1 = 0 AND d2 < 10";
+ private static final String QUERY_FILTER_AND = " WHERE d1__COLUMN_NAME = 0
AND __d2 < 10";
// StarTree supports OR predicates only on a single dimension
- private static final String QUERY_FILTER_OR = " WHERE d1 > 10 OR d1 < 50";
- private static final String QUERY_FILTER_COMPLEX_OR_MULTIPLE_DIMENSIONS = "
WHERE d2 < 95 AND (d1 > 10 OR d1 < 50)";
+ private static final String QUERY_FILTER_OR = " WHERE d1__COLUMN_NAME > 10
OR d1__COLUMN_NAME < 50";
+ private static final String QUERY_FILTER_COMPLEX_OR_MULTIPLE_DIMENSIONS =
+ " WHERE __d2 < 95 AND (d1__COLUMN_NAME > 10 OR d1__COLUMN_NAME < 50)";
private static final String
QUERY_FILTER_COMPLEX_AND_MULTIPLE_DIMENSIONS_THREE_PREDICATES =
- " WHERE d2 < 95 AND d2 > 25 AND (d1 > 10 OR d1 < 50)";
+ " WHERE __d2 < 95 AND __d2 > 25 AND (d1__COLUMN_NAME > 10 OR
d1__COLUMN_NAME < 50)";
private static final String
QUERY_FILTER_COMPLEX_OR_MULTIPLE_DIMENSIONS_THREE_PREDICATES =
- " WHERE (d2 > 95 OR d2 < 25) AND (d1 > 10 OR d1 < 50)";
- private static final String QUERY_FILTER_COMPLEX_OR_SINGLE_DIMENSION = "
WHERE d1 = 95 AND (d1 > 90 OR d1 < 100)";
+ " WHERE (__d2 > 95 OR __d2 < 25) AND (d1__COLUMN_NAME > 10 OR
d1__COLUMN_NAME < 50)";
+ private static final String QUERY_FILTER_COMPLEX_OR_SINGLE_DIMENSION =
+ " WHERE d1__COLUMN_NAME = 95 AND (d1__COLUMN_NAME > 90 OR
d1__COLUMN_NAME < 100)";
// Unsupported filters
- private static final String QUERY_FILTER_OR_MULTIPLE_DIMENSIONS = " WHERE d1
> 10 OR d2 < 50";
- private static final String QUERY_FILTER_OR_ON_AND = " WHERE (d1 > 10 AND d1
< 50) OR d1 < 50";
- private static final String QUERY_FILTER_OR_ON_NOT = " WHERE (NOT d1 > 10)
OR d1 < 50";
+ private static final String QUERY_FILTER_OR_MULTIPLE_DIMENSIONS = " WHERE
d1__COLUMN_NAME > 10 OR __d2 < 50";
+ private static final String QUERY_FILTER_OR_ON_AND =
+ " WHERE (d1__COLUMN_NAME > 10 AND d1__COLUMN_NAME < 50) OR
d1__COLUMN_NAME < 50";
+ private static final String QUERY_FILTER_OR_ON_NOT = " WHERE (NOT
d1__COLUMN_NAME > 10) OR d1__COLUMN_NAME < 50";
// Always false filters
- private static final String QUERY_FILTER_ALWAYS_FALSE = " WHERE d1 > 100";
- private static final String QUERY_FILTER_OR_ALWAYS_FALSE = " WHERE d1 > 100
OR d1 < 0";
+ private static final String QUERY_FILTER_ALWAYS_FALSE = " WHERE
d1__COLUMN_NAME > 100";
+ private static final String QUERY_FILTER_OR_ALWAYS_FALSE = " WHERE
d1__COLUMN_NAME > 100 OR d1__COLUMN_NAME < 0";
- private static final String QUERY_GROUP_BY = " GROUP BY d2";
- private static final String FILTER_AGG_CLAUSE = " FILTER(WHERE d1 > 10)";
+ private static final String QUERY_GROUP_BY = " GROUP BY __d2";
+ private static final String FILTER_AGG_CLAUSE = " FILTER(WHERE
d1__COLUMN_NAME > 10)";
private ValueAggregator _valueAggregator;
private DataType _aggregatedValueType;
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeIndexMapUtils.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeIndexMapUtils.java
index 7934879253..7ac45d3b18 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeIndexMapUtils.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeIndexMapUtils.java
@@ -199,8 +199,12 @@ public class StarTreeIndexMapUtils {
}
// Convert metric (function-column pair) to stored name for
backward-compatibility
if (column.contains(AggregationFunctionColumnPair.DELIMITER)) {
- AggregationFunctionColumnPair functionColumnPair =
AggregationFunctionColumnPair.fromColumnName(column);
- column =
AggregationFunctionColumnPair.resolveToStoredType(functionColumnPair).toColumnName();
+ try {
+ AggregationFunctionColumnPair functionColumnPair =
AggregationFunctionColumnPair.fromColumnName(column);
+ column =
AggregationFunctionColumnPair.resolveToStoredType(functionColumnPair).toColumnName();
+ } catch (Exception e) {
+ // Ignoring this exception for columns that are not metric
(function-column pair)
+ }
}
indexKey = new IndexKey(IndexType.FORWARD_INDEX, column);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
