This is an automated email from the ASF dual-hosted git repository.
tuglu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new fafc7a0ffd5 feat: add tdigest max compression config param (#19310)
fafc7a0ffd5 is described below
commit fafc7a0ffd57e12603a6cbea9dc56c1937c5628e
Author: jtuglu1 <[email protected]>
AuthorDate: Tue Apr 14 15:01:10 2026 -0700
feat: add tdigest max compression config param (#19310)
Provide a way to pin the maximum available compression on tdigest operators
in order to bound resources.
---
.../extensions-contrib/tdigestsketch-quantiles.md | 12 +++-
.../aggregation/tdigestsketch/TDigestConfig.java | 66 ++++++++++++++++++++++
.../TDigestSketchAggregatorFactory.java | 30 ++++++++--
.../tdigestsketch/TDigestSketchModule.java | 2 +
.../sql/TDigestGenerateSketchSqlAggregator.java | 14 ++++-
.../sql/TDigestSketchQuantileSqlAggregator.java | 14 ++++-
.../TDigestSketchAggregatorFactoryTest.java | 20 ++++++-
.../tdigestsketch/TDigestSketchAggregatorTest.java | 59 +++++++++++++++++--
.../sql/TDigestSketchSqlAggregatorTest.java | 43 +++++++-------
9 files changed, 220 insertions(+), 40 deletions(-)
diff --git a/docs/development/extensions-contrib/tdigestsketch-quantiles.md
b/docs/development/extensions-contrib/tdigestsketch-quantiles.md
index 101368445d8..65d33c9b9ee 100644
--- a/docs/development/extensions-contrib/tdigestsketch-quantiles.md
+++ b/docs/development/extensions-contrib/tdigestsketch-quantiles.md
@@ -41,6 +41,14 @@ To use this aggregator, make sure you
[include](../../configuration/extensions.m
druid.extensions.loadList=["druid-tdigestsketch"]
```
+### Configuration
+
+The following runtime property can be set in `runtime.properties`:
+
+|Property|Description|Default|
+|--------|-----------|-------|
+|`druid.tdigest.maxCompression`|Upper bound on the `compression` parameter
accepted by the `tDigestSketch` aggregator and the
`TDIGEST_GENERATE_SKETCH`/`TDIGEST_QUANTILE` SQL functions. When a query
specifies a compression value exceeding this limit, the value is silently
capped to `maxCompression`. Has no effect when unset.|None (no cap)|
+
### Aggregator
The result of the aggregation is a T-Digest sketch that is built ingesting
numeric values from the raw data or from
@@ -80,7 +88,7 @@ Example:
|type|This String should always be "tDigestSketch"|yes|
|name|A String for the output (result) name of the calculation.|yes|
|fieldName|A String for the name of the input field containing raw numeric
values or pre-generated T-Digest sketches.|yes|
-|compression|Parameter that determines the accuracy and size of the sketch.
Higher compression means higher accuracy but more space to store sketches.|no,
defaults to 100|
+|compression|Parameter that determines the accuracy and size of the sketch.
Higher compression means higher accuracy but more space to store sketches.
Capped by `druid.tdigest.maxCompression` when set.|no, defaults to 100|
### Post Aggregators
@@ -159,6 +167,7 @@ Once you load the T-Digest extension, you can use the
following SQL functions.
Builds a T-Digest sketch on values produced by an expression.
Compression parameter (default value 100) determines the accuracy and size of
the sketch.
Higher compression provides higher accuracy but requires more storage space.
+The compression value is capped by `druid.tdigest.maxCompression` when set.
* **Syntax**: `TDIGEST_GENERATE_SKETCH(expr, [compression])`
* **Default**: Empty Base64-encoded T-Digest sketch string
@@ -169,6 +178,7 @@ Higher compression provides higher accuracy but requires
more storage space.
Builds a T-Digest sketch on values produced by an expression and returns the
value for the quantile.
Compression parameter (default value 100) determines the accuracy and size of
the sketch.
Higher compression provides higher accuracy but requires more storage space.
+The compression value is capped by `druid.tdigest.maxCompression` when set.
* **Syntax**: `TDIGEST_QUANTILE(expr, quantileFraction, [compression])`
* **Default**: `Double.NaN`
diff --git
a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestConfig.java
b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestConfig.java
new file mode 100644
index 00000000000..232cd54db83
--- /dev/null
+++
b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestConfig.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.query.aggregation.tdigestsketch;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import javax.annotation.Nullable;
+
+public class TDigestConfig
+{
+ @JsonProperty("maxCompression")
+ @Nullable
+ private final Integer maxCompression;
+
+ @JsonCreator
+ public TDigestConfig(@JsonProperty("maxCompression") @Nullable Integer
maxCompression)
+ {
+ this.maxCompression = maxCompression;
+ }
+
+ @Nullable
+ public Integer getMaxCompression()
+ {
+ return maxCompression;
+ }
+
+ public static Builder builder()
+ {
+ return new Builder();
+ }
+
+ public static class Builder
+ {
+ @Nullable
+ private Integer maxCompression;
+
+ public Builder maxCompression(@Nullable Integer maxCompression)
+ {
+ this.maxCompression = maxCompression;
+ return this;
+ }
+
+ public TDigestConfig build()
+ {
+ return new TDigestConfig(maxCompression);
+ }
+ }
+}
diff --git
a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java
b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java
index da753777e03..655d7d84402 100644
---
a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java
+++
b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java
@@ -19,11 +19,14 @@
package org.apache.druid.query.aggregation.tdigestsketch;
+import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import com.tdunning.math.stats.MergingDigest;
import com.tdunning.math.stats.TDigest;
+import org.apache.druid.common.config.Configs;
+import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.query.aggregation.AggregateCombiner;
import org.apache.druid.query.aggregation.Aggregator;
import org.apache.druid.query.aggregation.AggregatorFactory;
@@ -60,6 +63,7 @@ import java.util.Objects;
@JsonTypeName(TDigestSketchAggregatorFactory.TYPE_NAME)
public class TDigestSketchAggregatorFactory extends AggregatorFactory
{
+ private static final Logger log = new
Logger(TDigestSketchAggregatorFactory.class);
// Default compression
public static final int DEFAULT_COMPRESSION = 100;
@@ -74,6 +78,8 @@ public class TDigestSketchAggregatorFactory extends
AggregatorFactory
@Nonnull
private final byte cacheTypeId;
+ private final TDigestConfig tDigestConfig;
+
public static final String TYPE_NAME = "tDigestSketch";
public static final ColumnType TYPE = ColumnType.ofComplex(TYPE_NAME);
@@ -81,22 +87,34 @@ public class TDigestSketchAggregatorFactory extends
AggregatorFactory
public TDigestSketchAggregatorFactory(
@JsonProperty("name") final String name,
@JsonProperty("fieldName") final String fieldName,
- @JsonProperty("compression") @Nullable final Integer compression
+ @JsonProperty("compression") @Nullable final Integer compression,
+ @JacksonInject final TDigestConfig tDigestConfig
)
{
- this(name, fieldName, compression,
AggregatorUtil.TDIGEST_BUILD_SKETCH_CACHE_TYPE_ID);
+ this(name, fieldName, compression,
AggregatorUtil.TDIGEST_BUILD_SKETCH_CACHE_TYPE_ID, tDigestConfig);
}
TDigestSketchAggregatorFactory(
final String name,
final String fieldName,
@Nullable final Integer compression,
- final byte cacheTypeId
+ final byte cacheTypeId,
+ final TDigestConfig tDigestConfig
)
{
this.name = Objects.requireNonNull(name, "Must have a valid, non-null
aggregator name");
this.fieldName = Objects.requireNonNull(fieldName, "Parameter fieldName
must be specified");
- this.compression = compression == null ? DEFAULT_COMPRESSION : compression;
+ this.tDigestConfig = tDigestConfig;
+ if (tDigestConfig.getMaxCompression() != null && compression != null &&
compression > tDigestConfig.getMaxCompression()) {
+ log.warn(
+ "Compression value [%d] is greater than the max allowed compression
value [%d]. Setting it to max allowed value",
+ compression,
+ tDigestConfig.getMaxCompression()
+ );
+ this.compression = tDigestConfig.getMaxCompression();
+ } else {
+ this.compression = Configs.valueOrDefault(compression,
DEFAULT_COMPRESSION);
+ }
this.cacheTypeId = cacheTypeId;
}
@@ -149,7 +167,7 @@ public class TDigestSketchAggregatorFactory extends
AggregatorFactory
@Override
public AggregatorFactory getCombiningFactory()
{
- return new TDigestSketchAggregatorFactory(name, name, compression);
+ return new TDigestSketchAggregatorFactory(name, name, compression,
tDigestConfig);
}
@Override
@@ -224,7 +242,7 @@ public class TDigestSketchAggregatorFactory extends
AggregatorFactory
@Override
public AggregatorFactory withName(String newName)
{
- return new TDigestSketchAggregatorFactory(newName, getFieldName(),
getCompression(), cacheTypeId);
+ return new TDigestSketchAggregatorFactory(newName, getFieldName(),
getCompression(), cacheTypeId, tDigestConfig);
}
@Override
diff --git
a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchModule.java
b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchModule.java
index 1a5150fdf74..94d25a6228d 100644
---
a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchModule.java
+++
b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchModule.java
@@ -26,6 +26,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.inject.Binder;
import com.tdunning.math.stats.MergingDigest;
+import org.apache.druid.guice.JsonConfigProvider;
import org.apache.druid.initialization.DruidModule;
import
org.apache.druid.query.aggregation.tdigestsketch.sql.TDigestGenerateSketchSqlAggregator;
import
org.apache.druid.query.aggregation.tdigestsketch.sql.TDigestSketchQuantileSqlAggregator;
@@ -66,6 +67,7 @@ public class TDigestSketchModule implements DruidModule
public void configure(Binder binder)
{
registerSerde();
+ JsonConfigProvider.bind(binder, "druid.tdigest", TDigestConfig.class);
SqlBindings.addAggregator(binder,
TDigestSketchQuantileSqlAggregator.class);
SqlBindings.addAggregator(binder,
TDigestGenerateSketchSqlAggregator.class);
}
diff --git
a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestGenerateSketchSqlAggregator.java
b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestGenerateSketchSqlAggregator.java
index 5604622755a..cb879df275d 100644
---
a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestGenerateSketchSqlAggregator.java
+++
b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestGenerateSketchSqlAggregator.java
@@ -19,6 +19,7 @@
package org.apache.druid.query.aggregation.tdigestsketch.sql;
+import com.google.inject.Inject;
import org.apache.calcite.rel.core.AggregateCall;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
@@ -28,6 +29,7 @@ import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.util.Optionality;
import org.apache.druid.query.aggregation.AggregatorFactory;
+import org.apache.druid.query.aggregation.tdigestsketch.TDigestConfig;
import
org.apache.druid.query.aggregation.tdigestsketch.TDigestSketchAggregatorFactory;
import org.apache.druid.query.aggregation.tdigestsketch.TDigestSketchUtils;
import org.apache.druid.segment.column.ColumnType;
@@ -48,6 +50,13 @@ public class TDigestGenerateSketchSqlAggregator implements
SqlAggregator
{
private static final SqlAggFunction FUNCTION_INSTANCE = new
TDigestGenerateSketchSqlAggregator.TDigestGenerateSketchSqlAggFunction();
private static final String NAME = "TDIGEST_GENERATE_SKETCH";
+ private final TDigestConfig tDigestConfig;
+
+ @Inject
+ public TDigestGenerateSketchSqlAggregator(TDigestConfig tDigestConfig)
+ {
+ this.tDigestConfig = tDigestConfig;
+ }
@Override
public SqlAggFunction calciteFunction()
@@ -116,14 +125,15 @@ public class TDigestGenerateSketchSqlAggregator
implements SqlAggregator
aggregatorFactory = new TDigestSketchAggregatorFactory(
name,
input.getDirectColumn(),
- compression
+ compression,
+ tDigestConfig
);
} else {
String virtualColumnName =
virtualColumnRegistry.getOrCreateVirtualColumnForExpression(
input,
ColumnType.FLOAT
);
- aggregatorFactory = new TDigestSketchAggregatorFactory(name,
virtualColumnName, compression);
+ aggregatorFactory = new TDigestSketchAggregatorFactory(name,
virtualColumnName, compression, tDigestConfig);
}
return Aggregation.create(aggregatorFactory);
diff --git
a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchQuantileSqlAggregator.java
b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchQuantileSqlAggregator.java
index 7cc0bdfe8c5..613155e03c0 100644
---
a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchQuantileSqlAggregator.java
+++
b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchQuantileSqlAggregator.java
@@ -20,6 +20,7 @@
package org.apache.druid.query.aggregation.tdigestsketch.sql;
import com.google.common.collect.ImmutableList;
+import com.google.inject.Inject;
import org.apache.calcite.rel.core.AggregateCall;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
@@ -33,6 +34,7 @@ import org.apache.calcite.util.Optionality;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator;
+import org.apache.druid.query.aggregation.tdigestsketch.TDigestConfig;
import
org.apache.druid.query.aggregation.tdigestsketch.TDigestSketchAggregatorFactory;
import
org.apache.druid.query.aggregation.tdigestsketch.TDigestSketchToQuantilePostAggregator;
import org.apache.druid.query.aggregation.tdigestsketch.TDigestSketchUtils;
@@ -53,6 +55,13 @@ public class TDigestSketchQuantileSqlAggregator implements
SqlAggregator
{
private static final SqlAggFunction FUNCTION_INSTANCE = new
TDigestSketchQuantileSqlAggFunction();
private static final String NAME = "TDIGEST_QUANTILE";
+ private final TDigestConfig tDigestConfig;
+
+ @Inject
+ public TDigestSketchQuantileSqlAggregator(TDigestConfig tDigestConfig)
+ {
+ this.tDigestConfig = tDigestConfig;
+ }
@Override
public SqlAggFunction calciteFunction()
@@ -134,14 +143,15 @@ public class TDigestSketchQuantileSqlAggregator
implements SqlAggregator
aggregatorFactory = new TDigestSketchAggregatorFactory(
sketchName,
input.getDirectColumn(),
- compression
+ compression,
+ tDigestConfig
);
} else {
String virtualColumnName =
virtualColumnRegistry.getOrCreateVirtualColumnForExpression(
input,
ColumnType.FLOAT
);
- aggregatorFactory = new TDigestSketchAggregatorFactory(sketchName,
virtualColumnName, compression);
+ aggregatorFactory = new TDigestSketchAggregatorFactory(sketchName,
virtualColumnName, compression, tDigestConfig);
}
return Aggregation.create(
diff --git
a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactoryTest.java
b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactoryTest.java
index f28fb9ad7b7..80ef6c8ecb5 100644
---
a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactoryTest.java
+++
b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactoryTest.java
@@ -43,7 +43,7 @@ public class TDigestSketchAggregatorFactoryTest
.granularity(Granularities.HOUR)
.aggregators(
new CountAggregatorFactory("count"),
- new TDigestSketchAggregatorFactory("tdigest", "col", null)
+ new TDigestSketchAggregatorFactory("tdigest", "col", null,
TDigestConfig.builder().maxCompression(200).build())
)
.postAggregators(
new FieldAccessPostAggregator("tdigest-access", "tdigest"),
@@ -66,8 +66,24 @@ public class TDigestSketchAggregatorFactoryTest
@Test
public void testWithName()
{
- TDigestSketchAggregatorFactory factory = new
TDigestSketchAggregatorFactory("tdigest", "col", null);
+ TDigestSketchAggregatorFactory factory = new
TDigestSketchAggregatorFactory("tdigest", "col", null,
TDigestConfig.builder().maxCompression(200).build());
Assert.assertEquals(factory, factory.withName("tdigest"));
Assert.assertEquals("newTest", factory.withName("newTest").getName());
}
+
+ @Test
+ public void testCompressionCappedAtMaxCompression()
+ {
+ TDigestConfig config = TDigestConfig.builder().maxCompression(150).build();
+ TDigestSketchAggregatorFactory factory = new
TDigestSketchAggregatorFactory("tdigest", "col", 300, config);
+ Assert.assertEquals(150, factory.getCompression());
+ }
+
+ @Test
+ public void testCompressionBelowMaxCompressionUnchanged()
+ {
+ TDigestConfig config = TDigestConfig.builder().maxCompression(150).build();
+ TDigestSketchAggregatorFactory factory = new
TDigestSketchAggregatorFactory("tdigest", "col", 100, config);
+ Assert.assertEquals(100, factory.getCompression());
+ }
}
diff --git
a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorTest.java
b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorTest.java
index 43927bae9a9..c95b8a7068e 100644
---
a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorTest.java
+++
b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorTest.java
@@ -19,6 +19,7 @@
package org.apache.druid.query.aggregation.tdigestsketch;
+import com.fasterxml.jackson.databind.InjectableValues;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.druid.data.input.ColumnsFilter;
import org.apache.druid.data.input.InputRowSchema;
@@ -65,6 +66,10 @@ public class TDigestSketchAggregatorTest extends
InitializedNullHandlingTest
TDigestSketchModule module = new TDigestSketchModule();
helper = AggregationTestHelper.createGroupByQueryAggregationTestHelper(
module.getJacksonModules(), config, tempFolder);
+ InjectableValues currentInjectableValues =
helper.getObjectMapper().getInjectableValues();
+ InjectableValues.Std currentInjectableValuesStd = (InjectableValues.Std)
currentInjectableValues;
+ currentInjectableValuesStd.addValue(TDigestConfig.class.getName(),
TDigestConfig.builder().build());
+ helper.getObjectMapper().setInjectableValues(currentInjectableValuesStd);
}
@Parameterized.Parameters(name = "{0}")
@@ -82,8 +87,12 @@ public class TDigestSketchAggregatorTest extends
InitializedNullHandlingTest
public void serializeDeserializeFactoryWithFieldName() throws Exception
{
ObjectMapper objectMapper = new DefaultObjectMapper();
+ objectMapper.setInjectableValues(
+ new InjectableValues.Std()
+ .addValue(TDigestConfig.class.getName(),
TDigestConfig.builder().build())
+ );
new
TDigestSketchModule().getJacksonModules().forEach(objectMapper::registerModule);
- TDigestSketchAggregatorFactory factory = new
TDigestSketchAggregatorFactory("name", "filedName", 128);
+ TDigestSketchAggregatorFactory factory = new
TDigestSketchAggregatorFactory("name", "filedName", 128,
TDigestConfig.builder().build());
AggregatorFactory other = objectMapper.readValue(
objectMapper.writeValueAsString(factory),
@@ -93,6 +102,44 @@ public class TDigestSketchAggregatorTest extends
InitializedNullHandlingTest
Assert.assertEquals(factory, other);
}
+ @Test
+ public void deserializedFactoryCompressionCappedAtMaxCompression() throws
Exception
+ {
+ ObjectMapper objectMapper = new DefaultObjectMapper();
+ objectMapper.setInjectableValues(
+ new InjectableValues.Std()
+ .addValue(TDigestConfig.class.getName(),
TDigestConfig.builder().maxCompression(150).build())
+ );
+ new
TDigestSketchModule().getJacksonModules().forEach(objectMapper::registerModule);
+ TDigestSketchAggregatorFactory factory = new
TDigestSketchAggregatorFactory("name", "fieldName", 300,
TDigestConfig.builder().maxCompression(150).build());
+
+ TDigestSketchAggregatorFactory deserialized =
(TDigestSketchAggregatorFactory) objectMapper.readValue(
+ objectMapper.writeValueAsString(factory),
+ AggregatorFactory.class
+ );
+
+ Assert.assertEquals(150, deserialized.getCompression());
+ }
+
+ @Test
+ public void deserializedFactoryCompressionBelowMaxCompressionUnchanged()
throws Exception
+ {
+ ObjectMapper objectMapper = new DefaultObjectMapper();
+ objectMapper.setInjectableValues(
+ new InjectableValues.Std()
+ .addValue(TDigestConfig.class.getName(),
TDigestConfig.builder().maxCompression(150).build())
+ );
+ new
TDigestSketchModule().getJacksonModules().forEach(objectMapper::registerModule);
+ TDigestSketchAggregatorFactory factory = new
TDigestSketchAggregatorFactory("name", "fieldName", 100,
TDigestConfig.builder().maxCompression(150).build());
+
+ TDigestSketchAggregatorFactory deserialized =
(TDigestSketchAggregatorFactory) objectMapper.readValue(
+ objectMapper.writeValueAsString(factory),
+ AggregatorFactory.class
+ );
+
+ Assert.assertEquals(100, deserialized.getCompression());
+ }
+
@Test
public void buildingSketchesAtIngestionTime() throws Exception
{
@@ -109,7 +156,7 @@ public class TDigestSketchAggregatorTest extends
InitializedNullHandlingTest
DelimitedInputFormat.forColumns(
List.of("timestamp", "sequenceNumber", "product", "value")
),
- List.of(new TDigestSketchAggregatorFactory("sketch", "value", 200)),
+ List.of(new TDigestSketchAggregatorFactory("sketch", "value", 200,
TDigestConfig.builder().build())),
0, // minTimestamp
Granularities.NONE,
10, // maxRowCount
@@ -117,7 +164,7 @@ public class TDigestSketchAggregatorTest extends
InitializedNullHandlingTest
.setDataSource("test_datasource")
.setGranularity(Granularities.ALL)
.setDimensions(Collections.emptyList())
- .setAggregatorSpecs(new
TDigestSketchAggregatorFactory("merged_sketch", "sketch", 200))
+ .setAggregatorSpecs(new
TDigestSketchAggregatorFactory("merged_sketch", "sketch", 200,
TDigestConfig.builder().build()))
.setPostAggregatorSpecs(
new TDigestSketchToQuantilesPostAggregator(
"quantiles",
@@ -162,7 +209,7 @@ public class TDigestSketchAggregatorTest extends
InitializedNullHandlingTest
.setDataSource("test_datasource")
.setGranularity(Granularities.ALL)
.setDimensions(Collections.emptyList())
- .setAggregatorSpecs(new
TDigestSketchAggregatorFactory("sketch", "value", 200))
+ .setAggregatorSpecs(new
TDigestSketchAggregatorFactory("sketch", "value", 200,
TDigestConfig.builder().build()))
.setPostAggregatorSpecs(
new TDigestSketchToQuantilesPostAggregator(
"quantiles",
@@ -200,7 +247,7 @@ public class TDigestSketchAggregatorTest extends
InitializedNullHandlingTest
DelimitedInputFormat.forColumns(
List.of("timestamp", "product", "sketch")
),
- List.of(new TDigestSketchAggregatorFactory("first_level_merge_sketch",
"sketch", 200)),
+ List.of(new TDigestSketchAggregatorFactory("first_level_merge_sketch",
"sketch", 200, TDigestConfig.builder().build())),
0, // minTimestamp
Granularities.NONE,
10, // maxRowCount
@@ -209,7 +256,7 @@ public class TDigestSketchAggregatorTest extends
InitializedNullHandlingTest
.setGranularity(Granularities.ALL)
.setDimensions(Collections.emptyList())
.setAggregatorSpecs(
- new
TDigestSketchAggregatorFactory("second_level_merge_sketch",
"first_level_merge_sketch", 200)
+ new
TDigestSketchAggregatorFactory("second_level_merge_sketch",
"first_level_merge_sketch", 200, TDigestConfig.builder().build())
)
.setPostAggregatorSpecs(
new TDigestSketchToQuantilesPostAggregator(
diff --git
a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java
b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java
index a67fe1c4fda..c18d0ad7f43 100644
---
a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java
+++
b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java
@@ -31,6 +31,7 @@ import
org.apache.druid.query.aggregation.DoubleSumAggregatorFactory;
import org.apache.druid.query.aggregation.FilteredAggregatorFactory;
import org.apache.druid.query.aggregation.PostAggregator;
import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator;
+import org.apache.druid.query.aggregation.tdigestsketch.TDigestConfig;
import
org.apache.druid.query.aggregation.tdigestsketch.TDigestSketchAggregatorFactory;
import org.apache.druid.query.aggregation.tdigestsketch.TDigestSketchModule;
import
org.apache.druid.query.aggregation.tdigestsketch.TDigestSketchToQuantilePostAggregator;
@@ -81,7 +82,6 @@ public class TDigestSketchSqlAggregatorTest extends
BaseCalciteQueryTest
)
{
TDigestSketchModule.registerSerde();
-
final QueryableIndex index =
IndexBuilder.create(jsonMapper)
.tmpDir(tempDirProducer.newTempFolder())
@@ -94,7 +94,8 @@ public class TDigestSketchSqlAggregatorTest extends
BaseCalciteQueryTest
new TDigestSketchAggregatorFactory(
"qsketch_m1",
"m1",
- 128
+ 128,
+ TDigestConfig.builder().build()
)
)
.withRollup(false)
@@ -131,7 +132,7 @@ public class TDigestSketchSqlAggregatorTest extends
BaseCalciteQueryTest
.intervals(new
MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity())))
.granularity(Granularities.ALL)
.aggregators(ImmutableList.of(
- new TDigestSketchAggregatorFactory("a0", "m1", 200)
+ new TDigestSketchAggregatorFactory("a0", "m1", 200,
TDigestConfig.builder().build())
))
.context(QUERY_CONTEXT_DEFAULT)
.build()
@@ -162,13 +163,13 @@ public class TDigestSketchSqlAggregatorTest extends
BaseCalciteQueryTest
.granularity(Granularities.ALL)
.aggregators(ImmutableList.of(
new TDigestSketchAggregatorFactory("a0:agg", "m1",
-
TDigestSketchAggregatorFactory.DEFAULT_COMPRESSION
+
TDigestSketchAggregatorFactory.DEFAULT_COMPRESSION,
TDigestConfig.builder().build()
),
new TDigestSketchAggregatorFactory("a1:agg", "m1",
- 200
+ 200,
TDigestConfig.builder().build()
),
new TDigestSketchAggregatorFactory("a2:agg", "m1",
- 300
+ 300,
TDigestConfig.builder().build()
)
))
.postAggregators(
@@ -201,7 +202,7 @@ public class TDigestSketchSqlAggregatorTest extends
BaseCalciteQueryTest
.intervals(new
MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity())))
.granularity(Granularities.ALL)
.aggregators(ImmutableList.of(
- new TDigestSketchAggregatorFactory("a0", "m1", 200)
+ new TDigestSketchAggregatorFactory("a0", "m1", 200,
TDigestConfig.builder().build())
))
.context(QUERY_CONTEXT_DEFAULT)
.build()
@@ -238,7 +239,7 @@ public class TDigestSketchSqlAggregatorTest extends
BaseCalciteQueryTest
)
)
.aggregators(ImmutableList.of(
- new TDigestSketchAggregatorFactory("a0", "v0", 200)
+ new TDigestSketchAggregatorFactory("a0", "v0", 200,
TDigestConfig.builder().build())
))
.context(QUERY_CONTEXT_DEFAULT)
.build()
@@ -267,7 +268,7 @@ public class TDigestSketchSqlAggregatorTest extends
BaseCalciteQueryTest
.intervals(new
MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity())))
.granularity(Granularities.ALL)
.aggregators(ImmutableList.of(
- new TDigestSketchAggregatorFactory("a0", "m1",
TDigestSketchAggregatorFactory.DEFAULT_COMPRESSION)
+ new TDigestSketchAggregatorFactory("a0", "m1",
TDigestSketchAggregatorFactory.DEFAULT_COMPRESSION,
TDigestConfig.builder().build())
))
.context(QUERY_CONTEXT_DEFAULT)
.build()
@@ -306,7 +307,7 @@ public class TDigestSketchSqlAggregatorTest extends
BaseCalciteQueryTest
.intervals(new
MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity())))
.granularity(Granularities.ALL)
.aggregators(ImmutableList.of(
- new TDigestSketchAggregatorFactory("a0:agg",
"qsketch_m1", 100)
+ new TDigestSketchAggregatorFactory("a0:agg",
"qsketch_m1", 100, TDigestConfig.builder().build())
))
.postAggregators(
new TDigestSketchToQuantilePostAggregator("a0",
makeFieldAccessPostAgg("a0:agg"), 0.1f),
@@ -349,7 +350,7 @@ public class TDigestSketchSqlAggregatorTest extends
BaseCalciteQueryTest
.setDimensions(new
DefaultDimensionSpec("dim1", "d0"))
.setAggregatorSpecs(
ImmutableList.of(
- new
TDigestSketchAggregatorFactory("a0", "m1", 200)
+ new
TDigestSketchAggregatorFactory("a0", "m1", 200, TDigestConfig.builder().build())
)
)
.setContext(QUERY_CONTEXT_DEFAULT)
@@ -360,7 +361,7 @@ public class TDigestSketchSqlAggregatorTest extends
BaseCalciteQueryTest
.setGranularity(Granularities.ALL)
.setAggregatorSpecs(
ImmutableList.of(
- new TDigestSketchAggregatorFactory("_a0:agg",
"a0", 100)
+ new TDigestSketchAggregatorFactory("_a0:agg",
"a0", 100, TDigestConfig.builder().build())
)
)
.setPostAggregatorSpecs(
@@ -401,7 +402,7 @@ public class TDigestSketchSqlAggregatorTest extends
BaseCalciteQueryTest
.intervals(new
MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity())))
.granularity(Granularities.ALL)
.aggregators(ImmutableList.of(
- new TDigestSketchAggregatorFactory("a0:agg", "m1", null)
+ new TDigestSketchAggregatorFactory("a0:agg", "m1", null,
TDigestConfig.builder().build())
))
.postAggregators(
new TDigestSketchToQuantilePostAggregator("a0",
makeFieldAccessPostAgg("a0:agg"), 0.0f),
@@ -431,13 +432,13 @@ public class TDigestSketchSqlAggregatorTest extends
BaseCalciteQueryTest
.granularity(Granularities.ALL)
.aggregators(ImmutableList.of(
new TDigestSketchAggregatorFactory("a0:agg", "m1",
-
TDigestSketchAggregatorFactory.DEFAULT_COMPRESSION
+
TDigestSketchAggregatorFactory.DEFAULT_COMPRESSION,
TDigestConfig.builder().build()
),
new TDigestSketchAggregatorFactory("a1:agg", "m1",
- 200
+ 200,
TDigestConfig.builder().build()
),
new TDigestSketchAggregatorFactory("a2:agg", "m1",
- 300
+ 300,
TDigestConfig.builder().build()
)
))
.postAggregators(
@@ -479,7 +480,7 @@ public class TDigestSketchSqlAggregatorTest extends
BaseCalciteQueryTest
ExprMacroTable.nil()
)
)
- .aggregators(new TDigestSketchAggregatorFactory("a0:agg",
"v0", 100))
+ .aggregators(new TDigestSketchAggregatorFactory("a0:agg",
"v0", 100, TDigestConfig.builder().build()))
.postAggregators(
new TDigestSketchToQuantilePostAggregator(
"a0",
@@ -524,8 +525,8 @@ public class TDigestSketchSqlAggregatorTest extends
BaseCalciteQueryTest
.filters(equality("dim2", 0L, ColumnType.LONG))
.granularity(Granularities.ALL)
.aggregators(ImmutableList.of(
- new TDigestSketchAggregatorFactory("a0", "m1",
TDigestSketchAggregatorFactory.DEFAULT_COMPRESSION),
- new TDigestSketchAggregatorFactory("a1:agg",
"qsketch_m1", 100)
+ new TDigestSketchAggregatorFactory("a0", "m1",
TDigestSketchAggregatorFactory.DEFAULT_COMPRESSION,
TDigestConfig.builder().build()),
+ new TDigestSketchAggregatorFactory("a1:agg",
"qsketch_m1", 100, TDigestConfig.builder().build())
))
.postAggregators(
new TDigestSketchToQuantilePostAggregator("a1",
makeFieldAccessPostAgg("a1:agg"), 0.1f)
@@ -561,11 +562,11 @@ public class TDigestSketchSqlAggregatorTest extends
BaseCalciteQueryTest
.setAggregatorSpecs(
aggregators(
new FilteredAggregatorFactory(
- new TDigestSketchAggregatorFactory("a0",
"m1", TDigestSketchAggregatorFactory.DEFAULT_COMPRESSION),
+ new TDigestSketchAggregatorFactory("a0",
"m1", TDigestSketchAggregatorFactory.DEFAULT_COMPRESSION,
TDigestConfig.builder().build()),
equality("dim1", "nonexistent",
ColumnType.STRING)
),
new FilteredAggregatorFactory(
- new
TDigestSketchAggregatorFactory("a1:agg", "qsketch_m1", 100),
+ new
TDigestSketchAggregatorFactory("a1:agg", "qsketch_m1", 100,
TDigestConfig.builder().build()),
equality("dim1", "nonexistent",
ColumnType.STRING)
)
)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
