This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 0a4b7722ab8 minor: update some deps and reduce thread pressure during
test runs (#19248)
0a4b7722ab8 is described below
commit 0a4b7722ab83070942319d1ab6c523747bd89da7
Author: Zoltan Haindrich <[email protected]>
AuthorDate: Fri Apr 3 07:06:33 2026 +0200
minor: update some deps and reduce thread pressure during test runs (#19248)
---
.../{collect_jstacks => collect_jvm_diagnostics} | 9 ++-
.github/workflows/worker.yml | 4 +-
distribution/bin/check-licenses.py | 1 +
embedded-tests/pom.xml | 2 +-
extensions-contrib/redis-cache/pom.xml | 2 -
licenses.yaml | 36 ++++++----
pom.xml | 83 ++++++++++++++--------
.../druid/quidem/DruidAvaticaTestDriver.java | 10 ++-
.../druid/sql/calcite/SqlTestFrameworkConfig.java | 32 +++++----
9 files changed, 115 insertions(+), 64 deletions(-)
diff --git a/.github/scripts/collect_jstacks
b/.github/scripts/collect_jvm_diagnostics
similarity index 78%
rename from .github/scripts/collect_jstacks
rename to .github/scripts/collect_jvm_diagnostics
index 7f4d2a6a160..9f3818ca97b 100755
--- a/.github/scripts/collect_jstacks
+++ b/.github/scripts/collect_jvm_diagnostics
@@ -16,13 +16,16 @@
# limitations under the License.
#--------------------------------------------------------------------
-echo "@ capturing threaddumps every minute for all java processes" >&2
+echo "@ capturing jvm diagnostics every minute for all java processes" >&2
+
+mkdir -p target/ci-debug
while :;do
sleep 60
t=`date +%s`
- echo "@ capturing threaddumps $t" >&2
+ echo "@ capturing jvm diagnostics $t" >&2
pgrep java | while read pid;do
- jstack $pid > target/jstack.$pid.$t
+ jstack $pid > target/ci-debug/jstack.$pid.$t
+ jcmd $pid GC.heap_info > target/ci-debug/heap.$pid.$t 2>&1
done
done
diff --git a/.github/workflows/worker.yml b/.github/workflows/worker.yml
index bde0d206ee4..6eb428b5750 100644
--- a/.github/workflows/worker.yml
+++ b/.github/workflows/worker.yml
@@ -84,7 +84,7 @@ jobs:
set -x
./.github/scripts/setup_test_profiling_env.sh ${{ inputs.jdk }}
run_id=$GITHUB_RUN_ID run_number=$GITHUB_RUN_NUMBER
run_attempt=$GITHUB_RUN_ATTEMPT key=${{ inputs.key }}
event_ref=$GITHUB_EVENT_REF run_url=$GITHUB_RUN_URL >> $GITHUB_ENV
echo "HASH=$(echo -n "${{ inputs.key }}" | sha256sum | cut -c-8)" >>
$GITHUB_ENV
- ./.github/scripts/collect_jstacks &
+ ./.github/scripts/collect_jvm_diagnostics &
- name: 'Execute: ${{ inputs.script }}'
run: ${{ inputs.script }}
@@ -118,4 +118,4 @@ jobs:
**/core.[0-9]*
**/TEST-*.xml
**/target/jacoco*.exec
- **/target/jstack*
+ **/target/ci-debug/**
diff --git a/distribution/bin/check-licenses.py
b/distribution/bin/check-licenses.py
index 7f373b11a94..7c85494862c 100755
--- a/distribution/bin/check-licenses.py
+++ b/distribution/bin/check-licenses.py
@@ -264,6 +264,7 @@ def build_compatible_license_names():
compatible_licenses['BSD-3-Clause'] = 'BSD-3-Clause License'
compatible_licenses['Unicode/ICU License'] = 'Unicode/ICU License'
+ compatible_licenses['Unicode-3.0'] = 'Unicode/ICU License'
compatible_licenses['SIL Open Font License 1.1'] = 'SIL Open Font License
1.1'
diff --git a/embedded-tests/pom.xml b/embedded-tests/pom.xml
index acccca71644..215c051fd47 100644
--- a/embedded-tests/pom.xml
+++ b/embedded-tests/pom.xml
@@ -748,7 +748,7 @@
<dependency>
<groupId>org.apache.httpcomponents.client5</groupId>
<artifactId>httpclient5</artifactId>
- <version>5.5</version>
+ <version>5.5.1</version>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/extensions-contrib/redis-cache/pom.xml
b/extensions-contrib/redis-cache/pom.xml
index 2c5f95b1348..44265f3755f 100644
--- a/extensions-contrib/redis-cache/pom.xml
+++ b/extensions-contrib/redis-cache/pom.xml
@@ -49,7 +49,6 @@
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
- <version>5.1.2</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
@@ -106,7 +105,6 @@
<dependency>
<groupId>com.github.fppt</groupId>
<artifactId>jedis-mock</artifactId>
- <version>1.1.12</version>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/licenses.yaml b/licenses.yaml
index 2d20bc227eb..5d89655d0f4 100644
--- a/licenses.yaml
+++ b/licenses.yaml
@@ -321,7 +321,7 @@ name: Jackson
license_category: binary
module: java-core
license_name: Apache License version 2.0
-version: 2.19.2
+version: 2.20.2
libraries:
- com.fasterxml.jackson.core: jackson-core
- com.fasterxml.jackson.core: jackson-annotations
@@ -360,11 +360,21 @@ notice: |
---
+name: Jackson
+license_category: binary
+module: java-core
+license_name: Apache License version 2.0
+version: "2.20"
+libraries:
+ - com.fasterxml.jackson.core: jackson-annotations
+
+---
+
name: Jackson
license_category: binary
module: extensions-contrib/druid-deltalake-extensions
license_name: Apache License version 2.0
-version: 2.19.2
+version: 2.20.2
libraries:
- com.fasterxml.jackson.core: jackson-databind
notice: |
@@ -390,7 +400,7 @@ name: JavaBeans Activation Framework API JAR
license_category: binary
module: java-core
license_name: Eclipse Distribution License 1.0
-version: 1.2.1
+version: 1.2.2
copyright: Oracle and/or its affiliates.
license_file_path: licenses/bin/jakarta.EDL1
libraries:
@@ -424,7 +434,7 @@ name: Error Prone Annotations
license_category: binary
module: java-core
license_name: Apache License version 2.0
-version: 2.35.1
+version: 2.41.0
libraries:
- com.google.errorprone: error_prone_annotations
@@ -1211,7 +1221,7 @@ name: org.yaml snakeyaml
license_category: binary
module: extensions-core/kubernetes-overlord-extensions
license_name: Apache License version 2.0
-version: 2.4
+version: 2.5
libraries:
- org.yaml: snakeyaml
@@ -1241,7 +1251,7 @@ name: org.bouncycastle bcprov-jdk18on
license_category: binary
module: extensions-core/kubernetes-extensions
license_name: MIT License
-version: "1.81"
+version: "1.82"
libraries:
- org.bouncycastle: bcprov-jdk18on
- org.bouncycastle: bcprov-ext-jdk18on
@@ -1434,7 +1444,7 @@ name: Netty
license_category: binary
module: java-core
license_name: Apache License version 2.0
-version: 4.2.6.Final
+version: 4.2.12.Final
libraries:
- io.netty: netty-buffer
- io.netty: netty-codec
@@ -2637,7 +2647,7 @@ name: RoaringBitmap
license_category: binary
module: java-core
license_name: Apache License version 2.0
-version: 0.9.49
+version: 1.6.13
libraries:
- org.roaringbitmap: RoaringBitmap
- org.roaringbitmap: shims
@@ -3024,7 +3034,7 @@ libraries:
---
name: Jackson Dataformat Yaml
-version: 2.19.2
+version: 2.20.2
license_category: binary
module: extensions/druid-avro-extensions
license_name: Apache License version 2.0
@@ -3179,7 +3189,7 @@ name: org.yaml snakeyaml
license_category: binary
module: extensions/druid-protobuf-extensions
license_name: Apache License version 2.0
-version: 2.0
+version: 2.5
libraries:
- org.yaml: snakeyaml
@@ -4140,7 +4150,7 @@ name: aircompressor
license_category: binary
module: extensions/druid-orc-extensions
license_name: Apache License version 2.0
-version: "0.21"
+version: "2.0.2"
libraries:
- io.airlift: aircompressor
@@ -4642,7 +4652,7 @@ name: Netty
license_category: binary
module: extensions/druid-azure-extensions
license_name: Apache License version 2.0
-version: 2.0.73.Final
+version: 2.0.75.Final
libraries:
- io.netty: netty-tcnative-boringssl-static
- io.netty: netty-tcnative-classes
@@ -5215,7 +5225,7 @@ name: jakarta.activation
license_category: binary
module: extensions/druid-avro-extensions
license_name: Eclipse Distribution License 1.0
-version: 1.2.2
+version: 2.0.1
libraries:
- com.sun.activation: jakarta.activation
diff --git a/pom.xml b/pom.xml
index be5e1fcf8df..8aadc56f459 100644
--- a/pom.xml
+++ b/pom.xml
@@ -96,7 +96,7 @@
<datasketches.memory.version>2.2.0</datasketches.memory.version>
<derby.version>10.14.2.0</derby.version>
<dropwizard.metrics.version>4.2.22</dropwizard.metrics.version>
- <errorprone.version>2.35.1</errorprone.version>
+ <errorprone.version>2.41.0</errorprone.version>
<fabric8.version>7.6.0</fabric8.version>
<fastutil.version>8.5.4</fastutil.version>
<guava.version>32.1.3-jre</guava.version>
@@ -104,13 +104,13 @@
<hamcrest.version>2.2</hamcrest.version>
<jetty.version>12.0.30</jetty.version>
<jersey.version>1.19.4</jersey.version>
- <jackson.version>2.19.2</jackson.version>
+ <jackson.version>2.20.2</jackson.version>
<codehaus.jackson.version>1.9.13</codehaus.jackson.version>
<log4j.version>2.25.3</log4j.version>
<mysql.version>8.2.0</mysql.version>
<mariadb.version>2.7.3</mariadb.version>
<netty3.version>3.10.6.Final</netty3.version>
- <netty4.version>4.2.6.Final</netty4.version>
+ <netty4.version>4.2.12.Final</netty4.version>
<postgresql.version>42.7.2</postgresql.version>
<protobuf.version>3.25.8</protobuf.version>
<resilience4j.version>1.3.1</resilience4j.version>
@@ -460,12 +460,12 @@
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk18on</artifactId>
- <version>1.81</version>
+ <version>1.82</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk18on</artifactId>
- <version>1.81</version>
+ <version>1.82</version>
</dependency>
<!-- Transitive dependency of hive-common in druid-kerberos,
druid-ranger-security and
druid-iceberg-extension -->
@@ -489,6 +489,21 @@
<artifactId>logging-interceptor</artifactId>
<version>${okhttp.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.squareup.okio</groupId>
+ <artifactId>okio</artifactId>
+ <version>3.16.4</version>
+ </dependency>
+ <dependency>
+ <groupId>redis.clients</groupId>
+ <artifactId>jedis</artifactId>
+ <version>7.0.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.github.fppt</groupId>
+ <artifactId>jedis-mock</artifactId>
+ <version>1.1.12</version>
+ </dependency>
<dependency>
<groupId>io.kubernetes</groupId>
<artifactId>client-java-api</artifactId>
@@ -1019,6 +1034,16 @@
<artifactId>httpclient</artifactId>
<version>${httpclient.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents.client5</groupId>
+ <artifactId>httpclient5</artifactId>
+ <version>5.5.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents.core5</groupId>
+ <artifactId>httpcore5</artifactId>
+ <version>5.3.6</version>
+ </dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
@@ -1088,7 +1113,17 @@
<dependency>
<groupId>org.roaringbitmap</groupId>
<artifactId>RoaringBitmap</artifactId>
- <version>0.9.49</version>
+ <version>1.6.13</version>
+ </dependency>
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ <version>2.5</version>
+ </dependency>
+ <dependency>
+ <groupId>io.airlift</groupId>
+ <artifactId>aircompressor</artifactId>
+ <version>2.0.2</version>
</dependency>
<dependency>
<groupId>org.checkerframework</groupId>
@@ -1150,6 +1185,16 @@
<artifactId>activation</artifactId>
<version>1.1.1</version>
</dependency>
+ <dependency>
+ <groupId>com.sun.activation</groupId>
+ <artifactId>jakarta.activation</artifactId>
+ <version>2.0.1</version>
+ </dependency>
+ <dependency>
+ <groupId>jakarta.activation</groupId>
+ <artifactId>jakarta.activation-api</artifactId>
+ <version>1.2.2</version>
+ </dependency>
<dependency>
<groupId>commons-pool</groupId>
<artifactId>commons-pool</artifactId>
@@ -1412,7 +1457,7 @@
<dependency>
<groupId>org.reflections</groupId>
<artifactId>reflections</artifactId>
- <version>0.9.12</version>
+ <version>0.10.2</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -1836,35 +1881,14 @@
<!-- upgrading these later -->
<exclude>org.apache.zookeeper:zookeeper</exclude>
<exclude>com.google.guava:guava</exclude>
-
<exclude>com.fasterxml.jackson.core:jackson-annotations</exclude>
-
<exclude>com.fasterxml.jackson.core:jackson-databind</exclude>
-
<exclude>com.fasterxml.jackson.core:jackson-core</exclude>
-
<exclude>com.fasterxml.jackson.datatype:jackson-datatype-jsr310</exclude>
-
<exclude>com.fasterxml.jackson.dataformat:jackson-dataformat-yaml</exclude>
-
<exclude>jakarta.activation:jakarta.activation-api</exclude>
<exclude>com.google.inject.extensions:guice-assistedinject</exclude>
<exclude>com.fasterxml.jackson.module:jackson-module-guice</exclude>
<exclude>com.google.inject:guice</exclude>
- <exclude>redis.clients:jedis</exclude>
-
<exclude>org.reflections:reflections</exclude>
-
<exclude>org.roaringbitmap:RoaringBitmap</exclude>
-
<exclude>io.airlift:aircompressor</exclude>
-
<exclude>com.squareup.okio:okio</exclude>
-
<exclude>org.apache.httpcomponents.client5:httpclient5</exclude>
-
<exclude>com.sun.activation:jakarta.activation</exclude>
- <exclude>org.yaml:snakeyaml</exclude>
-
<exclude>io.netty:netty-transport-native-kqueue</exclude>
-
<exclude>io.netty:netty-transport-native-epoll</exclude>
-
<exclude>io.netty:netty-handler-proxy</exclude>
-
<exclude>io.netty:netty-handler</exclude>
<exclude>com.google.http-client:google-http-client-jackson2</exclude>
<exclude>com.google.http-client:google-http-client</exclude>
<exclude>com.google.api.grpc:proto-google-common-protos</exclude>
-
<exclude>org.bouncycastle:bcpkix-jdk18on</exclude>
-
<exclude>io.netty:netty-codec-http</exclude>
<exclude>io.confluent:kafka-schema-registry-client</exclude>
<exclude>org.apache.kafka:kafka-clients</exclude>
-
<exclude>com.google.errorprone:error_prone_annotations</exclude>
</excludes>
<excludedScopes>
<excludedScope>provided</excludedScope>
@@ -2108,7 +2132,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
- <version>3.11.0</version>
+ <version>3.14.1</version>
<inherited>true</inherited>
<configuration>
<release>${maven.compiler.release}</release>
@@ -2159,6 +2183,7 @@
<arg>-XDcompilePolicy=simple</arg>
<arg>-Xplugin:ErrorProne
-XepExcludedPaths:.*/target/generated-(test-)?sources/.*
-XepDisableWarningsInGeneratedCode -Xep:ClassCanBeStatic:ERROR
-Xep:PreconditionsInvalidPlaceholder:ERROR -Xep:MissingOverride:ERROR
-Xep:DefaultCharset:ERROR -Xep:QualifierOrScopeOnInjectMethod:ERROR
-Xep:AssistedInjectAndInjectOnSameConstructor -Xep:AutoFactoryAtInject
-Xep:ClassName -Xep:ComparisonContractViolated -Xep:DepAnn -Xep:EmptyIf
-Xep:InjectInvalidTargetingOnScopi [...]
<!-- Error Prone requires exemptions for Java
>= 16, see https://errorprone.info/docs/installation#maven -->
+ <arg>--should-stop=ifError=FLOW</arg>
<arg>-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED</arg>
<arg>-J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED</arg>
<arg>-J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED</arg>
diff --git
a/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java
b/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java
index a586358e126..b917a5d7bb5 100644
--- a/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java
+++ b/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java
@@ -43,10 +43,11 @@ import
org.apache.druid.sql.calcite.util.SqlTestFramework.QueryComponentSupplier
import org.apache.druid.sql.hook.DruidHookDispatcher;
import org.apache.http.client.utils.URIBuilder;
import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
import java.io.Closeable;
import java.io.IOException;
-import java.net.InetSocketAddress;
import java.sql.Connection;
import java.sql.Driver;
import java.sql.DriverManager;
@@ -150,7 +151,12 @@ public class DruidAvaticaTestDriver implements Driver
AvaticaJettyServer(final DruidMeta druidMeta, DruidConnectionExtras
druidConnectionExtras) throws Exception
{
this.druidMeta = druidMeta;
- server = new Server(new InetSocketAddress("localhost", 0));
+ QueuedThreadPool threadPool = new QueuedThreadPool(10, 2, 1000);
+ server = new Server(threadPool);
+ ServerConnector connector = new ServerConnector(server);
+ connector.setHost("localhost");
+ connector.setPort(0);
+ server.addConnector(connector);
server.setHandler(getAvaticaHandler(druidMeta));
server.start();
url = StringUtils.format(
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java
b/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java
index 59b2c8e0683..f950f933a7b 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java
@@ -44,10 +44,8 @@ import org.junit.jupiter.api.extension.BeforeEachCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.reflections.Configuration;
import org.reflections.Reflections;
-import org.reflections.scanners.SubTypesScanner;
-import org.reflections.util.ClasspathHelper;
+import org.reflections.scanners.Scanners;
import org.reflections.util.ConfigurationBuilder;
-import org.reflections.util.FilterBuilder;
import javax.annotation.Nonnull;
@@ -66,6 +64,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@ -244,6 +243,8 @@ public class SqlTestFrameworkConfig
public static class SqlTestFrameworkConfigStore implements Closeable
{
+ private static final int MAX_CACHED_CONFIGS = 32;
+
private final Function<QueryComponentSupplier, QueryComponentSupplier>
queryComponentSupplierWrapper;
public SqlTestFrameworkConfigStore(
@@ -252,13 +253,24 @@ public class SqlTestFrameworkConfig
this.queryComponentSupplierWrapper = queryComponentSupplierWrapper;
}
- Map<SqlTestFrameworkConfig, ConfigurationInstance> configMap = new
HashMap<>();
+ Map<SqlTestFrameworkConfig, ConfigurationInstance> configMap = new
LinkedHashMap<>(16, 0.75f, true)
+ {
+ @Override
+ protected boolean removeEldestEntry(Map.Entry<SqlTestFrameworkConfig,
ConfigurationInstance> eldest)
+ {
+ if (size() > MAX_CACHED_CONFIGS) {
+ eldest.getValue().close();
+ return true;
+ }
+ return false;
+ }
+ };
public ConfigurationInstance getConfigurationInstance(
SqlTestFrameworkConfig config) throws Exception
{
ConfigurationInstance ret = configMap.get(config);
- if (!configMap.containsKey(config)) {
+ if (ret == null) {
ret = new ConfigurationInstance(config, queryComponentSupplierWrapper);
configMap.put(config, ret);
}
@@ -514,13 +526,9 @@ public class SqlTestFrameworkConfig
public Set<Class<? extends QueryComponentSupplier>> load(String pkg)
{
Configuration cfg = new ConfigurationBuilder()
- .setScanners(new SubTypesScanner(true))
- .setUrls(ClasspathHelper.forJavaClassPath())
- .filterInputsBy(
- new FilterBuilder()
- .includePackage(pkg)
- .and(s -> s.contains("ComponentSupplier"))
- );
+ .setScanners(Scanners.SubTypes)
+ .forPackage(pkg.isEmpty() ? "org.apache.druid" : pkg)
+ .filterInputsBy(s -> s.contains("ComponentSupplier"));
final Set<Class<? extends QueryComponentSupplier>>
baseComponentClazzes =
new Reflections(cfg).getSubTypesOf(QueryComponentSupplier.class);
LinkedHashSet<Class<? extends QueryComponentSupplier>> retVal = new
LinkedHashSet<>(baseComponentClazzes);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
