This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 0a936211fb5 feat: use aws-crt by default for async s3 operations
(#19249)
0a936211fb5 is described below
commit 0a936211fb5236ba7e37831b180cd9587bd504eb
Author: Zoltan Haindrich <[email protected]>
AuthorDate: Thu Apr 2 16:43:36 2026 +0200
feat: use aws-crt by default for async s3 operations (#19249)
---
docs/development/extensions-core/s3.md | 1 +
embedded-tests/pom.xml | 2 +-
.../druid/testing/embedded/minio/S3PerfTest.java | 311 +++++++++++++++++++++
extensions-core/s3-extensions/pom.xml | 5 +
.../druid/storage/s3/S3StorageDruidModule.java | 48 +++-
.../apache/druid/storage/s3/S3TransferConfig.java | 17 ++
licenses.yaml | 14 +
pom.xml | 5 +
.../testing/embedded/EmbeddedDruidServer.java | 13 +-
website/.spelling | 4 +
10 files changed, 411 insertions(+), 9 deletions(-)
diff --git a/docs/development/extensions-core/s3.md
b/docs/development/extensions-core/s3.md
index 6ad71adc74d..f56861e613e 100644
--- a/docs/development/extensions-core/s3.md
+++ b/docs/development/extensions-core/s3.md
@@ -57,6 +57,7 @@ To use S3 for Deep Storage, you must supply [connection
information](#configurat
|`druid.storage.transfer.useTransferManager`| If true, use AWS S3 Transfer
Manager to upload segments to S3.|true|
|`druid.storage.transfer.minimumUploadPartSize`| Minimum size (in bytes) of
each part in a multipart upload.|20971520 (20 MB)|
|`druid.storage.transfer.multipartUploadThreshold`| The file size threshold
(in bytes) above which a file upload is converted into a multipart upload
instead of a single PUT request.| 20971520 (20 MB)|
+|`druid.storage.transfer.asyncHttpClientType`| Async HTTP client
implementation used by the S3 Transfer Manager. Accepted values: `crt` (Amazon
CRT) or `netty` (Netty NIO).|`crt`|
## Configuration
diff --git a/embedded-tests/pom.xml b/embedded-tests/pom.xml
index 43b0c68de92..acccca71644 100644
--- a/embedded-tests/pom.xml
+++ b/embedded-tests/pom.xml
@@ -861,7 +861,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
- <excludedGroups>docker-test</excludedGroups>
+ <excludedGroups>docker-test,perf</excludedGroups>
</configuration>
</plugin>
<plugin>
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/S3PerfTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/S3PerfTest.java
new file mode 100644
index 00000000000..4753c50ab14
--- /dev/null
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/S3PerfTest.java
@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.minio;
+
+import org.apache.druid.common.aws.AWSModule;
+import org.apache.druid.indexing.common.task.IndexTask;
+import org.apache.druid.indexing.common.task.TaskBuilder;
+import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.storage.s3.S3StorageDruidModule;
+import org.apache.druid.testing.embedded.EmbeddedBroker;
+import org.apache.druid.testing.embedded.EmbeddedClusterApis;
+import org.apache.druid.testing.embedded.EmbeddedCoordinator;
+import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
+import org.apache.druid.testing.embedded.EmbeddedHistorical;
+import org.apache.druid.testing.embedded.EmbeddedIndexer;
+import org.apache.druid.testing.embedded.EmbeddedOverlord;
+import org.apache.druid.testing.embedded.EmbeddedRouter;
+import org.apache.druid.testing.embedded.junit5.EmbeddedClusterTestBase;
+import org.joda.time.DateTime;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * S3 segment-upload performance tests.
+ *
+ * <p>Each {@link Nested} inner class runs a full embedded cluster with a
different
+ * storage backend / HTTP client combination:
+ * <ul>
+ * <li>MinIO container — CRT (default), Netty NIO, synchronous PutObject</li>
+ * <li>Real AWS S3 (opt-in via {@value BUCKET_PROPERTY}) — same three
variants</li>
+ * </ul>
+ *
+ * <p>Real-S3 tests are skipped unless {@code -Ds3.test.bucket=<bucket>} is
set.
+ * Data is written under {@code perf-test/<run-timestamp>/} and must be
cleaned up manually.
+ */
+@Tag("perf")
+public class S3PerfTest
+{
+ private static final Logger log = new Logger(S3PerfTest.class);
+
+ private static final int TASK_COUNT = 5;
+ private static final int ROWS_PER_TASK = 200_000;
+ private static final long MS_PER_DAY = 86_400_000L;
+
+ private static final String BUCKET_PROPERTY = "s3.test.bucket";
+ private static final String REGION_PROPERTY = "s3.test.region";
+ private static final String DEFAULT_REGION = "us-east-1";
+
+ abstract static class PerfTestBase extends EmbeddedClusterTestBase
+ {
+ private final List<File> tempFiles = new ArrayList<>();
+
+ final EmbeddedOverlord overlord = new EmbeddedOverlord();
+ final EmbeddedCoordinator coordinator = new EmbeddedCoordinator();
+ final EmbeddedIndexer indexer = new
EmbeddedIndexer().addProperty("druid.worker.capacity", "10");
+ final EmbeddedHistorical historical = new EmbeddedHistorical();
+ final EmbeddedBroker broker = new EmbeddedBroker();
+
+ EmbeddedDruidCluster newBaseCluster()
+ {
+ return EmbeddedDruidCluster.withEmbeddedDerbyAndZookeeper()
+ .useLatchableEmitter()
+ .useDefaultTimeoutForLatchableEmitter(300);
+ }
+
+ EmbeddedDruidCluster addServers(EmbeddedDruidCluster cluster)
+ {
+ return cluster.addServer(overlord)
+ .addServer(coordinator)
+ .addServer(indexer)
+ .addServer(historical)
+ .addServer(broker)
+ .addServer(new EmbeddedRouter());
+ }
+
+ @BeforeAll
+ void generateInputFiles(@TempDir Path tempDir) throws IOException
+ {
+ final DateTime baseDay = DateTimes.of("2025-01-01");
+ for (int i = 0; i < TASK_COUNT; i++) {
+ final File f = generateLargeCsvFile(tempDir, baseDay.plusDays(i),
ROWS_PER_TASK);
+ tempFiles.add(f);
+ log.info("Generated %s (%d MB)", f.getName(), f.length() / (1024 *
1024));
+ }
+ }
+
+ @Test
+ @Timeout(value = 20, unit = TimeUnit.MINUTES)
+ public void test_concurrentUploads()
+ {
+ final List<String> taskIds = new ArrayList<>();
+ for (int i = 0; i < TASK_COUNT; i++) {
+ taskIds.add(EmbeddedClusterApis.newTaskId(dataSource));
+ }
+
+ log.info("Starting %d concurrent tasks (%d rows each)", TASK_COUNT,
ROWS_PER_TASK);
+ final long startNanos = System.nanoTime();
+
+ for (int i = 0; i < taskIds.size(); i++) {
+ final String taskId = taskIds.get(i);
+ final File csvFile = tempFiles.get(i);
+ final IndexTask task = TaskBuilder
+ .ofTypeIndex()
+ .isoTimestampColumn("time")
+ .csvInputFormatWithColumns("time", "item", "value", "description")
+ .localInputSourceWithFiles(csvFile)
+ .segmentGranularity("DAY")
+ .dimensions()
+ .dataSource(dataSource)
+ .withId(taskId);
+ cluster.callApi().onLeaderOverlord(o -> o.runTask(taskId, task));
+ }
+
+ for (String taskId : taskIds) {
+ cluster.callApi().waitForTaskToSucceed(taskId, overlord);
+ }
+
+ final long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() -
startNanos);
+ log.info(
+ "[%s] tasks=[%d] rowsPerTask=[%d] fileSizeMb=[%d] elapsed=[%d ms]",
+ getClass().getSimpleName(), TASK_COUNT, ROWS_PER_TASK,
tempFiles.get(0).length() / (1024 * 1024), elapsedMs
+ );
+ }
+ }
+
+ abstract static class MinIOPerfBase extends PerfTestBase
+ {
+ abstract void configureCluster(EmbeddedDruidCluster cluster);
+
+ @Override
+ public EmbeddedDruidCluster createCluster()
+ {
+ final EmbeddedDruidCluster cluster =
addServers(newBaseCluster().addResource(new MinIOStorageResource()));
+ configureCluster(cluster);
+ return cluster;
+ }
+ }
+
+ /** MinIO + Amazon CRT async HTTP client (default). */
+ @Nested
+ static class MinIO_Crt extends MinIOPerfBase
+ {
+ @Override
+ void configureCluster(EmbeddedDruidCluster cluster)
+ {
+ }
+ }
+
+ /** MinIO + Netty NIO async HTTP client. */
+ @Nested
+ static class MinIO_Netty extends MinIOPerfBase
+ {
+ @Override
+ void configureCluster(EmbeddedDruidCluster cluster)
+ {
+ cluster.addCommonProperty("druid.storage.transfer.asyncHttpClientType",
"netty");
+ }
+ }
+
+ /** MinIO + synchronous PutObject (transfer manager disabled). */
+ @Nested
+ static class MinIO_Sync extends MinIOPerfBase
+ {
+ @Override
+ void configureCluster(EmbeddedDruidCluster cluster)
+ {
+ cluster.addCommonProperty("druid.storage.transfer.useTransferManager",
"false");
+ }
+ }
+
+ abstract static class RealS3PerfBase extends PerfTestBase
+ {
+ abstract void configureCluster(EmbeddedDruidCluster cluster);
+
+ @Override
+ public EmbeddedDruidCluster createCluster()
+ {
+ final String bucket = System.getProperty(BUCKET_PROPERTY);
+ Assumptions.assumeTrue(
+ bucket != null,
+ "Skipping real S3 perf test: set -D" + BUCKET_PROPERTY + "=<bucket>
to enable"
+ );
+
+ final String region = System.getProperty(REGION_PROPERTY,
DEFAULT_REGION);
+ final String baseKey = "perf-test/" + System.currentTimeMillis();
+
+ final EmbeddedDruidCluster cluster = newBaseCluster()
+ .addExtension(S3StorageDruidModule.class)
+ .addExtension(AWSModule.class);
+
+ cluster.addCommonProperty("druid.storage.type", "s3");
+ cluster.addCommonProperty("druid.storage.bucket", bucket);
+ cluster.addCommonProperty("druid.storage.baseKey", baseKey);
+ cluster.addCommonProperty("druid.indexer.logs.type", "s3");
+ cluster.addCommonProperty("druid.indexer.logs.s3Bucket", bucket);
+ cluster.addCommonProperty("druid.indexer.logs.s3Prefix", baseKey +
"/logs");
+ cluster.addCommonProperty("druid.s3.endpoint.signingRegion", region);
+
+ configureCluster(cluster);
+ addServers(cluster);
+ return cluster;
+ }
+ }
+
+ /** Real S3 + Amazon CRT async HTTP client (default). */
+ @Nested
+ static class RealS3_Crt extends RealS3PerfBase
+ {
+ @Override
+ void configureCluster(EmbeddedDruidCluster cluster)
+ {
+ }
+ }
+
+ /** Real S3 + Netty NIO async HTTP client. */
+ @Nested
+ static class RealS3_Netty extends RealS3PerfBase
+ {
+ @Override
+ void configureCluster(EmbeddedDruidCluster cluster)
+ {
+ cluster.addCommonProperty("druid.storage.transfer.asyncHttpClientType",
"netty");
+ }
+ }
+
+ /** Real S3 + synchronous PutObject (transfer manager disabled). */
+ @Nested
+ static class RealS3_Sync extends RealS3PerfBase
+ {
+ @Override
+ void configureCluster(EmbeddedDruidCluster cluster)
+ {
+ cluster.addCommonProperty("druid.storage.transfer.useTransferManager",
"false");
+ }
+ }
+
+ private static File generateLargeCsvFile(Path dir, DateTime day, int
rowCount) throws IOException
+ {
+ final File file = dir.resolve("perf-" + day.toString("yyyy-MM-dd") +
".csv").toFile();
+ final Random rng = new Random(day.getMillis());
+ try (BufferedWriter w = Files.newBufferedWriter(file.toPath(),
StandardCharsets.UTF_8)) {
+ for (int i = 0; i < rowCount; i++) {
+ w.write(rowTimestamp(day, i, rowCount));
+ w.write(',');
+ w.write("item_");
+ w.write(Integer.toString(i));
+ w.write(',');
+ w.write(Long.toString(randomValue(rng)));
+ w.write(',');
+ w.write(randomDescription(rng));
+ w.write('\n');
+ }
+ }
+ return file;
+ }
+
+ private static String rowTimestamp(DateTime day, int row, int rowCount)
+ {
+ return day.plusMillis((int) (row * MS_PER_DAY / rowCount)).toString();
+ }
+
+ private static long randomValue(Random rng)
+ {
+ return (rng.nextLong() >>> 1) % 1_000_000L;
+ }
+
+ /** 13 × 16 hex chars = 208 chars of random data, resistant to compression.
*/
+ private static String randomDescription(Random rng)
+ {
+ final StringBuilder sb = new StringBuilder(208);
+ for (int j = 0; j < 13; j++) {
+ sb.append(String.format(Locale.ROOT, "%016x", rng.nextLong()));
+ }
+ return sb.toString();
+ }
+}
diff --git a/extensions-core/s3-extensions/pom.xml
b/extensions-core/s3-extensions/pom.xml
index 74b64c43c2b..0989ee41293 100644
--- a/extensions-core/s3-extensions/pom.xml
+++ b/extensions-core/s3-extensions/pom.xml
@@ -112,6 +112,11 @@
<artifactId>s3-transfer-manager</artifactId>
<version>${aws.sdk.v2.version}</version>
</dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>aws-crt-client</artifactId>
+ <version>${aws.sdk.v2.version}</version>
+ </dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>netty-nio-client</artifactId>
diff --git
a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3StorageDruidModule.java
b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3StorageDruidModule.java
index 54d275f7818..27fc537e34b 100644
---
a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3StorageDruidModule.java
+++
b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3StorageDruidModule.java
@@ -36,10 +36,13 @@ import org.apache.druid.guice.Binders;
import org.apache.druid.guice.JsonConfigProvider;
import org.apache.druid.guice.LazySingleton;
import org.apache.druid.initialization.DruidModule;
+import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.logger.Logger;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.http.apache.ApacheHttpClient;
import software.amazon.awssdk.http.apache.ProxyConfiguration;
+import software.amazon.awssdk.http.async.SdkAsyncHttpClient;
+import software.amazon.awssdk.http.crt.AwsCrtAsyncHttpClient;
import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3AsyncClient;
@@ -191,15 +194,12 @@ public class S3StorageDruidModule implements DruidModule
};
// Create async client supplier for S3TransferManager
+ final AsyncHttpClientType asyncHttpClientType =
+
AsyncHttpClientType.fromString(storageConfig.getS3TransferConfig().getAsyncHttpClientType());
final Supplier<S3AsyncClient> s3AsyncClientSupplier = () -> {
- NettyNioAsyncHttpClient.Builder asyncHttpClientBuilder =
NettyNioAsyncHttpClient.builder()
-
.connectionTimeout(Duration.ofMillis(clientConfig.getConnectionTimeoutMillis()))
-
.readTimeout(Duration.ofMillis(clientConfig.getSocketTimeoutMillis()))
- .maxConcurrency(clientConfig.getMaxConnections());
-
S3AsyncClientBuilder s3AsyncClientBuilder = S3AsyncClient.builder()
.credentialsProvider(provider)
- .httpClientBuilder(asyncHttpClientBuilder)
+ .httpClientBuilder(asyncHttpClientType.buildBuilder(clientConfig))
.forcePathStyle(clientConfig.isEnablePathStyleAccess())
.crossRegionAccessEnabled(clientConfig.isCrossRegionAccessEnabled())
.multipartEnabled(true);
@@ -221,6 +221,42 @@ public class S3StorageDruidModule implements DruidModule
.setS3StorageConfig(storageConfig);
}
+ public enum AsyncHttpClientType
+ {
+ CRT {
+ @Override
+ public SdkAsyncHttpClient.Builder<?> buildBuilder(AWSClientConfig
clientConfig)
+ {
+ // AwsCrtAsyncHttpClient.Builder does not expose readTimeout in SDK
2.40.0.
+ return AwsCrtAsyncHttpClient.builder()
+
.connectionTimeout(Duration.ofMillis(clientConfig.getConnectionTimeoutMillis()))
+
.maxConcurrency(clientConfig.getMaxConnections());
+ }
+ },
+ NETTY {
+ @Override
+ public SdkAsyncHttpClient.Builder<?> buildBuilder(AWSClientConfig
clientConfig)
+ {
+ return NettyNioAsyncHttpClient.builder()
+
.connectionTimeout(Duration.ofMillis(clientConfig.getConnectionTimeoutMillis()))
+
.readTimeout(Duration.ofMillis(clientConfig.getSocketTimeoutMillis()))
+
.maxConcurrency(clientConfig.getMaxConnections());
+ }
+ };
+
+ public abstract SdkAsyncHttpClient.Builder<?> buildBuilder(AWSClientConfig
clientConfig);
+
+ public static AsyncHttpClientType fromString(String value)
+ {
+ for (AsyncHttpClientType type : values()) {
+ if (type.name().equals(StringUtils.upperCase(value))) {
+ return type;
+ }
+ }
+ throw new ISE("Invalid druid.storage.transfer.asyncHttpClientType[%s].
Must be 'crt' or 'netty'.", value);
+ }
+ }
+
@Nullable
private static URI buildEndpointOverride(AWSEndpointConfig endpointConfig,
boolean useHttps)
{
diff --git
a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3TransferConfig.java
b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3TransferConfig.java
index 4df62b83342..5cd13e20b4e 100644
---
a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3TransferConfig.java
+++
b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3TransferConfig.java
@@ -38,6 +38,13 @@ public class S3TransferConfig
@Min(1)
private long multipartUploadThreshold = 20 * 1024 * 1024L;
+ /**
+ * Async HTTP client implementation to use with the S3 transfer manager.
+ * Accepted values: {@code "crt"} (Amazon CRT, default) or {@code "netty"}
(Netty NIO).
+ */
+ @JsonProperty
+ private String asyncHttpClientType = "crt";
+
public void setUseTransferManager(boolean useTransferManager)
{
this.useTransferManager = useTransferManager;
@@ -53,6 +60,11 @@ public class S3TransferConfig
this.multipartUploadThreshold = multipartUploadThreshold;
}
+ public void setAsyncHttpClientType(String asyncHttpClientType)
+ {
+ this.asyncHttpClientType = asyncHttpClientType;
+ }
+
public boolean isUseTransferManager()
{
return useTransferManager;
@@ -68,4 +80,9 @@ public class S3TransferConfig
return multipartUploadThreshold;
}
+ public String getAsyncHttpClientType()
+ {
+ return asyncHttpClientType;
+ }
+
}
diff --git a/licenses.yaml b/licenses.yaml
index befab3095a0..2d20bc227eb 100644
--- a/licenses.yaml
+++ b/licenses.yaml
@@ -247,6 +247,7 @@ libraries:
- software.amazon.awssdk: arns
- software.amazon.awssdk: auth
- software.amazon.awssdk: aws-core
+ - software.amazon.awssdk: aws-crt-client
- software.amazon.awssdk: aws-query-protocol
- software.amazon.awssdk: aws-xml-protocol
- software.amazon.awssdk: checksums
@@ -303,6 +304,19 @@ notice: |
---
+name: AWS CRT
+license_category: binary
+module: extensions-core/s3-extensions
+license_name: Apache License version 2.0
+version: 0.40.1
+libraries:
+ - software.amazon.awssdk.crt: aws-crt
+notice: |
+ AWS Common Runtime for Java
+ Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+---
+
name: Jackson
license_category: binary
module: java-core
diff --git a/pom.xml b/pom.xml
index b1c3daf8327..be5e1fcf8df 100644
--- a/pom.xml
+++ b/pom.xml
@@ -415,6 +415,11 @@
<artifactId>airline</artifactId>
<version>2.8.4</version>
</dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>aws-crt-client</artifactId>
+ <version>${aws.sdk.v2.version}</version>
+ </dependency>
<dependency>
<groupId>net.minidev</groupId>
<artifactId>json-smart</artifactId>
diff --git
a/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedDruidServer.java
b/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedDruidServer.java
index 5f2ff75fe82..2c4a6185a83 100644
---
a/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedDruidServer.java
+++
b/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedDruidServer.java
@@ -73,8 +73,12 @@ public abstract class EmbeddedDruidServer<T extends
EmbeddedDruidServer<T>> impl
);
beforeStartHooks.add(
(cluster, self) -> {
- // Add properties for temporary directories used by the servers
- final String logsDirectory =
cluster.getTestFolder().getOrCreateFolder("indexer-logs").getAbsolutePath();
+ // Add properties for temporary directories used by the servers.
+ // If task.logs.dir is set, write indexer logs there so they survive
TestFolder cleanup.
+ final String taskLogsDir = System.getProperty("task.logs.dir");
+ final String logsDirectory = taskLogsDir != null
+ ? taskLogsDir
+ :
cluster.getTestFolder().getOrCreateFolder("indexer-logs").getAbsolutePath();
final String taskDirectory =
cluster.getTestFolder().newFolder().getAbsolutePath();
final String storageDirectory =
cluster.getTestFolder().getOrCreateFolder("deep-store").getAbsolutePath();
log.info(
@@ -91,6 +95,11 @@ public abstract class EmbeddedDruidServer<T extends
EmbeddedDruidServer<T>> impl
self.addProperty("druid.indexer.logs.directory", logsDirectory);
self.addProperty("druid.storage.storageDirectory", storageDirectory);
+ // When task.logs.dir is set, force file-based task logging
+ if (taskLogsDir != null) {
+ self.addProperty("druid.indexer.logs.type", "file");
+ }
+
// Add properties for RuntimeInfoModule
self.addProperty(RuntimeInfoModule.SERVER_MEMORY_PROPERTY,
String.valueOf(serverMemory));
self.addProperty(RuntimeInfoModule.SERVER_DIRECT_MEMORY_PROPERTY,
String.valueOf(serverDirectMemory));
diff --git a/website/.spelling b/website/.spelling
index ccb105a010a..a5b1e7ec5fa 100644
--- a/website/.spelling
+++ b/website/.spelling
@@ -2628,6 +2628,7 @@ TableSpec
- ../docs/development/extensions-contrib/prometheus.md
TTL
+
- ../docs/development/extensions-contrib/consul.md
minimum
than
@@ -2635,3 +2636,6 @@ than
maxWatchRetries
cleartext
nginx
+
+- ../docs/development/extensions-core/s3.md
+NIO
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
