This is an automated email from the ASF dual-hosted git repository. gwphua pushed a commit to branch concurrent-linked-queue-for-writes in repository https://gitbox.apache.org/repos/asf/druid.git
commit 7f978ed1de4ff9377be170b16f05e60f0a967e35 Merge: 2f1eebd6704 fd771c06c0e Author: GWphua <[email protected]> AuthorDate: Thu Sep 4 10:43:27 2025 +0800 Merge branch 'master' into concurrent-linked-queue-for-writes .github/workflows/cron-job-its.yml | 14 - .github/workflows/reusable-revised-its.yml | 46 - .github/workflows/revised-its.yml | 41 +- .github/workflows/standard-its.yml | 13 - .../unit-and-integration-tests-unified.yml | 48 - .../benchmark/IndexedTableJoinCursorBenchmark.java | 2 +- .../druid/benchmark/JoinAndLookupBenchmark.java | 8 +- docs/development/extensions-core/k8s-jobs.md | 1 + embedded-tests/pom.xml | 12 + .../testing/embedded/auth/BasicAuthMSQTest.java | 89 +- .../indexing/KafkaSchemaRegistryResource.java | 24 +- .../testing/embedded/indexing/MoreResources.java | 66 ++ .../embedded/msq/BaseRealtimeQueryTest.java | 21 +- .../msq/EmbeddedDurableShuffleStorageTest.java | 2 +- .../testing/embedded/msq/EmbeddedMSQApis.java | 13 +- .../embedded/msq/EmbeddedMSQRealtimeQueryTest.java | 11 +- .../msq/EmbeddedMSQRealtimeUnnestQueryTest.java | 9 +- .../msq/MSQKeyStatisticsSketchMergeModeTest.java | 189 ++++ .../embedded/msq/MSQLocalDurableStorage.java | 52 + .../embedded/msq/MSQWorkerFaultToleranceTest.java | 135 +++ .../testing/embedded/msq/MultiStageQueryTest.java | 154 +++ .../embedded/query/QueryVirtualStorageTest.java | 239 ++++ .../testing/embedded/query/UnionQueryTest.java | 293 +++++ .../embedded/server/CoordinatorPauseTest.java | 119 ++ .../server/HttpEmitterEventCollectorTest.java | 4 +- .../k8s/overlord/KubernetesPeonLifecycle.java | 37 +- .../overlord/KubernetesPeonLifecycleFactory.java | 8 +- .../k8s/overlord/KubernetesTaskRunnerConfig.java | 23 + .../k8s/overlord/KubernetesTaskRunnerFactory.java | 7 +- .../k8s/overlord/KubernetesPeonLifecycleTest.java | 167 ++- .../druid/k8s/overlord/KubernetesWorkItemTest.java | 21 +- .../apache/druid/guice/ClusterTestingModule.java | 25 +- .../testing/cluster/ClusterTestingTaskConfig.java | 20 +- .../testing/cluster/task/FaultyOverlordClient.java | 87 +- .../druid/guice/ClusterTestingModuleTest.java | 24 +- .../druid/indexing/common/task/CompactionTask.java | 32 +- .../indexing/input/DruidSegmentInputEntity.java | 5 +- .../apache/druid/indexing/overlord/TaskQueue.java | 34 +- .../overlord/hrtr/HttpRemoteTaskRunner.java | 14 +- .../druid/indexing/overlord/hrtr/WorkerHolder.java | 3 + .../shuffle/LocalIntermediaryDataManager.java | 92 +- .../common/task/CompactionTaskRunTest.java | 4 + .../indexing/common/task/CompactionTaskTest.java | 20 +- .../druid/indexing/common/task/IndexTaskTest.java | 2 + .../AbstractMultiPhaseParallelIndexingTest.java | 3 +- .../indexing/input/DruidSegmentReaderTest.java | 12 + .../overlord/TaskQueueConcurrencyTest.java | 160 ++- .../indexing/overlord/hrtr/WorkerHolderTest.java | 15 +- .../DruidExactCountBitmap/docker-compose.py | 36 - integration-tests-ex/cases/pom.xml | 52 - .../DruidExactCountBitmapTest.java | 205 ---- .../testsEx/categories/DruidExactCountBitmap.java | 24 - .../testsEx/categories/MultiStageQueryWithMM.java | 24 - .../org/apache/druid/testsEx/categories/Query.java | 24 - .../msq/ITKeyStatisticsSketchMergeMode.java | 268 ----- .../msq/ITMultiStageQueryWorkerFaultTolerance.java | 173 --- .../apache/druid/testsEx/msq/MultiStageQuery.java | 266 ----- .../apache/druid/testsEx/query/UnionQueryTest.java | 207 ---- .../resources/druid-exact-count-bitmap/data.json | 10 - .../kafka_supervisor_template.json | 74 -- .../druid-exact-count-bitmap/queries.json | 115 -- .../cases/src/test/resources/query/union_data.json | 10 - .../query/union_kafka_supervisor_template.json | 69 -- .../src/test/resources/query/union_queries.json | 566 ---------- integration-tests-ex/image/docker-build.sh | 18 - .../tools/ServerManagerForQueryErrorTest.java | 8 +- .../ServerManagerForQueryErrorTest.java | 7 +- .../tests/indexer/ITTestCoordinatorPausedTest.java | 94 -- .../msq/dart/controller/DartControllerContext.java | 4 +- .../msq/dart/controller/DartWorkerManager.java | 13 +- .../msq/dart/worker/DartDataSegmentProvider.java | 10 +- .../apache/druid/msq/exec/ControllerContext.java | 10 +- .../org/apache/druid/msq/exec/ControllerImpl.java | 50 +- .../druid/msq/exec/TaskDataSegmentProvider.java | 28 +- .../org/apache/druid/msq/exec/WorkerManager.java | 12 +- .../msq/indexing/IndexerControllerContext.java | 5 +- .../druid/msq/indexing/IndexerWorkerContext.java | 2 +- .../apache/druid/msq/indexing/MSQWorkerTask.java | 1 + .../druid/msq/indexing/MSQWorkerTaskLauncher.java | 153 ++- .../druid/msq/querykit/BaseLeafFrameProcessor.java | 4 +- .../msq/dart/controller/DartWorkerManagerTest.java | 34 +- .../org/apache/druid/msq/exec/MSQTasksTest.java | 3 +- .../msq/exec/TaskDataSegmentProviderTest.java | 196 +--- ...ts.java => MSQWorkerTaskLauncherRetryTest.java} | 289 +++-- .../msq/indexing/MSQWorkerTaskLauncherTest.java | 18 +- .../druid/msq/test/CalciteMSQTestsHelper.java | 3 +- .../org/apache/druid/msq/test/MSQTestBase.java | 7 +- .../druid/msq/test/MSQTestControllerContext.java | 5 +- .../druid/msq/test/MSQTestSegmentManager.java | 2 +- .../apache/druid/java/util/metrics/CgroupUtil.java | 2 +- .../druid/query/DataSegmentAndDescriptor.java | 33 +- .../query/expression/NestedDataExpressions.java | 2 + .../org/apache/druid/segment/CursorBuildSpec.java | 21 +- .../segment/ReferenceCountedSegmentProvider.java | 143 +-- .../segment/ReferenceCountingCloseableObject.java | 8 +- .../apache/druid/segment/SegmentMapFunction.java | 16 +- .../org/apache/druid/segment/SegmentReference.java | 74 ++ .../segment/UnnestColumnValueSelectorCursor.java | 5 +- .../apache/druid/segment/UnnestCursorFactory.java | 159 ++- .../druid/segment/UnnestDimensionCursor.java | 7 +- .../segment/join/HashJoinSegmentCursorFactory.java | 31 +- .../segment/loading/AcquireSegmentAction.java | 91 ++ .../projections/ProjectionMatchBuilder.java | 20 - .../druid/segment/projections/Projections.java | 67 +- .../apache/druid/query/QueryRunnerTestHelper.java | 13 +- .../RestrictAllTablesPolicyEnforcerTest.java | 5 +- .../timeboundary/TimeBoundaryQueryRunnerTest.java | 18 +- .../apache/druid/segment/IndexMergerTestBase.java | 3 + .../segment/ReferenceCountingSegmentTest.java | 2 +- .../java/org/apache/druid/segment/TestHelper.java | 10 + .../java/org/apache/druid/segment/TestIndex.java | 15 +- .../UnnestColumnValueSelectorCursorTest.java | 60 +- .../druid/segment/UnnestCursorFactoryTest.java | 197 +++- .../join/BaseHashJoinSegmentCursorFactoryTest.java | 2 +- .../join/HashJoinSegmentCursorFactoryTest.java | 12 +- .../druid/segment/join/HashJoinSegmentTest.java | 4 +- .../druid/segment/join/PostJoinCursorTest.java | 2 +- .../druid/segment/projections/ProjectionsTest.java | 5 + .../apache/druid/client/cache/CachePopulator.java | 3 +- .../apache/druid/segment/loading/CacheEntry.java | 52 + .../segment/loading/CacheEntryIdentifier.java | 14 +- ...stBytesUsedStorageLocationSelectorStrategy.java | 2 +- .../loading/SegmentCacheEntryIdentifier.java | 44 +- .../druid/segment/loading/SegmentCacheManager.java | 143 ++- .../druid/segment/loading/SegmentLoaderConfig.java | 34 +- .../segment/loading/SegmentLocalCacheManager.java | 1115 ++++++++++++------- .../druid/segment/loading/StorageLocation.java | 907 +++++++++++++-- .../segment/loading/VirtualPlaceholderSegment.java | 130 +++ .../apache/druid/segment/realtime/FireHydrant.java | 10 +- .../appenderator/AppenderatorsManager.java | 6 +- .../druid/server/LocalQuerySegmentWalker.java | 6 +- .../org/apache/druid/server/SegmentManager.java | 201 ++-- .../server/{coordination => }/ServerManager.java | 321 ++++-- ...strapper.java => SegmentCacheBootstrapper.java} | 13 +- .../server/coordinator/duty/CompactSegments.java | 1 + .../coordinator/duty/CoordinatorDutyGroup.java | 3 +- .../server/coordinator/duty/DutyGroupStatus.java | 16 +- .../druid/server/http/CoordinatorDutyStatus.java | 6 +- .../druid/server/http/HistoricalResource.java | 12 +- .../segment/loading/NoopSegmentCacheManager.java | 23 +- .../SegmentLocalCacheManagerConcurrencyTest.java | 796 +++++++++++++- .../loading/SegmentLocalCacheManagerTest.java | 1154 +++++++++----------- .../StorageLocationSelectorStrategyTest.java | 44 +- .../druid/segment/loading/StorageLocationTest.java | 511 +++++++-- .../druid/server/ClientQuerySegmentWalkerTest.java | 61 +- .../org/apache/druid/server/QueryStackTests.java | 6 +- .../apache/druid/server/SegmentManagerTest.java | 17 +- .../server/SpecificSegmentsQuerySegmentWalker.java | 29 +- .../server/TestClusterQuerySegmentWalker.java | 48 +- ...java => SegmentCacheBootstrapperCacheTest.java} | 25 +- ...Test.java => SegmentCacheBootstrapperTest.java} | 22 +- .../coordination/SegmentLoadDropHandlerTest.java | 6 - ...egmentManagerBroadcastJoinIndexedTableTest.java | 3 +- .../SegmentManagerThreadSafetyTest.java | 3 +- .../server/coordination/ServerManagerTest.java | 17 +- .../druid/test/utils/TestSegmentCacheManager.java | 86 +- .../main/java/org/apache/druid/cli/CliBroker.java | 4 +- .../java/org/apache/druid/cli/CliHistorical.java | 6 +- .../main/java/org/apache/druid/cli/CliIndexer.java | 4 +- .../main/java/org/apache/druid/cli/CliPeon.java | 4 +- .../testing/embedded/EmbeddedDruidServer.java | 2 +- .../embedded/junit5/EmbeddedClusterTestBase.java | 5 +- .../druid/sql/calcite/util/SqlTestFramework.java | 1 + 163 files changed, 7212 insertions(+), 5250 deletions(-) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
