This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a change to branch xiangfu0/mse-proto-segment-list
in repository https://gitbox.apache.org/repos/asf/pinot.git
discard 106eae5c210 Add a JMH benchmark for the segment list encodings
discard cfe955ab2da Name the dispatcher flag after the config it follows
discard d5a7d538763 Fall back to disabled when the cluster-config key is
cleared
discard 8a0df331b95 Rename the proto segment list config to say that it is a
toggle
discard 0cfdb8d8836 Make the proto segment list encoding a plain live cluster
config
discard e3d35557810 Make the segment list encoding mode a live cluster config
discard 0f4e47688a3 Auto-enable the proto segment list encoding from server
versions
discard 56567d16b63 Stop JSON-encoding MSE leaf-stage segment lists on the
broker compile path
add 5c5a7940755 Hash the full value in fixed-byte BYTES and BIG_DECIMAL
dictionary batch murmur3 reads (#19634)
add fe8cd0261ec Change SegmentMetadata.getCrc and getDataCrc to return
long (#19629)
add 0306987954e Keep orphaned multipart temp files out of java.io.tmpdir
(#19627)
add bcb0d466e9d [MSE] Build ORDER BY on SortOperator instead of
SortedMailboxReceiveOperator (#19412)
add d30d683ba7f Bump com.dynatrace.hash4j:hash4j from 0.30.0 to 0.31.0
(#19641)
add c6efbfb4177 Bump joda-time:joda-time from 2.14.3 to 2.14.4 (#19640)
add 5725f884739 Bump com.github.luben:zstd-jni from 1.5.7-17 to 1.5.7-18
(#19639)
add cfd14629af2 Fix re-reading a huge value with the same
VarByteChunkForwardIndexReaderV4 context (#19638)
add 06c8f7e2ac2 Upgrade Apache parent to 40 and migrate to native Surefire
3.6 (#19633)
add 33d88e480ba Remove unused ValueReader.getPaddedString API (#19635)
add f6edf7d5776 Stop JSON-encoding MSE leaf-stage segment lists on the
broker compile path
add 191075d0060 Auto-enable the proto segment list encoding from server
versions
add 14b00e5e5a3 Make the segment list encoding mode a live cluster config
add 48327b7c484 Make the proto segment list encoding a plain live cluster
config
add fbc0d383e9a Rename the proto segment list config to say that it is a
toggle
add 40203d793e5 Fall back to disabled when the cluster-config key is
cleared
add 51c410bd6ba Name the dispatcher flag after the config it follows
add b08930a5a7e Add a JMH benchmark for the segment list encodings
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (106eae5c210)
\
N -- N -- N refs/heads/xiangfu0/mse-proto-segment-list
(b08930a5a7e)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.mvn/jvm.config | 3 +-
pinot-common/pom.xml | 26 +-
.../metadata/segment/SegmentZKMetadataUtils.java | 6 +-
.../pinot/common/utils/NativeTestNGSuiteTest.java | 230 +++++
.../pinot/common/utils/OrderedTestNGSuite.java | 49 +
pinot-controller/pom.xml | 85 +-
.../api/ControllerAdminApiApplication.java | 58 ++
.../api/resources/ControllerFilePathProvider.java | 12 +
.../PinotSegmentUploadDownloadRestletResource.java | 19 +-
.../pinot/controller/api/upload/ZKOperator.java | 2 +-
.../realtime/provisioning/MemoryEstimator.java | 4 +-
.../pinot/controller/ControllerTestSetup.java | 2 +-
.../api/ControllerAdminApiApplicationTest.java | 216 ++++
.../api/ControllerFilePathProviderTest.java | 48 +
.../api/PinotSegmentRestletResourceTest.java | 4 +-
...otSegmentUploadDownloadRestletResourceTest.java | 39 +
.../controller/api/upload/ZKOperatorTest.java | 20 +-
.../LineageDeleteInterleavingIntegrationTest.java | 2 +-
.../PinotLLCRealtimeSegmentManagerTest.java | 14 +-
.../controller/suites/ControllerStatefulSuite.java | 28 +-
.../suites/ControllerStatelessSuite.java | 28 +-
.../controller/utils/SegmentMetadataMockUtils.java | 32 +-
.../validation/ValidationManagerTest.java | 2 +-
pinot-controller/testng-statefull.xml | 39 -
pinot-controller/testng-stateless.xml | 41 -
.../core/data/manager/BaseTableDataManager.java | 17 +-
...ableDataManagerEnqueueSegmentToReplaceTest.java | 2 +-
.../data/manager/BaseTableDataManagerTest.java | 24 +-
.../offline/DimensionTableDataManagerTest.java | 2 +-
.../realtime/RealtimeSegmentDataManagerTest.java | 2 +-
...centileSmartTDigestAggregationFunctionTest.java | 49 -
...igestHighThresholdAggregationFunctionTest.java} | 8 +-
...gestSmallThresholdAggregationFunctionTest.java} | 31 +-
pinot-integration-tests/pom.xml | 65 +-
.../GroupByEnableTrimOptionIntegrationTest.java | 4 +-
.../tests/OfflineClusterIntegrationTest.java | 2 +-
.../tests/custom/GroupByOptionsTest.java | 18 +-
.../tests/custom/GroupByTrimmingTest.java | 10 +-
.../tests/suites/CustomClusterSuite.java | 27 +-
.../integration/tests/suites/KinesisSuite.java | 41 +
.../tests/suites/MultiNodesOfflineSuite.java | 38 +
.../tests/suites/SharedHybridSuite.java | 43 +
.../tests/suites/SharedKafkaRealtimeSuite.java | 45 +
.../custom-cluster-integration-test-suite.xml | 28 -
.../resources/kinesis-integration-test-suite.xml | 29 -
.../multi-nodes-offline-integration-test-suite.xml | 36 -
.../shared-hybrid-integration-test-suite.xml | 31 -
...hared-kafka-realtime-integration-test-suite.xml | 33 -
.../perf/BenchmarkMseSortImplementations.java | 182 ++++
.../tasks/BaseSingleSegmentConversionExecutor.java | 2 +-
.../pinot/plugin/minion/tasks/MinionTaskUtils.java | 8 +-
.../UpsertCompactionTaskExecutor.java | 6 +-
.../UpsertCompactMergeTaskExecutor.java | 11 +-
.../BaseSingleSegmentConversionExecutorTest.java | 4 +-
.../plugin/minion/tasks/MinionTaskUtilsTest.java | 46 +-
.../UpsertCompactMergeTaskExecutorTest.java | 44 +-
.../PinotAggregateExchangeNodeInsertRule.java | 11 +-
.../rel/rules/PinotSortExchangeNodeInsertRule.java | 9 +-
.../rules/PinotWindowExchangeNodeInsertRule.java | 49 +-
.../pinot/query/QueryPlannerRuleOptionsTest.java | 12 +-
.../src/test/resources/queries/GroupByPlans.json | 8 +-
.../src/test/resources/queries/JoinPlans.json | 4 +-
.../src/test/resources/queries/OrderByPlans.json | 22 +-
.../test/resources/queries/PinotHintablePlans.json | 8 +-
.../resources/queries/WindowFunctionPlans.json | 1066 +++++++++++---------
.../query/runtime/operator/FullSortOperator.java | 106 ++
.../query/runtime/operator/LimitSortOperator.java | 107 ++
.../pinot/query/runtime/operator/SortOperator.java | 255 +++--
.../operator/SortedMailboxReceiveOperator.java | 13 +
.../query/runtime/operator/TopNSortOperator.java | 109 ++
.../query/runtime/plan/PlanNodeToOpChain.java | 2 +-
.../operator/OperatorBufferReleaseTest.java | 44 +-
.../query/runtime/operator/SortOperatorTest.java | 221 +++-
.../local/data/manager/SegmentDataManager.java | 2 +-
.../immutable/ImmutableSegmentLoader.java | 2 +-
.../local/io/util/FixedByteValueReaderWriter.java | 6 +-
.../pinot/segment/local/io/util/ValueReader.java | 25 +-
.../local/io/util/VarLengthValueReader.java | 26 +-
.../index/readers/BaseImmutableDictionary.java | 20 +-
.../index/readers/BigDecimalDictionary.java | 7 +-
.../segment/index/readers/BytesDictionary.java | 7 +-
.../segment/index/readers/StringDictionary.java | 7 +-
.../forward/VarByteChunkForwardIndexReaderV4.java | 3 +-
.../SegmentCrcVirtualColumnProvider.java | 23 +-
.../converter/RealtimeSegmentConverterTest.java | 2 +-
.../segment/index/SegmentMetadataImplTest.java | 2 +-
.../SegmentV1V2ToV3FormatConverterTest.java | 2 +-
.../segment/index/creator/VarByteChunkV4Test.java | 66 ++
.../local/segment/index/loader/LoaderTest.java | 2 +-
.../index/readers/ImmutableDictionaryTest.java | 97 ++
.../SegmentMetadataVirtualColumnProviderTest.java | 2 +-
.../apache/pinot/segment/spi/SegmentMetadata.java | 7 +-
.../spi/index/metadata/SegmentMetadataImpl.java | 8 +-
.../spi/loader/SegmentDirectoryLoaderContext.java | 11 +-
.../pinot/server/api/resources/TablesResource.java | 16 +-
.../server/predownload/PredownloadSegmentInfo.java | 2 +-
.../pinot/server/api/TablesResourceTest.java | 441 ++++----
pom.xml | 47 +-
98 files changed, 3149 insertions(+), 1627 deletions(-)
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/utils/NativeTestNGSuiteTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/utils/OrderedTestNGSuite.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/api/ControllerAdminApiApplicationTest.java
copy
pinot-common/src/main/java/org/apache/pinot/sql/parsers/parser/SqlPhysicalExplain.java
=>
pinot-controller/src/test/java/org/apache/pinot/controller/suites/ControllerStatefulSuite.java
(55%)
copy
pinot-common/src/main/java/org/apache/pinot/sql/parsers/parser/SqlPhysicalExplain.java
=>
pinot-controller/src/test/java/org/apache/pinot/controller/suites/ControllerStatelessSuite.java
(55%)
delete mode 100644 pinot-controller/testng-statefull.xml
delete mode 100644 pinot-controller/testng-stateless.xml
copy
pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/{PercentileAggregationFunctionTest.java
=> PercentileSmartTDigestHighThresholdAggregationFunctionTest.java} (71%)
copy
pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/{PercentileKLLAggregationFunctionTest.java
=> PercentileSmartTDigestSmallThresholdAggregationFunctionTest.java} (62%)
copy
pinot-broker/src/main/java/org/apache/pinot/broker/cursors/JsonResponseSerde.java
=>
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/suites/CustomClusterSuite.java
(59%)
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/suites/KinesisSuite.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/suites/MultiNodesOfflineSuite.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/suites/SharedHybridSuite.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/suites/SharedKafkaRealtimeSuite.java
delete mode 100644
pinot-integration-tests/src/test/resources/custom-cluster-integration-test-suite.xml
delete mode 100644
pinot-integration-tests/src/test/resources/kinesis-integration-test-suite.xml
delete mode 100644
pinot-integration-tests/src/test/resources/multi-nodes-offline-integration-test-suite.xml
delete mode 100644
pinot-integration-tests/src/test/resources/shared-hybrid-integration-test-suite.xml
delete mode 100644
pinot-integration-tests/src/test/resources/shared-kafka-realtime-integration-test-suite.xml
create mode 100644
pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkMseSortImplementations.java
create mode 100644
pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/FullSortOperator.java
create mode 100644
pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/LimitSortOperator.java
create mode 100644
pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/TopNSortOperator.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
