github-advanced-security[bot] commented on code in PR #19718: URL: https://github.com/apache/druid/pull/19718#discussion_r3628002229
########## benchmarks/src/test/java/org/apache/druid/sql/calcite/schema/SysSegmentsTableBenchmark.java: ########## @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.schema; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import org.apache.calcite.DataContext; +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.linq4j.QueryProvider; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.druid.client.BrokerSegmentWatcherConfig; +import org.apache.druid.client.InternalQueryConfig; +import org.apache.druid.client.TimelineServerView; +import org.apache.druid.client.coordinator.NoopCoordinatorClient; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.CloseableIterators; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.parsers.CloseableIterator; +import org.apache.druid.segment.join.JoinableFactory; +import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig; +import org.apache.druid.server.QueryLifecycleFactory; +import org.apache.druid.server.SegmentManager; +import org.apache.druid.server.metrics.NoopServiceEmitter; +import org.apache.druid.server.security.AllowAllAuthorizer; +import org.apache.druid.server.security.AuthenticationResult; +import org.apache.druid.server.security.Authorizer; +import org.apache.druid.server.security.AuthorizerMapper; +import org.apache.druid.server.security.Escalator; +import org.apache.druid.sql.calcite.planner.CatalogResolver; +import org.apache.druid.sql.calcite.planner.PlannerContext; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentStatusInCluster; +import org.apache.druid.timeline.partition.LinearShardSpec; +import org.easymock.EasyMock; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +/** + * Benchmarks {@link SystemSchema.SegmentsTable#scan} to quantify the effect of pushing a + * {@code datasource} equality predicate down into the sys.segments scan. + * + * <p>{@link #fullScan} runs with no filter (every segment is materialized, the pre-pushdown + * behavior). {@link #filteredScanSingleDataSource} runs {@code WHERE datasource = ...}, which after + * the push-down only authorizes and builds rows for the matching datasource. The gap between the two + * is the win for the datasource-filtered UI/API queries. + * + * <p>The available-segment cache is left empty so the benchmark isolates the published-segment path, + * which dominates cost on large clusters. The cache is disabled so each invocation re-fetches from a + * synthetic Coordinator client and exercises the real {@link MetadataSegmentView#getSegments(Set)} + * filter rather than a warmed snapshot. + */ +@State(Scope.Benchmark) +@Fork(value = 1, jvmArgsAppend = {"-Xmx12g"}) +@Warmup(iterations = 3, time = 3) +@Measurement(iterations = 5, time = 3) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +public class SysSegmentsTableBenchmark +{ + @Param({"1000000", "10000000"}) + private int numSegments; + + @Param({"100", "1000"}) + private int numDataSources; + + private SystemSchema.SegmentsTable segmentsTable; + private List<SegmentStatusInCluster> publishedSegments; + private DataContext dataContext; + private List<RexNode> noFilter; + private List<RexNode> singleDataSourceFilter; + + /** + * An empty {@link BrokerSegmentMetadataCache}: the benchmark never announces segments to it, so the + * available-segment side of the scan contributes nothing and the published path is measured in + * isolation. Mirrors the null-arg construction proven in DruidSchemaInternRowSignatureBenchmark. + */ + private static class EmptyBrokerSegmentMetadataCache extends BrokerSegmentMetadataCache + { + EmptyBrokerSegmentMetadataCache() + { + super( + EasyMock.mock(QueryLifecycleFactory.class), + EasyMock.mock(TimelineServerView.class), + BrokerSegmentMetadataCacheConfig.create(), + EasyMock.mock(Escalator.class), + EasyMock.mock(InternalQueryConfig.class), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataFactory( + EasyMock.mock(JoinableFactory.class), + EasyMock.mock(SegmentManager.class) + ), + new NoopCoordinatorClient(), + CentralizedDatasourceSchemaConfig.create() + ); + } + } + + @Setup(Level.Trial) + public void setup() + { + final List<SegmentStatusInCluster> segments = new ArrayList<>(numSegments); + for (int i = 0; i < numSegments; i++) { + final String dataSource = StringUtils.format("datasource_%d", i % numDataSources); + final int dayOffset = i / numDataSources; + final DataSegment segment = + DataSegment.builder() Review Comment: ## CodeQL / Deprecated method or constructor invocation Invoking [DataSegment.builder](1) should be avoided because it has been deprecated. [Show more details](https://github.com/apache/druid/security/code-scanning/11353) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
