This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a change to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
from be4e981b42 [DIRTY] fix maven package error
new f66b85f0d2 KYLIN-5521 fix JoinsGraph
new 54940c9a9d KYLIN-5521 fix LeftOrInner Join
new 2cf578b0fa KYLIN-5522 fix local debug mode
new 520e7c67ea KYLIN-5523 computed column as join key & partition column
new 2502c15632 KYLIN-5521 fix JoinsGraph
new 2a152803bb mirror: fix complie
new f929bd876e KYLIN-5524 Supports CONCAT for variable arguments
new 09ea457278 KYLIN-5525 fix error when model init error in multi threads
new 4a889f08ab KYLIN-5526 fix unique queue async query count more than
setting
new 5657afc7f2 KYLIN-5523 [FOLLOW UP] fix some function of cc as join key
or filter column
new 59a2e21034 KYLIN-5527 Data count check for segment layout building
new 00e52f4b3f [DIRTY] Rewrite spark InferFiltersFromConstraints
new 842a0601a6 KYLIN-5528 support collect sparder gc
new 620a6ad5bf KYLIN-5529 Support adding "basic agg index" or "basic table
index" separately
new 597a9367ec KYLIN-5530 remove repartition write KYLIN-5530 Optimized
snapshot builds KYLIN-5530 Flat Table Repartition before writing data source
tables/directories
new 80a87f9d53 [DIRTY] Fix a build job is submitted in yarn cluster mode,
it cannot be written into the driver log
new 2e60700e52 KYLIN-5521 [FOLLOW UP] fix LeftOrInner Join & unstable UT
new e39b34cb46 KYLIN-5531 remove redundancy code
new 2e83b6e8a5 mirror: fix ut & compile
new 96d806eae8 KYLIN-5532 When calcite optimize, can cancel query search
new 2e29f00053 KYLIN-5533 add model list lite api
new df6ef0a5ae KYLIN-5534 Fix the inconsistent behavior of exporting tds
between normal users and admin
new 625c4b6c7e KYLIN-5521 [FOLLOW UP] Fix JoinsGraph toString method
StackOverFlow
new 9bf63e0183 KYLIN-5535 Fix the username suffix matching
new 5d251e3139 KYLIN-5536 Limit the segment range of MAX query to improve
query performance
new c8a464d2ca KYLIN-5533 [FOLLOW UP] fix BeanUtils copyProperties NPE
new 23bf641c01 KYLIN-5523 [FOLLOW UP] Fix the error caused by using
ceil/floor functions in model filtering conditions.
new 10a180422c mirror: fix snyk, upgrade tomcat-embed-core from 9.0.69 to
9.0.71 fix snyk, upgrade commons-fileupload from 1.3.3 to 1.5
new 9af54c0484 KYLIN-5534 [FOLLOW UP] Set 'kylin.model.skip-check-tds' to
true
new fb90074de2 KYLIN-5523 [FOLLOW UP] Fix using a computed column as the
join key leads to a schema change failure
new 6309697701 KYLIN-5523 [FOLLOW UP] fix generate flat table sql
new cb3f4e604f KYLIN-5523 [FOLLOW UP] expand the filter condition
expression to ensure compatibility
new b10a87b65d KYLIN-5530 [FOLLOW UP] fix config
kylin.engine.persist-flat-use-snapshot-enabled
new e0599cbaed [DIRTY] release kyspark 4.6.6.0
new 111723d7a4 KYLIN-5537 Upgrade version, resolve vulnerability
new e19d2918c5 mirror: fix UT for snapshot building
new f508aa3439 mirror: remove unused import
new 1d1a9d0d60 [DIRTY] Fix Sonar
The 38 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
pom.xml | 12 +-
.../src/main/resources/config/init.properties | 2 +-
.../kylin/rest/controller/NBasicController.java | 2 +-
.../apache/kylin/rest/KylinPrepareEnvListener.java | 2 +-
.../kylin/rest/aspect/InsensitiveNameAspect.java | 2 +-
.../apache/kylin/rest/broadcaster/Broadcaster.java | 2 +-
.../rest/config/initialize/AclTCRListener.java | 2 +-
.../rest/config/initialize/CacheCleanListener.java | 2 +-
.../rest/config/initialize/MetricsRegistry.java | 2 +-
.../initialize/TableSchemaChangeListener.java | 2 +-
.../ResourceGroupEntityValidator.java | 2 +-
.../ResourceGroupKylinInstanceValidator.java | 2 +-
.../ResourceGroupMappingInfoValidator.java | 2 +-
.../interceptor/RepeatableRequestBodyFilter.java | 2 +-
.../interceptor/ResourceGroupCheckerFilter.java | 2 +-
.../apache/kylin/rest/monitor/MonitorReporter.java | 2 +-
.../org/apache/kylin/rest/request/UserRequest.java | 2 +-
.../apache/kylin/rest/service/AccessService.java | 2 +-
.../kylin/rest/service/AsyncTaskService.java | 2 +-
.../apache/kylin/rest/service/BasicService.java | 2 +-
.../apache/kylin/rest/service/LdapUserService.java | 2 +-
.../rest/service/MockQuerySmartSupporter.java} | 18 +-
.../kylin/rest/service/OpenUserGroupService.java | 2 +-
.../apache/kylin/rest/service/ProjectService.java | 2 +-
.../apache/kylin/rest/service/SystemService.java | 7 +-
.../apache/kylin/rest/service/UserAclService.java | 2 +-
.../java/org/apache/kylin/tool/util/ToolUtil.java | 2 +-
.../kylin/rest/service/AccessServiceTest.java | 4 +-
.../kylin/rest/service/AclTCRServiceTest.java | 2 +-
.../java/org/apache/kylin/common/KapConfig.java | 8 +
.../java/org/apache/kylin/common/KylinConfig.java | 2 +-
.../org/apache/kylin/common/KylinConfigBase.java | 59 +-
.../java/org/apache/kylin/common/KylinVersion.java | 2 +-
.../java/org/apache/kylin/common/StorageURL.java | 2 +-
.../apache/kylin/common/debug/BackdoorToggles.java | 2 +-
.../org/apache/kylin/common/msg/CnMessage.java | 2 +-
.../java/org/apache/kylin/common/msg/Message.java | 2 +-
.../kylin/common/persistence/StringEntity.java | 2 +-
.../persistence/metadata/JdbcMetadataStore.java | 2 +-
.../transaction/AuditLogReplayWorker.java | 2 +-
.../kylin/common/util/DaemonThreadFactory.java | 2 +-
.../org/apache/kylin/common/util/DateFormat.java | 2 +-
.../org/apache/kylin/common/util/EncryptUtil.java | 2 +-
.../org/apache/kylin/common/util/HadoopUtil.java | 2 +-
.../org/apache/kylin/common/util/JsonUtil.java | 1 +
.../kylin/common/util/OrderedProperties.java | 2 +-
.../org/apache/kylin/common/util/SSHClient.java | 2 +-
.../util/{StringUtil.java => StringHelper.java} | 104 +++-
.../java/org/apache/kylin/common/util/Unsafe.java | 2 +-
.../org/apache/kylin/common/util/ZipFileUtils.java | 2 +-
.../org/apache/kylin/rest/util/PagingUtil.java | 2 +-
.../kylin-backward-compatibility.properties | 1 +
.../src/main/resources/kylin-defaults0.properties | 2 +-
.../apache/kylin/common/KylinConfigBaseTest.java | 18 +
.../common/util/NLocalFileMetadataTestCase.java | 4 +-
.../apache/kylin/common/util/StringHelperTest.java | 104 ++++
.../apache/kylin/engine/spark/ExecutableUtils.java | 2 +-
.../apache/kylin/job/constant/JobStatusEnum.java | 6 +
.../org/apache/kylin/job/dao/ExecutablePO.java | 2 +-
.../apache/kylin/job/engine/JobEngineConfig.java | 2 +-
.../kylin/job/execution/AbstractExecutable.java | 28 +-
.../kylin/job/execution/DefaultExecutable.java | 45 +-
.../job/execution/DefaultExecutableOnModel.java | 10 +-
.../job/execution/EmailNotificationContent.java | 18 +-
.../kylin/job/execution/ExecutableParams.java | 2 +-
.../kylin/job/execution/ExecutableState.java | 11 +-
.../apache/kylin/job/execution/ExecuteResult.java | 23 +-
.../kylin/job/execution/NExecutableManager.java | 12 +
.../org/apache/kylin/job/JoinedFlatTableTest.java | 242 --------
.../job/execution/NExecutableManagerTest.java | 23 +-
.../job/impl/threadpool/NDefaultSchedulerTest.java | 2 +-
.../apache/kylin/dimension/DimensionEncoding.java | 6 +-
.../apache/kylin/metadata/acl/AclTCRManager.java | 2 +-
.../metadata/cube/cuboid/AggIndexMatcher.java | 2 +-
.../kylin/metadata/cube/cuboid/IndexMatcher.java | 2 +-
.../metadata/cube/cuboid/KECuboidSchedulerV1.java | 2 +-
.../kylin/metadata/cube/model/IndexPlan.java | 28 +-
.../kylin/metadata/cube/model/NBatchConstants.java | 1 +
.../kylin/metadata/cube/model/NDataLayout.java | 19 +-
.../cube/model/NDataLoadingRangeManager.java | 2 +-
.../kylin/metadata/cube/model/NDataSegDetails.java | 33 +-
.../kylin/metadata/cube/model/NDataSegment.java | 65 +-
.../cube/model/NDataflowCapabilityChecker.java | 2 +-
.../metadata/cube/model/NDataflowManager.java | 4 +-
.../metadata/cube/model/NIndexPlanManager.java | 2 +-
.../metadata/cube/model/SegmentPartition.java | 2 +-
.../cube/storage/ProjectStorageInfoCollector.java | 2 +-
.../kylin/metadata/datatype/BooleanSerializer.java | 2 +-
.../apache/kylin/metadata/datatype/DataType.java | 2 +-
.../apache/kylin/metadata/epoch/EpochManager.java | 2 +-
.../metadata/filter/function/LikeMatchers.java | 2 +-
.../insensitive/ModelInsensitiveRequest.java | 2 +-
.../insensitive/ProjectInsensitiveRequest.java | 2 +-
.../insensitive/UserInsensitiveRequest.java | 2 +-
.../kylin/metadata/model/ColExcludedChecker.java | 4 +-
.../apache/kylin/metadata/model/ColumnDesc.java | 109 ++--
.../kylin/metadata/model/ComputedColumnDesc.java | 2 +-
.../org/apache/kylin/metadata/model/ISegment.java | 2 +
.../kylin/metadata/model/JoinedFlatTable.java | 371 -----------
.../apache/kylin/metadata/model/JoinsGraph.java | 607 ------------------
.../apache/kylin/metadata/model/MeasureDesc.java | 2 +-
.../kylin/metadata/model/ModelDimensionDesc.java | 4 +-
.../metadata/model/ModelJoinRelationTypeEnum.java | 2 +-
.../kylin/metadata/model/MultiPartitionDesc.java | 2 +-
.../apache/kylin/metadata/model/NDataModel.java | 30 +-
.../kylin/metadata/model/NDataModelManager.java | 2 +-
.../metadata/model/NTableMetadataManager.java | 2 +-
.../apache/kylin/metadata/model/PartitionDesc.java | 10 +-
.../org/apache/kylin/metadata/model/TblColRef.java | 34 +-
.../metadata/model/alias/ExpressionComparator.java | 2 +-
.../model/graph/DefaultJoinEdgeMatcher.java | 69 +++
.../apache/kylin/metadata/model/graph/Edge.java | 148 +++++
.../metadata/model/graph/IJoinEdgeMatcher.java} | 14 +-
.../kylin/metadata/model/graph/JoinsGraph.java | 515 ++++++++++++++++
.../model/schema/AffectedModelContext.java | 2 +-
.../metadata/model/schema/ModelEdgeCollector.java | 2 +-
.../kylin/metadata/model/tool/CalciteParser.java | 12 +-
.../metadata/model/util/ComputedColumnUtil.java | 78 +--
.../kylin/metadata/project/ProjectInstance.java | 6 +-
.../metadata/query/JdbcQueryHistoryStore.java | 2 +-
.../realization/RealizationRuntimeException.java} | 15 +-
.../kylin/metadata/realization/SQLDigest.java | 2 +-
.../recommendation/candidate/RawRecItemTable.java | 2 +-
.../recommendation/entity/CCRecItemV2.java | 2 +-
.../metadata/recommendation/ref/OptRecV2.java | 2 +-
.../metadata/sourceusage/SourceUsageManager.java | 2 +-
.../kylin/metadata/streaming/KafkaConfig.java | 2 +-
.../kylin/metadata/user/NKylinUserManager.java | 20 +-
.../kylin/rest/security/ExternalAclProvider.java | 2 +-
.../apache/kylin/rest/security/UserAclManager.java | 2 +-
.../kylin/source/adhocquery/IPushDownRunner.java | 18 +-
.../cube/planner/algorithm/AlgorithmTestBase.java | 6 +-
.../metadata/epoch/EpochUpdateLockManagerTest.java | 2 +-
.../kylin/metadata/model/JoinsGraphTest.java | 106 +++-
.../kylin/metadata/model/MockJoinGraphBuilder.java | 1 +
.../model/util/ComputedColumnUtilTest.java | 4 +-
.../metadata/project/NProjectManagerTest.java | 2 +-
.../kylin/metadata/user/NKylinUserManagerTest.java | 12 +
.../kylin/metrics/HdfsCapacityMetricsTest.java | 4 +-
.../apache/kylin/common/metrics/MetricsGroup.java | 2 +-
.../common/metrics/MetricsInfluxdbReporter.java | 2 +-
.../src/main/resources/config/init.properties | 2 +-
.../kylin/rest/controller/BaseController.java | 2 +-
.../kylin/rest/controller/JobController.java | 2 +-
.../kylin/rest/controller/SampleController.java | 6 +-
.../kylin/rest/controller/SegmentController.java | 2 +-
.../rest/controller/open/OpenSampleController.java | 7 +-
.../open/OpenStreamingJobController.java | 2 +-
.../rest/controller/v2/SegmentControllerV2.java | 2 +-
.../rest/controller/SampleControllerTest.java | 2 +-
.../controller/StreamingJobControllerTest.java | 2 +-
.../controller/open/OpenSampleControllerTest.java | 12 +-
.../open/OpenStreamingJobControllerTest.java | 2 +-
.../kylin/rest/response/ExecutableResponse.java | 8 +-
.../rest/response/ExecutableStepResponse.java | 3 +
.../org/apache/kylin/rest/service/JobService.java | 14 +-
.../kylin/rest/service/ModelBuildService.java | 2 +-
.../apache/kylin/rest/service/SnapshotService.java | 101 ++-
.../kylin/rest/service/TableSamplingService.java | 2 +-
.../apache/kylin/rest/service/JobServiceTest.java | 20 +-
.../kylin/rest/service/ModelServiceBuildTest.java | 16 +-
.../org/apache/kylin/rest/service/StageTest.java | 3 +
.../sdk/datasource/PushDownRunnerSDKImpl.java | 4 +-
.../sdk/datasource/adaptor/DefaultAdaptor.java | 2 +-
.../sdk/datasource/framework/JdbcConnector.java | 2 +-
.../datasource/framework/conv/ParamNodeParser.java | 2 +-
.../datasource/framework/conv/SqlConverter.java | 2 +-
.../apache/kylin/rest/service/SparkDDLService.java | 2 +-
.../kylin/rest/service/SparkSourceService.java | 22 +-
.../apache/kylin/rest/service/TableService.java | 55 +-
.../metadata/_global/project/index_build_test.json | 40 ++
.../3ec47efc-573a-9304-4405-8e05ae184322.json | 70 +++
.../b2f206e1-7a15-c94a-20f5-f608d550ead6.json | 69 +++
.../3ec47efc-573a-9304-4405-8e05ae184322.json | 204 ++++++
.../3ec47efc-573a-9304-4405-8e05ae184322.json | 241 ++++++++
.../index_build_test/table/SSB.CUSTOMER.json | 77 +++
.../index_build_test/table/SSB.LINEORDER.json | 131 ++++
.../rest/config/initialize/JobSyncListener.java | 2 +-
.../rest/config/initialize/JobSchedulerTest.java | 2 +-
.../org/apache/kylin/event/SchemaChangeTest.java | 5 +-
.../kylin/metadata/model/AntiFlatCheckerTest.java | 4 +-
.../metadata/model/ColExcludedCheckerTest.java | 4 +-
.../kylin/newten/NBadQueryAndPushDownTest.java | 6 +-
.../org/apache/kylin/newten/TableIndexTest.java | 19 +
.../java/org/apache/kylin/query/KylinTestBase.java | 16 +-
.../java/org/apache/kylin/query/NKapQueryTest.java | 2 +-
.../rest/controller/NProjectControllerTest.java | 2 +-
.../java/org/apache/kylin/util/ExecAndComp.java | 3 +-
.../rest/controller/open/OpenModelController.java | 5 +-
.../rest/controller/NIndexPlanController.java | 7 +-
.../kylin/rest/controller/NModelController.java | 13 +-
.../kylin/rest/controller/NProjectController.java | 2 +-
.../kylin/rest/controller/NTableController.java | 24 +-
.../kylin/rest/controller/NUserController.java | 2 +-
.../rest/controller/NUserGroupController.java | 2 +-
.../rest/controller/v2/NAccessControllerV2.java | 2 +-
.../controller/open/OpenModelControllerTest.java | 4 +-
.../rest/controller/NModelControllerTest.java | 70 ++-
.../rest/controller/NTableControllerTest.java | 12 +-
.../kylin/rest/controller/NUserControllerTest.java | 2 +-
.../apache/kylin/rest/model/FuzzyKeySearcher.java | 2 +-
.../apache/kylin/rest/request/ModelRequest.java | 6 +
.../apache/kylin/rest/response/IndexResponse.java | 4 +
.../rest/response/NDataModelLiteResponse.java | 129 ++++
.../kylin/rest/response/NDataModelResponse.java | 18 +
.../kylin/rest/response/NDataSegmentResponse.java | 4 +-
.../response/SynchronizedCommentsResponse.java | 2 +-
.../kylin/rest/service/AbstractModelService.java | 2 +-
.../kylin/rest/service/BaseIndexUpdateHelper.java | 34 +-
.../kylin/rest/service/FusionIndexService.java | 24 +-
.../kylin/rest/service/FusionModelService.java | 2 +-
.../kylin/rest/service/IndexPlanService.java | 52 +-
.../kylin/rest/service/ModelSemanticHelper.java | 43 +-
.../apache/kylin/rest/service/ModelService.java | 206 ++++---
.../apache/kylin/rest/service/ModelTdsService.java | 22 +-
...{ModelQueryParams.java => IndexPlanParams.java} | 36 +-
.../rest/service/params/ModelQueryParams.java | 2 +
.../rest/service/params/PaginationParams.java} | 22 +-
.../org/apache/kylin/rest/util/ModelUtils.java | 2 +-
.../org/apache/kylin/tool/bisync/BISyncTool.java | 4 +-
.../apache/kylin/tool/bisync/SyncModelBuilder.java | 5 -
.../rest/response/NDataModelResponseTest.java | 106 ++++
.../apache/kylin/rest/service/BaseIndexTest.java | 49 +-
.../kylin/rest/service/FusionModelServiceTest.java | 2 +-
.../service/ModelServiceSemanticUpdateTest.java | 55 +-
.../kylin/rest/service/ModelServiceTest.java | 122 ++--
.../service/ModelTdsServiceColumnNameTest.java | 1 +
.../kylin/rest/service/ModelTdsServiceTest.java | 69 ++-
.../kylin/rest/service/ProjectServiceTest.java | 6 +-
.../kylin/rest/service/TableServiceTest.java | 134 ++--
.../kylin/tool/bisync/SyncModelBuilderTest.java | 2 +-
.../tool/bisync/tableau/TableauDatasourceTest.java | 5 +-
.../src/main/resources/config/init.properties | 2 +-
.../org/apache/kylin/query/util/EscapeParser.jj | 42 +-
.../org/apache/kylin/query/SlowQueryDetector.java | 9 +-
.../apache/kylin/query/relnode/OLAPContext.java | 4 +-
.../kylin/query/routing/RealizationChooser.java | 7 +-
.../kylin/query/routing/RealizationPruner.java | 177 ++++--
.../org/apache/kylin/query/schema/OLAPTable.java | 6 +-
.../query/security/AccessDeniedException.java | 2 +-
.../org/apache/kylin/query/security/RowFilter.java | 2 +-
.../apache/kylin/query/util/AsyncQueryUtil.java | 2 +-
.../org/apache/kylin/query/util/EscapeDialect.java | 4 +
.../apache/kylin/query/util/EscapeFunction.java | 16 +
.../query/util/ModelViewSqlNodeComparator.java | 2 +-
.../org/apache/kylin/query/util/PushDownUtil.java | 412 +++++++++----
.../apache/kylin/query/util/QueryAliasMatcher.java | 55 +-
.../org/apache/kylin/query/util/QueryUtil.java | 431 +++----------
.../rest/controller/NAsyncQueryController.java | 10 +-
.../kylin/rest/controller/NQueryController.java | 4 +-
.../controller/SparkMetricsControllerTest.java | 2 +-
.../apache/kylin/rest/response/SQLResponseV2.java | 2 +-
.../kylin/rest/service/ModelQueryService.java | 2 +-
.../apache/kylin/rest/service/MonitorService.java | 2 +-
.../kylin/rest/service/QueryHistoryService.java | 2 +-
.../apache/kylin/rest/service/QueryService.java | 16 +-
.../kylin/rest/util/AsyncQueryRequestLimits.java | 8 +-
.../kylin/rest/util/QueryCacheSignatureUtil.java | 2 +-
.../org/apache/kylin/rest/util/QueryUtils.java | 2 +-
.../kylin/rest/service/ModelQueryServiceTest.java | 18 +-
.../kylin/rest/service/ModelServiceQueryTest.java | 103 ++--
.../rest/service/QueryHistoryServiceTest.java | 2 +-
.../kylin/rest/service/QueryServiceTest.java | 40 +-
src/query/pom.xml | 9 +-
.../optrule/AbstractAggCaseWhenFunctionRule.java | 18 +-
.../optrule/CountDistinctCaseWhenFunctionRule.java | 17 +-
.../query/optrule/KapAggFilterTransposeRule.java | 4 +-
.../kap/query/optrule/KapAggJoinTransposeRule.java | 4 +-
.../kap/query/optrule/KapAggProjectMergeRule.java | 4 +-
.../query/optrule/KapAggProjectTransposeRule.java | 4 +-
.../query/optrule/KapCountDistinctJoinRule.java | 4 +-
.../kap/query/optrule/KapSumCastTransposeRule.java | 19 +-
.../main/java/org/apache/kylin/query/QueryCli.java | 78 ---
.../kylin/query/engine/AsyncQueryApplication.java | 2 +-
.../apache/kylin/query/engine/AsyncQueryJob.java | 2 +-
.../kylin/query/engine/QueryRoutingEngine.java | 16 +-
.../kylin/query/udf/stringUdf/ConcatUDF.java | 45 +-
.../org/apache/kylin/query/util/RuleUtils.java | 150 +++++
.../kylin/query/engine/AsyncQueryJobTest.java | 2 +-
.../kylin/query/engine/SelectRealizationTest.java | 7 +-
.../kylin/query/engine/view/ModelViewTest.java | 2 +-
.../kylin/query/schema/KylinSqlValidatorTest.java | 2 +-
.../org/apache/kylin/query/udf/StringUDFTest.java | 21 +-
.../apache/kylin/query/util/CCOnRealModelTest.java | 11 +-
.../query/util/EscapeTransformerCalciteTest.java | 22 +
.../query/util/EscapeTransformerSparkSqlTest.java | 11 +
.../apache/kylin/query/util/PushDownUtilTest.java | 251 +++++++-
.../org/apache/kylin/query/util/QueryUtilTest.java | 344 +++++------
.../kap/secondstorage/SecondStorageLockTest.java | 6 +-
.../kap/secondstorage/tdvt/TDVTHiveTest.java | 4 +-
.../kap/clickhouse/ClickHouseStorage.java | 2 +-
.../kyligence/kap/clickhouse/job/ClickHouse.java | 15 +-
.../kap/clickhouse/job/ClickHouseLoad.java | 2 +-
.../kap/clickhouse/job/ClickhouseLoadFileLoad.java | 2 +-
.../kap/clickhouse/job/HadoopMockUtil.java | 2 +-
.../management/OpenSecondStorageEndpoint.java | 2 +-
.../management/SecondStorageEndpoint.java | 2 +-
.../service/ModelServiceWithSecondStorageTest.java | 4 +-
.../io/kyligence/kap/secondstorage/ddl/DDL.java | 2 +-
.../kap/secondstorage/ddl/exp/TableIdentifier.java | 2 +-
.../kap/secondstorage/metadata/Manager.java | 2 +-
.../org/apache/kylin/rest/QueryNodeFilter.java | 2 +-
.../apache/kylin/rest/ZookeeperClusterManager.java | 2 +-
.../org/apache/kylin/rest/config/CorsConfig.java | 2 +-
.../rest/discovery/KylinServiceDiscoveryCache.java | 4 +-
.../discovery/KylinServiceDiscoveryClient.java | 2 +-
.../src/main/resources/config/init.properties | 2 +-
.../source/hive/BeelineOptionsProcessorTest.java | 2 +-
src/spark-project/engine-spark/pom.xml | 5 +
.../engine/spark/application/SparkApplication.java | 46 +-
.../builder/PartitionDictionaryBuilderHelper.java | 2 +-
.../spark/job/DefaultSparkBuildJobHandler.java | 2 +-
.../spark/job/ExecutableAddCuboidHandler.java | 48 +-
.../kylin/engine/spark/job/NSparkCubingStep.java | 42 +-
.../kylin/engine/spark/job/NSparkExecutable.java | 6 +-
.../job/SparkCleanupTransactionalTableStep.java | 2 +-
.../spark/merger/AfterBuildResourceMerger.java | 4 +-
.../merger/AfterMergeOrRefreshResourceMerger.java | 4 +-
.../kylin/engine/spark/mockup/CsvSource.java | 2 +-
.../kylin/engine/spark/mockup/CsvTableReader.java | 2 +-
.../engine/spark/source/NSparkDataSource.java | 2 +-
.../spark/source/NSparkMetadataExplorer.java | 2 +-
.../spark/stats/analyzer/TableAnalyzerJob.java | 2 +-
.../spark/stats/utils/DateTimeCheckUtils.java | 2 +-
.../engine/spark/utils/ComputedColumnEvalUtil.java | 52 +-
.../spark/utils/HiveTransactionTableHelper.java | 2 +-
.../engine/spark/utils/SparkJobFactoryUtils.java | 2 +-
.../engine/spark/builder/CreateFlatTable.scala | 135 +---
.../engine/spark/builder/DFBuilderHelper.scala | 12 +-
.../spark/builder/DictionaryBuilderHelper.java | 2 +-
.../engine/spark/builder/PartitionFlatTable.scala | 69 ---
.../engine/spark/builder/SegmentFlatTable.scala | 683 ---------------------
.../engine/spark/builder/SnapshotBuilder.scala | 11 +-
.../kylin/engine/spark/job/BuildJobInfos.scala | 21 +-
.../apache/kylin/engine/spark/job/DFMergeJob.java | 2 +-
.../kylin/engine/spark/job/FlatTableHelper.scala | 16 +-
.../kylin/engine/spark/job/PartitionExec.scala | 4 +
.../engine/spark/job/RDPartitionBuildExec.scala | 35 +-
.../engine/spark/job/RDSegmentBuildExec.scala | 32 +-
.../kylin/engine/spark/job/RDSegmentBuildJob.java | 11 +-
.../kylin/engine/spark/job/SegmentBuildJob.java | 16 +-
.../kylin/engine/spark/job/SegmentExec.scala | 43 +-
.../apache/kylin/engine/spark/job/SegmentJob.java | 4 +
.../kylin/engine/spark/job/SegmentMergeJob.java | 5 +-
.../kylin/engine/spark/job/SnapshotBuildJob.java | 2 +-
.../kylin/engine/spark/job/stage/StageExec.scala | 113 ++--
.../engine/spark/job/stage/build/BuildLayer.scala | 23 +-
.../engine/spark/job/stage/build/BuildStage.scala | 11 +-
.../job/stage/build/FlatTableAndDictBase.scala | 139 +++--
.../job/stage/build/GatherFlatTableStats.scala | 2 -
.../spark/job/stage/build/GenerateFlatTable.scala | 145 ++++-
.../stage/build/MaterializedFactTableView.scala | 24 +-
.../build/partition/PartitionBuildLayer.scala | 23 +-
.../build/partition/PartitionBuildStage.scala | 6 -
.../partition/PartitionFlatTableAndDictBase.scala | 22 +
.../PartitionMaterializedFactTableView.scala | 30 +-
.../engine/spark/job/stage/merge/MergeStage.scala | 2 +-
.../merge/partition/PartitionMergeStage.scala | 2 +-
.../spark/smarter/IndexDependencyParser.scala | 55 +-
.../engine/spark/job/NSparkExecutableTest.java | 15 +-
.../GenerateFlatTableWithSparkSessionTest.java | 444 ++++++++++++++
.../spark/builder/TestDimensionTableStat.scala | 13 +-
.../engine/spark/builder/TestFlatTable.scala} | 41 +-
.../engine/spark/builder/TestInferFilters.scala | 18 +-
.../spark/builder/TestSegmentFlatTable.scala | 35 +-
.../engine/spark/job/TestRDSegmentBuildExec.scala | 111 ++++
.../PartitionFlatTableAndDictBaseTest.scala | 88 +++
.../org/apache/kylin/source/jdbc/JdbcSource.java | 2 +-
.../query/pushdown/PushDownRunnerJdbcImpl.java | 4 +-
.../query/pushdown/PushDownRunnerSparkImpl.java | 3 +-
.../asyncprofiler/AsyncProfiling.scala | 2 +-
.../QueryAsyncProfilerDriverPlugin.scala | 2 +-
.../QueryAsyncProfilerSparkPlugin.scala | 2 +-
.../query/plugin/diagnose/DiagnoseConstant.scala} | 25 +-
.../diagnose/DiagnoseDriverPlugin.scala} | 50 +-
.../plugin/diagnose/DiagnoseExecutorPlugin.scala | 124 ++++
.../query/plugin/diagnose/DiagnoseHelper.scala | 62 ++
.../diagnose/DiagnoseSparkPlugin.scala} | 9 +-
.../kylin/query/runtime/ExpressionConverter.scala | 2 +-
.../scala/org/apache/spark/sql/KylinSession.scala | 56 +-
.../scala/org/apache/spark/sql/SparderEnv.scala | 36 +-
.../pushdown/PushDownRunnerSparkImplTest.java | 15 +-
.../SparkPluginWithMeta.scala} | 4 +-
.../asyncprofiler/AsyncProfilingTest.scala | 5 +-
.../QueryAsyncProfilerSparkPluginTest.scala | 2 +-
.../QueryProfilerDriverPluginTest.scala} | 5 +-
.../diagnose/DiagnoseExecutorPluginTest.scala | 109 ++++
.../query/plugin/diagnose/DiagnosePluginTest.scala | 82 +++
.../kylin/engine/spark/utils/Repartitioner.java | 15 +-
.../kylin/engine/spark/job/NSparkCubingUtil.java | 13 +-
.../apache/spark/dict/NGlobalDictHDFSStore.java | 2 +-
.../datasource/storage/LayoutFormatWriter.scala | 98 +++
.../sql/datasource/storage/StorageStore.scala | 51 +-
.../RewriteInferFiltersFromConstraints.scala | 101 +++
.../asyncprofiler/AsyncProfilerToolTest.java | 2 +-
.../RewriteInferFiltersFromConstraintsSuite.scala | 320 ++++++++++
.../scala/io/kyligence/kap/common/SSSource.scala | 89 +++
.../scala/org/apache/kylin/common/SSSource.scala | 5 +-
.../kylin/it/TestQueryAndBuildFunSuite.scala | 1 +
.../kylin/rest/service/StreamingJobService.java | 2 +-
.../org/apache/kylin/kafka/util/KafkaUtils.java | 2 +-
.../streaming/jobs/AbstractSparkJobLauncher.java | 2 +-
.../streaming/manager/StreamingJobManager.java | 2 +-
.../apache/kylin/streaming/rest/RestSupport.java | 2 +-
.../kylin/streaming/CreateStreamingFlatTable.scala | 24 +-
.../kylin/tool/AbstractInfoExtractorTool.java | 2 +-
.../java/org/apache/kylin/tool/AuditLogTool.java | 2 +-
.../org/apache/kylin/tool/ClickhouseDiagTool.java | 4 +-
.../main/java/org/apache/kylin/tool/ConfTool.java | 2 +-
.../apache/kylin/tool/DumpHadoopSystemProps.java | 2 +-
.../org/apache/kylin/tool/JobDiagInfoTool.java | 2 +-
.../java/org/apache/kylin/tool/KylinLogTool.java | 2 +-
.../org/apache/kylin/tool/daemon/KapGuardian.java | 2 +-
.../kylin/tool/garbage/DataflowCleanerCLI.java | 2 +-
.../apache/kylin/tool/obf/KylinConfObfuscator.java | 2 +-
.../kylin/tool/upgrade/CheckProjectModeCLI.java | 2 +-
.../kylin/tool/upgrade/DeleteFavoriteQueryCLI.java | 2 +-
.../apache/kylin/tool/upgrade/RenameEntity.java | 2 +-
.../kylin/tool/upgrade/RenameUserResourceTool.java | 2 +-
.../apache/kylin/tool/upgrade/UpdateModelCLI.java | 2 +-
.../kylin/tool/upgrade/UpdateProjectCLI.java | 2 +-
.../kylin/tool/upgrade/UpdateSessionTableCLI.java | 2 +-
.../kylin/tool/upgrade/UpdateUserGroupCLI.java | 2 +-
.../kylin/tool/util/HadoopConfExtractor.java | 2 +-
.../org/apache/kylin/tool/util/ServerInfoUtil.java | 2 +-
.../org/apache/kylin/tool/StorageCleanerTest.java | 2 +-
.../kylin/tool/garbage/SnapshotCleanerTest.java | 2 +-
.../apache/kylin/tool/general/CryptToolTest.java | 2 +-
.../tool/upgrade/RenameUserResourceToolTest.java | 2 +-
429 files changed, 7592 insertions(+), 4788 deletions(-)
copy
src/{query/src/main/java/org/apache/kylin/query/udf/stringUdf/ConcatUDF.java =>
common-service/src/main/java/org/apache/kylin/rest/service/MockQuerySmartSupporter.java}
(68%)
rename
src/core-common/src/main/java/org/apache/kylin/common/util/{StringUtil.java =>
StringHelper.java} (63%)
create mode 100644
src/core-common/src/test/java/org/apache/kylin/common/util/StringHelperTest.java
delete mode 100644
src/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
delete mode 100644
src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinedFlatTable.java
delete mode 100644
src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsGraph.java
create mode 100644
src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/DefaultJoinEdgeMatcher.java
create mode 100644
src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/Edge.java
copy
src/{query/src/main/java/org/apache/kylin/query/udf/stringUdf/ConcatUDF.java =>
core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/IJoinEdgeMatcher.java}
(70%)
create mode 100644
src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java
copy
src/{query/src/main/java/org/apache/kylin/query/udf/stringUdf/ConcatUDF.java =>
core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRuntimeException.java}
(70%)
create mode 100644
src/examples/test_case_data/localmeta/metadata/_global/project/index_build_test.json
create mode 100644
src/examples/test_case_data/localmeta/metadata/index_build_test/dataflow/3ec47efc-573a-9304-4405-8e05ae184322.json
create mode 100644
src/examples/test_case_data/localmeta/metadata/index_build_test/dataflow_details/3ec47efc-573a-9304-4405-8e05ae184322/b2f206e1-7a15-c94a-20f5-f608d550ead6.json
create mode 100644
src/examples/test_case_data/localmeta/metadata/index_build_test/index_plan/3ec47efc-573a-9304-4405-8e05ae184322.json
create mode 100644
src/examples/test_case_data/localmeta/metadata/index_build_test/model_desc/3ec47efc-573a-9304-4405-8e05ae184322.json
create mode 100644
src/examples/test_case_data/localmeta/metadata/index_build_test/table/SSB.CUSTOMER.json
create mode 100644
src/examples/test_case_data/localmeta/metadata/index_build_test/table/SSB.LINEORDER.json
create mode 100644
src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelLiteResponse.java
copy
src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/{ModelQueryParams.java
=> IndexPlanParams.java} (62%)
copy
src/{query/src/main/java/org/apache/kylin/query/udf/stringUdf/ConcatUDF.java =>
modeling-service/src/main/java/org/apache/kylin/rest/service/params/PaginationParams.java}
(69%)
delete mode 100644 src/query/src/main/java/org/apache/kylin/query/QueryCli.java
create mode 100644
src/query/src/main/java/org/apache/kylin/query/util/RuleUtils.java
delete mode 100644
src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/PartitionFlatTable.scala
delete mode 100644
src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SegmentFlatTable.scala
create mode 100644
src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/stage/build/GenerateFlatTableWithSparkSessionTest.java
copy
src/spark-project/engine-spark/src/{main/scala/org/apache/kylin/engine/spark/job/stage/build/MaterializedFactTableView.scala
=> test/scala/org/apache/kylin/engine/spark/builder/TestFlatTable.scala} (57%)
create mode 100644
src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/job/TestRDSegmentBuildExec.scala
create mode 100644
src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/job/stage/build/partition/PartitionFlatTableAndDictBaseTest.scala
rename src/spark-project/sparder/src/main/scala/org/apache/kylin/query/{ =>
plugin}/asyncprofiler/AsyncProfiling.scala (98%)
copy src/spark-project/sparder/src/main/scala/org/apache/kylin/query/{ =>
plugin}/asyncprofiler/QueryAsyncProfilerDriverPlugin.scala (97%)
copy src/spark-project/sparder/src/main/scala/org/apache/kylin/query/{ =>
plugin}/asyncprofiler/QueryAsyncProfilerSparkPlugin.scala (95%)
copy
src/{query/src/main/java/org/apache/kylin/query/udf/stringUdf/ConcatUDF.java =>
spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseConstant.scala}
(69%)
rename
src/spark-project/sparder/src/main/scala/org/apache/kylin/query/{asyncprofiler/QueryAsyncProfilerDriverPlugin.scala
=> plugin/diagnose/DiagnoseDriverPlugin.scala} (51%)
create mode 100644
src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseExecutorPlugin.scala
create mode 100644
src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseHelper.scala
rename
src/spark-project/sparder/src/main/scala/org/apache/kylin/query/{asyncprofiler/QueryAsyncProfilerSparkPlugin.scala
=> plugin/diagnose/DiagnoseSparkPlugin.scala} (72%)
rename
src/spark-project/sparder/src/test/scala/org/apache/kylin/query/{asyncprofiler/AsyncPluginWithMeta.scala
=> plugin/SparkPluginWithMeta.scala} (94%)
rename src/spark-project/sparder/src/test/scala/org/apache/kylin/query/{ =>
plugin}/asyncprofiler/AsyncProfilingTest.scala (94%)
rename src/spark-project/sparder/src/test/scala/org/apache/kylin/query/{ =>
plugin}/asyncprofiler/QueryAsyncProfilerSparkPluginTest.scala (96%)
rename
src/spark-project/sparder/src/test/scala/org/apache/kylin/query/{asyncprofiler/QueryAsyncProfilerDriverPluginTest.scala
=> plugin/asyncprofiler/QueryProfilerDriverPluginTest.scala} (92%)
create mode 100644
src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseExecutorPluginTest.scala
create mode 100644
src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/diagnose/DiagnosePluginTest.scala
create mode 100644
src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/datasource/storage/LayoutFormatWriter.scala
create mode 100644
src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/execution/datasource/RewriteInferFiltersFromConstraints.scala
create mode 100644
src/spark-project/spark-common/src/test/scala/org/apache/spark/sql/execution/datasource/RewriteInferFiltersFromConstraintsSuite.scala
create mode 100644
src/spark-project/spark-it/src/test/scala/io/kyligence/kap/common/SSSource.scala
