This is an automated email from the ASF dual-hosted git repository.
liyang pushed a change to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
from f6cc0ce32c KYLIN-5820,added pagination and add count distinct
function in batch measure dialog
new 8ab87c93b9 KYLIN-5775 upgrade testcontainers to 1.16.2
new 5d781d1917 KYLIN-5776 Support parameter type is string for
timestampadd function
new a67e854a1d KYLIN-5777 Change spark.yarn.submit.file.replication from 1
to 3
new 9f269b527f KYLIN-5779 Without project permission，API "/access/acls"
throws NPE
new 8929bb820c KYLIN-5780 Dimension capacity of aggGroup does not work
new cb2a11d503 KYLIN-5781 Adjust the query result order of sql with union
new 1b63eefbc9 KYLIN-5782 Duplicated join keys result empty query result
new 8aaa87cbb3 KYLIN-5783 Round query result of decimal zero by config
new 40a10818a0 Add some code to compatible with the enterprise version
new 1367e9ccff KYLIN-5784 Optimization for query history filters
new d7d8098598 KYLIN-5785 Support implicit query of split function
new dd16079c42 KYLIN-5787 Use t-digest as spark percentile_approx function
new 9db3502f80 KYLIN-5774 Optimize Calcite plan to convert spark logical
plan
new 775efe62b5 KYLIN-5788 Enhance global dict on flat table encoding stage
logging & retry
new ca370d2dad KYLIN-5789 Clean sparder history and spark history
automatically
new 2695c5da83 KYLIN-5790 Security of kafka-clients
new 30c4f28a16 KYLIN-5786 Add a write lock when merging v3 dict
new 4ed3729312 KYLIN-5778 Upgrade spark to 3.2.0-kylin-4.6.14.0
The 18 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
pom.xml | 10 +-
.../src/main/resources/config/init.properties | 2 +-
.../src/main/resources/kylinSecurity.xml | 1 +
.../kylin/rest/controller/NSystemController.java | 18 +-
.../src/main/resources/kylinSecurity.xml | 1 +
.../apache/kylin/rest/service/AccessService.java | 6 +-
.../apache/kylin/rest/service/ProjectService.java | 6 +-
.../apache/kylin/rest/service/SystemService.java | 9 +
.../kylin/rest/service/AclTCRServiceTest.java | 4 +
.../src/test/resources/kylinSecurity.xml | 1 +
.../org/apache/kylin/common/KylinConfigBase.java | 19 ++-
.../common/exception/code/ErrorCodeServer.java | 3 +
.../src/main/resources/kylin-defaults0.properties | 3 +-
.../resources/kylin_error_msg_conf_cn.properties | 3 +
.../resources/kylin_error_msg_conf_en.properties | 3 +
.../kylin_error_suggestion_conf_cn.properties | 3 +
.../kylin_error_suggestion_conf_en.properties | 3 +
.../main/resources/kylin_errorcode_conf.properties | 3 +
.../measure/percentile/PercentileCounter.java | 1 +
.../apache/kylin/metadata/model/NDataModel.java | 9 +
.../metadata/query/JdbcQueryHistoryStore.java | 19 ++-
.../kylin/metadata/query/QueryHistoryDAO.java | 2 +
.../kylin/metadata/query/RDBMSQueryHistoryDAO.java | 5 +
.../metadata/query/util/QueryHisStoreUtil.java | 5 +
.../recommendation/entity/DimensionRecItemV2.java | 17 +-
.../metadata/query/RDBMSQueryHistoryDaoTest.java | 17 +-
.../src/main/resources/config/init.properties | 2 +-
.../src/main/resources/kylinSecurity.xml | 1 +
.../kylin/rest/controller/SegmentController.java | 6 +-
.../rest/controller/SegmentControllerTest.java | 43 ++++-
.../org/apache/kylin/rest/KafkaCondition.java} | 20 +--
.../apache/kylin/rest/service/KafkaService.java | 8 +-
.../apache/kylin/rest/service/TableService.java | 2 +-
.../apache/kylin/rest/service/ScheduleService.java | 60 +++++--
.../kylin/rest/service/ScheduleServiceTest.java | 25 ++-
.../kylin/newten/EnhancedAggPushDownTest.java | 2 +-
...itInfoTest.java => SpecialColumnNamesTest.java} | 68 +++-----
.../apache/kylin/query/engine/QueryExecTest.java | 59 +------
.../java/org/apache/kylin/util/ExecAndComp.java | 42 +++--
.../_global/project/special_column_names.json} | 2 +-
.../c41390c5-b35d-4db3-b167-029874b85a2c.json} | 2 +-
.../c41390c5-b35d-4db3-b167-029874b85a2c.json} | 2 +-
.../c41390c5-b35d-4db3-b167-029874b85a2c.json} | 10 +-
.../special_column_names}/table/SSB.CUSTOMER.json | 6 +-
.../table/SSB.P_LINEORDER.json | 10 +-
.../kylin/rest/controller/KafkaController.java | 2 +-
.../kylin/rest/controller/NModelController.java | 11 +-
.../rest/controller/NModelControllerTest.java | 107 +++++++++++-
.../kylin/rest/service/FusionIndexService.java | 10 ++
.../kylin/rest/service/IndexPlanService.java | 2 +
.../org/apache/kylin/rest/util/ModelUtils.java | 36 +++-
.../kylin/rest/service/FusionIndexServiceTest.java | 23 +++
.../src/main/resources/config/init.properties | 2 +-
.../src/main/resources/kylinSecurity.xml | 1 +
.../org/apache/kylin/query/relnode/KapJoinRel.java | 17 +-
.../apache/kylin/query/relnode/OLAPJoinRel.java | 48 +++---
.../kylin/rest/service/QueryHistoryService.java | 27 +--
.../rest/service/QueryHistoryServiceTest.java | 6 +-
.../kylin/query/engine/AsyncQueryApplication.java | 5 +
.../kylin/query/engine/QueryRoutingEngine.java | 12 --
.../org/apache/kylin/query/udf/SparkStringUDF.java | 6 +-
.../query/engine/AsyncQueryApplicationTest.java | 3 +
.../org/apache/kylin/rest/QueryNodeFilter.java | 2 +
.../src/main/resources/config/init.properties | 2 +-
.../spark/builder/v3dict/DictionaryBuilder.scala | 63 ++++---
.../builder/v3dict/GlobalDictionarySuite.scala | 12 +-
.../kylin/query/pushdown/SparkSqlClient.scala | 2 +
.../kylin/query/runtime/CalciteToSparkPlaner.scala | 126 +++++---------
.../kylin/query/runtime/ExpressionConverter.scala | 2 +
.../kylin/query/runtime/SparderRexVisitor.scala | 37 ++--
.../apache/kylin/query/runtime/SparkEngine.java | 8 +-
.../kylin/query/runtime/plan/AggregatePlan.scala | 98 ++++++-----
.../kylin/query/runtime/plan/FilterPlan.scala | 25 +--
.../apache/kylin/query/runtime/plan/JoinPlan.scala | 58 ++++---
.../kylin/query/runtime/plan/LimitPlan.scala | 25 ++-
.../kylin/query/runtime/plan/MinusPlan.scala | 14 +-
.../kylin/query/runtime/plan/ProjectPlan.scala | 19 ++-
.../kylin/query/runtime/plan/ResultPlan.scala | 3 +-
.../apache/kylin/query/runtime/plan/SortPlan.scala | 52 +++---
.../kylin/query/runtime/plan/TableScanPlan.scala | 118 ++++++-------
.../kylin/query/runtime/plan/UnionPlan.scala | 30 ++--
.../kylin/query/runtime/plan/ValuesPlan.scala | 15 +-
.../kylin/query/runtime/plan/WindowPlan.scala | 29 ++--
.../apache/kylin/query/util/RuntimeHelper.scala | 31 ++--
.../kylin/query/util/SparderDerivedUtil.scala | 52 +++---
.../scala/org/apache/spark/sql/KapFunctions.scala | 8 +-
.../apache/spark/sql/KylinDataFrameManager.scala | 19 ++-
.../scala/org/apache/spark/sql/SparderEnv.scala | 3 +
.../org/apache/spark/sql/SparkOperation.scala | 64 +++++--
.../sql/catalyst/expressions/KapExpresssions.scala | 2 +-
.../sql/execution/utils/SchemaProcessor.scala | 36 ++--
.../spark/sql/manager/SparderLookupManager.scala | 69 ++++----
.../org/apache/spark/sql/udf/DictEncodeImpl.scala | 6 +-
.../org/apache/spark/sql/udf/UdfManager.scala | 9 +-
.../kylin/query/pushdown/SparderTypeUtilTest.java | 16 ++
.../kylin/query/sql/KylinDataFrameManagerTest.java | 32 ++--
.../query/runtime/plan/SegmentEmptyTest.scala | 38 ++---
.../org/apache/spark/dict/NBucketDictionary.java | 7 +-
.../apache/spark/dict/NGlobalDictHDFSStore.java | 92 +++++++++-
.../org/apache/spark/dict/NGlobalDictMetaInfo.java | 6 +-
.../org/apache/spark/dict/NGlobalDictStore.java | 2 +-
.../org/apache/spark/dict/NGlobalDictionaryV2.java | 6 +-
.../org/apache/spark/sql/SparderTypeUtil.scala | 11 +-
.../org/apache/spark/sql/SparkInternalAgent.scala | 0
.../sql/datasource/storage/StorageStore.scala | 73 ++++----
.../org/apache/spark/sql/udaf/Percentile.scala | 5 +
.../scala/org/apache/kylin/common/JobSupport.scala | 7 +-
.../apache/kylin/helper/MetadataToolHelper.java | 7 +-
.../org/apache/kylin/helper/RoutineToolHelper.java | 41 ++++-
.../java/org/apache/kylin/tool/AuditLogTool.java | 18 +-
.../org/apache/kylin/tool/MaintainModeTool.java | 21 +--
.../constant/StringConstant.java} | 16 +-
.../tool/garbage/CleanTaskExecutorService.java | 14 +-
.../kylin/tool/garbage/ExecutableCleaner.java | 6 +-
.../apache/kylin/tool/garbage/GarbageCleaner.java | 49 +-----
.../apache/kylin/tool/garbage/IndexCleaner.java | 6 +-
.../apache/kylin/tool/garbage/MetadataCleaner.java | 47 ++++-
.../apache/kylin/tool/garbage/SnapshotCleaner.java | 6 +-
.../kylin/tool/garbage/SourceUsageCleaner.java | 7 +-
.../apache/kylin/tool/garbage/StorageCleaner.java | 190 ++++++++++++++++++---
.../apache/kylin/tool/routine/FastRoutineTool.java | 1 +
.../org/apache/kylin/tool/routine/RoutineTool.java | 31 +++-
.../kylin/tool/security/AdminUserInitCLI.java | 18 +-
.../kylin/tool/security/KylinPasswordResetCLI.java | 18 +-
.../org/apache/kylin/tool/StorageCleanerTest.java | 126 +++++++++++++-
.../garbage/CleanTaskExecutorServiceTests.java | 2 +-
.../kylin/tool/garbage/ExecutableCleanerTest.java | 8 +-
.../kylin/tool/garbage/SnapshotCleanerTest.java | 2 +-
.../kylin/tool/garbage/SourceUsageCleanerTest.java | 8 +-
.../kylin/tool/security/AdminUserInitCLITest.java | 8 +-
.../tool/security/KylinPasswordResetCLITest.java | 10 +-
.../appstatus_application_1677899901295_8490 | 0
..._1_application_1677899901295_8490_1690953331329 | 0
..._2_application_1677899901295_8490_1690953331329 | 0
..._1_application_1677899901295_5936_1690235831577 | 0
.../spark-history/application_1677899901295_0989 | 0
.../spark-history/application_1677899901295_8243 | 0
137 files changed, 1834 insertions(+), 1021 deletions(-)
copy src/{tool/src/main/java/org/apache/kylin/tool/YarnSparkLogExtractor.java
=> datasource-service/src/main/java/org/apache/kylin/rest/KafkaCondition.java}
(65%)
copy
src/kylin-it/src/test/java/org/apache/kylin/newten/{ExtractLimitInfoTest.java
=> SpecialColumnNamesTest.java} (58%)
copy
src/kylin-it/src/test/resources/ut_meta/{bloomfilter/metadata/_global/project/bloomfilter.json
=> special_column_names/metadata/_global/project/special_column_names.json}
(96%)
copy
src/kylin-it/src/test/resources/ut_meta/{bloomfilter/metadata/bloomfilter/dataflow/c41390c5-b93d-4db3-b167-029874b85a2c.json
=>
special_column_names/metadata/special_column_names/dataflow/c41390c5-b35d-4db3-b167-029874b85a2c.json}
(82%)
copy
src/kylin-it/src/test/resources/ut_meta/{bloomfilter/metadata/bloomfilter/index_plan/c41390c5-b93d-4db3-b167-029874b85a2c.json
=>
special_column_names/metadata/special_column_names/index_plan/c41390c5-b35d-4db3-b167-029874b85a2c.json}
(97%)
copy
src/kylin-it/src/test/resources/ut_meta/{bloomfilter/metadata/bloomfilter/model_desc/c41390c5-b93d-4db3-b167-029874b85a2c.json
=>
special_column_names/metadata/special_column_names/model_desc/c41390c5-b35d-4db3-b167-029874b85a2c.json}
(94%)
copy src/kylin-it/src/test/resources/ut_meta/{bloomfilter/metadata/bloomfilter
=> special_column_names/metadata/special_column_names}/table/SSB.CUSTOMER.json
(95%)
copy src/kylin-it/src/test/resources/ut_meta/{bloomfilter/metadata/bloomfilter
=>
special_column_names/metadata/special_column_names}/table/SSB.P_LINEORDER.json
(93%)
rename src/spark-project/{engine-spark =>
spark-common}/src/main/scala/org/apache/spark/sql/SparkInternalAgent.scala
(100%)
rename src/tool/src/main/java/org/apache/kylin/{helper/HelperConstants.java =>
tool/constant/StringConstant.java} (67%)
copy kystudio/static/.gitkeep =>
src/tool/src/test/resources/ut_storage/working-dir2/sparder-history/localhost_7070/eventlog_v2_application_1677899901295_4823#1690192675042/appstatus_application_1677899901295_8490
(100%)
copy kystudio/static/.gitkeep =>
src/tool/src/test/resources/ut_storage/working-dir2/sparder-history/localhost_7070/eventlog_v2_application_1677899901295_4823#1690192675042/events_1_application_1677899901295_8490_1690953331329
(100%)
copy kystudio/static/.gitkeep =>
src/tool/src/test/resources/ut_storage/working-dir2/sparder-history/localhost_7070/eventlog_v2_application_1677899901295_4823#1690192675042/events_2_application_1677899901295_8490_1690953331329
(100%)
copy kystudio/static/.gitkeep =>
src/tool/src/test/resources/ut_storage/working-dir2/sparder-history/localhost_7071/eventlog_v2_application_1677899901295_4824#1690192771380/events_1_application_1677899901295_5936_1690235831577
(100%)
copy kystudio/static/.gitkeep =>
src/tool/src/test/resources/ut_storage/working-dir2/spark-history/application_1677899901295_0989
(100%)
copy kystudio/static/.gitkeep =>
src/tool/src/test/resources/ut_storage/working-dir2/spark-history/application_1677899901295_8243
(100%)
