This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a change to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from ee1be8c KYLIN-4944 kylin4 Docker image is upgraded from centos6.9 to
centos7… (#1635)
new 0dc4c06 KYLIN-4966 Refresh the existing segment according to the new
cuboid list in kylin4
new 082df34 [KYLIN-4948] Provide an API to allow users to adjust cuboids
manually
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/kylin/cube/model/CubeBuildTypeEnum.java | 5 +
.../kylin/job/constant/ExecutableConstants.java | 1 +
.../org/apache/kylin/metadata/model/TableDesc.java | 1 -
.../spark/SparkBatchCubingEngineParquet.java | 3 +-
.../engine/spark/job/FilterRecommendCuboidJob.java | 104 ++++++
.../kylin/engine/spark/job/JobStepFactory.java | 10 +-
.../apache/kylin/engine/spark/job/JobStepType.java | 4 +-
.../engine/spark/job/NResourceDetectStep.java | 40 +-
.../NSparkBatchOptimizeJobCheckpointBuilder.java | 40 +-
.../spark/job/NSparkCleanupHdfsStorageStep.java | 90 +++++
.../kylin/engine/spark/job/NSparkCubingUtil.java | 7 +-
...esourceDetectStep.java => NSparkLocalStep.java} | 40 +-
...arkMergingJob.java => NSparkOptimizingJob.java} | 86 ++---
...rkCubingStep.java => NSparkOptimizingStep.java} | 35 +-
.../job/NSparkUpdateCubeInfoAfterOptimizeStep.java | 25 +-
.../kylin/engine/spark/job/OptimizeBuildJob.java} | 411 +++++++++------------
.../job/ResourceDetectBeforeOptimizingJob.java | 108 ++++++
.../engine/spark/utils/UpdateMetadataUtil.java | 84 ++++-
.../kylin/engine/spark/job/BuildJobInfos.scala | 39 ++
.../kylin/engine/spark/job/CubeBuildJob.java | 16 +-
.../engine/spark/job/CuboidStatisticsJob.scala | 9 +
.../kylin/engine/spark/job/LogJobInfoUtils.scala | 32 ++
.../engine/spark/job/ParentSourceChooser.scala | 33 +-
.../spark/job/ResourceDetectBeforeCubingJob.java | 8 +-
.../engine/spark/metadata/cube/ManagerHub.java | 13 +-
.../kylin/engine/spark/metadata/MetaData.scala | 4 +
.../engine/spark/metadata/MetadataConverter.scala | 21 +-
.../kylin/engine/spark2/NOptimizeJobTest.java | 151 ++++++++
.../kylin/rest/controller/CubeController.java | 60 +++
.../kylin/rest/request/JobOptimizeRequest2.java | 24 +-
.../org/apache/kylin/rest/service/JobService.java | 13 +-
31 files changed, 1051 insertions(+), 466 deletions(-)
create mode 100644
kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/FilterRecommendCuboidJob.java
copy
build-engine/src/main/java/org/apache/kylin/engine/mr/BatchOptimizeJobCheckpointBuilder.java
=>
kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/NSparkBatchOptimizeJobCheckpointBuilder.java
(78%)
create mode 100644
kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/NSparkCleanupHdfsStorageStep.java
copy
kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/{NResourceDetectStep.java
=> NSparkLocalStep.java} (68%)
copy
kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/{NSparkMergingJob.java
=> NSparkOptimizingJob.java} (60%)
copy
kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/{NSparkCubingStep.java
=> NSparkOptimizingStep.java} (89%)
copy
build-engine/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterCheckpointStep.java
=>
kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/NSparkUpdateCubeInfoAfterOptimizeStep.java
(89%)
copy
kylin-spark-project/kylin-spark-engine/src/main/{scala/org/apache/kylin/engine/spark/job/CubeBuildJob.java
=> java/org/apache/kylin/engine/spark/job/OptimizeBuildJob.java} (53%)
create mode 100644
kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/ResourceDetectBeforeOptimizingJob.java
create mode 100644
kylin-spark-project/kylin-spark-test/src/test/java/org/apache/kylin/engine/spark2/NOptimizeJobTest.java
copy query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java =>
server-base/src/main/java/org/apache/kylin/rest/request/JobOptimizeRequest2.java
(61%)
