[kylin] branch master updated: [KYLIN-5035] Fix Merge Cuboid Statistics EOFException.

[xxyu]

This is an automated email from the ASF dual-hosted git repository.

xxyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git


The following commit(s) were added to refs/heads/master by this push:
     new e587d3a  [KYLIN-5035] Fix Merge Cuboid Statistics EOFException.
e587d3a is described below

commit e587d3a7fee65b47227ecaae1a552d7a33d932cc
Author: yangjiang <yangji...@ebay.com>
AuthorDate: Fri Jul 16 14:27:37 2021 +0800

    [KYLIN-5035] Fix Merge Cuboid Statistics EOFException.
---
 .../apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidJob.java  | 3 +++
 .../kylin/engine/spark/SparkCalculateStatsFromBaseCuboidJob.java       | 2 ++
 2 files changed, 5 insertions(+)

diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidJob.java
index 4305a25..965967b 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidJob.java
@@ -73,6 +73,9 @@ public class CalculateStatsFromBaseCuboidJob extends 
AbstractHadoopJob {
             job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
             job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_ID, 
segmentID);
             
job.getConfiguration().set(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT, 
statistics_sampling_percent);
+            // use customized writer should disable mr.speculation
+            job.getConfiguration().set("mapreduce.reduce.speculative", 
"false");
+
             logger.info("Starting: " + job.getJobName());
 
             setJobClasspath(job, cube.getConfig());
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCalculateStatsFromBaseCuboidJob.java
 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCalculateStatsFromBaseCuboidJob.java
index 838efc0..bd60449 100644
--- 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCalculateStatsFromBaseCuboidJob.java
+++ 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCalculateStatsFromBaseCuboidJob.java
@@ -133,6 +133,8 @@ public class SparkCalculateStatsFromBaseCuboidJob extends 
AbstractApplication im
 
         SparkConf sparkConf = SparkUtil.setKryoSerializerInConf();
         sparkConf.setAppName("Kylin_Calculate_Statics_From_BaseCuboid_Data_" + 
cubeName + "_With_Spark");
+        // use customized writer should disable spark.speculation
+        sparkConf.set("spark.speculation", "false");
 
         KylinSparkJobListener jobListener = new KylinSparkJobListener();
         try (JavaSparkContext sc = new JavaSparkContext(sparkConf)) {