[kylin] branch main updated: KYLIN-5027 Fix query for the cube dose not build basecuboid

[xxyu]

This is an automated email from the ASF dual-hosted git repository.

xxyu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/kylin.git


The following commit(s) were added to refs/heads/main by this push:
     new 94b109f  KYLIN-5027 Fix query for the cube dose not build basecuboid
94b109f is described below

commit 94b109f363629a8829e9a90ac6f8853c6fd961b0
Author: yaqian.zhang <598593...@qq.com>
AuthorDate: Mon Aug 16 16:54:37 2021 +0800

    KYLIN-5027 Fix query for the cube dose not build basecuboid
---
 .../apache/kylin/cube/CubeCapabilityChecker.java   | 25 ++++++++++++++++++++--
 .../java/org/apache/kylin/cube/model/CubeDesc.java | 13 +++++++++++
 .../kylin/job/execution/ExecutableManager.java     |  5 ++++-
 3 files changed, 40 insertions(+), 3 deletions(-)

diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 5ba0a2d..5d96628 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -24,7 +24,6 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 import java.util.Arrays;
-import java.util.stream.Collectors;
 
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang.StringUtils;
@@ -183,11 +182,33 @@ public class CubeCapabilityChecker {
         } else {
             HashSet<TblColRef> aggResult = result;
             for (AggregationGroup aggGroup : cubeDesc.getAggregationGroups()) {
-                HashSet<TblColRef> tmpAggResult = (HashSet<TblColRef>) 
result.stream().filter(col -> 
!Arrays.asList(aggGroup.getIncludes()).contains(col.getCanonicalName())).collect(Collectors.toSet());
+                List<String> aggGroupColumn = 
Arrays.asList(aggGroup.getIncludes());
+                HashSet<TblColRef> tmpAggResult = new 
HashSet<>(dimensionColumns);
+                Iterator<TblColRef> iterator = result.iterator();
+                while (iterator.hasNext()) {
+                    TblColRef col = iterator.next();
+                    String colName = col.getCanonicalName();
+                    String[] colInfo = colName.split("\\.");
+                    // colName: database.table.column
+                    if (colInfo.length == 3) {
+                        colName = colInfo[1] + "." + colInfo[2];
+                    }
+                    // colName: tableAlias:database.table.column
+                    if (col.getTableAlias() != null) {
+                        colName = col.getTableAlias() + "." + colInfo[2];
+                    }
+                    // aggGroupColumn: 
[table1Alias.column1,table2Alias.column2,....]
+                    if (aggGroupColumn.contains(colName)) {
+                        tmpAggResult.remove(col);
+                    }
+                }
                 if (tmpAggResult.size() < aggResult.size()) {
                     aggResult = tmpAggResult;
                 }
             }
+            if (aggResult.size() > 0) {
+                aggResult.removeAll(cubeDesc.listDerivedDimensionColumns());
+            }
             result = aggResult;
         }
         return result;
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 1cc31d0..dc431ef 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -255,6 +255,19 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
     }
 
     /**
+     * @return derived dimension columns
+     */
+    public List<TblColRef> listDerivedDimensionColumns() {
+        List<TblColRef> result = new ArrayList<TblColRef>();
+        for (TblColRef col : dimensionColumns) {
+            if (isDerived(col)) {
+                result.add(col);
+            }
+        }
+        return result;
+    }
+
+    /**
      * @return dimension columns excluding derived
      */
     public List<TblColRef> listDimensionColumnsExcludingDerived(boolean 
alsoExcludeExtendedCol) {
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 66b17f5..5fb5fbd 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -570,7 +570,6 @@ public class ExecutableManager {
             if (endTime != 0) {
                 long interruptTime = System.currentTimeMillis() - endTime + 
job.getInterruptTime();
                 info = Maps.newHashMap(getJobOutput(jobId).getInfo());
-                
getJobOutput(jobId).getInfo().remove(AbstractExecutable.END_TIME);
                 info.put(AbstractExecutable.INTERRUPT_TIME, 
Long.toString(interruptTime));
                 info.remove(AbstractExecutable.END_TIME);
             }
@@ -683,6 +682,10 @@ public class ExecutableManager {
                     jobOutput.setInfo(info);
                 }
             }
+            if ((ExecutableState.ERROR.equals(oldStatus) || 
ExecutableState.STOPPED.equals(oldStatus))
+                    && ExecutableState.READY.equals(newStatus)) {
+                jobOutput.getInfo().remove(AbstractExecutable.END_TIME);
+            }
             if (output != null) {
                 if (output.length() > config.getJobOutputMaxSize()) {
                     output = output.substring(0, config.getJobOutputMaxSize());