This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/main by this push:
new 5312bac HOTFIX for spark3
5312bac is described below
commit 5312bac727e5d64bfa7be999de0d2bddc273a3b8
Author: Zhichao Zhang <[email protected]>
AuthorDate: Sat Aug 7 16:51:13 2021 +0800
HOTFIX for spark3
---
build/bin/kylin.sh | 13 +-
.../src/main/resources/kylin-defaults.properties | 15 +-
docker/dockerfile/standalone/conf/bin/kylin.sh | 201 +++++++--------------
.../kylin/spark/classloader/SparkClassLoader.java | 1 -
.../engine/spark/metadata/cube/PathManager.java | 2 +-
.../kylin/engine/spark/job/NSparkExecutable.java | 3 -
.../org/apache/spark/conf/rule/SparkConfRule.scala | 16 +-
.../org/apache/spark/utils/ResourceUtils.scala | 7 +-
.../scala/org/apache/spark/sql/KylinSession.scala | 17 +-
pom.xml | 1 +
.../kylin/rest/controller/CubeController.java | 10 +-
11 files changed, 119 insertions(+), 167 deletions(-)
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 77d84fb..9498f0b 100755
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -143,18 +143,12 @@ function retrieveDependency() {
# compose hadoop_dependencies
hadoop_dependencies=${hadoop_dependencies}:`hadoop classpath`
-# if [ -n "${hbase_dependency}" ]; then
-# hadoop_dependencies=${hadoop_dependencies}:${hbase_dependency}
-# fi
if [ -n "${hive_dependency}" ]; then
- hadoop_dependencies=${hadoop_dependencies}:${hive_dependency}
+ hadoop_dependencies=${hive_dependency}:${hadoop_dependencies}
fi
if [ -n "${kafka_dependency}" ]; then
hadoop_dependencies=${hadoop_dependencies}:${kafka_dependency}
fi
- if [ -n "${spark_dependency}" ]; then
- hadoop_dependencies=${hadoop_dependencies}:${spark_dependency}
- fi
# compose KYLIN_TOMCAT_CLASSPATH
tomcat_classpath=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*
@@ -305,6 +299,11 @@ function retrieveStartCommand() {
verbose "java opts is ${KYLIN_EXTRA_START_OPTS} ${KYLIN_TOMCAT_OPTS}"
verbose "java classpath is ${KYLIN_TOMCAT_CLASSPATH}"
classpathDebug ${KYLIN_TOMCAT_CLASSPATH}
+ # For Sparder
+ if [ -z ${HADOOP_CONF_DIR} ]
+ then
+ export HADOOP_CONF_DIR=${kylin_hadoop_conf_dir}
+ fi
$JAVA ${KYLIN_EXTRA_START_OPTS} ${KYLIN_TOMCAT_OPTS} -classpath
${KYLIN_TOMCAT_CLASSPATH} org.apache.catalina.startup.Bootstrap start >>
${KYLIN_HOME}/logs/kylin.out 2>&1 & echo $! > ${KYLIN_HOME}/pid &
}
diff --git a/core-common/src/main/resources/kylin-defaults.properties
b/core-common/src/main/resources/kylin-defaults.properties
index 3bff28c..c990ac2 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -18,7 +18,7 @@
### METADATA | ENV ###
# The metadata store has two implementations(RDBMS/HBase), while RDBMS is
recommended in Kylin 4.X
-# Please refer to
https://cwiki.apache.org/confluence/display/KYLIN/How+to+use+HBase+metastore+in+Kylin+4.0
if you prefer HBase
+# Please check https://kylin.apache.org/docs/tutorial/mysql_metastore.html
kylin.metadata.url=kylin_metadata@jdbc,url=jdbc:mysql://localhost:3306/kylin,username=XXXX,password=XXXXXX,maxActive=10,maxIdle=10
# metadata cache sync retry times
@@ -243,15 +243,16 @@ kylin.security.saml.context-path=/kylin
# Spark conf (default is in spark/conf/spark-defaults.conf)
kylin.engine.spark-conf.spark.master=yarn
-#kylin.engine.spark-conf.spark.submit.deployMode=client
+kylin.engine.spark-conf.spark.submit.deployMode=client
kylin.engine.spark-conf.spark.yarn.queue=default
#kylin.engine.spark-conf.spark.executor.cores=1
#kylin.engine.spark-conf.spark.executor.memory=4G
#kylin.engine.spark-conf.spark.executor.instances=1
#kylin.engine.spark-conf.spark.executor.memoryOverhead=1024M
-kylin.engine.spark-conf.spark.driver.cores=1
-kylin.engine.spark-conf.spark.driver.memory=1G
-kylin.engine.spark-conf.spark.shuffle.service.enabled=true
+#kylin.engine.spark-conf.spark.driver.cores=1
+#kylin.engine.spark-conf.spark.driver.memory=2G
+#kylin.engine.spark-conf.spark.driver.memoryOverhead=512M
+#kylin.engine.spark-conf.spark.shuffle.service.enabled=false
kylin.engine.spark-conf.spark.eventLog.enabled=true
kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-history
@@ -266,6 +267,7 @@
kylin.engine.spark-conf.spark.executor.extraJavaOptions=-Dfile.encoding=UTF-8 -D
# uncomment for HDP
#kylin.engine.spark-conf.spark.driver.extraJavaOptions=-Dhdp.version=current
#kylin.engine.spark-conf.spark.yarn.am.extraJavaOptions=-Dhdp.version=current
+kylin.engine.spark-conf.spark.driver.extraJavaOptions=-XX:+CrashOnOutOfMemoryError
### SPARK QUERY ENGINE CONFIGS (a.k.a. Sparder Context) ###
# Enlarge cores and memory to improve query performance in production env,
please check https://cwiki.apache.org/confluence/display/KYLIN/User+Manual+4.X
@@ -282,8 +284,9 @@ kylin.query.spark-conf.spark.executor.memoryOverhead=1G
kylin.query.spark-conf.spark.serializer=org.apache.spark.serializer.JavaSerializer
#kylin.query.spark-conf.spark.sql.shuffle.partitions=40
#kylin.query.spark-conf.spark.yarn.jars=hdfs://localhost:9000/spark2_jars/*
+kylin.query.spark-conf.spark.hadoop.yarn.timeline-service.enabled=false
-kylin.query.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current
-Dlog4j.configuration=spark-executor-log4j.properties -Dlog4j.debug
-Dkylin.hdfs.working.dir=${kylin.env.hdfs-working-dir}
-Dkylin.metadata.identifier=${kylin.metadata.url.identifier}
-Dkylin.spark.category=sparder -Dkylin.spark.project=${job.project}
+kylin.query.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current
-Dlog4j.configuration=spark-executor-log4j.properties -Dlog4j.debug
-Dkylin.hdfs.working.dir=${kylin.env.hdfs-working-dir}
-Dkylin.metadata.identifier=${kylin.metadata.url.identifier}
-Dkylin.spark.category=sparder
# uncomment for HDP
#kylin.query.spark-conf.spark.driver.extraJavaOptions=-Dhdp.version=current
#kylin.query.spark-conf.spark.yarn.am.extraJavaOptions=-Dhdp.version=current
diff --git a/docker/dockerfile/standalone/conf/bin/kylin.sh
b/docker/dockerfile/standalone/conf/bin/kylin.sh
index a691cec..9498f0b 100755
--- a/docker/dockerfile/standalone/conf/bin/kylin.sh
+++ b/docker/dockerfile/standalone/conf/bin/kylin.sh
@@ -143,20 +143,12 @@ function retrieveDependency() {
# compose hadoop_dependencies
hadoop_dependencies=${hadoop_dependencies}:`hadoop classpath`
-# if [ -n "${hbase_dependency}" ]; then
-# hadoop_dependencies=${hadoop_dependencies}:${hbase_dependency}
-# fi
if [ -n "${hive_dependency}" ]; then
- #hadoop_dependencies=${hadoop_dependencies}:${hive_dependency}
hadoop_dependencies=${hive_dependency}:${hadoop_dependencies}
fi
if [ -n "${kafka_dependency}" ]; then
hadoop_dependencies=${hadoop_dependencies}:${kafka_dependency}
fi
- if [ -n "${spark_dependency}" ]; then
- #hadoop_dependencies=${hadoop_dependencies}:${spark_dependency}
- hadoop_dependencies=${spark_dependency}:${hadoop_dependencies}
- fi
# compose KYLIN_TOMCAT_CLASSPATH
tomcat_classpath=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*
@@ -271,14 +263,8 @@ function runTool() {
exec $JAVA ${KYLIN_EXTRA_START_OPTS} ${KYLIN_TOOL_OPTS} -classpath
${KYLIN_TOOL_CLASSPATH} "$@"
}
-if [ "$2" == "--reload-dependency" ]
-then
- reload_dependency=1
-fi
-
# start command
-if [ "$1" == "start" ]
-then
+function retrieveStartCommand() {
if [ -f "${KYLIN_HOME}/pid" ]
then
PID=`cat $KYLIN_HOME/pid`
@@ -313,23 +299,15 @@ then
verbose "java opts is ${KYLIN_EXTRA_START_OPTS} ${KYLIN_TOMCAT_OPTS}"
verbose "java classpath is ${KYLIN_TOMCAT_CLASSPATH}"
classpathDebug ${KYLIN_TOMCAT_CLASSPATH}
+ # For Sparder
+ if [ -z ${HADOOP_CONF_DIR} ]
+ then
+ export HADOOP_CONF_DIR=${kylin_hadoop_conf_dir}
+ fi
$JAVA ${KYLIN_EXTRA_START_OPTS} ${KYLIN_TOMCAT_OPTS} -classpath
${KYLIN_TOMCAT_CLASSPATH} org.apache.catalina.startup.Bootstrap start >>
${KYLIN_HOME}/logs/kylin.out 2>&1 & echo $! > ${KYLIN_HOME}/pid &
+}
- echo ""
- echo "A new Kylin instance is started by $USER. To stop it, run 'kylin.sh
stop'"
- echo "Check the log at ${KYLIN_HOME}/logs/kylin.log"
- echo "Web UI is at http://${KYLIN_REST_ADDRESS}/kylin";
- exit 0
-
-# run command
-elif [ "$1" == "run" ]
-then
- retrieveStartCommand
- ${start_command}
-
-# stop command
-elif [ "$1" == "stop" ]
-then
+function retrieveStopCommand() {
if [ -f "${KYLIN_HOME}/pid" ]
then
PID=`cat $KYLIN_HOME/pid`
@@ -342,7 +320,7 @@ then
for ((i=0; i<$LOOP_COUNTER; i++))
do
- # wait to process stopped
+ # wait to process stopped
sleep $WAIT_TIME
if ps -p $PID > /dev/null ; then
echo "Stopping in progress. Will check after $WAIT_TIME
secs again..."
@@ -364,128 +342,71 @@ then
fi
fi
- # process is killed , remove pid file
+ # process is killed , remove pid file
rm -rf ${KYLIN_HOME}/pid
echo "Kylin with pid ${PID} has been stopped."
- exit 0
+ return 0
else
- quit "Kylin with pid ${PID} is not running"
+ echo "Kylin with pid ${PID} is not running"
+ return 1
fi
else
- quit "Kylin is not running"
+ return 1
fi
+}
-# streaming command
-elif [ "$1" == "streaming" ]
+if [ "$2" == "--reload-dependency" ]
then
- if [ $# -lt 2 ]
- then
- echo "Invalid input args $@"
- exit -1
- fi
- if [ "$2" == "start" ]
- then
- if [ -f "${KYLIN_HOME}/streaming_receiver_pid" ]
- then
- PID=`cat $KYLIN_HOME/streaming_receiver_pid`
- if ps -p $PID > /dev/null
- then
- echo "Kylin streaming receiver is running, stop it first"
- exit 1
- fi
- fi
- #retrive $hbase_dependency
- metadataUrl=`${dir}/get-properties.sh kylin.metadata.url`
- if [[ "${metadataUrl##*@}" == "hbase" ]]
- then
- source ${dir}/find-hbase-dependency.sh
- fi
- #retrive $KYLIN_EXTRA_START_OPTS
- if [ -f "${KYLIN_HOME}/conf/setenv.sh" ]
- then source ${KYLIN_HOME}/conf/setenv.sh
- fi
+ reload_dependency=1
+fi
- mkdir -p ${KYLIN_HOME}/ext
- HBASE_CLASSPATH=`hbase classpath`
- #echo "hbase class path:"$HBASE_CLASSPATH
-
STREAM_CLASSPATH=${KYLIN_HOME}/lib/streaming/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH}
-
- # KYLIN_EXTRA_START_OPTS is for customized settings, checkout
bin/setenv.sh
- $JAVA -cp $STREAM_CLASSPATH ${KYLIN_EXTRA_START_OPTS} \
- -Dlog4j.configuration=stream-receiver-log4j.properties\
- -DKYLIN_HOME=${KYLIN_HOME}\
- -Dkylin.hbase.dependency=${hbase_dependency} \
- org.apache.kylin.stream.server.StreamingReceiver $@ >
${KYLIN_HOME}/logs/streaming_receiver.out 2>&1 & echo $! >
${KYLIN_HOME}/streaming_receiver_pid &
- exit 0
- elif [ "$2" == "stop" ]
- then
- if [ ! -f "${KYLIN_HOME}/streaming_receiver_pid" ]
- then
- echo "Streaming receiver is not running, please check"
- exit 1
- fi
- PID=`cat ${KYLIN_HOME}/streaming_receiver_pid`
- if [ "$PID" = "" ]
- then
- echo "Streaming receiver is not running, please check"
- exit 1
- else
- echo "Stopping streaming receiver: $PID"
- WAIT_TIME=2
- LOOP_COUNTER=20
- if ps -p $PID > /dev/null
- then
- kill $PID
-
- for ((i=0; i<$LOOP_COUNTER; i++))
- do
- # wait to process stopped
- sleep $WAIT_TIME
- if ps -p $PID > /dev/null ; then
- echo "Stopping in progress. Will check after
$WAIT_TIME secs again..."
- continue;
- else
- break;
- fi
- done
-
- # if process is still around, use kill -9
- if ps -p $PID > /dev/null
- then
- echo "Initial kill failed, getting serious now..."
- kill -9 $PID
- sleep 1 #give kill -9 sometime to "kill"
- if ps -p $PID > /dev/null
- then
- quit "Warning, even kill -9 failed, giving up! Sorry..."
- fi
- fi
+# start command
+if [ "$1" == "start" ]
+then
+ retrieveStartCommand
- # process is killed , remove pid file
- rm -rf ${KYLIN_HOME}/streaming_receiver_pid
- echo "Kylin streaming receiver with pid ${PID} has been
stopped."
- exit 0
- else
- quit "Kylin streaming receiver with pid ${PID} is not running"
- fi
- fi
- elif [[ "$2" = org.apache.kylin.* ]]
+ echo ""
+ echo "A new Kylin instance is started by $USER. To stop it, run 'kylin.sh
stop'"
+ echo "Check the log at ${KYLIN_HOME}/logs/kylin.log"
+ echo "Web UI is at http://${KYLIN_REST_ADDRESS}/kylin";
+ exit 0
+
+# run command
+elif [ "$1" == "run" ]
+then
+ retrieveStartCommand
+ ${start_command}
+
+# stop command
+elif [ "$1" == "stop" ]
+then
+ retrieveStopCommand
+ if [[ $? == 0 ]]
then
- source ${KYLIN_HOME}/conf/setenv.sh
- HBASE_CLASSPATH=`hbase classpath`
- #echo "hbase class path:"$HBASE_CLASSPATH
-
STREAM_CLASSPATH=${KYLIN_HOME}/lib/streaming/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH}
-
- shift
- # KYLIN_EXTRA_START_OPTS is for customized settings, checkout
bin/setenv.sh
- $JAVA -cp $STREAM_CLASSPATH ${KYLIN_EXTRA_START_OPTS} \
- -Dlog4j.configuration=stream-receiver-log4j.properties\
- -DKYLIN_HOME=${KYLIN_HOME}\
- -Dkylin.hbase.dependency=${hbase_dependency} \
- "$@"
exit 0
+ else
+ quit "Kylin is not running"
fi
+# restart command
+elif [ "$1" == "restart" ]
+then
+ echo "Restarting kylin..."
+ echo "--> Stopping kylin first if it's running..."
+ retrieveStopCommand
+ if [[ $? != 0 ]]
+ then
+ echo "Kylin is not running, now start it"
+ fi
+ echo "--> Start kylin..."
+ retrieveStartCommand
+
+ echo ""
+ echo "A new Kylin instance is started by $USER. To stop it, run 'kylin.sh
stop'"
+ echo "Check the log at ${KYLIN_HOME}/logs/kylin.log"
+ echo "Web UI is at http://${KYLIN_REST_ADDRESS}/kylin";
+ exit 0
+
elif [ "$1" = "version" ]
then
runTool org.apache.kylin.common.KylinVersion
@@ -500,5 +421,5 @@ elif [[ "$1" = org.apache.kylin.* ]]
then
runTool "$@"
else
- quit "Usage: 'kylin.sh [-v] start' or 'kylin.sh [-v] stop'"
+ quit "Usage: 'kylin.sh [-v] start' or 'kylin.sh [-v] stop' or 'kylin.sh
[-v] restart'"
fi
diff --git
a/kylin-spark-project/kylin-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/SparkClassLoader.java
b/kylin-spark-project/kylin-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/SparkClassLoader.java
index d74fb26..2dc0292 100644
---
a/kylin-spark-project/kylin-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/SparkClassLoader.java
+++
b/kylin-spark-project/kylin-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/SparkClassLoader.java
@@ -121,7 +121,6 @@ public class SparkClassLoader extends URLClassLoader {
for (File jar : jars) {
addURL(jar.toURI().toURL());
}
-
}
@Override
diff --git
a/kylin-spark-project/kylin-spark-common/src/main/java/org/apache/kylin/engine/spark/metadata/cube/PathManager.java
b/kylin-spark-project/kylin-spark-common/src/main/java/org/apache/kylin/engine/spark/metadata/cube/PathManager.java
index 0d34451..e30b2c0 100644
---
a/kylin-spark-project/kylin-spark-common/src/main/java/org/apache/kylin/engine/spark/metadata/cube/PathManager.java
+++
b/kylin-spark-project/kylin-spark-common/src/main/java/org/apache/kylin/engine/spark/metadata/cube/PathManager.java
@@ -59,7 +59,7 @@ public final class PathManager {
* Delete segment path
*/
public static boolean deleteSegmentParquetStoragePath(CubeInstance cube,
String segmentName, String identifier) throws IOException {
- if (cube == null || StringUtils.isNoneBlank(segmentName)||
StringUtils.isNoneBlank(identifier)) {
+ if (cube == null || StringUtils.isBlank(segmentName) ||
StringUtils.isBlank(identifier)) {
return false;
}
String path = getSegmentParquetStoragePath(cube, segmentName,
identifier);
diff --git
a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
index fbfae2e..86e234a 100644
---
a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
+++
b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
@@ -400,9 +400,6 @@ public class NSparkExecutable extends AbstractExecutable {
String.format(Locale.ROOT, "%s:%s", APP_JAR_NAME,
Paths.get(kylinJobJar).getFileName().toString()) :
kylinJobJar);
- if (sparkConfs.containsKey("spark.sql.hive.metastore.jars")) {
- jars = jars + "," +
sparkConfs.get("spark.sql.hive.metastore.jars");
- }
String sparkUploadFiles =
config.sparkUploadFiles(isLocalMaster(sparkConfs), isYarnCluster);
if (StringUtils.isNotBlank(sparkUploadFiles)) {
sb.append("--files ").append(sparkUploadFiles).append(" ");
diff --git
a/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/spark/conf/rule/SparkConfRule.scala
b/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/spark/conf/rule/SparkConfRule.scala
index 403e872..781d1b0 100644
---
a/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/spark/conf/rule/SparkConfRule.scala
+++
b/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/spark/conf/rule/SparkConfRule.scala
@@ -107,8 +107,20 @@ class ExecutorInstancesRule extends SparkConfRule {
logInfo(s"The number of instances calculated by cuboids " +
s"is: $calculateExecutorInsByLayoutSize")
- val availableMem =
helper.getFetcher.fetchQueueAvailableResource(queue).available.memory
- val availableCore =
helper.getFetcher.fetchQueueAvailableResource(queue).available.vCores
+ var availableMem = Int.MaxValue
+ try {
+ availableMem =
helper.getFetcher.fetchQueueAvailableResource(queue).available.memory
+ } catch {
+ case throwable: Throwable =>
+ logWarning(s"Error when getting available memory from cluster, ignore
it.")
+ }
+ var availableCore = Int.MaxValue
+ try {
+ availableCore =
helper.getFetcher.fetchQueueAvailableResource(queue).available.vCores
+ } catch {
+ case throwable: Throwable =>
+ logWarning(s"Error when getting available cpus from cluster, ignore
it.")
+ }
val executorMem =
(Utils.byteStringAsMb(helper.getConf(SparkConfHelper.EXECUTOR_MEMORY)) +
Utils.byteStringAsMb(helper.getConf(SparkConfHelper.EXECUTOR_OVERHEAD)))
diff --git
a/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/spark/utils/ResourceUtils.scala
b/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/spark/utils/ResourceUtils.scala
index de6697b..edf4b21 100644
---
a/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/spark/utils/ResourceUtils.scala
+++
b/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/spark/utils/ResourceUtils.scala
@@ -67,8 +67,11 @@ object ResourceUtils extends Logging {
def checkResource(sparkConf: SparkConf, clusterInfo: ClusterInfoFetcher):
Boolean = {
val queue = sparkConf.get("spark.yarn.queue", "default")
- val driverMemory = (Utils.byteStringAsMb(sparkConf.get(DRIVER_MEMORY)) +
Utils.byteStringAsMb(sparkConf.get(DRIVER_OVERHEAD))).toInt
- val driverCores = sparkConf.get(DRIVER_CORES).toInt
+ var driverMemory = Utils.byteStringAsMb(sparkConf.get(DRIVER_MEMORY)).toInt
+ if (sparkConf.contains(DRIVER_OVERHEAD)) {
+ driverMemory = driverMemory +
Utils.byteStringAsMb(sparkConf.get(DRIVER_OVERHEAD)).toInt
+ }
+ val driverCores = sparkConf.get(DRIVER_CORES, "1").toInt
val queueAvailable =
minusDriverResource(clusterInfo.fetchQueueAvailableResource(queue),
driverMemory, driverCores)
val instances = sparkConf.get(EXECUTOR_INSTANCES).toInt
val executorMemory = (Utils.byteStringAsMb(sparkConf.get(EXECUTOR_MEMORY))
diff --git
a/kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/KylinSession.scala
b/kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/KylinSession.scala
index 892cf9c..5d25d9c 100644
---
a/kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/KylinSession.scala
+++
b/kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/KylinSession.scala
@@ -135,6 +135,14 @@ object KylinSession extends Logging {
private lazy val conf: KylinConfig = KylinConfig.getInstanceFromEnv
+ def setDistJarFiles(sparkConf: SparkConf, key: String, value: String):
Unit = {
+ if (sparkConf.contains(key)) {
+ sparkConf.set(key, value + "," + sparkConf.get(key))
+ } else {
+ sparkConf.set(key, value)
+ }
+ }
+
def initSparkConf(sparkConf: SparkConf): SparkConf = {
if (sparkConf.getBoolean("user.kylin.session", false)) {
return sparkConf
@@ -172,12 +180,13 @@ object KylinSession extends Logging {
if (!"true".equalsIgnoreCase(System.getProperty("spark.local"))) {
if (sparkConf.get("spark.master").startsWith("yarn")) {
- sparkConf.set("spark.yarn.dist.jars",
+ setDistJarFiles(sparkConf, "spark.yarn.dist.jars",
KylinConfig.getInstanceFromEnv.getKylinParquetJobJarPath)
- sparkConf.set("spark.yarn.dist.files", conf.sparkUploadFiles())
+ setDistJarFiles(sparkConf, "spark.yarn.dist.files",
conf.sparkUploadFiles())
} else {
- sparkConf.set("spark.jars", conf.sparderJars)
- sparkConf.set("spark.files", conf.sparkUploadFiles())
+ setDistJarFiles(sparkConf, "spark.jars",
+ KylinConfig.getInstanceFromEnv.getKylinParquetJobJarPath)
+ setDistJarFiles(sparkConf, "spark.files", conf.sparkUploadFiles())
}
val fileName = KylinConfig.getInstanceFromEnv.getKylinParquetJobJarPath
diff --git a/pom.xml b/pom.xml
index d8f73be..1b794fd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2034,6 +2034,7 @@
<spark.version>3.1.1</spark.version>
<spark.version.dir>spark31</spark.version.dir>
<jackson.version>2.10.0</jackson.version>
+ <slf4j.version>1.7.30</slf4j.version>
</properties>
<build>
<plugins>
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 23ad6a7..fff3faf 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -82,6 +82,7 @@ import org.apache.kylin.rest.service.ProjectService;
import org.apache.kylin.rest.service.QueryService;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.ValidateUtil;
+import org.apache.kylin.shaded.com.google.common.collect.Maps;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -174,7 +175,14 @@ public class CubeController extends BasicController {
@RequestMapping(value = "validEncodings", method = { RequestMethod.GET },
produces = { "application/json" })
@ResponseBody
public Map<String, Integer> getValidEncodings() {
- throw new UnsupportedOperationException("Unsupported in Kylin 4.0 .");
+ Map<String, Integer> encodings;
+ try {
+ encodings = DimensionEncodingFactory.getValidEncodings();
+ } catch (Exception e) {
+ logger.error("Error when getting valid encodings", e);
+ return Maps.newHashMap();
+ }
+ return encodings;
}
@RequestMapping(value = "/{cubeName}", method = { RequestMethod.GET },
produces = { "application/json" })
