xy720 commented on a change in pull request #4163:
URL: https://github.com/apache/incubator-doris/pull/4163#discussion_r459840070
##########
File path:
fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkEtlJobHandler.java
##########
@@ -92,12 +90,42 @@ public void submitEtlJob(long loadJobId, String loadLabel,
EtlJobConfig etlJobCo
// delete outputPath
deleteEtlOutputPath(etlJobConfig.outputPath, brokerDesc);
- // upload app resource and jobconfig to hdfs
+ // remote repository
+ SparkRepository.SparkArchive archive =
getRemoteArchive(resource.getRepositoryDir(), brokerDesc);
+ List<SparkRepository.SparkLibrary> libraries = archive.libraries;
+ Optional<SparkRepository.SparkLibrary> dppLibrary = libraries.stream().
+ filter(library -> library.libType ==
SparkRepository.SparkLibrary.LibType.DPP).findFirst();
+ Optional<SparkRepository.SparkLibrary> spark2xLibrary =
libraries.stream().
+ filter(library -> library.libType ==
SparkRepository.SparkLibrary.LibType.SPARK2X).findFirst();
+ if (!dppLibrary.isPresent() || !spark2xLibrary.isPresent()) {
+ throw new LoadException("failed to get library info from
repository");
+ }
+
+ // spark home
+ String spark_home = Config.spark_home_default_dir;
+ // etl config path
String configsHdfsDir = etlJobConfig.outputPath + "/" + JOB_CONFIG_DIR
+ "/";
- String appResourceHdfsPath = configsHdfsDir + APP_RESOURCE_NAME;
+ // etl config json path
String jobConfigHdfsPath = configsHdfsDir + CONFIG_FILE_NAME;
+ // spark submit app resource path
+ String appResourceHdfsPath = dppLibrary.get().remotePath;
+ // spark yarn archive path
+ String jobArchiveHdfsPath = spark2xLibrary.get().remotePath;
+ // spark yarn stage dir
+ String jobStageHdfsPath = resource.getWorkingDir();
+ LOG.info("configsHdfsDir={}, appResourceHdfsPath={},
jobConfigHdfsPath={}, jobArchiveHdfsPath={}",
+ configsHdfsDir, appResourceHdfsPath, jobConfigHdfsPath,
jobArchiveHdfsPath);
+
+ // update archive and stage configs here
+ Map<String, String> sparkConfigs = resource.getSparkConfigs();
+ if (Strings.isNullOrEmpty(sparkConfigs.get("spark.yarn.archive"))) {
+ sparkConfigs.put("spark.yarn.archive", jobArchiveHdfsPath);
+ }
+ if (Strings.isNullOrEmpty(sparkConfigs.get("spark.yarn.stage.dir"))) {
+ sparkConfigs.put("spark.yarn.stage.dir", jobStageHdfsPath);
Review comment:
A self-generated path by spark in hdfs to save temporary configuration
for spark application
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
