wuwenchi commented on code in PR #38946:
URL: https://github.com/apache/doris/pull/38946#discussion_r1721537201
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSplit.java:
##########
@@ -22,29 +22,51 @@
import org.apache.doris.datasource.TableFormatType;
import org.apache.hadoop.fs.Path;
+import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.table.source.DataSplit;
import org.apache.paimon.table.source.DeletionFile;
import org.apache.paimon.table.source.Split;
import java.util.List;
import java.util.Optional;
+import java.util.UUID;
public class PaimonSplit extends FileSplit {
private Split split;
private TableFormatType tableFormatType;
private Optional<DeletionFile> optDeletionFile;
+ private static final Path DUMMY_PATH = new Path("hdfs://dummyPath");
+
public PaimonSplit(Split split) {
- super(new Path("hdfs://dummyPath"), 0, 0, 0, null, null);
+ super(DUMMY_PATH, 0, 0, 0, null, null);
this.split = split;
this.tableFormatType = TableFormatType.PAIMON;
this.optDeletionFile = Optional.empty();
+
+ if (split instanceof DataSplit) {
+ List<DataFileMeta> dataFileMetas = ((DataSplit) split).dataFiles();
+ this.path = new Path("hdfs://" + dataFileMetas.get(0).fileName());
Review Comment:
Because it will be checked by location for be.
##########
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java:
##########
@@ -908,6 +908,25 @@ public class SessionVariable implements Serializable,
Writable {
setter = "setPipelineTaskNum")
public int parallelPipelineTaskNum = 0;
+
+ public enum IgnoreSplitType {
+ NONE,
+ IGNORE_JNI,
+ IGNORE_NATIVE
+ }
+
+ public static final String IGNORE_SPLIT_TYPE = "ignore_split_type";
+ @VariableMgr.VarAttr(name = IGNORE_SPLIT_TYPE,
+ checker = "checkIgnoreSplitType",
+ description = {"忽略指定类型的split", "Ignore splits of the specified
type"})
+ public String ignoreSplitType = IgnoreSplitType.NONE.toString();
+
+ public static final String USE_CONSISTENT_HASHING_FOR_EXTERNAL_SCAN =
"use_consistent_hash_for_external_scan";
+ @VariableMgr.VarAttr(name = USE_CONSISTENT_HASHING_FOR_EXTERNAL_SCAN,
+ description = {"对外表采用一致性hash的方式做split的分发",
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
