(doris) branch branch-c108335-hive-sql updated: [tmp] support skip.header.line.count

[morningman]

This is an automated email from the ASF dual-hosted git repository.

morningman pushed a commit to branch branch-c108335-hive-sql
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-c108335-hive-sql by 
this push:
     new 808ef61620c [tmp] support skip.header.line.count
808ef61620c is described below

commit 808ef61620c75f5726be62daea731bdd0ff736cf
Author: morningman <yun...@selectdb.com>
AuthorDate: Tue Apr 8 06:13:08 2025 -0700

    [tmp] support skip.header.line.count
---
 .../java/org/apache/doris/datasource/hive/HiveProperties.java    | 9 +++++++++
 .../org/apache/doris/datasource/hive/source/HiveScanNode.java    | 2 ++
 2 files changed, 11 insertions(+)

diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveProperties.java 
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveProperties.java
index 74f3dcc1a9d..b098b9d43b7 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveProperties.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveProperties.java
@@ -17,6 +17,7 @@
 
 package org.apache.doris.datasource.hive;
 
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableSet;
 import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.serde2.OpenCSVSerde;
@@ -131,6 +132,14 @@ public class HiveProperties {
         return 
HiveMetaStoreClientHelper.firstPresentOrDefault(DEFAULT_ESCAPE_CHAR, 
escapeChar);
     }
 
+    public static int getCsvSkipHeaderLine(Table table) {
+        String valueFromTbl = 
table.getParameters().get("skip.header.line.count");
+        if (!Strings.isNullOrEmpty(valueFromTbl)) {
+            return Integer.valueOf(valueFromTbl);
+        }
+        return 0;
+    }
+
     // Set properties to table
     public static void setTableProperties(Table table, Map<String, String> 
properties) {
         HashMap<String, String> serdeProps = new HashMap<>();
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
 
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
index 890f6147f33..7255a54de98 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
@@ -431,6 +431,8 @@ public class HiveScanNode extends FileQueryScanNode {
         Table table = hmsTable.getRemoteTable();
         // TODO: separate hive text table and OpenCsv table
         String serDeLib = table.getSd().getSerdeInfo().getSerializationLib();
+        int skipHeaderLine = HiveProperties.getCsvSkipHeaderLine(table);
+        fileAttributes.setSkipLines(skipHeaderLine);
         if 
(serDeLib.equals("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe")) {
             TFileTextScanRangeParams textParams = new 
TFileTextScanRangeParams();
             // set properties of LazySimpleSerDe


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org