morningman commented on code in PR #16983:
URL: https://github.com/apache/doris/pull/16983#discussion_r1121432177
##########
fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java:
##########
@@ -180,6 +193,74 @@ protected void parseProperties(Map<String, String>
validParams) throws AnalysisE
stripOuterArray =
Boolean.valueOf(validParams.get(STRIP_OUTER_ARRAY)).booleanValue();
numAsString =
Boolean.valueOf(validParams.get(NUM_AS_STRING)).booleanValue();
fuzzyParse =
Boolean.valueOf(validParams.get(FUZZY_PARSE)).booleanValue();
+
+ if (formatString.equals("csv") || formatString.equals("csv_with_names")
+ || formatString.equals("csv_with_names_and_types")) {
+ parseCsvSchema(csvSchema, validParams);
+ }
+ }
+
+ // public for unit test
+ public static void parseCsvSchema(List<Column> csvSchema, Map<String,
String> validParams)
+ throws AnalysisException {
+ String csvSchemaStr = validParams.get(CSV_SCHEMA);
+ if (Strings.isNullOrEmpty(csvSchemaStr)) {
+ return;
+ }
+ // the schema str is like: "k1:int;k2:bigint;k3:varchar(20)"
+ String[] schemaStrs = csvSchemaStr.split(";");
+ try {
+ for (String schemaStr : schemaStrs) {
+ String[] kv = schemaStr.replace(" ", "").split(":");
+ if (kv.length != 2) {
+ throw new AnalysisException("invalid csv schema: " +
csvSchemaStr);
+ }
+ Column column = null;
+ String name = kv[0].toLowerCase();
+ FeNameFormat.checkColumnName(name);
+ String type = kv[1].toLowerCase();
+ if (type.equals("tinyint")) {
+ column = new Column(name, PrimitiveType.TINYINT, true);
+ } else if (type.equals("smallint")) {
+ column = new Column(name, PrimitiveType.SMALLINT, true);
+ } else if (type.equals("int")) {
+ column = new Column(name, PrimitiveType.INT, true);
+ } else if (type.equals("bigint")) {
+ column = new Column(name, PrimitiveType.BIGINT, true);
+ } else if (type.equals("largeint")) {
+ column = new Column(name, PrimitiveType.LARGEINT, true);
+ } else if (type.equals("float")) {
+ column = new Column(name, PrimitiveType.FLOAT, true);
+ } else if (type.equals("double")) {
+ column = new Column(name, PrimitiveType.DOUBLE, true);
+ } else if (type.startsWith("decimal")) {
+ // regex decimal(p, s)
+ Pattern pattern =
Pattern.compile("decimal\\((\\d+),(\\d+)\\)");
+ Matcher matcher = pattern.matcher(type);
+ if (!matcher.find()) {
+ throw new AnalysisException("invalid decimal type: " +
type);
+ }
+ int precision = Integer.parseInt(matcher.group(1));
+ int scale = Integer.parseInt(matcher.group(2));
+ column = new Column(name,
ScalarType.createDecimalV3Type(precision, scale), false, null, true, null,
+ "");
+ } else if (type.equals("date")) {
+ column = new Column(name, ScalarType.createDateType(),
false, null, true, null, "");
+ } else if (type.equals("datetime")) {
+ column = new Column(name, ScalarType.createDatetimeType(),
false, null, true, null, "");
Review Comment:
done, all will use datetimev2
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
