This is an automated email from the ASF dual-hosted git repository.
fanjia pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 0555f8520b [Fix][Connector-V2] ArrowToSeatunnelRowReader
convertSeatunnelRowValue add handle Second TIMESTAMP type (#9393)
0555f8520b is described below
commit 0555f8520b0f191f5d37d12e73089aee8649d1c7
Author: dyp12 <[email protected]>
AuthorDate: Thu Jun 5 10:48:22 2025 +0800
[Fix][Connector-V2] ArrowToSeatunnelRowReader convertSeatunnelRowValue add
handle Second TIMESTAMP type (#9393)
Signed-off-by: dyp12 <[email protected]>
Co-authored-by: dyp12 <[email protected]>
---
.../arrow/converter/TimeStampSecConverter.java | 44 ++++++++++++++++++++++
.../arrow/reader/ArrowToSeatunnelRowReader.java | 14 +++++--
...atunnel.common.source.arrow.converter.Converter | 1 +
.../arrow/ArrowToSeatunnelRowReaderTest.java | 34 ++++++++++++++++-
.../source/arrow/SeaTunnelDataTypeHolder.java | 2 +
5 files changed, 90 insertions(+), 5 deletions(-)
diff --git
a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampSecConverter.java
b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampSecConverter.java
new file mode 100644
index 0000000000..a0a91b6dfe
--- /dev/null
+++
b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampSecConverter.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package
org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter;
+
+import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeStampSecVector;
+import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types;
+
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+
+public class TimeStampSecConverter implements Converter<TimeStampSecVector> {
+ @Override
+ public Object convert(int rowIndex, TimeStampSecVector fieldVector) {
+ if (fieldVector == null || fieldVector.isNull(rowIndex)) {
+ return null;
+ }
+ LocalDateTime localDateTime = fieldVector.getObject(rowIndex);
+ return localDateTime
+ .atZone(ZoneOffset.UTC)
+ .withZoneSameInstant(ZoneId.systemDefault())
+ .toLocalDateTime();
+ }
+
+ @Override
+ public boolean support(Types.MinorType type) {
+ return Types.MinorType.TIMESTAMPSEC == type;
+ }
+}
diff --git
a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java
b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java
index 5dbd56a369..5f5b5fec17 100644
---
a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java
+++
b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java
@@ -204,9 +204,17 @@ public class ArrowToSeatunnelRowReader implements
AutoCloseable {
}
case TIMESTAMP:
if (fieldValue instanceof Long) {
- return Instant.ofEpochMilli((Long) fieldValue)
- .atZone(ZoneId.systemDefault())
- .toLocalDateTime();
+ // this TIMESTAMP value may be SECOND not milliseconds
+ if (Types.MinorType.TIMESTAMPSEC == minorType
+ || Types.MinorType.TIMESTAMPSECTZ == minorType) {
+ return Instant.ofEpochSecond((Long) fieldValue)
+ .atZone(ZoneId.systemDefault())
+ .toLocalDateTime();
+ } else {
+ return Instant.ofEpochMilli((Long) fieldValue)
+ .atZone(ZoneId.systemDefault())
+ .toLocalDateTime();
+ }
} else if (fieldValue instanceof String) {
return LocalDateTime.parse((String) fieldValue,
DATETIME_FORMATTER);
} else if (fieldValue instanceof Text) {
diff --git
a/seatunnel-connectors-v2/connector-common/src/main/resources/META-INF/services/org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Converter
b/seatunnel-connectors-v2/connector-common/src/main/resources/META-INF/services/org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Converter
index aadc8ea5f5..310946f2ca 100644
---
a/seatunnel-connectors-v2/connector-common/src/main/resources/META-INF/services/org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Converter
+++
b/seatunnel-connectors-v2/connector-common/src/main/resources/META-INF/services/org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Converter
@@ -23,3 +23,4 @@
org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStam
org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampMilliConverter
org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampNanoConverter
org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.DateMilliConvertor
+org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampSecConverter
diff --git
a/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/ArrowToSeatunnelRowReaderTest.java
b/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/ArrowToSeatunnelRowReaderTest.java
index e84bee4939..91118e667f 100644
---
a/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/ArrowToSeatunnelRowReaderTest.java
+++
b/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/ArrowToSeatunnelRowReaderTest.java
@@ -36,6 +36,8 @@ import
org.apache.seatunnel.shade.org.apache.arrow.vector.SmallIntVector;
import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeMicroVector;
import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeStampMicroVector;
import
org.apache.seatunnel.shade.org.apache.arrow.vector.TimeStampMilliTZVector;
+import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeStampSecTZVector;
+import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeStampSecVector;
import org.apache.seatunnel.shade.org.apache.arrow.vector.TinyIntVector;
import org.apache.seatunnel.shade.org.apache.arrow.vector.VarBinaryVector;
import org.apache.seatunnel.shade.org.apache.arrow.vector.VarCharVector;
@@ -122,6 +124,8 @@ public class ArrowToSeatunnelRowReaderTest {
seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("date2", 0));
seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("array1", 0));
seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("array2", 0));
+ seaTunnelDataTypeHolder.add(new
SeaTunnelDataTypeHolder("timestampSec", 0));
+ seaTunnelDataTypeHolder.add(new
SeaTunnelDataTypeHolder("timestampSecTz", 0));
seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("map", 0));
}
@@ -165,10 +169,22 @@ public class ArrowToSeatunnelRowReaderTest {
// array int
vectors.add(ListVector.empty("array2", rootAllocator));
// map
+
+ // SECOND timestamp without timezone
+ vectors.add(new TimeStampSecVector("timestampSec", rootAllocator));
+ // SECOND timestamp with timezone
+ vectors.add(
+ new TimeStampSecTZVector(
+ Field.nullable(
+ "timestampSecTz",
+ new ArrowType.Timestamp(
+ TimeUnit.SECOND,
ZoneId.systemDefault().getId())),
+ rootAllocator));
}
// allocate storage
vectors.forEach(FieldVector::allocateNew);
long epochMilli =
localDateTime.atZone(zoneId).toInstant().toEpochMilli();
+ long epochSecond =
localDateTime.atZone(zoneId).toInstant().getEpochSecond();
byte byteStart = 'a';
@@ -219,6 +235,10 @@ public class ArrowToSeatunnelRowReaderTest {
} else if (vector instanceof DateDayVector) {
((DateDayVector) vector)
.setSafe(i, (int)
localDateTime.toLocalDate().toEpochDay());
+ } else if (vector instanceof TimeStampSecVector) {
+ ((TimeStampSecVector) vector).setSafe(i,
epochSecond);
+ } else if (vector instanceof TimeStampSecTZVector) {
+ ((TimeStampSecTZVector) vector).setSafe(i,
epochSecond);
}
}
});
@@ -352,12 +372,10 @@ public class ArrowToSeatunnelRowReaderTest {
List<Object> actualString3Data =
rows.stream().map(s ->
s.getField(11)).collect(Collectors.toList());
Assertions.assertEquals(stringData, actualString3Data);
-
// check timestamp with tz
List<Object> actualTimestamp2Data =
rows.stream().map(s ->
s.getField(12)).distinct().collect(Collectors.toList());
Assertions.assertEquals(Collections.singletonList(localDateTime),
actualTimestamp2Data);
-
// check time
List<Object> actualTimeDate =
rows.stream().map(s ->
s.getField(13)).distinct().collect(Collectors.toList());
@@ -381,6 +399,18 @@ public class ArrowToSeatunnelRowReaderTest {
List<Object> actualArrayTimestampData =
rows.stream().map(s ->
s.getField(17)).collect(Collectors.toList());
Assertions.assertIterableEquals(arrayData2,
actualArrayTimestampData);
+ // check SECOND timestamp without timezone
+ List<Object> actualTimestampSecData =
+ rows.stream().map(s ->
s.getField(18)).distinct().collect(Collectors.toList());
+ Assertions.assertEquals(
+ Collections.singletonList(localDateTime),
actualTimestampSecData);
+
+ // check SECOND timestamp with timezone
+ List<Object> actualTimestampSecTzData =
+ rows.stream().map(s ->
s.getField(19)).distinct().collect(Collectors.toList());
+ Assertions.assertEquals(
+ Collections.singletonList(localDateTime),
actualTimestampSecTzData);
+
// todo check map
// The java api has problems building MapVectors,and there are no
examples on the
// official website
diff --git
a/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/SeaTunnelDataTypeHolder.java
b/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/SeaTunnelDataTypeHolder.java
index 6f1c5579da..5a519d574a 100644
---
a/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/SeaTunnelDataTypeHolder.java
+++
b/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/SeaTunnelDataTypeHolder.java
@@ -65,6 +65,8 @@ public class SeaTunnelDataTypeHolder {
return new DecimalType(10, 2);
case "timestamp1":
case "timestamp2":
+ case "timestampSec":
+ case "timestampSecTz":
return LocalTimeType.LOCAL_DATE_TIME_TYPE;
case "time":
return LocalTimeType.LOCAL_TIME_TYPE;
