This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-12919-535d4237dd4b6ae92b14a9cbd790acf8d81cc71d in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit 4ee722e99fa498738748babb1eb3a53e01a2da06 Author: FatMan <[email protected]> AuthorDate: Tue Nov 10 04:40:00 2026 +0000 [Fix][Connector-V2][Jdbc] Read PostgreSQL binary numeric Infinity as null (#12919) Co-authored-by: chen <[email protected]> --- .../dialect/psql/PostgresJdbcRowConverter.java | 62 ++++++++++++++++-- .../dialect/psql/PostgresJdbcRowConverterTest.java | 74 ++++++++++++++++++++++ .../connectors/seatunnel/jdbc/JdbcPostgresIT.java | 6 +- ...urce_and_sink_numeric_special_value_binary.conf | 2 +- 4 files changed, 134 insertions(+), 10 deletions(-) diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresJdbcRowConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresJdbcRowConverter.java index 0252f7507f..de6c0fe2dd 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresJdbcRowConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresJdbcRowConverter.java @@ -69,6 +69,14 @@ import static org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.ps @Slf4j public class PostgresJdbcRowConverter extends AbstractJdbcRowConverter { + /** A binary PostgreSQL {@code numeric} special value consists of the four header words. */ + private static final int BINARY_NUMERIC_HEADER_LENGTH = 8; + + /** Sign words of the binary {@code numeric} infinities, see PostgreSQL {@code numeric.c}. */ + private static final int BINARY_NUMERIC_POSITIVE_INFINITY = 0xD000; + + private static final int BINARY_NUMERIC_NEGATIVE_INFINITY = 0xF000; + private final boolean numericSpecialValueAsNull; /** @@ -220,24 +228,64 @@ public class PostgresJdbcRowConverter extends AbstractJdbcRowConverter { * <p>PostgreSQL allows the special numeric values {@code NaN}, {@code Infinity} and {@code * -Infinity} (supported since PostgreSQL 14), which have no {@code BigDecimal} representation. * The pgjdbc driver fails to convert them with {@code PSQLException: Bad value for type - * BigDecimal : NaN} (text protocol) or with a {@code ClassCastException} (binary protocol, - * where the driver returns a {@code Double}). When {@code numeric_special_value_as_null} is + * BigDecimal : NaN} (text protocol), or in the binary protocol with a {@code + * ClassCastException} for {@code NaN} (the driver returns a {@code Double}) and an {@code + * IllegalArgumentException} for the infinities. When {@code numeric_special_value_as_null} is * enabled, such values are read as {@code null} and a warning is logged for the first * occurrence per column, so that a single special value does not abort the whole job. Otherwise * the original error is rethrown, which keeps the default behavior unchanged. */ private BigDecimal getDecimalValue(ResultSet rs, int columnIndex) throws SQLException { - BigDecimal value = - JdbcFieldTypeUtils.getBigDecimal(rs, columnIndex, numericSpecialValueAsNull); + BigDecimal value; + try { + value = JdbcFieldTypeUtils.getBigDecimal(rs, columnIndex, numericSpecialValueAsNull); + } catch (IllegalArgumentException e) { + // Binary protocol only: pgjdbc cannot decode the sign word of Infinity and -Infinity + if (numericSpecialValueAsNull && isBinaryNumericInfinity(rs, columnIndex)) { + logNumericSpecialValue(rs, columnIndex); + return null; + } + throw e; + } if (value == null && numericSpecialValueAsNull - && JdbcFieldTypeUtils.isNumericSpecialValue(rs, columnIndex) - && shouldLogNumericSpecialValue(columnIndex)) { + && JdbcFieldTypeUtils.isNumericSpecialValue(rs, columnIndex)) { + logNumericSpecialValue(rs, columnIndex); + } + return value; + } + + private void logNumericSpecialValue(ResultSet rs, int columnIndex) { + if (shouldLogNumericSpecialValue(columnIndex)) { log.warn( "The numeric special value at column '{}' cannot be represented as BigDecimal, reading it as null. Further special values in this column will not be logged", getColumnLabel(rs, columnIndex)); } - return value; + } + + /** + * Checks whether the raw bytes of the column are a binary PostgreSQL {@code numeric} {@code + * Infinity} or {@code -Infinity}. The binary format starts with four 16-bit words: digit count, + * weight, sign and display scale (see {@code numeric.c}); an infinity has no digits and the + * sign word {@code 0xD000} or {@code 0xF000}. The text value cannot be used here, because in + * the binary protocol {@code getString} decodes the same bytes and fails the same way. + */ + static boolean isBinaryNumericInfinity(ResultSet rs, int columnIndex) { + byte[] bytes; + try { + bytes = rs.getBytes(columnIndex); + } catch (SQLException e) { + // The raw value cannot be read, so it cannot be identified as an infinity + return false; + } + if (bytes == null || bytes.length != BINARY_NUMERIC_HEADER_LENGTH) { + return false; + } + int digitCount = ((bytes[0] & 0xFF) << 8) | (bytes[1] & 0xFF); + int sign = ((bytes[4] & 0xFF) << 8) | (bytes[5] & 0xFF); + return digitCount == 0 + && (sign == BINARY_NUMERIC_POSITIVE_INFINITY + || sign == BINARY_NUMERIC_NEGATIVE_INFINITY); } /** diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresJdbcRowConverterTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresJdbcRowConverterTest.java index f76ff61b22..f9299c1243 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresJdbcRowConverterTest.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresJdbcRowConverterTest.java @@ -354,6 +354,80 @@ public class PostgresJdbcRowConverterTest { () -> new PostgresJdbcRowConverter().toInternal(rs, tableSchema)); } + @Test + public void testToInternalWithBinaryNumericInfinityAsNull() throws SQLException { + // In the binary protocol pgjdbc cannot decode the sign word of Infinity and -Infinity + TableSchema tableSchema = createTableSchema("decimal_col", new DecimalType(10, 2), null); + for (int sign : new int[] {0xD000, 0xF000}) { + ResultSet rs = mockBinaryNumeric(binaryNumericHeader(0, sign)); + + SeaTunnelRow row = new PostgresJdbcRowConverter(true).toInternal(rs, tableSchema); + + Assertions.assertEquals(1, row.getField(0)); + Assertions.assertNull(row.getField(1), "sign " + Integer.toHexString(sign)); + } + } + + @Test + public void testToInternalKeepsBinaryNumericInfinityErrorWithoutOption() throws SQLException { + TableSchema tableSchema = createTableSchema("decimal_col", new DecimalType(10, 2), null); + ResultSet rs = mockBinaryNumeric(binaryNumericHeader(0, 0xD000)); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> new PostgresJdbcRowConverter().toInternal(rs, tableSchema)); + } + + @Test + public void testToInternalRethrowsBinaryNumericErrorForOrdinaryValue() throws SQLException { + // One digit word: not an infinity even though the driver failed to decode the value + TableSchema tableSchema = createTableSchema("decimal_col", new DecimalType(10, 2), null); + ResultSet rs = mockBinaryNumeric(new byte[] {0, 1, 0, 0, (byte) 0xD0, 0, 0, 0, 0, 1}); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> new PostgresJdbcRowConverter(true).toInternal(rs, tableSchema)); + } + + @Test + public void testIsBinaryNumericInfinity() throws SQLException { + Assertions.assertTrue( + PostgresJdbcRowConverter.isBinaryNumericInfinity( + mockBinaryNumeric(binaryNumericHeader(0, 0xD000)), 2)); + Assertions.assertTrue( + PostgresJdbcRowConverter.isBinaryNumericInfinity( + mockBinaryNumeric(binaryNumericHeader(0, 0xF000)), 2)); + // A digit word, the NaN sign (decoded as a Double by the driver) or a positive sign + Assertions.assertFalse( + PostgresJdbcRowConverter.isBinaryNumericInfinity( + mockBinaryNumeric(binaryNumericHeader(1, 0xD000)), 2)); + Assertions.assertFalse( + PostgresJdbcRowConverter.isBinaryNumericInfinity( + mockBinaryNumeric(binaryNumericHeader(0, 0xC000)), 2)); + Assertions.assertFalse( + PostgresJdbcRowConverter.isBinaryNumericInfinity( + mockBinaryNumeric(binaryNumericHeader(0, 0x0000)), 2)); + } + + /** A NUMERIC column at index 2 whose binary value the driver cannot decode. */ + private ResultSet mockBinaryNumeric(byte[] rawValue) throws SQLException { + ResultSet rs = mock(ResultSet.class); + setupMockResultSet(rs, "INT4", "NUMERIC", 1, null); + IllegalArgumentException invalidSign = + new IllegalArgumentException("invalid sign in \"numeric\" value"); + when(rs.getBigDecimal(2)).thenThrow(invalidSign); + when(rs.getString(2)).thenThrow(invalidSign); + when(rs.getBytes(2)).thenReturn(rawValue); + return rs; + } + + /** Builds the 8-byte header of a binary PostgreSQL numeric value without digit words. */ + private static byte[] binaryNumericHeader(int digitCount, int sign) { + return new byte[] { + (byte) (digitCount >> 8), (byte) digitCount, 0, 0, (byte) (sign >> 8), (byte) sign, 0, 0 + }; + } + @Test public void testNumericSpecialValueIsLoggedOncePerColumn() { PostgresJdbcRowConverter converter = new PostgresJdbcRowConverter(true); diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcPostgresIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcPostgresIT.java index a654f841e6..66f218557c 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcPostgresIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcPostgresIT.java @@ -521,8 +521,10 @@ public class JdbcPostgresIT extends TestSuiteBase implements TestResource { /** * Verifies that the numeric special values are also read as null when the driver uses the - * binary transfer protocol, where the driver fails with a ClassCastException instead of a - * PSQLException. + * binary transfer protocol, where the driver fails with a ClassCastException for NaN and an + * IllegalArgumentException for Infinity and -Infinity instead of a PSQLException. The driver + * only switches to binary results for server-prepared statements, so the job sets + * prepareThreshold=-1 to prepare the statement before its first execution. */ @TestTemplate public void testNumericSpecialValueAsNullWithBinaryTransfer(TestContainer container) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/resources/jdbc_postgres_source_and_sink_numeric_special_value_binary.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/resources/jdbc_postgres_source_and_sink_numeric_special_value_binary.conf index e954c39453..12e8ce3ec4 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/resources/jdbc_postgres_source_and_sink_numeric_special_value_binary.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/resources/jdbc_postgres_source_and_sink_numeric_special_value_binary.conf @@ -22,7 +22,7 @@ env { source{ jdbc{ - url = "jdbc:postgresql://postgresql:5432/test?loggerLevel=OFF&binaryTransfer=true" + url = "jdbc:postgresql://postgresql:5432/test?loggerLevel=OFF&binaryTransfer=true&prepareThreshold=-1" driver = "org.postgresql.Driver" username = "test" password = "test"
