reswqa commented on code in PR #123:
URL:
https://github.com/apache/flink-connector-pulsar/pull/123#discussion_r3535879318
##########
.github/workflows/push_pr.yml:
##########
@@ -27,15 +27,20 @@ concurrency:
jobs:
compile_and_test:
strategy:
+ fail-fast: false
Review Comment:
Could we remove this?
##########
.github/workflows/weekly.yml:
##########
@@ -29,20 +29,29 @@ jobs:
compile_and_test:
if: github.repository_owner == 'apache'
strategy:
+ fail-fast: false
matrix:
+ # Track the latest snapshot of every supported Flink 2.x line so we
+ # catch breaking changes in Flink before they ship a release.
flink_branches: [{
- flink: 1.20-SNAPSHOT,
+ flink: 2.0-SNAPSHOT,
branch: main
}, {
- flink: 1.20.4,
- branch: v4.2
+ flink: 2.1-SNAPSHOT,
+ branch: main
+ }, {
+ flink: 2.2-SNAPSHOT,
+ branch: main
+ }, {
+ flink: 2.3-SNAPSHOT,
+ branch: main
}
]
uses: apache/flink-connector-shared-utils/.github/workflows/ci.yml@ci_utils
with:
flink_version: ${{ matrix.flink_branches.flink }}
connector_branch: ${{ matrix.flink_branches.branch }}
- jdk_version: ${{ matrix.flink_branches.jdk || '8, 11, 17' }}
Review Comment:
jdk8 is needed for `v4.2` testing.
##########
.github/workflows/weekly.yml:
##########
@@ -29,20 +29,29 @@ jobs:
compile_and_test:
if: github.repository_owner == 'apache'
strategy:
+ fail-fast: false
matrix:
+ # Track the latest snapshot of every supported Flink 2.x line so we
+ # catch breaking changes in Flink before they ship a release.
flink_branches: [{
- flink: 1.20-SNAPSHOT,
+ flink: 2.0-SNAPSHOT,
branch: main
}, {
- flink: 1.20.4,
- branch: v4.2
Review Comment:
I think we should keep the testing for `v4.2`, because the weekly ci is only
triggered from `main` branch's `weekly.yml`.
##########
flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/testutils/function/ControlSource.java:
##########
@@ -1,5 +1,5 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
+ * Licensed to the Apache Software Foundation (ASF)
Review Comment:
Why should we change the license text?
##########
flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/testutils/function/ControlSource.java:
##########
@@ -55,17 +51,15 @@
/**
* This source is used for testing in Pulsar sink. We would generate a fix
number of records by the
- * topic name and message index.
+ * topic name and message index. It wraps a {@link DataGeneratorSource} so the
connector test code
+ * does not depend on the legacy {@code SourceFunction} API in Flink 2.x.
Review Comment:
```suggestion
* topic name and message index.
```
##########
flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarTableITCase.java:
##########
@@ -147,12 +150,22 @@ void pulsarSourceSink(String format) throws Exception {
}
}
+ // CAST(TIME AS VARCHAR) shows a trailing ".0" for the fractional part
on
+ // Flink 2.2+ but not on 2.0/2.1. Strip the all-zero fraction from the
+ // standalone TIME field so the assertion is stable across the whole CI
+ // matrix (2.0.2 / 2.1.3 / 2.2.1 / 2.3.0). TIMESTAMP fields are not
+ // affected: they carry a date prefix and the regex requires ",
HH:mm:ss".
Review Comment:
```suggestion
// CAST(TIME AS VARCHAR) shows a trailing ".0" for the fractional
part on
// Flink 2.2+ but not on 2.0/2.1. Strip the all-zero fraction from
the
// standalone TIME field so the assertion is stable across the whole
CI
// matrix (2.0.x ~ 2.3.x). TIMESTAMP fields are not
// affected: they carry a date prefix and the regex requires ",
HH:mm:ss".
```
##########
flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarTableITCase.java:
##########
@@ -147,12 +150,22 @@ void pulsarSourceSink(String format) throws Exception {
}
}
+ // CAST(TIME AS VARCHAR) shows a trailing ".0" for the fractional part
on
+ // Flink 2.2+ but not on 2.0/2.1. Strip the all-zero fraction from the
+ // standalone TIME field so the assertion is stable across the whole CI
+ // matrix (2.0.2 / 2.1.3 / 2.2.1 / 2.3.0). TIMESTAMP fields are not
+ // affected: they carry a date prefix and the regex requires ",
HH:mm:ss".
Review Comment:
Can we figure out which PR's modification caused this difference? Let's
paste it(e.g. FLINK-XXXXX) down here.
##########
.github/workflows/push_pr.yml:
##########
@@ -27,15 +27,20 @@ concurrency:
jobs:
compile_and_test:
strategy:
+ fail-fast: false
matrix:
- flink: [ 1.20.4 ]
- jdk: [ '8, 11, 17' ]
+ # Per-push CI tests only the latest Flink release (2.3.0).
Cross-version
+ # compatibility (2.0.x – 2.2.x) is covered by weekly.yml against the
+ # snapshot branches. The affected IT cases have been verified locally
+ # against 2.0.2 / 2.1.3 / 2.2.1 / 2.3.0 — see PR description.
Review Comment:
I think this comment can be removed.
##########
flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/TableDataTypeUtils.java:
##########
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.FieldsDataType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.LogicalTypeRoot;
+import org.apache.flink.table.types.logical.RowType;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+/**
+ * Utility methods for working with table {@link DataType}s.
+ *
+ * <p>This class contains methods that were removed from Flink's {@code
DataTypeUtils} in Flink 2.x.
+ * They are copied here to maintain the connector's functionality.
+ */
+@Internal
+public class TableDataTypeUtils {
+
+ private TableDataTypeUtils() {}
+
+ /**
+ * Removes a prefix from all field names in a row data type.
+ *
+ * @param dataType The row data type whose field names should be stripped.
+ * @param prefix The prefix to remove from each field name.
+ * @return A new data type with the prefix removed from field names.
+ */
+ public static DataType stripRowPrefix(DataType dataType, String prefix) {
+ if (!dataType.getLogicalType().is(LogicalTypeRoot.ROW)) {
+ throw new IllegalArgumentException("Row data type expected.");
+ }
+
+ final RowType rowType = (RowType) dataType.getLogicalType();
+ final List<String> newFieldNames =
+ rowType.getFieldNames().stream()
+ .map(
+ s -> {
+ if (s.startsWith(prefix)) {
+ return s.substring(prefix.length());
+ }
+ return s;
+ })
+ .collect(Collectors.toList());
+ final LogicalType newRowType = renameRowFields(rowType, newFieldNames);
+ return new FieldsDataType(
+ newRowType, dataType.getConversionClass(),
dataType.getChildren());
+ }
+
+ /**
+ * Renames the fields in a {@link RowType}.
+ *
+ * @param rowType The row type to rename fields in.
+ * @param newFieldNames The new field names to use.
+ * @return A new row type with the renamed fields.
+ */
+ public static RowType renameRowFields(RowType rowType, List<String>
newFieldNames) {
Review Comment:
ok, let's keep it as is.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
