This is an automated email from the ASF dual-hosted git repository. wanghailin pushed a commit to branch dev in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push: new 082814da1f [Fix][Connector-starrocks] Fix drop column bug for starrocks (#8216) 082814da1f is described below commit 082814da1fd0c9ded80e19fe1ced1a64e2d89b29 Author: 峰峰 <dufeng1...@126.com> AuthorDate: Mon Dec 9 20:07:16 2024 +0800 [Fix][Connector-starrocks] Fix drop column bug for starrocks (#8216) --- .../seatunnel/starrocks/util/SchemaUtils.java | 2 +- .../starrocks/StarRocksSchemaChangeIT.java | 24 ++++++++++++++++++++++ .../src/test/resources/ddl/drop_columns.sql | 14 ------------- ...olumns.sql => drop_columns_validate_schema.sql} | 16 +-------------- 4 files changed, 26 insertions(+), 30 deletions(-) diff --git a/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/util/SchemaUtils.java b/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/util/SchemaUtils.java index a4a654dd2b..4e3a589085 100644 --- a/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/util/SchemaUtils.java +++ b/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/util/SchemaUtils.java @@ -236,7 +236,7 @@ public class SchemaUtils { String selectColumnSQL = String.format( "SELECT %s FROM %s WHERE 1 != 1", - quoteIdentifier(column), tablePath.getTableName()); + quoteIdentifier(column), tablePath.getFullName()); try (Statement statement = connection.createStatement()) { return statement.execute(selectColumnSQL); } catch (SQLException e) { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/java/org/apache/seatunnel/e2e/connector/starrocks/StarRocksSchemaChangeIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/java/org/apache/seatunnel/e2e/connector/starrocks/StarRocksSchemaChangeIT.java index 66d98ceb80..f74291e179 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/java/org/apache/seatunnel/e2e/connector/starrocks/StarRocksSchemaChangeIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/java/org/apache/seatunnel/e2e/connector/starrocks/StarRocksSchemaChangeIT.java @@ -191,6 +191,9 @@ public class StarRocksSchemaChangeIT extends TestSuiteBase implements TestResour assertSchemaEvolutionForAddColumns( DATABASE, SOURCE_TABLE, SINK_TABLE, mysqlConnection, starRocksConnection); + assertSchemaEvolutionForDropColumns( + DATABASE, SOURCE_TABLE, SINK_TABLE, mysqlConnection, starRocksConnection); + // savepoint 1 Assertions.assertEquals(0, container.savepointJob(jobId).getExitCode()); @@ -303,6 +306,27 @@ public class StarRocksSchemaChangeIT extends TestSuiteBase implements TestResour }); } + private void assertSchemaEvolutionForDropColumns( + String database, + String sourceTable, + String sinkTable, + Connection sourceConnection, + Connection sinkConnection) { + + // case1 add columns with cdc data at same time + shopDatabase.setTemplateName("drop_columns_validate_schema.sql").createAndInitialize(); + await().atMost(60000, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> + Assertions.assertIterableEquals( + query( + String.format(QUERY_COLUMNS, database, sourceTable), + sourceConnection), + query( + String.format(QUERY_COLUMNS, database, sinkTable), + sinkConnection))); + } + private void assertTableStructureAndData( String database, String sourceTable, diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/resources/ddl/drop_columns.sql b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/resources/ddl/drop_columns.sql index 5c3b7d1f54..9464e02e1d 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/resources/ddl/drop_columns.sql +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/resources/ddl/drop_columns.sql @@ -21,20 +21,6 @@ CREATE DATABASE IF NOT EXISTS `shop`; use shop; -alter table products drop column add_column4,drop column add_column6; -insert into products -values (137,"scooter","Small 2-wheel scooter",3.14,'xx',1,1.1), - (138,"car battery","12V car battery",8.1,'xx',2,1.2), - (139,"12-pack drill bits","12-pack of drill bits with sizes ranging from #40 to #3",0.8,'xx',3,1.3), - (140,"hammer","12oz carpenter's hammer",0.75,'xx',4,1.4), - (141,"hammer","14oz carpenter's hammer",0.875,'xx',5,1.5), - (142,"hammer","16oz carpenter's hammer",1.0,'xx',6,1.6), - (143,"rocks","box of assorted rocks",5.3,'xx',7,1.7), - (144,"jacket","water resistent black wind breaker",0.1,'xx',8,1.8), - (145,"spare tire","24 inch spare tire",22.2,'xx',9,1.9); -update products set name = 'dailai' where id in (140,141,142); -delete from products where id < 137; - alter table products drop column add_column1,drop column add_column3; insert into products diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/resources/ddl/drop_columns.sql b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/resources/ddl/drop_columns_validate_schema.sql similarity index 72% copy from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/resources/ddl/drop_columns.sql copy to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/resources/ddl/drop_columns_validate_schema.sql index 5c3b7d1f54..262006bd81 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/resources/ddl/drop_columns.sql +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/resources/ddl/drop_columns_validate_schema.sql @@ -33,18 +33,4 @@ values (137,"scooter","Small 2-wheel scooter",3.14,'xx',1,1.1), (144,"jacket","water resistent black wind breaker",0.1,'xx',8,1.8), (145,"spare tire","24 inch spare tire",22.2,'xx',9,1.9); update products set name = 'dailai' where id in (140,141,142); -delete from products where id < 137; - - -alter table products drop column add_column1,drop column add_column3; -insert into products -values (146,"scooter","Small 2-wheel scooter",3.14,1), - (147,"car battery","12V car battery",8.1,2), - (148,"12-pack drill bits","12-pack of drill bits with sizes ranging from #40 to #3",0.8,3), - (149,"hammer","12oz carpenter's hammer",0.75,4), - (150,"hammer","14oz carpenter's hammer",0.875,5), - (151,"hammer","16oz carpenter's hammer",1.0,6), - (152,"rocks","box of assorted rocks",5.3,7), - (153,"jacket","water resistent black wind breaker",0.1,8), - (154,"spare tire","24 inch spare tire",22.2,9); -update products set name = 'dailai' where id > 143; +delete from products where id < 137; \ No newline at end of file