This is an automated email from the ASF dual-hosted git repository.
zhonghongsheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 33a75039b76 Scaling IT optimization (#17702)
33a75039b76 is described below
commit 33a75039b7615d25f9392ca624b6da3526eb5bee
Author: azexcy <101622833+aze...@users.noreply.github.com>
AuthorDate: Mon May 16 19:30:40 2022 +0800
Scaling IT optimization (#17702)
* Add more print when error happened
* Replace addResource template
* Scaling increment task optimization
* Fix codestyle
* Spilt add resource
---
.../ShardingSphereConfigurationException.java | 10 +++++++
.../datanode/SingleTableDataNodeLoader.java | 6 ++--
.../data/pipeline/cases/base/BaseITCase.java | 22 ++++++++-------
.../data/pipeline/cases/base/BaseMySQLITCase.java | 8 +++---
.../pipeline/cases/base/BaseOpenGaussITCase.java | 9 +++---
.../pipeline/cases/base/BasePostgreSQLITCase.java | 8 +++---
.../data/pipeline/cases/base/BaseTaskRunnable.java | 20 ++++++++------
.../pipeline/cases/command/CommonSQLCommand.java | 6 ++++
...nnable.java => MySQLIncrementTaskRunnable.java} | 21 ++++++++++----
...e.java => PostgreSQLIncrementTaskRunnable.java} | 16 +++++++----
.../pipeline/cases/mysql/MySQLManualScalingIT.java | 5 ++--
.../cases/openguass/OpenGaussManualScalingIT.java | 3 +-
.../postgresql/PostgreSQLManualScalingIT.java | 5 ++--
.../framework/param/ScalingParameterized.java | 16 +++++------
.../src/test/resources/env/common/command.xml | 32 ++++++++++++++++++----
.../manual/mysql/integer_primary_key/sql.xml | 6 ++--
.../manual/postgresql/integer_primary_key/sql.xml | 6 ++--
17 files changed, 129 insertions(+), 70 deletions(-)
diff --git
a/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/config/exception/ShardingSphereConfigurationException.java
b/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/config/exception/ShardingSphereConfigurationException.java
index 76ccd980afd..b47ca3555c5 100644
---
a/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/config/exception/ShardingSphereConfigurationException.java
+++
b/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/config/exception/ShardingSphereConfigurationException.java
@@ -34,6 +34,16 @@ public final class ShardingSphereConfigurationException
extends RuntimeException
super(String.format(errorMessage, args));
}
+ /**
+ * Constructs an exception with error message and cause.
+ *
+ * @param errorMessage formatted error message
+ * @param cause the cause of this exception
+ */
+ public ShardingSphereConfigurationException(final String errorMessage,
final Throwable cause) {
+ super(errorMessage, cause);
+ }
+
/**
* Constructs an exception with cause exception.
*
diff --git
a/shardingsphere-kernel/shardingsphere-single-table/shardingsphere-single-table-core/src/main/java/org/apache/shardingsphere/singletable/datanode/SingleTableDataNodeLoader.java
b/shardingsphere-kernel/shardingsphere-single-table/shardingsphere-single-table-core/src/main/java/org/apache/shardingsphere/singletable/datanode/SingleTableDataNodeLoader.java
index 7144d6643e8..6720ce544f1 100644
---
a/shardingsphere-kernel/shardingsphere-single-table/shardingsphere-single-table-core/src/main/java/org/apache/shardingsphere/singletable/datanode/SingleTableDataNodeLoader.java
+++
b/shardingsphere-kernel/shardingsphere-single-table/shardingsphere-single-table-core/src/main/java/org/apache/shardingsphere/singletable/datanode/SingleTableDataNodeLoader.java
@@ -75,7 +75,7 @@ public final class SingleTableDataNodeLoader {
private static Map<String, Collection<DataNode>> load(final String
databaseName, final DatabaseType databaseType, final String dataSourceName,
final DataSource
dataSource, final Collection<String> excludedTables) {
- Map<String, Collection<String>> schemaTableNames =
loadSchemaTableNames(databaseName, databaseType, dataSource);
+ Map<String, Collection<String>> schemaTableNames =
loadSchemaTableNames(databaseName, databaseType, dataSource, dataSourceName);
Map<String, Collection<DataNode>> result = new LinkedHashMap<>();
for (Entry<String, Collection<String>> entry :
schemaTableNames.entrySet()) {
for (String each : entry.getValue()) {
@@ -102,11 +102,11 @@ public final class SingleTableDataNodeLoader {
return false;
}
- private static Map<String, Collection<String>> loadSchemaTableNames(final
String databaseName, final DatabaseType databaseType, final DataSource
dataSource) {
+ private static Map<String, Collection<String>> loadSchemaTableNames(final
String databaseName, final DatabaseType databaseType, final DataSource
dataSource, final String dataSourceName) {
try {
return SchemaMetaDataLoader.loadSchemaTableNames(databaseName,
databaseType, dataSource);
} catch (final SQLException ex) {
- throw new ShardingSphereConfigurationException("Can not load
table: %s", ex.getMessage());
+ throw new ShardingSphereConfigurationException(String.format("Can
not load table, databaseName: %s, dataSourceName: %s", databaseName,
dataSourceName), ex);
}
}
}
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java
index d2a065d4971..6c0d5223a64 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java
@@ -68,8 +68,6 @@ import static org.junit.Assert.assertTrue;
@Getter(AccessLevel.PROTECTED)
public abstract class BaseITCase {
- protected static final String ADD_RESOURCE_TEMPLATE = "ADD RESOURCE %s
(URL='%s',USER=%s,PASSWORD=%s)";
-
protected static final JdbcUrlAppender JDBC_URL_APPENDER = new
JdbcUrlAppender();
private static final IntegrationTestEnvironment ENV =
IntegrationTestEnvironment.getInstance();
@@ -122,17 +120,21 @@ public abstract class BaseITCase {
return result;
}
- protected void addResource(final Connection connection) throws
SQLException {
- addResource(connection, "root", "root");
+ protected void addSourceResource(final Connection connection, final String
username, final String password) throws SQLException {
+ Properties queryProps = createQueryProperties();
+ String addSourceResource =
commonSQLCommand.getSourceAddResourceTemplate().replace("${user}",
username).replace("${password}", password)
+ .replace("${ds0}",
JDBC_URL_APPENDER.appendQueryProperties(getActualJdbcUrlTemplate("ds_0"),
queryProps))
+ .replace("${ds1}",
JDBC_URL_APPENDER.appendQueryProperties(getActualJdbcUrlTemplate("ds_1"),
queryProps));
+ connection.createStatement().execute(addSourceResource);
}
- protected void addResource(final Connection connection, final String
username, final String password) throws SQLException {
+ protected void addTargetSourceResource(final String username, final String
password) {
Properties queryProps = createQueryProperties();
-
connection.createStatement().execute(String.format(ADD_RESOURCE_TEMPLATE,
"ds_0",
JDBC_URL_APPENDER.appendQueryProperties(getActualJdbcUrlTemplate("ds_0"),
queryProps), username, password));
-
connection.createStatement().execute(String.format(ADD_RESOURCE_TEMPLATE,
"ds_1",
JDBC_URL_APPENDER.appendQueryProperties(getActualJdbcUrlTemplate("ds_1"),
queryProps), username, password));
-
connection.createStatement().execute(String.format(ADD_RESOURCE_TEMPLATE,
"ds_2",
JDBC_URL_APPENDER.appendQueryProperties(getActualJdbcUrlTemplate("ds_2"),
queryProps), username, password));
-
connection.createStatement().execute(String.format(ADD_RESOURCE_TEMPLATE,
"ds_3",
JDBC_URL_APPENDER.appendQueryProperties(getActualJdbcUrlTemplate("ds_3"),
queryProps), username, password));
-
connection.createStatement().execute(String.format(ADD_RESOURCE_TEMPLATE,
"ds_4",
JDBC_URL_APPENDER.appendQueryProperties(getActualJdbcUrlTemplate("ds_4"),
queryProps), username, password));
+ String addTargetResource =
commonSQLCommand.getTargetAddResourceTemplate().replace("${user}",
username).replace("${password}", password)
+ .replace("${ds2}",
JDBC_URL_APPENDER.appendQueryProperties(getActualJdbcUrlTemplate("ds_2"),
queryProps))
+ .replace("${ds3}",
JDBC_URL_APPENDER.appendQueryProperties(getActualJdbcUrlTemplate("ds_3"),
queryProps))
+ .replace("${ds4}",
JDBC_URL_APPENDER.appendQueryProperties(getActualJdbcUrlTemplate("ds_4"),
queryProps));
+ getJdbcTemplate().execute(addTargetResource);
}
private String getActualJdbcUrlTemplate(final String databaseName) {
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseMySQLITCase.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseMySQLITCase.java
index f4fbac260f0..9c09210b25d 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseMySQLITCase.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseMySQLITCase.java
@@ -22,7 +22,7 @@ import lombok.SneakyThrows;
import org.apache.shardingsphere.infra.database.type.DatabaseType;
import org.apache.shardingsphere.infra.database.type.dialect.MySQLDatabaseType;
import
org.apache.shardingsphere.integration.data.pipeline.cases.command.ExtraSQLCommand;
-import
org.apache.shardingsphere.integration.data.pipeline.cases.common.SimpleIncrementTaskRunnable;
+import
org.apache.shardingsphere.integration.data.pipeline.cases.common.MySQLIncrementTaskRunnable;
import
org.apache.shardingsphere.integration.data.pipeline.framework.helper.ScalingTableSQLHelper;
import
org.apache.shardingsphere.integration.data.pipeline.framework.param.ScalingParameterized;
import org.apache.shardingsphere.sharding.spi.KeyGenerateAlgorithm;
@@ -49,17 +49,17 @@ public abstract class BaseMySQLITCase extends BaseITCase {
}
@SneakyThrows(SQLException.class)
- protected void addResource() {
+ protected void addSourceResource() {
Properties queryProps = createQueryProperties();
// TODO if use jdbcurl like "jdbc:mysql:localhost:3307/sharding_db",
will throw exception show "Datasource or ShardingSphere rule does not exist"
try (Connection connection =
DriverManager.getConnection(JDBC_URL_APPENDER.appendQueryProperties(getComposedContainer().getProxyJdbcUrl(""),
queryProps), "root", "root")) {
connection.createStatement().execute("USE sharding_db");
- addResource(connection);
+ addSourceResource(connection, "root", "root");
}
}
protected void startIncrementTask(final KeyGenerateAlgorithm
keyGenerateAlgorithm) {
- setIncreaseTaskThread(new Thread(new
SimpleIncrementTaskRunnable(getJdbcTemplate(), extraSQLCommand,
keyGenerateAlgorithm)));
+ setIncreaseTaskThread(new Thread(new
MySQLIncrementTaskRunnable(getJdbcTemplate(), extraSQLCommand,
keyGenerateAlgorithm)));
getIncreaseTaskThread().start();
}
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseOpenGaussITCase.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseOpenGaussITCase.java
index d6d7e0508f2..e0d017563e6 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseOpenGaussITCase.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseOpenGaussITCase.java
@@ -22,7 +22,7 @@ import lombok.SneakyThrows;
import org.apache.shardingsphere.infra.database.type.DatabaseType;
import
org.apache.shardingsphere.infra.database.type.dialect.OpenGaussDatabaseType;
import
org.apache.shardingsphere.integration.data.pipeline.cases.command.ExtraSQLCommand;
-import
org.apache.shardingsphere.integration.data.pipeline.cases.common.SimpleIncrementTaskRunnable;
+import
org.apache.shardingsphere.integration.data.pipeline.cases.common.PostgreSQLIncrementTaskRunnable;
import
org.apache.shardingsphere.integration.data.pipeline.framework.helper.ScalingTableSQLHelper;
import
org.apache.shardingsphere.integration.data.pipeline.framework.param.ScalingParameterized;
import org.apache.shardingsphere.sharding.spi.KeyGenerateAlgorithm;
@@ -49,16 +49,17 @@ public abstract class BaseOpenGaussITCase extends
BaseITCase {
sqlHelper = new ScalingTableSQLHelper(DATABASE_TYPE, extraSQLCommand,
getJdbcTemplate());
}
+ // TODO add source resource should be common,after all problem be solved.
@SneakyThrows(SQLException.class)
- protected void addResource() {
+ protected void addSourceResource() {
Properties queryProps = createQueryProperties();
try (Connection connection =
DriverManager.getConnection(JDBC_URL_APPENDER.appendQueryProperties(getComposedContainer().getProxyJdbcUrl("sharding_db"),
queryProps), "root", "root")) {
- addResource(connection, "gaussdb", "Root@123");
+ addSourceResource(connection, "gaussdb", "Root@123");
}
}
protected void startIncrementTask(final KeyGenerateAlgorithm
keyGenerateAlgorithm) {
- setIncreaseTaskThread(new Thread(new
SimpleIncrementTaskRunnable(getJdbcTemplate(), extraSQLCommand,
keyGenerateAlgorithm)));
+ setIncreaseTaskThread(new Thread(new
PostgreSQLIncrementTaskRunnable(getJdbcTemplate(), extraSQLCommand,
keyGenerateAlgorithm)));
getIncreaseTaskThread().start();
}
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BasePostgreSQLITCase.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BasePostgreSQLITCase.java
index a439ac8ca0b..f16535d0cad 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BasePostgreSQLITCase.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BasePostgreSQLITCase.java
@@ -22,7 +22,7 @@ import lombok.SneakyThrows;
import org.apache.shardingsphere.infra.database.type.DatabaseType;
import
org.apache.shardingsphere.infra.database.type.dialect.PostgreSQLDatabaseType;
import
org.apache.shardingsphere.integration.data.pipeline.cases.command.ExtraSQLCommand;
-import
org.apache.shardingsphere.integration.data.pipeline.cases.common.SimpleIncrementTaskRunnable;
+import
org.apache.shardingsphere.integration.data.pipeline.cases.common.PostgreSQLIncrementTaskRunnable;
import
org.apache.shardingsphere.integration.data.pipeline.framework.helper.ScalingTableSQLHelper;
import
org.apache.shardingsphere.integration.data.pipeline.framework.param.ScalingParameterized;
import org.apache.shardingsphere.sharding.spi.KeyGenerateAlgorithm;
@@ -49,15 +49,15 @@ public abstract class BasePostgreSQLITCase extends
BaseITCase {
}
@SneakyThrows(SQLException.class)
- protected void addResource() {
+ protected void addSourceResource() {
Properties queryProps = createQueryProperties();
try (Connection connection =
DriverManager.getConnection(JDBC_URL_APPENDER.appendQueryProperties(getComposedContainer().getProxyJdbcUrl("sharding_db"),
queryProps), "root", "root")) {
- addResource(connection);
+ addSourceResource(connection, "root", "root");
}
}
protected void startIncrementTask(final KeyGenerateAlgorithm
keyGenerateAlgorithm) {
- setIncreaseTaskThread(new Thread(new
SimpleIncrementTaskRunnable(getJdbcTemplate(), extraSQLCommand,
keyGenerateAlgorithm)));
+ setIncreaseTaskThread(new Thread(new
PostgreSQLIncrementTaskRunnable(getJdbcTemplate(), extraSQLCommand,
keyGenerateAlgorithm)));
getIncreaseTaskThread().start();
}
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseTaskRunnable.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseTaskRunnable.java
index cfb166a770f..d363d791a09 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseTaskRunnable.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseTaskRunnable.java
@@ -17,17 +17,18 @@
package org.apache.shardingsphere.integration.data.pipeline.cases.base;
-import lombok.AllArgsConstructor;
import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
import
org.apache.shardingsphere.integration.data.pipeline.cases.command.ExtraSQLCommand;
import org.apache.shardingsphere.sharding.spi.KeyGenerateAlgorithm;
import org.springframework.jdbc.core.JdbcTemplate;
import java.time.Instant;
-import java.util.concurrent.ThreadLocalRandom;
@Getter
-@AllArgsConstructor
+@RequiredArgsConstructor
+@Slf4j
public abstract class BaseTaskRunnable implements Runnable {
private final JdbcTemplate jdbcTemplate;
@@ -36,9 +37,11 @@ public abstract class BaseTaskRunnable implements Runnable {
private final KeyGenerateAlgorithm keyGenerateAlgorithm;
- protected abstract Object[] getOrderInsertDate();
+ protected abstract Object[] getOrderInsertData();
- protected abstract Object[] getOrderInsertItemDate();
+ protected abstract Object[] getOrderInsertItemData();
+
+ protected abstract Object[] getOrderUpdateData(Object primaryKey);
/**
* Insert order.
@@ -46,7 +49,7 @@ public abstract class BaseTaskRunnable implements Runnable {
* @return primary key of insert data
*/
public Object insertOrder() {
- Object[] orderInsertDate = getOrderInsertDate();
+ Object[] orderInsertDate = getOrderInsertData();
jdbcTemplate.update(extraSQLCommand.getInsertOrder(), orderInsertDate);
return orderInsertDate[0];
}
@@ -57,7 +60,7 @@ public abstract class BaseTaskRunnable implements Runnable {
* @return primary key of insert data
*/
public Object insertOrderItem() {
- Object[] orderInsertItemDate = getOrderInsertItemDate();
+ Object[] orderInsertItemDate = getOrderInsertItemData();
jdbcTemplate.update(extraSQLCommand.getInsertOrderItem(),
orderInsertItemDate);
return orderInsertItemDate[0];
}
@@ -68,8 +71,7 @@ public abstract class BaseTaskRunnable implements Runnable {
* @param primaryKey primary key
*/
public void updateOrderByPrimaryKey(final Object primaryKey) {
- jdbcTemplate.update(extraSQLCommand.getUpdateOrderById(), "updated" +
Instant.now().getEpochSecond(), null, primaryKey);
- jdbcTemplate.update(extraSQLCommand.getUpdateOrderById(), "updated" +
Instant.now().getEpochSecond(), ThreadLocalRandom.current().nextInt(0, 100),
primaryKey);
+ jdbcTemplate.update(extraSQLCommand.getUpdateOrderById(),
getOrderUpdateData(primaryKey));
}
/**
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/command/CommonSQLCommand.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/command/CommonSQLCommand.java
index 3b50e42a6ec..d89c3af1fd2 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/command/CommonSQLCommand.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/command/CommonSQLCommand.java
@@ -52,4 +52,10 @@ public final class CommonSQLCommand {
@XmlElement(name = "auto-alter-table-rule")
private String autoAlterTableRule;
+
+ @XmlElement(name = "source-add-resource-template")
+ private String sourceAddResourceTemplate;
+
+ @XmlElement(name = "target-add-resource-template")
+ private String targetAddResourceTemplate;
}
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/common/SimpleIncrementTaskRunnable.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/common/MySQLIncrementTaskRunnable.java
similarity index 72%
copy from
shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/common/SimpleIncrementTaskRunnable.java
copy to
shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/common/MySQLIncrementTaskRunnable.java
index 61a5d32ed74..529687a30d0 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/common/SimpleIncrementTaskRunnable.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/common/MySQLIncrementTaskRunnable.java
@@ -23,12 +23,13 @@ import
org.apache.shardingsphere.integration.data.pipeline.cases.command.ExtraSQ
import org.apache.shardingsphere.sharding.spi.KeyGenerateAlgorithm;
import org.springframework.jdbc.core.JdbcTemplate;
+import java.time.Instant;
import java.util.concurrent.ThreadLocalRandom;
@Slf4j
-public final class SimpleIncrementTaskRunnable extends BaseTaskRunnable {
+public final class MySQLIncrementTaskRunnable extends BaseTaskRunnable {
- public SimpleIncrementTaskRunnable(final JdbcTemplate jdbcTemplate, final
ExtraSQLCommand extraSQLCommand, final KeyGenerateAlgorithm
keyGenerateAlgorithm) {
+ public MySQLIncrementTaskRunnable(final JdbcTemplate jdbcTemplate, final
ExtraSQLCommand extraSQLCommand, final KeyGenerateAlgorithm
keyGenerateAlgorithm) {
super(jdbcTemplate, extraSQLCommand, keyGenerateAlgorithm);
}
@@ -43,20 +44,30 @@ public final class SimpleIncrementTaskRunnable extends
BaseTaskRunnable {
deleteOrderItemByPrimaryKey(orderItemPrimaryKey);
} else {
updateOrderByPrimaryKey(orderPrimaryKey);
+ setFieldsToNull(orderPrimaryKey);
updateOrderItemByPrimaryKey(orderItemPrimaryKey);
}
executeCount++;
- log.info("Simple increment task runnable execute successfully.");
}
+ log.info("MySQL increment task runnable execute successfully.");
+ }
+
+ private void setFieldsToNull(final Object primaryKey) {
+ getJdbcTemplate().update(" UPDATE t_order SET t_unsigned_int = null
WHERE id = ?", primaryKey);
}
@Override
- protected Object[] getOrderInsertDate() {
+ protected Object[] getOrderInsertData() {
return new Object[]{getKeyGenerateAlgorithm().generateKey(),
ThreadLocalRandom.current().nextInt(0, 6),
ThreadLocalRandom.current().nextInt(0, 6)};
}
@Override
- protected Object[] getOrderInsertItemDate() {
+ protected Object[] getOrderInsertItemData() {
return new Object[]{getKeyGenerateAlgorithm().generateKey(),
ThreadLocalRandom.current().nextInt(0, 6),
ThreadLocalRandom.current().nextInt(0, 6), "OK"};
}
+
+ @Override
+ protected Object[] getOrderUpdateData(final Object primaryKey) {
+ return new Object[]{"updated" + Instant.now().getEpochSecond(),
ThreadLocalRandom.current().nextInt(0, 100), primaryKey};
+ }
}
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/common/SimpleIncrementTaskRunnable.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/common/PostgreSQLIncrementTaskRunnable.java
similarity index 79%
rename from
shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/common/SimpleIncrementTaskRunnable.java
rename to
shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/common/PostgreSQLIncrementTaskRunnable.java
index 61a5d32ed74..f457c88e64e 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/common/SimpleIncrementTaskRunnable.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/common/PostgreSQLIncrementTaskRunnable.java
@@ -23,12 +23,13 @@ import
org.apache.shardingsphere.integration.data.pipeline.cases.command.ExtraSQ
import org.apache.shardingsphere.sharding.spi.KeyGenerateAlgorithm;
import org.springframework.jdbc.core.JdbcTemplate;
+import java.time.Instant;
import java.util.concurrent.ThreadLocalRandom;
@Slf4j
-public final class SimpleIncrementTaskRunnable extends BaseTaskRunnable {
+public final class PostgreSQLIncrementTaskRunnable extends BaseTaskRunnable {
- public SimpleIncrementTaskRunnable(final JdbcTemplate jdbcTemplate, final
ExtraSQLCommand extraSQLCommand, final KeyGenerateAlgorithm
keyGenerateAlgorithm) {
+ public PostgreSQLIncrementTaskRunnable(final JdbcTemplate jdbcTemplate,
final ExtraSQLCommand extraSQLCommand, final KeyGenerateAlgorithm
keyGenerateAlgorithm) {
super(jdbcTemplate, extraSQLCommand, keyGenerateAlgorithm);
}
@@ -46,17 +47,22 @@ public final class SimpleIncrementTaskRunnable extends
BaseTaskRunnable {
updateOrderItemByPrimaryKey(orderItemPrimaryKey);
}
executeCount++;
- log.info("Simple increment task runnable execute successfully.");
}
+ log.info("PostgreSQL increment task runnable execute successfully.");
}
@Override
- protected Object[] getOrderInsertDate() {
+ protected Object[] getOrderInsertData() {
return new Object[]{getKeyGenerateAlgorithm().generateKey(),
ThreadLocalRandom.current().nextInt(0, 6),
ThreadLocalRandom.current().nextInt(0, 6)};
}
@Override
- protected Object[] getOrderInsertItemDate() {
+ protected Object[] getOrderInsertItemData() {
return new Object[]{getKeyGenerateAlgorithm().generateKey(),
ThreadLocalRandom.current().nextInt(0, 6),
ThreadLocalRandom.current().nextInt(0, 6), "OK"};
}
+
+ @Override
+ protected Object[] getOrderUpdateData(final Object primaryKey) {
+ return new Object[]{"updated" + Instant.now().getEpochSecond(),
primaryKey};
+ }
}
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/mysql/MySQLManualScalingIT.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/mysql/MySQLManualScalingIT.java
index 65a36974f8e..1c45e88b36b 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/mysql/MySQLManualScalingIT.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/mysql/MySQLManualScalingIT.java
@@ -63,10 +63,10 @@ public final class MySQLManualScalingIT extends
BaseMySQLITCase {
@Before
public void setUp() throws InterruptedException {
- addResource();
+ addSourceResource();
initShardingAlgorithm();
// TODO wait for algorithm init
- TimeUnit.SECONDS.sleep(2);
+ TimeUnit.SECONDS.sleep(3);
createScalingRule();
}
@@ -80,6 +80,7 @@ public final class MySQLManualScalingIT extends
BaseMySQLITCase {
getSqlHelper().initTableData(true);
startIncrementTask(new SnowflakeKeyGenerateAlgorithm());
assertOriginalSourceSuccess();
+ addTargetSourceResource("root", "root");
getJdbcTemplate().execute(getCommonSQLCommand().getAutoAlterTableRule());
String jobId = String.valueOf(getJdbcTemplate().queryForMap("SHOW
SCALING LIST").get("id"));
getIncreaseTaskThread().join(60 * 1000L);
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/openguass/OpenGaussManualScalingIT.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/openguass/OpenGaussManualScalingIT.java
index b56fc20fa1a..5c65147cdb1 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/openguass/OpenGaussManualScalingIT.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/openguass/OpenGaussManualScalingIT.java
@@ -61,7 +61,7 @@ public final class OpenGaussManualScalingIT extends
BaseOpenGaussITCase {
@Before
public void setUp() throws InterruptedException {
- addResource();
+ addSourceResource();
initShardingAlgorithm();
// TODO wait for algorithm init
TimeUnit.SECONDS.sleep(2);
@@ -81,6 +81,7 @@ public final class OpenGaussManualScalingIT extends
BaseOpenGaussITCase {
getJdbcTemplate().batchUpdate(getExtraSQLCommand().getInsertOrderItem(),
dataPair.getRight());
startIncrementTask(new SnowflakeKeyGenerateAlgorithm());
assertOriginalSourceSuccess();
+ addTargetSourceResource("gaussdb", "Root@123");
getJdbcTemplate().execute(getCommonSQLCommand().getAutoAlterTableRule());
String jobId = String.valueOf(getJdbcTemplate().queryForMap("SHOW
SCALING LIST").get("id"));
getIncreaseTaskThread().join(60 * 1000L);
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/postgresql/PostgreSQLManualScalingIT.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/postgresql/PostgreSQLManualScalingIT.java
index 1ff39f0bc22..169bbd43702 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/postgresql/PostgreSQLManualScalingIT.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/postgresql/PostgreSQLManualScalingIT.java
@@ -58,10 +58,10 @@ public final class PostgreSQLManualScalingIT extends
BasePostgreSQLITCase {
@Before
public void setUp() throws InterruptedException {
- addResource();
+ addSourceResource();
initShardingAlgorithm();
// TODO wait for algorithm init
- TimeUnit.SECONDS.sleep(2);
+ TimeUnit.SECONDS.sleep(3);
createScalingRule();
createSchema("test");
}
@@ -75,6 +75,7 @@ public final class PostgreSQLManualScalingIT extends
BasePostgreSQLITCase {
getSqlHelper().initTableData(true);
startIncrementTask(new SnowflakeKeyGenerateAlgorithm());
assertOriginalSourceSuccess();
+ addTargetSourceResource("root", "root");
getJdbcTemplate().execute(getCommonSQLCommand().getAutoAlterTableRule());
String jobId = String.valueOf(getJdbcTemplate().queryForMap("SHOW
SCALING LIST").get("id"));
getIncreaseTaskThread().join(60 * 1000L);
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/framework/param/ScalingParameterized.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/framework/param/ScalingParameterized.java
index 69b9840c2a0..e3c5c0f185c 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/framework/param/ScalingParameterized.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/framework/param/ScalingParameterized.java
@@ -17,19 +17,17 @@
package org.apache.shardingsphere.integration.data.pipeline.framework.param;
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
import org.apache.shardingsphere.infra.database.type.DatabaseType;
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
+@Getter
+@RequiredArgsConstructor
public final class ScalingParameterized {
- private DatabaseType databaseType;
+ private final DatabaseType databaseType;
- private String dockerImageName;
+ private final String dockerImageName;
- private String scenario;
+ private final String scenario;
}
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/common/command.xml
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/common/command.xml
index 612057343bd..39cf89cd8a8 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/common/command.xml
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/common/command.xml
@@ -86,11 +86,31 @@
)
</auto-alter-table-rule>
- <simple-insert-order>
- INSERT INTO t_order ( id, order_id, user_id) VALUES (?, ?, ?)
- </simple-insert-order>
+ <source-add-resource-template>
+ ADD RESOURCE ds_0 (
+ URL="${ds0}",
+ USER=${user},
+ PASSWORD=${password}
+ ), ds_1 (
+ URL="${ds1}",
+ USER=${user},
+ PASSWORD=${password}
+ )
+ </source-add-resource-template>
- <insert-order-item>
- INSERT INTO t_order_item(item_id, order_id, user_id, status)
VALUES(?,?,?,?)
- </insert-order-item>
+ <target-add-resource-template>
+ ADD RESOURCE ds_2 (
+ URL="${ds2}",
+ USER=${user},
+ PASSWORD=${password}
+ ),ds_3 (
+ URL="${ds3}",
+ USER=${user},
+ PASSWORD=${password}
+ ),ds_4 (
+ URL="${ds4}",
+ USER=${user},
+ PASSWORD=${password}
+ )
+ </target-add-resource-template>
</command>
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/scenario/manual/mysql/integer_primary_key/sql.xml
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/scenario/manual/mysql/integer_primary_key/sql.xml
index ddb1734d791..d497c549081 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/scenario/manual/mysql/integer_primary_key/sql.xml
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/scenario/manual/mysql/integer_primary_key/sql.xml
@@ -20,7 +20,7 @@
`id` BIGINT NOT NULL COMMENT 'pk id',
`order_id` INT NOT NULL,
`user_id` INT NOT NULL,
- `t_varchar` VARCHAR ( 255 ) NULL,
+ `status` VARCHAR ( 255 ) NULL,
`t_tinyint` TINYINT ( 1 ) NULL,
`t_timestamp` TIMESTAMP NULL,
`t_datetime` datetime NULL ON UPDATE CURRENT_TIMESTAMP,
@@ -51,7 +51,7 @@
id,
order_id,
user_id,
- t_varchar,
+ status,
t_tinyint,
t_timestamp,
t_datetime,
@@ -75,7 +75,7 @@
</insert-order-item>
<update-order-by-id>
- UPDATE t_order SET t_varchar = ?,t_unsigned_int = ? WHERE id = ?
+ UPDATE t_order SET status = ?,t_unsigned_int = ? WHERE id = ?
</update-order-by-id>
<update-order-item-by-id>
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/scenario/manual/postgresql/integer_primary_key/sql.xml
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/scenario/manual/postgresql/integer_primary_key/sql.xml
index db800537266..67e70819739 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/scenario/manual/postgresql/integer_primary_key/sql.xml
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/scenario/manual/postgresql/integer_primary_key/sql.xml
@@ -20,7 +20,7 @@
id int8 NOT NULL,
order_id int4 NOT NULL,
user_id int NOT NULL,
- t_varchar varchar(50) DEFAULT NULL,
+ status varchar(50) DEFAULT NULL,
PRIMARY KEY (id)
)
</create-table-order>
@@ -36,7 +36,7 @@
</create-table-order-item>
<full-insert-order>
- INSERT INTO test.t_order(id, order_id, user_id, t_varchar) VALUES
(?,?,?,?)
+ INSERT INTO test.t_order(id, order_id, user_id, status) VALUES
(?,?,?,?)
</full-insert-order>
<insert-order>
@@ -48,7 +48,7 @@
</insert-order-item>
<update-order-by-id>
- UPDATE test.t_order SET t_varchar = ? WHERE id = ?
+ UPDATE test.t_order SET status = ? WHERE id = ?
</update-order-by-id>
<update-order-item-by-id>
