This is an automated email from the ASF dual-hosted git repository.
zhonghongsheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new aaf406e9d68 Improve stop scaling when running increment task. (#19096)
aaf406e9d68 is described below
commit aaf406e9d688d2ad963fc2e926103a6cc6c56b47
Author: azexcy <[email protected]>
AuthorDate: Thu Jul 14 09:43:12 2022 +0800
Improve stop scaling when running increment task. (#19096)
* Improve stop scaling when running increment task.
* Scaling IT case add stop/start case
* Fix code styles
* re run ci
---
.../scenario/rulealtered/RuleAlteredJobCenter.java | 1 +
.../rulealtered/RuleAlteredJobPreparer.java | 2 +-
.../pipeline/cases/base/BaseExtraSQLITCase.java | 11 +++---
.../data/pipeline/cases/base/BaseITCase.java | 22 ++++++++++--
.../pipeline/cases/command/CommonSQLCommand.java | 11 +++---
.../pipeline/cases/command/ExtraSQLCommand.java | 4 ---
.../cases/general/MySQLGeneralScalingIT.java | 13 +++-----
.../cases/general/PostgreSQLGeneralScalingIT.java | 24 ++++---------
.../cases/primarykey/TextPrimaryKeyScalingIT.java | 30 +++++------------
.../pipeline/env/IntegrationTestEnvironment.java | 24 +++++++++++++
.../container/database/MySQLContainer.java | 7 +---
.../container/database/OpenGaussContainer.java | 8 ++---
.../container/database/PostgreSQLContainer.java | 5 ---
.../src/test/resources/env/common/command.xml | 39 ++++++++--------------
14 files changed, 91 insertions(+), 110 deletions(-)
diff --git
a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/RuleAlteredJobCenter.java
b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/RuleAlteredJobCenter.java
index dfbdcedaafd..5c8ace8b80e 100644
---
a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/RuleAlteredJobCenter.java
+++
b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/RuleAlteredJobCenter.java
@@ -62,5 +62,6 @@ public final class RuleAlteredJobCenter {
return;
}
job.stop();
+ JOB_MAP.remove(jobId);
}
}
diff --git
a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/RuleAlteredJobPreparer.java
b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/RuleAlteredJobPreparer.java
index 5252c9e2469..a6cae479220 100644
---
a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/RuleAlteredJobPreparer.java
+++
b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/RuleAlteredJobPreparer.java
@@ -89,7 +89,7 @@ public final class RuleAlteredJobPreparer {
try {
initIncrementalTasks(jobContext);
if (jobContext.isStopping()) {
- throw new PipelineJobPrepareFailedException("Job stopping,
jobId=" + jobContext.getJobId());
+ throw new PipelineIgnoredException("Job stopping, jobId=" +
jobContext.getJobId());
}
initInventoryTasks(jobContext);
log.info("prepare, jobId={}, shardingItem={}, inventoryTasks={},
incrementalTasks={}",
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseExtraSQLITCase.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseExtraSQLITCase.java
index 21e7d3013bc..b605a7e41af 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseExtraSQLITCase.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseExtraSQLITCase.java
@@ -21,9 +21,9 @@ import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import
org.apache.shardingsphere.integration.data.pipeline.cases.command.ExtraSQLCommand;
import
org.apache.shardingsphere.integration.data.pipeline.framework.param.ScalingParameterized;
+import
org.apache.shardingsphere.integration.data.pipeline.util.DatabaseTypeUtil;
import javax.xml.bind.JAXB;
-import java.util.List;
import java.util.Objects;
import static org.junit.Assert.assertFalse;
@@ -49,10 +49,11 @@ public abstract class BaseExtraSQLITCase extends BaseITCase
{
executeWithLog(extraSQLCommand.getCreateTableOrder());
}
- protected void createTableIndexList() {
- List<String> createTableIndexList =
extraSQLCommand.getCreateTableIndexList();
- for (String each : createTableIndexList) {
- executeWithLog(each);
+ protected void createTableIndexList(final String schema) {
+ if (DatabaseTypeUtil.isPostgreSQL(getDatabaseType())) {
+ executeWithLog(String.format("CREATE INDEX IF NOT EXISTS
idx_user_id ON %s.t_order ( user_id )", schema));
+ } else if (DatabaseTypeUtil.isOpenGauss(getDatabaseType())) {
+ executeWithLog(String.format("CREATE INDEX idx_user_id ON
%s.t_order ( user_id )", schema));
}
}
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java
index 2d57e8e82b0..997ed08869a 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java
@@ -59,6 +59,8 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -85,6 +87,8 @@ public abstract class BaseITCase {
protected static final String DS_4 = "scaling_it_4";
+ protected static final Executor SCALING_EXECUTOR =
Executors.newFixedThreadPool(5);
+
@Rule
@Getter(AccessLevel.NONE)
public ScalingWatcher scalingWatcher;
@@ -125,7 +129,7 @@ public abstract class BaseITCase {
jdbcInfo = new JdbcInfoEntity(databaseContainer.getUsername(),
databaseContainer.getPassword(), databaseContainer.getPort());
} else {
jdbcInfo = ENV.getActualDatabaseJdbcInfo(getDatabaseType());
- jdbcUrl = DataSourceEnvironment.getURL(databaseType, "localhost",
jdbcInfo.getPort());
+ jdbcUrl = DataSourceEnvironment.getURL(databaseType, "localhost",
jdbcInfo.getPort(), DatabaseTypeUtil.isOpenGauss(databaseType) ? "postgres" :
"");
try (Connection connection = DriverManager.getConnection(jdbcUrl,
jdbcInfo.getUsername(), jdbcInfo.getPassword())) {
for (String each : Arrays.asList(DS_0, DS_1, DS_2, DS_3,
DS_4)) {
try {
@@ -321,6 +325,14 @@ public abstract class BaseITCase {
executeWithLog(String.format("STOP SCALING SOURCE WRITING %s", jobId));
}
+ protected void stopScaling(final String jobId) {
+ executeWithLog(String.format("STOP SCALING %s", jobId), 5);
+ }
+
+ protected void startScaling(final String jobId) {
+ executeWithLog(String.format("START SCALING %s", jobId));
+ }
+
protected void applyScaling(final String jobId) {
executeWithLog(String.format("APPLY SCALING %s", jobId));
}
@@ -347,6 +359,10 @@ public abstract class BaseITCase {
log.info("jobId: {}", jobId);
Map<String, String> actualStatusMap = new HashMap<>(2, 1);
String showScalingStatus = String.format("SHOW SCALING STATUS %s",
jobId);
+ SCALING_EXECUTOR.execute(() -> {
+ stopScaling(jobId);
+ startScaling(jobId);
+ });
for (int i = 0; i < 15; i++) {
List<Map<String, Object>> showScalingStatusResMap =
queryForListWithLog(showScalingStatus);
log.info("{}: {}", showScalingStatus, showScalingStatusResMap);
@@ -367,9 +383,9 @@ public abstract class BaseITCase {
break;
}
assertBeforeApplyScalingMetadataCorrectly();
- TimeUnit.SECONDS.sleep(2);
+ ThreadUtil.sleep(4, TimeUnit.SECONDS);
}
-
assertThat(actualStatusMap.values().stream().filter(StringUtils::isNotBlank).collect(Collectors.toSet()).size(),
is(1));
+
assertThat(actualStatusMap.values().stream().filter(StringUtils::isNotBlank).collect(Collectors.toSet()),
is(Collections.singleton(JobStatus.EXECUTE_INCREMENTAL_TASK.name())));
}
protected void assertCheckScalingSuccess(final String jobId) {
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/command/CommonSQLCommand.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/command/CommonSQLCommand.java
index bbd0ae9c217..6cdbad8abf2 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/command/CommonSQLCommand.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/command/CommonSQLCommand.java
@@ -47,14 +47,11 @@ public final class CommonSQLCommand {
@XmlElement(name = "alter-sharding-algorithm")
private String alterShardingAlgorithm;
- @XmlElement(name = "alter-sharding-table-rule")
- private String alterShardingTableRule;
+ @XmlElement(name = "alter-order-with-item-auto-table-rule")
+ private String alterOrderWithItemAutoTableRule;
- @XmlElement(name = "auto-alter-order-with-item-sharding-table-rule")
- private String autoAlterOrderWithItemShardingTableRule;
-
- @XmlElement(name = "auto-alter-order-sharding-table-rule")
- private String autoAlterOrderShardingTableRule;
+ @XmlElement(name = "alter-order-auto-table-rule")
+ private String alterOrderAutoTableRule;
@XmlElement(name = "source-add-resource-template")
private String sourceAddResourceTemplate;
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/command/ExtraSQLCommand.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/command/ExtraSQLCommand.java
index ad3ba38cfd9..7e61813433b 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/command/ExtraSQLCommand.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/command/ExtraSQLCommand.java
@@ -23,7 +23,6 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
-import java.util.List;
@Data
@XmlRootElement(name = "command")
@@ -42,9 +41,6 @@ public final class ExtraSQLCommand {
@XmlElement(name = "full-insert-order-item")
private String fullInsertOrderItem;
- @XmlElement(name = "create-table-index")
- private List<String> createTableIndexList;
-
@XmlElement(name = "update-table-order-status")
private String updateTableOrderStatus;
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/MySQLGeneralScalingIT.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/MySQLGeneralScalingIT.java
index 3ace86a2432..1e6bc86f8d0 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/MySQLGeneralScalingIT.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/MySQLGeneralScalingIT.java
@@ -18,7 +18,6 @@
package org.apache.shardingsphere.integration.data.pipeline.cases.general;
import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.shardingsphere.infra.database.type.dialect.MySQLDatabaseType;
import
org.apache.shardingsphere.integration.data.pipeline.cases.base.BaseExtraSQLITCase;
@@ -60,13 +59,9 @@ public final class MySQLGeneralScalingIT extends
BaseExtraSQLITCase {
if (ENV.getItEnvType() == ScalingITEnvTypeEnum.NONE) {
return result;
}
- if (ENV.getItEnvType() == ScalingITEnvTypeEnum.DOCKER) {
- for (String version : ENV.getMysqlVersions()) {
- result.add(new ScalingParameterized(new MySQLDatabaseType(),
version, "env/scenario/general/mysql.xml"));
- }
- }
- if (ENV.getItEnvType() == ScalingITEnvTypeEnum.NATIVE &&
StringUtils.equalsIgnoreCase(ENV.getNativeDatabaseType(), "MySQL")) {
- result.add(new ScalingParameterized(new MySQLDatabaseType(), "",
"env/scenario/general/mysql.xml"));
+ MySQLDatabaseType databaseType = new MySQLDatabaseType();
+ for (String version : ENV.listDatabaseDockerImageNames(databaseType)) {
+ result.add(new ScalingParameterized(databaseType, version,
"env/scenario/general/mysql.xml"));
}
return result;
}
@@ -90,7 +85,7 @@ public final class MySQLGeneralScalingIT extends
BaseExtraSQLITCase {
}
addTargetResource();
startIncrementTask(new MySQLIncrementTask(getJdbcTemplate(),
keyGenerateAlgorithm, true));
-
executeWithLog(getCommonSQLCommand().getAutoAlterOrderWithItemShardingTableRule());
+
executeWithLog(getCommonSQLCommand().getAlterOrderWithItemAutoTableRule());
String jobId = getScalingJobId();
waitScalingFinished(jobId);
assertCheckScalingSuccess(jobId);
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/PostgreSQLGeneralScalingIT.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/PostgreSQLGeneralScalingIT.java
index bf0ab7f9afd..811446d0cc5 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/PostgreSQLGeneralScalingIT.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/PostgreSQLGeneralScalingIT.java
@@ -18,7 +18,6 @@
package org.apache.shardingsphere.integration.data.pipeline.cases.general;
import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import
org.apache.shardingsphere.infra.database.type.dialect.OpenGaussDatabaseType;
import
org.apache.shardingsphere.infra.database.type.dialect.PostgreSQLDatabaseType;
@@ -61,21 +60,11 @@ public final class PostgreSQLGeneralScalingIT extends
BaseExtraSQLITCase {
if (ENV.getItEnvType() == ScalingITEnvTypeEnum.NONE) {
return result;
}
- if (ENV.getItEnvType() == ScalingITEnvTypeEnum.DOCKER) {
- for (String dockerImageName : ENV.getPostgresVersions()) {
- result.add(new ScalingParameterized(new
PostgreSQLDatabaseType(), dockerImageName,
"env/scenario/general/postgresql.xml"));
- }
- for (String dockerImageName : ENV.getOpenGaussVersions()) {
- result.add(new ScalingParameterized(new
OpenGaussDatabaseType(), dockerImageName,
"env/scenario/general/postgresql.xml"));
- }
+ for (String dockerImageName : ENV.listDatabaseDockerImageNames(new
PostgreSQLDatabaseType())) {
+ result.add(new ScalingParameterized(new PostgreSQLDatabaseType(),
dockerImageName, "env/scenario/general/postgresql.xml"));
}
- if (ENV.getItEnvType() == ScalingITEnvTypeEnum.NATIVE) {
- if (StringUtils.equalsIgnoreCase(ENV.getNativeDatabaseType(),
"PostgreSQL")) {
- result.add(new ScalingParameterized(new
PostgreSQLDatabaseType(), "", "env/scenario/general/postgresql.xml"));
- }
- if (StringUtils.equalsIgnoreCase(ENV.getNativeDatabaseType(),
"openGauss")) {
- result.add(new ScalingParameterized(new
OpenGaussDatabaseType(), "", "env/scenario/general/postgresql.xml"));
- }
+ for (String dockerImageName : ENV.listDatabaseDockerImageNames(new
OpenGaussDatabaseType())) {
+ result.add(new ScalingParameterized(new OpenGaussDatabaseType(),
dockerImageName, "env/scenario/general/postgresql.xml"));
}
return result;
}
@@ -90,8 +79,9 @@ public final class PostgreSQLGeneralScalingIT extends
BaseExtraSQLITCase {
createOrderTableRule();
createOrderItemTableRule();
createOrderTable();
- createTableIndexList();
createOrderItemTable();
+ // TODO wait kernel support create index if not exists
+ // createTableIndexList("test");
executeWithLog("COMMENT ON COLUMN test.t_order.user_id IS 'user id';");
SnowflakeKeyGenerateAlgorithm keyGenerateAlgorithm = new
SnowflakeKeyGenerateAlgorithm();
Pair<List<Object[]>, List<Object[]>> dataPair =
ScalingCaseHelper.generateFullInsertData(keyGenerateAlgorithm,
parameterized.getDatabaseType(), 3000);
@@ -99,7 +89,7 @@ public final class PostgreSQLGeneralScalingIT extends
BaseExtraSQLITCase {
getJdbcTemplate().batchUpdate(getExtraSQLCommand().getFullInsertOrderItem(),
dataPair.getRight());
addTargetResource();
startIncrementTask(new PostgreSQLIncrementTask(getJdbcTemplate(), new
SnowflakeKeyGenerateAlgorithm(), "test", true));
-
executeWithLog(getCommonSQLCommand().getAutoAlterOrderWithItemShardingTableRule());
+
executeWithLog(getCommonSQLCommand().getAlterOrderWithItemAutoTableRule());
String jobId = getScalingJobId();
waitScalingFinished(jobId);
assertCheckScalingSuccess(jobId);
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/primarykey/TextPrimaryKeyScalingIT.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/primarykey/TextPrimaryKeyScalingIT.java
index 322460e9daf..0b69ad032a0 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/primarykey/TextPrimaryKeyScalingIT.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/primarykey/TextPrimaryKeyScalingIT.java
@@ -18,7 +18,6 @@
package org.apache.shardingsphere.integration.data.pipeline.cases.primarykey;
import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
import org.apache.shardingsphere.infra.database.type.dialect.MySQLDatabaseType;
import
org.apache.shardingsphere.infra.database.type.dialect.OpenGaussDatabaseType;
import
org.apache.shardingsphere.infra.database.type.dialect.PostgreSQLDatabaseType;
@@ -55,27 +54,14 @@ public class TextPrimaryKeyScalingIT extends
BaseExtraSQLITCase {
if (ENV.getItEnvType() == ScalingITEnvTypeEnum.NONE) {
return result;
}
- if (ENV.getItEnvType() == ScalingITEnvTypeEnum.DOCKER) {
- for (String version : ENV.getMysqlVersions()) {
- result.add(new ScalingParameterized(new MySQLDatabaseType(),
version, "env/scenario/primarykey/text_primary_key/mysql.xml"));
- }
- for (String version : ENV.getPostgresVersions()) {
- result.add(new ScalingParameterized(new
PostgreSQLDatabaseType(), version,
"env/scenario/primarykey/text_primary_key/postgresql.xml"));
- }
- for (String version : ENV.getOpenGaussVersions()) {
- result.add(new ScalingParameterized(new
OpenGaussDatabaseType(), version,
"env/scenario/primarykey/text_primary_key/postgresql.xml"));
- }
+ for (String version : ENV.listDatabaseDockerImageNames(new
MySQLDatabaseType())) {
+ result.add(new ScalingParameterized(new MySQLDatabaseType(),
version, "env/scenario/primarykey/text_primary_key/mysql.xml"));
}
- if (ENV.getItEnvType() == ScalingITEnvTypeEnum.NATIVE) {
- if (StringUtils.equalsIgnoreCase(ENV.getNativeDatabaseType(),
"MySQL")) {
- result.add(new ScalingParameterized(new MySQLDatabaseType(),
"", "env/scenario/primarykey/text_primary_key/mysql.xml"));
- }
- if (StringUtils.equalsIgnoreCase(ENV.getNativeDatabaseType(),
"PostgreSQL")) {
- result.add(new ScalingParameterized(new
PostgreSQLDatabaseType(), "",
"env/scenario/primarykey/text_primary_key/postgresql.xml"));
- }
- if (StringUtils.equalsIgnoreCase(ENV.getNativeDatabaseType(),
"openGauss")) {
- result.add(new ScalingParameterized(new
OpenGaussDatabaseType(), "",
"env/scenario/primarykey/text_primary_key/postgresql.xml"));
- }
+ for (String version : ENV.listDatabaseDockerImageNames(new
PostgreSQLDatabaseType())) {
+ result.add(new ScalingParameterized(new PostgreSQLDatabaseType(),
version, "env/scenario/primarykey/text_primary_key/postgresql.xml"));
+ }
+ for (String version : ENV.listDatabaseDockerImageNames(new
OpenGaussDatabaseType())) {
+ result.add(new ScalingParameterized(new OpenGaussDatabaseType(),
version, "env/scenario/primarykey/text_primary_key/postgresql.xml"));
}
return result;
}
@@ -90,7 +76,7 @@ public class TextPrimaryKeyScalingIT extends
BaseExtraSQLITCase {
createOrderTable();
batchInsertOrder();
addTargetResource();
-
executeWithLog(getCommonSQLCommand().getAutoAlterOrderShardingTableRule());
+ executeWithLog(getCommonSQLCommand().getAlterOrderAutoTableRule());
String jobId = getScalingJobId();
waitScalingFinished(jobId);
assertCheckScalingSuccess(jobId);
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/env/IntegrationTestEnvironment.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/env/IntegrationTestEnvironment.java
index 146af6029a9..f9a4dfdbd7d 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/env/IntegrationTestEnvironment.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/env/IntegrationTestEnvironment.java
@@ -28,6 +28,7 @@ import
org.apache.shardingsphere.integration.data.pipeline.env.enums.ScalingITEn
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Properties;
import java.util.stream.Collectors;
@@ -175,4 +176,27 @@ public final class IntegrationTestEnvironment {
public static IntegrationTestEnvironment getInstance() {
return INSTANCE;
}
+
+ /**
+ * List database docker image names.
+ *
+ * @param databaseType database type.
+ * @return database docker image names
+ */
+ public List<String> listDatabaseDockerImageNames(final DatabaseType
databaseType) {
+ // Native mode needn't use docker image, just return a list which
contain one item
+ if (getItEnvType() == ScalingITEnvTypeEnum.NATIVE) {
+ return Collections.singletonList("");
+ }
+ switch (databaseType.getType()) {
+ case "MySQL":
+ return mysqlVersions;
+ case "PostgreSQL":
+ return postgresVersions;
+ case "openGauss":
+ return openGaussVersions;
+ default:
+ throw new UnsupportedOperationException("Unsupported database
type: " + databaseType.getType());
+ }
+ }
}
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/framework/container/database/MySQLContainer.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/framework/container/database/MySQLContainer.java
index aaf86d9ab2d..3118aee9c26 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/framework/container/database/MySQLContainer.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/framework/container/database/MySQLContainer.java
@@ -19,8 +19,6 @@ package
org.apache.shardingsphere.integration.data.pipeline.framework.container.
import org.apache.shardingsphere.infra.database.type.DatabaseType;
import org.apache.shardingsphere.infra.database.type.dialect.MySQLDatabaseType;
-import
org.apache.shardingsphere.integration.data.pipeline.env.IntegrationTestEnvironment;
-import
org.apache.shardingsphere.integration.data.pipeline.env.enums.ScalingITEnvTypeEnum;
import org.apache.shardingsphere.test.integration.env.DataSourceEnvironment;
import
org.apache.shardingsphere.test.integration.framework.container.wait.JDBCConnectionWaitStrategy;
import org.testcontainers.containers.BindMode;
@@ -48,10 +46,7 @@ public final class MySQLContainer extends DatabaseContainer {
addEnv("MYSQL_ROOT_PASSWORD", username);
addEnv("MYSQL_ROOT_HOST", "%");
withClasspathResourceMapping("/env/mysql/my.cnf", "/etc/mysql/my.cnf",
BindMode.READ_ONLY);
- withExposedPorts(getPort());
- if (ScalingITEnvTypeEnum.NATIVE ==
IntegrationTestEnvironment.getInstance().getItEnvType()) {
- addFixedExposedPort(port, port);
- }
+ withExposedPorts(port);
setWaitStrategy(new JDBCConnectionWaitStrategy(() ->
DriverManager.getConnection(DataSourceEnvironment.getURL(DATABASE_TYPE,
"localhost", getFirstMappedPort()), "root", "root")));
}
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/framework/container/database/OpenGaussContainer.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/framework/container/database/OpenGaussContainer.java
index 4e5a736cb0e..a9acbc33c43 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/framework/container/database/OpenGaussContainer.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/framework/container/database/OpenGaussContainer.java
@@ -20,8 +20,6 @@ package
org.apache.shardingsphere.integration.data.pipeline.framework.container.
import org.apache.commons.lang3.StringUtils;
import org.apache.shardingsphere.infra.database.type.DatabaseType;
import
org.apache.shardingsphere.infra.database.type.dialect.OpenGaussDatabaseType;
-import
org.apache.shardingsphere.integration.data.pipeline.env.IntegrationTestEnvironment;
-import
org.apache.shardingsphere.integration.data.pipeline.env.enums.ScalingITEnvTypeEnum;
import org.apache.shardingsphere.test.integration.env.DataSourceEnvironment;
import
org.apache.shardingsphere.test.integration.framework.container.wait.JDBCConnectionWaitStrategy;
import org.testcontainers.containers.BindMode;
@@ -41,8 +39,11 @@ public final class OpenGaussContainer extends
DatabaseContainer {
private final int port = 5432;
+ private String dockerImageName = "";
+
public OpenGaussContainer(final String dockerImageName) {
super(DATABASE_TYPE, dockerImageName);
+ this.dockerImageName = dockerImageName;
}
@Override
@@ -52,9 +53,6 @@ public final class OpenGaussContainer extends
DatabaseContainer {
withClasspathResourceMapping("/env/postgresql/postgresql.conf",
"/usr/local/opengauss/share/postgresql/postgresql.conf.sample",
BindMode.READ_ONLY);
withPrivilegedMode(true);
withExposedPorts(port);
- if (ScalingITEnvTypeEnum.NATIVE ==
IntegrationTestEnvironment.getInstance().getItEnvType()) {
- addFixedExposedPort(port, port);
- }
setWaitStrategy(new JDBCConnectionWaitStrategy(() ->
DriverManager.getConnection(DataSourceEnvironment.getURL(DATABASE_TYPE,
"localhost", getFirstMappedPort(), "postgres"),
username, password)));
}
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/framework/container/database/PostgreSQLContainer.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/framework/container/database/PostgreSQLContainer.java
index a2aaa93b9ec..424a3f0a5fe 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/framework/container/database/PostgreSQLContainer.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/framework/container/database/PostgreSQLContainer.java
@@ -19,8 +19,6 @@ package
org.apache.shardingsphere.integration.data.pipeline.framework.container.
import org.apache.shardingsphere.infra.database.type.DatabaseType;
import
org.apache.shardingsphere.infra.database.type.dialect.PostgreSQLDatabaseType;
-import
org.apache.shardingsphere.integration.data.pipeline.env.IntegrationTestEnvironment;
-import
org.apache.shardingsphere.integration.data.pipeline.env.enums.ScalingITEnvTypeEnum;
import org.apache.shardingsphere.test.integration.env.DataSourceEnvironment;
import
org.apache.shardingsphere.test.integration.framework.container.wait.JDBCConnectionWaitStrategy;
import org.testcontainers.containers.BindMode;
@@ -49,9 +47,6 @@ public final class PostgreSQLContainer extends
DatabaseContainer {
addEnv("POSTGRES_PASSWORD", password);
withClasspathResourceMapping("/env/postgresql/postgresql.conf",
"/etc/postgresql/postgresql.conf", BindMode.READ_ONLY);
withExposedPorts(port);
- if (ScalingITEnvTypeEnum.NATIVE ==
IntegrationTestEnvironment.getInstance().getItEnvType()) {
- addFixedExposedPort(port, port);
- }
setWaitStrategy(new JDBCConnectionWaitStrategy(() ->
DriverManager.getConnection(DataSourceEnvironment.getURL(DATABASE_TYPE,
"localhost", getFirstMappedPort(), "postgres"),
username, password)));
}
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/common/command.xml
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/common/command.xml
index 0916aafc663..2d0c98c9ea5 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/common/command.xml
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/common/command.xml
@@ -32,21 +32,21 @@
</create-order-item-sharding-algorithm>
<create-order-table-rule>
- CREATE SHARDING TABLE RULE t_order (
- DATANODES("ds_${0..1}.t_order_${0..1}"),
-
DATABASE_STRATEGY(TYPE=standard,SHARDING_COLUMN=user_id,SHARDING_ALGORITHM=database_inline),
-
TABLE_STRATEGY(TYPE=standard,SHARDING_COLUMN=order_id,SHARDING_ALGORITHM=t_order_inline),
+ CREATE SHARDING TABLE RULE t_order(
+ RESOURCES(ds_0,ds_1),
+ SHARDING_COLUMN=order_id,
+ TYPE(NAME=hash_mod,PROPERTIES("sharding-count"=4)),
KEY_GENERATE_STRATEGY(COLUMN=order_id,TYPE(NAME=snowflake))
)
</create-order-table-rule>
<create-order-item-table-rule>
- CREATE SHARDING TABLE RULE t_order_item (
- DATANODES("ds_${0..1}.t_order_item_${0..1}"),
-
DATABASE_STRATEGY(TYPE=standard,SHARDING_COLUMN=user_id,SHARDING_ALGORITHM=database_inline),
-
TABLE_STRATEGY(TYPE=standard,SHARDING_COLUMN=order_id,SHARDING_ALGORITHM=t_order_item_inline),
+ CREATE SHARDING TABLE RULE t_order_item(
+ RESOURCES(ds_0,ds_1),
+ SHARDING_COLUMN=order_id,
+ TYPE(NAME=hash_mod,PROPERTIES("sharding-count"=4)),
KEY_GENERATE_STRATEGY(COLUMN=order_id,TYPE(NAME=snowflake))
- )
+ );
</create-order-item-table-rule>
<alter-sharding-algorithm>
@@ -54,20 +54,7 @@
(TYPE(NAME=INLINE,PROPERTIES("algorithm-expression"="ds_${user_id % 3
+ 2}")))
</alter-sharding-algorithm>
- <alter-sharding-table-rule>
- ALTER SHARDING TABLE RULE t_order (
- DATANODES("ds_${2..4}.t_order_${0..1}"),
-
DATABASE_STRATEGY(TYPE=standard,SHARDING_COLUMN=user_id,SHARDING_ALGORITHM=database_inline),
-
TABLE_STRATEGY(TYPE=standard,SHARDING_COLUMN=order_id,SHARDING_ALGORITHM=t_order_inline),
- KEY_GENERATE_STRATEGY(COLUMN=order_id,TYPE(NAME=snowflake))
- ), t_order_item (
- DATANODES("ds_${2..4}.t_order_item_${0..1}"),
-
DATABASE_STRATEGY(TYPE=standard,SHARDING_COLUMN=user_id,SHARDING_ALGORITHM=database_inline),
-
TABLE_STRATEGY(TYPE=standard,SHARDING_COLUMN=order_id,SHARDING_ALGORITHM=t_order_item_inline),
- KEY_GENERATE_STRATEGY(COLUMN=order_id,TYPE(NAME=snowflake)))
- </alter-sharding-table-rule>
-
- <auto-alter-order-with-item-sharding-table-rule>
+ <alter-order-with-item-auto-table-rule>
ALTER SHARDING TABLE RULE t_order(
RESOURCES(ds_2, ds_3, ds_4),
SHARDING_COLUMN=order_id,
@@ -79,16 +66,16 @@
TYPE(NAME=hash_mod,PROPERTIES("sharding-count"=6)),
KEY_GENERATE_STRATEGY(COLUMN=order_id,TYPE(NAME=snowflake))
)
- </auto-alter-order-with-item-sharding-table-rule>
+ </alter-order-with-item-auto-table-rule>
- <auto-alter-order-sharding-table-rule>
+ <alter-order-auto-table-rule>
ALTER SHARDING TABLE RULE t_order(
RESOURCES(ds_2, ds_3, ds_4),
SHARDING_COLUMN=order_id,
TYPE(NAME=hash_mod,PROPERTIES("sharding-count"=6)),
KEY_GENERATE_STRATEGY(COLUMN=order_id,TYPE(NAME=snowflake))
)
- </auto-alter-order-sharding-table-rule>
+ </alter-order-auto-table-rule>
<source-add-resource-template>
ADD RESOURCE ds_0 (
