This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new e88d0fc75e PHOENIX-7702 Backward compatibility test framework support
for phoenix 5.2 and hbase 2.6 (#2287)
e88d0fc75e is described below
commit e88d0fc75e6dc48de1fa2d24de48753da36d119f
Author: Viraj Jasani <[email protected]>
AuthorDate: Thu Sep 18 10:40:09 2025 -0700
PHOENIX-7702 Backward compatibility test framework support for phoenix 5.2
and hbase 2.6 (#2287)
---
.../end2end/BackwardCompatibilityTestUtil.java | 141 +++++++++++++++------
.../it/resources/compatible_client_versions.json | 55 +++++++-
.../src/it/resources/scripts/execute_query.sh | 67 ++++++++--
3 files changed, 206 insertions(+), 57 deletions(-)
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityTestUtil.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityTestUtil.java
index 7b0d14870f..d72e415085 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityTestUtil.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityTestUtil.java
@@ -43,6 +43,7 @@ import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
+import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.regex.Matcher;
@@ -57,10 +58,15 @@ import
org.apache.phoenix.coprocessorclient.MetaDataProtocol;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.util.PropertiesUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
public final class BackwardCompatibilityTestUtil {
+
+ private static final Logger LOGGER =
LoggerFactory.getLogger(BackwardCompatibilityTestUtil.class);
+
public static final String SQL_DIR = "sql_files/";
public static final String RESULTS_AND_GOLD_FILES_DIR = "gold_files/";
public static final String COMPATIBLE_CLIENTS_JSON =
"compatible_client_versions.json";
@@ -115,7 +121,7 @@ public final class BackwardCompatibilityTestUtil {
private BackwardCompatibilityTestUtil() {
}
- public static List<MavenCoordinates> computeClientVersions() throws
Exception {
+ public static List<ClientVersionGroup> computeClientVersionGroups() throws
Exception {
String hbaseVersion = VersionInfo.getVersion();
Pattern p = Pattern.compile("\\d+\\.\\d+");
Matcher m = p.matcher(hbaseVersion);
@@ -123,7 +129,7 @@ public final class BackwardCompatibilityTestUtil {
if (m.find()) {
hbaseProfile = m.group();
}
- List<MavenCoordinates> clientVersions = Lists.newArrayList();
+ List<ClientVersionGroup> clientVersionGroups = Lists.newArrayList();
ObjectMapper mapper = new ObjectMapper();
mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
try (InputStream inputStream =
@@ -131,26 +137,41 @@ public final class BackwardCompatibilityTestUtil {
assertNotNull(inputStream);
JsonNode jsonNode = mapper.readTree(inputStream);
JsonNode HBaseProfile = jsonNode.get(hbaseProfile);
+ List<MavenCoordinates> artifacts = new ArrayList<>();
for (final JsonNode clientVersion : HBaseProfile) {
- clientVersions.add(mapper.treeToValue(clientVersion,
MavenCoordinates.class));
+ artifacts.add(mapper.treeToValue(clientVersion,
MavenCoordinates.class));
+ }
+ if (!artifacts.isEmpty()) {
+ MavenCoordinates primary = artifacts.get(0);
+ clientVersionGroups.add(new ClientVersionGroup(primary, artifacts));
}
}
- return clientVersions;
+ return clientVersionGroups;
}
- // Executes the queries listed in the operation file with a given client
version
- public static void executeQueryWithClientVersion(MavenCoordinates
clientVersion, String operation,
- String zkQuorum) throws Exception {
+ public static List<MavenCoordinates> computeClientVersions() throws
Exception {
+ List<ClientVersionGroup> groups = computeClientVersionGroups();
+ List<MavenCoordinates> result = new ArrayList<>();
+ for (ClientVersionGroup group : groups) {
+ result.add(group.getPrimaryArtifact());
+ }
+ return result;
+ }
+
+ public static void executeQueryWithClientVersionGroup(ClientVersionGroup
clientVersionGroup,
+ String operation, String zkQuorum) throws Exception {
List<String> cmdParams = Lists.newArrayList();
cmdParams.add(BASH);
- // Note that auto-commit is true for queries executed via SQLline
URL fileUrl =
BackwardCompatibilityIT.class.getClassLoader().getResource(EXECUTE_QUERY_SH);
assertNotNull(fileUrl);
cmdParams.add(new File(fileUrl.getFile()).getAbsolutePath());
cmdParams.add(zkQuorum);
- cmdParams.add(clientVersion.getGroupId());
- cmdParams.add(clientVersion.getArtifactId());
- cmdParams.add(clientVersion.getVersion());
+ cmdParams.add(String.valueOf(clientVersionGroup.getAllArtifacts().size()));
+ for (MavenCoordinates artifact : clientVersionGroup.getAllArtifacts()) {
+ cmdParams.add(artifact.getGroupId());
+ cmdParams.add(artifact.getArtifactId());
+ cmdParams.add(artifact.getVersion());
+ }
fileUrl = BackwardCompatibilityIT.class.getClassLoader()
.getResource(SQL_DIR + operation + SQL_EXTENSION);
@@ -171,44 +192,34 @@ public final class BackwardCompatibilityTestUtil {
ProcessBuilder pb = new ProcessBuilder(cmdParams);
final Process p = pb.start();
final StringBuffer sb = new StringBuffer();
- // Capture the output stream if any from the execution of the script
- Thread outputStreamThread = new Thread() {
- @Override
- public void run() {
- try (
- BufferedReader reader = new BufferedReader(new
InputStreamReader(p.getInputStream()))) {
- String line;
- while ((line = reader.readLine()) != null) {
- sb.append(line);
- }
- } catch (final Exception e) {
- sb.append(e.getMessage());
+ Thread outputStreamThread = new Thread(() -> {
+ try (BufferedReader reader = new BufferedReader(new
InputStreamReader(p.getInputStream()))) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ sb.append(line);
}
+ } catch (final Exception e) {
+ sb.append(e.getMessage());
}
- };
+ });
outputStreamThread.start();
- // Capture the error stream if any from the execution of the script
- Thread errorStreamThread = new Thread() {
- @Override
- public void run() {
- try (
- BufferedReader reader = new BufferedReader(new
InputStreamReader(p.getErrorStream()))) {
- String line;
- while ((line = reader.readLine()) != null) {
- sb.append(line);
- }
- } catch (final Exception e) {
- sb.append(e.getMessage());
+ Thread errorStreamThread = new Thread(() -> {
+ try (BufferedReader reader = new BufferedReader(new
InputStreamReader(p.getErrorStream()))) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ sb.append(line);
}
+ } catch (final Exception e) {
+ sb.append(e.getMessage());
}
- };
+ });
errorStreamThread.start();
p.waitFor();
String resultDump = "PLACEHOLDER. Could not read result file";
try {
resultDump = new String(Files.readAllBytes(Paths.get(resultFilePath)));
} catch (Exception e) {
- // We return the placeholder
+ LOGGER.error(resultFilePath + " could not be read", e);
}
assertEquals(
String.format(
@@ -217,6 +228,31 @@ public final class BackwardCompatibilityTestUtil {
0, p.exitValue());
}
+ public static void executeQueryWithClientVersion(MavenCoordinates
clientVersion, String operation,
+ String zkQuorum) throws Exception {
+ List<ClientVersionGroup> groups = computeClientVersionGroups();
+ ClientVersionGroup targetGroup = null;
+
+ for (ClientVersionGroup group : groups) {
+ if (
+
group.getPrimaryArtifact().getArtifactId().equals(clientVersion.getArtifactId())
+ &&
group.getPrimaryArtifact().getVersion().equals(clientVersion.getVersion())
+ ) {
+ targetGroup = group;
+ break;
+ }
+ }
+
+ if (targetGroup != null) {
+ executeQueryWithClientVersionGroup(targetGroup, operation, zkQuorum);
+ } else {
+ List<MavenCoordinates> artifacts = Lists.newArrayList();
+ artifacts.add(clientVersion);
+ ClientVersionGroup group = new ClientVersionGroup(clientVersion,
artifacts);
+ executeQueryWithClientVersionGroup(group, operation, zkQuorum);
+ }
+ }
+
public static void checkForPreConditions(MavenCoordinates
compatibleClientVersion,
Configuration conf) throws Exception {
// For the first code cut of any major version, there wouldn't be any
backward compatible
@@ -384,4 +420,33 @@ public final class BackwardCompatibilityTestUtil {
}
}
+
+ /**
+ * Represents a group of Maven artifacts that should be used together for a
client version
+ */
+ public static class ClientVersionGroup {
+
+ private final MavenCoordinates primaryArtifact;
+ private final List<MavenCoordinates> allArtifacts;
+
+ public ClientVersionGroup(MavenCoordinates primaryArtifact,
+ List<MavenCoordinates> allArtifacts) {
+ this.primaryArtifact = primaryArtifact;
+ this.allArtifacts = allArtifacts;
+ }
+
+ public MavenCoordinates getPrimaryArtifact() {
+ return primaryArtifact;
+ }
+
+ public List<MavenCoordinates> getAllArtifacts() {
+ return allArtifacts;
+ }
+
+ @Override
+ public String toString() {
+ return "ClientVersionGroup{primary=" + primaryArtifact + ", artifacts="
+ allArtifacts.size()
+ + "}";
+ }
+ }
}
diff --git a/phoenix-core/src/it/resources/compatible_client_versions.json
b/phoenix-core/src/it/resources/compatible_client_versions.json
index ecfa8b017a..e48386a44c 100644
--- a/phoenix-core/src/it/resources/compatible_client_versions.json
+++ b/phoenix-core/src/it/resources/compatible_client_versions.json
@@ -1,9 +1,50 @@
{
- "_comment": "Lists all phoenix compatible client versions against the
current branch version for a given hbase profile If hbase profile is 1.3,
phoenix client versions 4.14.3 and 4.15.0 are tested against current branch
version",
- "2.1": [ {"artifactId":"phoenix-client-hbase-2.1", "version":"5.1.0"} ],
- "2.2": [ {"artifactId":"phoenix-client-hbase-2.2", "version":"5.1.0"} ],
- "2.3": [ {"artifactId":"phoenix-client-hbase-2.3", "version":"5.1.0"} ],
- "2.4": [ {"artifactId":"phoenix-client-hbase-2.4", "version":"5.1.2"} ],
- "2.5": [ {"artifactId":"phoenix-client-hbase-2.5", "version":"5.1.3"} ],
- "2.6": [ ]
+ "_comment": "Lists all phoenix compatible client versions against the
current branch version for a given hbase profile.",
+ "2.4": [
+ {
+ "artifactId": "phoenix-client-hbase-2.4",
+ "version": "5.1.2"
+ }
+ ],
+ "2.5": [
+ {
+ "artifactId": "phoenix-client-hbase-2.5",
+ "version": "5.1.3"
+ }
+ ],
+ "2.6": [
+ {
+ "artifactId": "phoenix-client-embedded-hbase-2.6",
+ "version": "5.2.1"
+ },
+ {
+ "artifactId": "phoenix-core-client",
+ "version": "5.2.1"
+ },
+ {
+ "artifactId": "sqlline",
+ "version": "1.9.0",
+ "groupId": "sqlline"
+ },
+ {
+ "artifactId": "jline-terminal",
+ "version": "3.12.1",
+ "groupId": "org.jline"
+ },
+ {
+ "artifactId": "jline-reader",
+ "version": "3.12.1",
+ "groupId": "org.jline"
+ },
+ {
+ "artifactId": "jline-builtins",
+ "version": "3.12.1",
+ "groupId": "org.jline"
+ },
+ {
+ "artifactId": "log4j-1.2-api",
+ "version": "2.18.0",
+ "groupId": "org.apache.logging.log4j"
+ }
+ ]
}
diff --git a/phoenix-core/src/it/resources/scripts/execute_query.sh
b/phoenix-core/src/it/resources/scripts/execute_query.sh
index 116851cf29..393a4fe684 100644
--- a/phoenix-core/src/it/resources/scripts/execute_query.sh
+++ b/phoenix-core/src/it/resources/scripts/execute_query.sh
@@ -20,13 +20,27 @@
# This script is intended to run the sql queries in a file with the given
client version
zk_url=$1
-client_group_id=$2
-client_artifact_id=$3
-client_version=$4
-sqlfile=$5
-resultfile=$6
-tmp_dir=$7
-maven_home=$8
+artifact_count=$2
+
+artifacts=()
+param_index=3
+for ((i=0; i<artifact_count; i++)); do
+ group_id=${!param_index}
+ ((param_index++))
+ artifact_id=${!param_index}
+ ((param_index++))
+ version=${!param_index}
+ ((param_index++))
+ artifacts+=("$group_id:$artifact_id:$version")
+done
+
+sqlfile=${!param_index}
+((param_index++))
+resultfile=${!param_index}
+((param_index++))
+tmp_dir=${!param_index}
+((param_index++))
+maven_home=${!param_index}
if [ -n $maven_home ]; then
export PATH=$maven_home/bin:$PATH
@@ -37,12 +51,41 @@ if [ -z ${error_code+x} ]; then
security_manager='-Djava.security.manager=allow'
fi
-mvn -B dependency:get
-Dartifact=${client_group_id}:${client_artifact_id}:${client_version}
-mvn -B dependency:copy
-Dartifact=${client_group_id}:${client_artifact_id}:${client_version} \
--DoutputDirectory=$tmp_dir
+echo "Downloading ${#artifacts[@]} artifacts..."
+for artifact in "${artifacts[@]}"; do
+ echo "Downloading $artifact"
+ mvn -B dependency:get -Dartifact=$artifact
+ if [ $? -ne 0 ]; then
+ echo "Failed to download artifact $artifact"
+ exit 1
+ fi
+
+ mvn -B dependency:copy -Dartifact=$artifact -DoutputDirectory=$tmp_dir
+ if [ $? -ne 0 ]; then
+ echo "Failed to copy artifact $artifact"
+ exit 1
+ fi
+done
+
+classpath="."
+for jar in $tmp_dir/*.jar; do
+ classpath="$classpath:$jar"
+done
+
+echo "Using classpath: $classpath"
+
+phoenix_props="-Dphoenix.query.timeoutMs=60000
-Dphoenix.query.keepAliveMs=60000"
+
+phoenix_props="$phoenix_props -Dlog4j.logger.org.apache.phoenix=DEBUG"
+phoenix_props="$phoenix_props -Dlog4j.logger.org.apache.hadoop.hbase=DEBUG"
+phoenix_props="$phoenix_props -Dlog4j.logger.sqlline=DEBUG"
+phoenix_props="$phoenix_props -Dlog4j.logger.org.apache.zookeeper=INFO"
+
+echo "Attempting to connect to Phoenix with URL: jdbc:phoenix:$zk_url"
+echo "Using classpath: $classpath"
+echo "Phoenix properties: $phoenix_props"
-phoenix_client_jar=$tmp_dir/${client_artifact_id}-${client_version}.jar
-java -cp ".:$phoenix_client_jar" $security_manager sqlline.SqlLine \
+java -cp "$classpath" $security_manager $phoenix_props sqlline.SqlLine \
-d org.apache.phoenix.jdbc.PhoenixDriver -u jdbc:phoenix:$zk_url -n none -p
none \
--color=false --fastConnect=true --outputformat=csv \
--silent=true --verbose=false --isolation=TRANSACTION_READ_COMMITTED
--run=$sqlfile &> $resultfile
