This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 526114bce [#4263] fix(CI): Split Hive Ranger container instance in the
IT (#4267)
526114bce is described below
commit 526114bcee95896355faed3bf115ecbe40cef71a
Author: Xun Liu <x...@datastrato.com>
AuthorDate: Mon Jul 29 17:30:58 2024 +0800
[#4263] fix(CI): Split Hive Ranger container instance in the IT (#4267)
### What changes were proposed in this pull request?
Split the hive container and the hive container with the ranger plug-in
enabled.
### Why are the changes needed?
Fix: #4263
### Does this PR introduce _any_ user-facing change?
N/A
### How was this patch tested?
CI passed.
---
build.gradle.kts | 6 +--
gradle.properties | 2 +-
.../integration/test/container/ContainerSuite.java | 63 ++++++++++++++++++----
.../test/authorization/ranger/RangerHiveIT.java | 10 ++--
4 files changed, 62 insertions(+), 19 deletions(-)
diff --git a/build.gradle.kts b/build.gradle.kts
index 2fcba8a8b..d04fe1a0d 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -771,10 +771,10 @@ fun printDockerCheckInfo() {
}
println("------------------ Check Docker environment ---------------------")
- println("Docker server status ............................................
[${if (dockerRunning) "running" else "stop"}]")
+ println("Docker server status ............................................
[${if (dockerRunning) "running" else "\u001B[31mstop\u001B[0m"}]")
if (OperatingSystem.current().isMacOsX()) {
- println("mac-docker-connector status .....................................
[${if (macDockerConnector) "running" else "stop"}]")
- println("OrbStack status .................................................
[${if (dockerRunning && isOrbStack) "yes" else "no"}]")
+ println("mac-docker-connector status .....................................
[${if (macDockerConnector) "running" else "\u001B[31mstop\u001B[0m"}]")
+ println("OrbStack status .................................................
[${if (dockerRunning && isOrbStack) "yes" else "\u001B[31mno\u001B[0m"}]")
}
val dockerTest = project.extra["dockerTest"] as? Boolean ?: false
diff --git a/gradle.properties b/gradle.properties
index 5b36e4623..b4deeef33 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -40,4 +40,4 @@ defaultScalaVersion = 2.12
pythonVersion = 3.8
# skipDockerTests is used to skip the tests that require Docker to be running.
-skipDockerTests = false
+skipDockerTests = true
diff --git
a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java
b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java
index 4afaaeffd..00286d045 100644
---
a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java
+++
b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java
@@ -18,6 +18,8 @@
*/
package org.apache.gravitino.integration.test.container;
+import static
org.apache.gravitino.integration.test.container.RangerContainer.DOCKER_ENV_RANGER_SERVER_URL;
+
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.RemoveNetworkCmd;
import com.github.dockerjava.api.model.Info;
@@ -32,6 +34,7 @@ import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
+import java.util.Objects;
import org.apache.gravitino.integration.test.util.CloseableGroup;
import org.apache.gravitino.integration.test.util.TestDatabaseName;
import org.slf4j.Logger;
@@ -52,6 +55,9 @@ public class ContainerSuite implements Closeable {
private static Network network = null;
private static volatile HiveContainer hiveContainer;
+
+ // Enable the Ranger plugin in the Hive container
+ private static volatile HiveContainer hiveRangerContainer;
private static volatile TrinoContainer trinoContainer;
private static volatile TrinoITContainers trinoITContainers;
private static volatile RangerContainer rangerContainer;
@@ -98,29 +104,60 @@ public class ContainerSuite implements Closeable {
}
public void startHiveContainer() {
- startHiveContainer(
- ImmutableMap.<String, String>builder().put("HADOOP_USER_NAME",
"anonymous").build());
+ if (hiveContainer == null) {
+ synchronized (ContainerSuite.class) {
+ if (hiveContainer == null) {
+ // Start Hive container
+ HiveContainer.Builder hiveBuilder =
+ HiveContainer.builder()
+ .withHostName("gravitino-ci-hive")
+ .withEnvVars(
+ ImmutableMap.<String, String>builder()
+ .put("HADOOP_USER_NAME", "anonymous")
+ .build())
+ .withNetwork(network);
+ HiveContainer container = closer.register(hiveBuilder.build());
+ container.start();
+ hiveContainer = container;
+ }
+ }
+ }
}
/**
- * To start the Hive container, you can to specify environment variables:
HIVE_RUNTIME_VERSION:
- * Hive version, currently support `hive2`(default) and `hive3`
DOCKER_ENV_RANGER_SERVER_URL:
- * Ranger server URL DOCKER_ENV_RANGER_HIVE_REPOSITORY_NAME: Ranger Hive
repository name
- * DOCKER_ENV_RANGER_HDFS_REPOSITORY_NAME: Ranger HDFS repository name
+ * To start and enable Ranger plugin in Hive container, <br>
+ * you can specify environment variables: <br>
+ * 1. HIVE_RUNTIME_VERSION: Hive version, currently only support `hive3`, We
can support `hive2`
+ * in the future <br>
+ * 2. DOCKER_ENV_RANGER_SERVER_URL: Ranger server URL <br>
+ * 3. DOCKER_ENV_RANGER_HIVE_REPOSITORY_NAME: Ranger Hive repository name
<br>
+ * 4. DOCKER_ENV_RANGER_HDFS_REPOSITORY_NAME: Ranger HDFS repository name
<br>
*/
- public void startHiveContainer(Map<String, String> envVars) {
- if (hiveContainer == null) {
+ public void startHiveRangerContainer(Map<String, String> envVars) {
+ // If you want to enable Hive Ranger plugin, you need both set the
`RANGER_SERVER_URL` and
+ // `RANGER_HIVE_REPOSITORY_NAME` environment variables.
+ // If you want to enable HDFS Ranger plugin, you need both set the
`RANGER_SERVER_URL` and
+ // `RANGER_HDFS_REPOSITORY_NAME` environment variables.
+ if (envVars == null
+ || (!Objects.equals(envVars.get(HiveContainer.HIVE_RUNTIME_VERSION),
HiveContainer.HIVE3))
+ || (!envVars.containsKey(DOCKER_ENV_RANGER_SERVER_URL)
+ ||
(!envVars.containsKey(RangerContainer.DOCKER_ENV_RANGER_HIVE_REPOSITORY_NAME)
+ &&
!envVars.containsKey(RangerContainer.DOCKER_ENV_RANGER_HDFS_REPOSITORY_NAME))))
{
+ throw new IllegalArgumentException("Error environment variables for Hive
Ranger container");
+ }
+
+ if (hiveRangerContainer == null) {
synchronized (ContainerSuite.class) {
- if (hiveContainer == null) {
+ if (hiveRangerContainer == null) {
// Start Hive container
HiveContainer.Builder hiveBuilder =
HiveContainer.builder()
- .withHostName("gravitino-ci-hive")
+ .withHostName("gravitino-ci-hive-ranger")
.withEnvVars(envVars)
.withNetwork(network);
HiveContainer container = closer.register(hiveBuilder.build());
container.start();
- hiveContainer = container;
+ hiveRangerContainer = container;
}
}
}
@@ -327,6 +364,10 @@ public class ContainerSuite implements Closeable {
return hiveContainer;
}
+ public HiveContainer getHiveRangerContainer() {
+ return hiveRangerContainer;
+ }
+
public void startRangerContainer() {
if (rangerContainer == null) {
synchronized (ContainerSuite.class) {
diff --git
a/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/RangerHiveIT.java
b/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/RangerHiveIT.java
index 19e84f9d8..db861699e 100644
---
a/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/RangerHiveIT.java
+++
b/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/RangerHiveIT.java
@@ -51,7 +51,7 @@ public class RangerHiveIT extends RangerIT {
public static void setup() {
RangerIT.setup();
- containerSuite.startHiveContainer(
+ containerSuite.startHiveRangerContainer(
new HashMap<>(
ImmutableMap.of(
HiveContainer.HIVE_RUNTIME_VERSION,
@@ -68,8 +68,10 @@ public class RangerHiveIT extends RangerIT {
HiveContainer.HADOOP_USER_NAME,
adminUser)));
-
createRangerHdfsRepository(containerSuite.getHiveContainer().getContainerIpAddress(),
true);
-
createRangerHiveRepository(containerSuite.getHiveContainer().getContainerIpAddress(),
true);
+ createRangerHdfsRepository(
+ containerSuite.getHiveRangerContainer().getContainerIpAddress(), true);
+ createRangerHiveRepository(
+ containerSuite.getHiveRangerContainer().getContainerIpAddress(), true);
allowAnyoneAccessHDFS();
allowAnyoneAccessInformationSchema();
@@ -77,7 +79,7 @@ public class RangerHiveIT extends RangerIT {
String url =
String.format(
"jdbc:hive2://%s:%d/default",
- containerSuite.getHiveContainer().getContainerIpAddress(),
+ containerSuite.getHiveRangerContainer().getContainerIpAddress(),
HiveContainer.HIVE_SERVICE_PORT);
try {
Class.forName("org.apache.hive.jdbc.HiveDriver");
