This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino-playground.git
The following commit(s) were added to refs/heads/main by this push:
new d38a70e [#102] improvement: improve startup script (#103)
d38a70e is described below
commit d38a70e3ede4f22bfc1d3229d1919db5d5059984
Author: JUN <oren....@gmail.com>
AuthorDate: Wed Jan 8 12:22:00 2025 +0800
[#102] improvement: improve startup script (#103)
### What changes were proposed in this pull request?
1. Removed the debug option `set -x`.
2. Fixed warnings identified by IntelliJ, such as changing `cd
"${playground_dir}" >/dev/null` to `cd "${playground_dir}" >/dev/null ||
exit 1` to prevent failures when `cd` fails.
3. Updated `docker run --pull always hello-world >/dev/null 2>&1` to
`docker run --rm --pull always hello-world:latest >/dev/null 2>&1` to
avoid pulling multiple images and ensure the container is removed after
it finishes.
4. Changed naming conventions and log formats for consistency.
5. Added disk, RAM, and CPU checks.
6. Added the `--skip-checks` option for a quicker startup.
7. Corrected the log suffix format from `%Y%m%d%H%m%s` to
`%Y%m%d%H%M%s`.
8. Removed the unnecessary `Confirm the requirement is available in your
OS [Y/n]:`.
9. Combined port check status into one line instead of multiple lines.
### Why are the changes needed?
To improve the user experience.
fix #102
fix #113
### Does this PR introduce _any_ user-facing change?
Added a new option `--skip-checks|-s` for a quicker startup.
### How was this patch tested?
Tested with the following commands:
- `./playground start`
- `./playground status`
- `./playground stop`
- `./playground start --skip-checks`
- `./playground start -s`
These tests were run to ensure the script works as intended.
---
README.md | 19 +++---
playground.sh | 210 +++++++++++++++++++++++++++++++++++++++++-----------------
2 files changed, 158 insertions(+), 71 deletions(-)
diff --git a/README.md b/README.md
index 13c3f13..d167063 100644
--- a/README.md
+++ b/README.md
@@ -48,8 +48,6 @@ The playground runs several services. The TCP ports used may
clash with existing
## Playground usage
-
-
### One curl command launch playground
```shell
/bin/bash -c "$(curl -fsSL
https://raw.githubusercontent.com/apache/gravitino-playground/HEAD/install.sh)"
@@ -64,15 +62,16 @@ cd gravitino-playground
### Start
-```
+```shell
./playground.sh start
```
### Check status
-```shell
+```shell
./playground.sh status
```
-### Stop playground
+
+### Stop
```shell
./playground.sh stop
```
@@ -85,12 +84,12 @@ cd gravitino-playground
```shell
docker exec -it playground-trino bash
-````
+```
2. Open the Trino CLI in the container.
```shell
-trino@container_id:/$ trino
+trino
```
## Using Jupyter Notebook
@@ -112,7 +111,7 @@ docker exec -it playground-spark bash
2. Open the Spark SQL client in the container.
```shell
-spark@container_id:/$ cd /opt/spark && /bin/bash bin/spark-sql
+cd /opt/spark && /bin/bash bin/spark-sql
```
## Monitoring Gravitino
@@ -269,7 +268,7 @@ insert into customers (customer_id, customer_name,
customer_email) values (12,'J
```
2. Login Trino container and execute the steps.
-You can get all the customers from both the Hive and Iceberg table.
+ You can get all the customers from both the Hive and Iceberg table.
```shell
docker exec -it playground-trino bash
@@ -324,7 +323,7 @@ Gravitino supports to provide the ability of access control
for Hive tables usin
For example, there are a manager and staffs in your company. Manager creates a
Hive catalog and create different roles.
The manager can give different roles to different staffs.
-You can run the command
+You can run the command
```shell
./playground.sh start --enable-ranger
diff --git a/playground.sh b/playground.sh
index e354729..e26c111 100755
--- a/playground.sh
+++ b/playground.sh
@@ -18,23 +18,31 @@
# under the License.
#
-set -x
-
playground_dir="$(dirname "${BASH_SOURCE-$0}")"
playground_dir="$(
- cd "${playground_dir}" >/dev/null
+ cd "${playground_dir}" >/dev/null || exit 1
pwd
)"
+playgroundRuntimeName="gravitino-playground"
+requiredDiskSpaceGB=25
+requiredRamGB=8
+requiredCpuCores=2
+requiredPorts=(6080 8090 9001 3307 19000 19083 60070 13306 15342 18080 18888
19090 13000)
+dockerComposeCommand=""
+
testDocker() {
- echo "INFO: Testing Docker environment by running hello-world..."
- docker run --pull always hello-world:latest >/dev/null 2>&1
+ echo "[INFO] Testing Docker environment by running hello-world..."
+ # Use `always` to test network connection
+ docker run --rm --pull always hello-world:linux >/dev/null 2>&1
+
if [ $? -eq 0 ]; then
- echo "INFO: Docker is working correctly!"
+ echo "[INFO] Docker check passed: Docker is working correctly!"
else
- echo "ERROR: There was an issue running the hello-world container. Please
check your Docker installation."
+ echo "[ERROR] Docker check failed: There was an issue running the
hello-world container. Please check your Docker installation."
exit 1
fi
+
for containerId in $(docker ps -a | grep hello-world | awk '{print $1}'); do
docker rm $containerId
done
@@ -43,97 +51,177 @@ testDocker() {
done
}
-checkCompose() {
- isExist=$(which docker-compose)
- if [ $isExist ]; then
- true # Placeholder, do nothing
+checkDockerCompose() {
+ dockerComposeCommand=""
+ if command -v docker >/dev/null 2>&1 && command -v docker compose >/dev/null
2>&1; then
+ dockerComposeCommand="docker compose"
+ elif command -v docker-compose >/dev/null 2>&1; then
+ dockerComposeCommand="docker-compose"
+ else
+ echo "[ERROR] Docker compose check failed: There was an issue running the
docker compose command. Please check your docker compose installation."
+ exit 1
+ fi
+ echo "[INFO] Docker compose check passed: Docker compose is working
correctly using ${dockerComposeCommand} command!"
+}
+
+checkPlaygroundNotRunning() {
+ if ${dockerComposeCommand} ls | grep -q "${playgroundRuntimeName}"; then
+ echo "[ERROR] Playground runtime is already running. Please stop it first."
+ exit 1
+ fi
+}
+
+checkPlaygroundRunning() {
+ if ! ${dockerComposeCommand} ls | grep -q "${playgroundRuntimeName}"; then
+ echo "[ERROR] Playground runtime is not running. Please start it first."
+ exit 1
+ fi
+}
+
+checkDockerDisk() {
+ # Step 1: Get Docker Root Directory
+ local dockerRootDir="$(docker info 2>/dev/null | grep "Docker Root Dir" |
awk '{print $NF}')"
+
+ # Step 2: Check if the Docker directory exists
+ if [ -z "${dockerRootDir}" ]; then
+ echo "[ERROR] Docker disk check failed: Docker is not running or Docker
Root Directory not found."
+ exit 1
+ fi
+
+ local availableSpaceKB
+
+ if [ -d "${dockerRootDir}" ]; then
+ # Check available space in the Docker directory's partition
+ availableSpaceKB=$(df -k "${dockerRootDir}" | awk 'NR==2 {print $4}')
+ else
+ # Check available space in the root partition if the directory doesn't
exist (special case for WSL)
+ availableSpaceKB=$(df -k / | awk 'NR==2 {print $4}')
+ fi
+
+ # Step 3: Check if available space is greater than required
+ local availableSpaceGB=$((${availableSpaceKB} / 1024 / 1024))
+
+ if [ "${availableSpaceGB}" -ge "${requiredDiskSpaceGB}" ]; then
+ echo "[INFO] Docker disk check passed: ${availableSpaceGB} GB available."
else
- echo "ERROR: No docker service environment found. Please install
docker-compose."
- exit
+ echo "[ERROR] Docker disk check failed: Only ${availableSpaceGB} GB
available, required ${requiredDiskSpaceGB} GB or more."
+ exit 1
fi
}
-checkPortInUse() {
- local port=$1
- if [[ "$(uname)" == "Darwin" ]]; then
- openPort=$(lsof -i :$port -sTCP:LISTEN)
- elif [[ "$(uname)" == "Linux" ]]; then
- echo "Checking ports with sudo permission ..."
- openPort=$(sudo lsof -i :$port -sTCP:LISTEN)
+checkDockerRam() {
+ local totalRamBytes=$(docker info --format '{{.MemTotal}}')
+ # Convert from bytes to GB
+ local totalRamGB=$((totalRamBytes / 1024 / 1024 / 1024))
+
+ if [ "${totalRamGB}" -ge "${requiredRamGB}" ]; then
+ echo "[INFO] Docker RAM check passed: ${totalRamGB} GB available."
+ else
+ echo "[ERROR] Docker RAM check failed: Only ${totalRamGB} GB available,
required ${requiredRamGB} GB or more."
+ exit 1
fi
- if [ -z "${openPort}" ]; then
- echo "INFO: Port $port is ok."
+}
+
+checkDockerCpu() {
+ local cpuCores=$(docker info --format '{{.NCPU}}')
+
+ if [ "${cpuCores}" -ge "${requiredCpuCores}" ]; then
+ echo "[INFO] Docker CPU check passed: ${cpuCores} cores available."
else
- echo "ERROR: Port $port is in use. Please check it."
+ echo "[ERROR] Docker CPU check failed: Only ${cpuCores} cores available,
required ${requiredCpuCores} cores or more."
+ exit 1
+ fi
+}
+
+checkPortsInUse() {
+ local usedPorts=()
+ local availablePorts=()
+
+ for port in "${requiredPorts[@]}"; do
+ if [[ "$(uname)" == "Darwin" ]]; then
+ openPort=$(lsof -i :${port} -sTCP:LISTEN)
+ # Use sudo only when necessary
+ elif [[ "$(uname)" == "Linux" ]]; then
+ openPort=$(sudo lsof -i :${port} -sTCP:LISTEN)
+ fi
+
+ if [ -z "${openPort}" ]; then
+ availablePorts+=("${port}")
+ else
+ usedPorts+=("${port}")
+ fi
+ done
+
+ echo "[INFO] Port status check results:"
+
+ if [ ${#availablePorts[@]} -gt 0 ]; then
+ echo "[INFO] Available ports: ${availablePorts[*]}"
+ fi
+
+ if [ ${#usedPorts[@]} -gt 0 ]; then
+ echo "[ERROR] Ports in use: ${usedPorts[*]}"
+ echo "[ERROR] Please check these ports."
exit 1
fi
}
start() {
- echo "INFO: Starting the playground..."
+ if [ "${enableRanger}" == true ]; then
+ echo "[INFO] Starting the playground with Ranger..."
+ else
+ echo "[INFO] Starting the playground..."
+ fi
+
+ echo "[INFO] The playground requires ${requiredCpuCores} CPU cores,
${requiredRamGB} GB of RAM, and ${requiredDiskSpaceGB} GB of disk storage to
operate efficiently."
+ checkPortsInUse
testDocker
- checkCompose
- ports=(8090 9001 3307 19000 19083 60070 13306 15342 18080 18888 19090 13000)
- for port in "${ports[@]}"; do
- checkPortInUse ${port}
- done
+ checkDockerCompose
+ checkPlaygroundNotRunning
+ checkDockerDisk
+ checkDockerRam
+ checkDockerCpu
cd ${playground_dir}
- echo "Preparing packages..."
+ echo "[INFO] Preparing packages..."
./init/spark/spark-dependency.sh
./init/gravitino/gravitino-dependency.sh
./init/jupyter/jupyter-dependency.sh
- logSuffix=$(date +%Y%m%d%H%m%s)
- if [ "$enableRanger" == true ]; then
- docker-compose -f docker-compose.yaml -f
docker-enable-ranger-hive-override.yaml up --detach
+ logSuffix=$(date +%Y%m%d%H%M%s)
+ if [ "${enableRanger}" == true ]; then
+ ${dockerComposeCommand} -f docker-compose.yaml -f
docker-enable-ranger-hive-override.yaml -p ${playgroundRuntimeName} up --detach
else
- docker-compose up --detach
+ ${dockerComposeCommand} -p ${playgroundRuntimeName} up --detach
fi
-
- docker compose logs -f >${playground_dir}/playground-${logSuffix}.log 2>&1 &
- echo "Check log details: ${playground_dir}/playground-${logSuffix}.log"
+ ${dockerComposeCommand} -p ${playgroundRuntimeName} logs -f
>${playground_dir}/playground-${logSuffix}.log 2>&1 &
+ echo "[INFO] Check log details:
${playground_dir}/playground-${logSuffix}.log"
}
status() {
- docker-compose ps -a
+ checkDockerCompose
+ checkPlaygroundRunning
+ ${dockerComposeCommand} ps -a
}
stop() {
- echo "INFO: Stopping the playground..."
+ checkDockerCompose
+ checkPlaygroundRunning
+ echo "[INFO] Stopping the playground..."
- docker-compose down
+ ${dockerComposeCommand} down
if [ $? -eq 0 ]; then
- echo "INFO: Playground stopped!"
+ echo "[INFO] Playground stopped!"
fi
}
case "$1" in
start)
- if [[ "$2" == "-y" ]]; then
- input="y"
- else
- echo "The playground requires 2 CPU cores, 8 GB of RAM, and 25 GB of disk
storage to operate efficiently."
- read -r -p "Confirm the requirement is available in your OS [Y/n]:" input
- fi
-
- if [[ "$2" == "--enable-ranger" || "$3" == "--enable-ranger" ]]; then
+ if [[ "$2" == "--enable-ranger" ]]; then
enableRanger=true
else
enableRanger=false
fi
-
- case $input in
- [yY][eE][sS] | [yY]) ;;
- [nN][oO] | [nN])
- exit 0
- ;;
- *)
- echo "ERROR: Invalid input!"
- exit 1
- ;;
- esac
start
;;
status)
@@ -143,7 +231,7 @@ stop)
stop
;;
*)
- echo "Usage: $0 [start | status | stop]"
+ echo "Usage: $0 <start|status|stop> [--enable-ranger]"
exit 1
;;
esac
