This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino-playground.git
The following commit(s) were added to refs/heads/main by this push:
new ab9dcfa [#85] Enhance playground stability and user experience (#87)
ab9dcfa is described below
commit ab9dcfaa6cb9bd429392bfc0f2a0ad9510b93619
Author: danhuawang <154112360+danhuaw...@users.noreply.github.com>
AuthorDate: Fri Oct 25 00:11:03 2024 +0800
[#85] Enhance playground stability and user experience (#87)
1. add health check: start_period
2. remove resource limit ,add resource requirements in readme
3. improve console output ,persist log file
4. add precheck for ports in use
5. add precheck for docker if works in current user
6. fix Trino unhealthy issue
7. replace folder /user/datastrato to /user/gravitino
8. fix /user/datastrato not exists
9. fix the warning: "MYSQL_ROOT_PASSWORD" is empty string
10. fix the 'version' warning message in console
11. add confirmation for resources requirements
12. change playground usage: ./playground.sh [start | status | stop]
[-y]
<img width="763" alt="image"
src="https://github.com/user-attachments/assets/44a9327d-d33d-4a51-b5d2-bb8581aeafbe";>
<img width="1576" alt="image"
src="https://github.com/user-attachments/assets/bcda5375-e381-4da7-b9d1-038c56e98bba";>
<img width="1851" alt="image"
src="https://github.com/user-attachments/assets/279f1f79-03cf-4af0-be68-e837cdf17f94";>
<img width="1050" alt="image"
src="https://github.com/user-attachments/assets/10dd7485-d1d0-497e-a57b-4d34925ae3f9";>
<img width="799" alt="image"
src="https://github.com/user-attachments/assets/64149bd5-79e1-4bc1-8384-7341cfb3d8d6";>
<img width="1839" alt="image"
src="https://github.com/user-attachments/assets/c9f3344b-27c8-448d-9a9c-b43ebc2a40fd";>
---
README.md | 26 +++--
docker-compose.yaml | 60 +++-------
healthcheck/gravitino-healthcheck.sh | 4 +-
init/common/common.sh | 58 ++++++++++
init/common/init_metalake_catalog.sh | 4 +-
.../gravitino/gravitino-dependency.sh | 37 +++---
init/gravitino/init.sh | 8 +-
init/hive/init.sh | 4 +-
init/jupyter/gravitino-fileset-example.ipynb | 10 +-
init/jupyter/gravitino-spark-trino-example.ipynb | 2 +-
init/spark/init.sh | 9 +-
init/spark/spark-dependency.sh | 46 ++++++++
init/trino/init.sh | 11 +-
playground.sh | 125 +++++++++++++++++++++
14 files changed, 302 insertions(+), 102 deletions(-)
diff --git a/README.md b/README.md
index c365b5b..a81d174 100644
--- a/README.md
+++ b/README.md
@@ -25,7 +25,11 @@ Depending on your network and computer, startup time may
take 3-5 minutes. Once
## Prerequisites
-Install Git and Docker Compose.
+Install Git, Docker, Docker Compose.
+
+## System Resource Requirements
+
+2 CPU cores, 8 GB RAM, 25 GB disk storage, MacOS or Linux OS (Verified
Ubuntu22.04 Ubuntu24.04 AmazonLinux).
## TCP ports used
@@ -40,25 +44,25 @@ The playground runs a number of services. The TCP ports
used may clash with exis
| playground-trino | 18080 |
| playground-jupyter | 18888 |
-## Start playground
+## Playground usage
+
+### Launch playground
-### Launch all components of playground
```shell
git clone g...@github.com:apache/gravitino-playground.git
cd gravitino-playground
-./launch-playground.sh
+./playground.sh start
```
-### Launch special component or components of playground
+### Check status
+```shell
+./playground.sh status
+```
+### Stop playground
```shell
-git clone g...@github.com:apache/gravitino-playground.git
-cd gravitino-playground
-./launch-playground.sh hive|gravitino|trino|postgresql|mysql|spark|jupyter
+./playground.sh stop
```
-Note. Components have dependencies, only launching one or several components
cannot experience
-the full functionality of the playground.
-
## Experiencing Apache Gravitino with Trino SQL
### Using Trino CLI in Docker Container
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 0b7e1e2..a4f2964 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -16,7 +16,6 @@
# specific language governing permissions and limitations
# under the License.
#
-version: '3.0'
services:
hive:
image: apache/gravitino-playground:hive-2.7.3
@@ -36,11 +35,7 @@ services:
interval: 10s
timeout: 60s
retries: 5
- deploy:
- resources:
- limits:
- cpus: "3"
- memory: 3G
+ start_period: 20s
gravitino:
image: apache/gravitino:0.6.1-incubating
@@ -57,16 +52,12 @@ services:
volumes:
- ./healthcheck:/tmp/healthcheck
- ./init/gravitino:/tmp/gravitino
- deploy:
- resources:
- limits:
- cpus: "0.5"
- memory: 500M
healthcheck:
test: ["CMD", "/tmp/healthcheck/gravitino-healthcheck.sh"]
interval: 5s
timeout: 60s
retries: 5
+ start_period: 20s
trino:
image: apache/gravitino-playground:trino-435-gravitino-0.6.1-incubating
@@ -83,16 +74,18 @@ services:
volumes:
- ./init/trino:/tmp/trino
- ./init/common:/tmp/common
+ - ./healthcheck:/tmp/healthcheck
depends_on:
hive :
condition: service_healthy
gravitino :
condition: service_healthy
- deploy:
- resources:
- limits:
- cpus: "1"
- memory: 2G
+ healthcheck:
+ test: ["CMD", "/tmp/healthcheck/trino-healthcheck.sh"]
+ interval: 10s
+ timeout: 60s
+ retries: 5
+ start_period: 20s
postgresql:
image: postgres:13
@@ -106,40 +99,31 @@ services:
- "15432:5432"
volumes:
- ./init/postgres:/docker-entrypoint-initdb.d/
- deploy:
- resources:
- limits:
- cpus: "1"
- memory: 500M
mysql:
image: mysql:8.0
container_name: playground-mysql
+ environment:
+ - MYSQL_ROOT_PASSWORD=mysql
+ - MYSQL_USER=mysql
+ - MYSQL_PASSWORD=mysql
+ - MYSQL_DATABASE=db
ports:
- "13306:3306"
volumes:
- ./init/mysql:/docker-entrypoint-initdb.d/
- environment:
- MYSQL_ROOT_PASSWORD: mysql
- MYSQL_USER: mysql
- MYSQL_PASSWORD: mysql
- MYSQL_DATABASE: db
command:
--default-authentication-plugin=mysql_native_password
--character-set-server=utf8mb4
--collation-server=utf8mb4_general_ci
--explicit_defaults_for_timestamp=true
--lower_case_table_names=1
- deploy:
- resources:
- limits:
- cpus: "1"
- memory: 500M
healthcheck:
- test: ["CMD-SHELL", "mysqladmin ping -h localhost -p
${MYSQL_ROOT_PASSWORD}"]
+ test: ["CMD-SHELL", "mysqladmin ping -h localhost -pmysql"]
interval: 5s
timeout: 60s
retries: 5
+ start_period: 20s
spark:
image: spark:3.4.1-scala2.12-java11-ubuntu
@@ -153,11 +137,6 @@ services:
- ./init/spark:/tmp/spark
- ./init/common:/tmp/common
- spark_jars:/opt/spark/jars
- deploy:
- resources:
- limits:
- cpus: "1"
- memory: 1G
jupyter:
image: jupyter/pyspark-notebook:spark-3.4.1
@@ -173,11 +152,6 @@ services:
condition: service_healthy
gravitino :
condition: service_healthy
- deploy:
- resources:
- limits:
- cpus: "0.5"
- memory: 500M
volumes:
- spark_jars:
+ spark_jars:
\ No newline at end of file
diff --git a/healthcheck/gravitino-healthcheck.sh
b/healthcheck/gravitino-healthcheck.sh
index 4f9e35f..eb324b7 100755
--- a/healthcheck/gravitino-healthcheck.sh
+++ b/healthcheck/gravitino-healthcheck.sh
@@ -25,7 +25,7 @@ success=false
while [ $attempt -lt $max_attempts ]; do
response=$(curl -X GET -H "Content-Type: application/json"
http://127.0.0.1:8090/api/version)
-
+
if echo "$response" | grep -q "\"code\":0"; then
success=true
break
@@ -33,7 +33,7 @@ while [ $attempt -lt $max_attempts ]; do
echo "Attempt $((attempt + 1)) failed..."
sleep 1
fi
-
+
((attempt++))
done
diff --git a/init/common/common.sh b/init/common/common.sh
new file mode 100644
index 0000000..49ed6f3
--- /dev/null
+++ b/init/common/common.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+download_and_verify() {
+ local jar_url=$1
+ local md5_url=$2
+ local download_dir=$3
+ local jar_file=$(basename "${jar_url}")
+ local md5_file="${jar_file}.md5"
+ echo "Downloading ${jar_file} to ${download_dir}/packages"
+ # If md5 file doesn't exist, then download it
+ if [ ! -f "${download_dir}/packages/${md5_file}" ]; then
+ curl -L -o "${download_dir}/packages/${md5_file}" "${md5_url}"
+ fi
+
+ # If jar file doesn't exist, then download it
+ if [ ! -f "${download_dir}/packages/${jar_file}" ]; then
+ curl -L -o "${download_dir}/packages/${jar_file}" "${jar_url}"
+ fi
+
+ local md5_command
+ if [[ "$(uname)" == "Darwin" ]]; then
+ # macOS
+ md5_command="md5 -q"
+ elif [[ "$(uname)" == "Linux" ]]; then
+ md5_command="md5sum"
+ else
+ break
+ fi
+
+ # Computer jar file md5
+ local_md5=$($md5_command "${download_dir}/packages/${jar_file}" | awk '{
print $1 }')
+ # Get md5 from md5 file
+ file_md5=$(cat "${download_dir}/packages/${md5_file}")
+
+ # Checksum verification
+ if [ "${local_md5}" != "${file_md5}" ]; then
+ echo "Use ${md5_file} to MD5 checksum ${jar_file} verification failed,
Please delete it."
+ exit 1
+ fi
+}
diff --git a/init/common/init_metalake_catalog.sh
b/init/common/init_metalake_catalog.sh
index 8a4a7be..9b2bed4 100644
--- a/init/common/init_metalake_catalog.sh
+++ b/init/common/init_metalake_catalog.sh
@@ -30,7 +30,6 @@ else
fi
fi
-
response=$(curl
http://gravitino:8090/api/metalakes/metalake_demo/catalogs/catalog_hive)
if echo "$response" | grep -q "\"code\":0"; then
true
@@ -65,7 +64,7 @@ if echo "$response" | grep -q "\"code\":0"; then
else
# Create Mysql catalog for experience Gravitino service
response=$(curl -X POST -H "Accept: application/vnd.gravitino.v1+json" -H
"Content-Type: application/json" -d '{ "name":"catalog_mysql",
"type":"RELATIONAL", "provider":"jdbc-mysql", "comment":"comment",
"properties":{ "jdbc-url":"jdbc:mysql://mysql:3306", "jdbc-user":"mysql",
"jdbc-password":"mysql", "jdbc-driver": "com.mysql.cj.jdbc.Driver" } }'
http://gravitino:8090/api/metalakes/metalake_demo/catalogs)
-
+
if echo "$response" | grep -q "catalog_mysql"; then
true # Placeholder, do nothing
else
@@ -87,4 +86,3 @@ else
exit 1
fi
fi
-
diff --git a/launch-playground.sh b/init/gravitino/gravitino-dependency.sh
similarity index 51%
rename from launch-playground.sh
rename to init/gravitino/gravitino-dependency.sh
index 1c220cb..c3a2a29 100755
--- a/launch-playground.sh
+++ b/init/gravitino/gravitino-dependency.sh
@@ -17,26 +17,21 @@
# specific language governing permissions and limitations
# under the License.
#
-# set -ex
-playground_dir="$(dirname "${BASH_SOURCE-$0}")"
-playground_dir="$(cd "${playground_dir}">/dev/null; pwd)"
-isExist=`which docker-compose`
-if [ $isExist ]
-then
- true # Placeholder, do nothing
-else
- echo "ERROR: No docker service environment found, please install
docker-compose first."
- exit
-fi
-
-components=""
-case "${1}" in
- *)
- components=$@
-esac
+gravitino_dir="$(dirname "${BASH_SOURCE-$0}")"
+gravitino_dir="$(
+ cd "${gravitino_dir}" >/dev/null
+ pwd
+)"
+. "${gravitino_dir}/../common/common.sh"
-cd ${playground_dir}
-docker-compose up ${components}
+if [[ ! -d "${gravitino_dir}/packages" ]]; then
+ mkdir -p "${gravitino_dir}/packages"
+fi
+# Prepare download packages
+MYSQL_CONNECTOR_JAVA_JAR="https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar";
+MYSQL_CONNECTOR_JAVA_MD5="${MYSQL_CONNECTOR_JAVA_JAR}.md5"
+download_and_verify "${MYSQL_CONNECTOR_JAVA_JAR}"
"${MYSQL_CONNECTOR_JAVA_MD5}" "${gravitino_dir}"
-# Clean Docker containers when you quit this script
-docker-compose down
+POSTGRESQL_JAR="https://repo1.maven.org/maven2/org/postgresql/postgresql/42.2.7/postgresql-42.2.7.jar";
+POSTGRESQL_MD5="${POSTGRESQL_JAR}.md5"
+download_and_verify "${POSTGRESQL_JAR}" "${POSTGRESQL_MD5}" "${gravitino_dir}"
diff --git a/init/gravitino/init.sh b/init/gravitino/init.sh
index 1c01423..d29d0e1 100644
--- a/init/gravitino/init.sh
+++ b/init/gravitino/init.sh
@@ -17,12 +17,12 @@
# under the License.
#
echo "Start to download the jar package of JDBC"
-wget
https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar
-O /root/gravitino/catalogs/jdbc-mysql/libs/mysql-connector-java-8.0.27.jar
-wget https://jdbc.postgresql.org/download/postgresql-42.7.0.jar -O
/root/gravitino/catalogs/jdbc-postgresql/libs/postgresql-42.7.0.jar
-cp /root/gravitino/catalogs/jdbc-postgresql/libs/postgresql-42.7.0.jar
/root/gravitino/catalogs/lakehouse-iceberg/libs
+cp /tmp/gravitino/packages/mysql-connector-java-8.0.27.jar
/root/gravitino/catalogs/jdbc-mysql/libs/mysql-connector-java-8.0.27.jar
+cp /tmp/gravitino/packages/postgresql-42.2.7.jar
/root/gravitino/catalogs/jdbc-postgresql/libs/postgresql-42.2.7.jar
+cp /root/gravitino/catalogs/jdbc-postgresql/libs/postgresql-42.2.7.jar
/root/gravitino/catalogs/lakehouse-iceberg/libs
cp /root/gravitino/catalogs/jdbc-mysql/libs/mysql-connector-java-8.0.27.jar
/root/gravitino/catalogs/lakehouse-iceberg/libs
-cp /root/gravitino/catalogs/jdbc-postgresql/libs/postgresql-42.7.0.jar
/root/gravitino/iceberg-rest-server/libs
+cp /root/gravitino/catalogs/jdbc-postgresql/libs/postgresql-42.2.7.jar
/root/gravitino/iceberg-rest-server/libs
cp /root/gravitino/catalogs/jdbc-mysql/libs/mysql-connector-java-8.0.27.jar
/root/gravitino/iceberg-rest-server/libs
cp /tmp/gravitino/gravitino.conf /root/gravitino/conf
echo "Finish downloading"
diff --git a/init/hive/init.sh b/init/hive/init.sh
index aea00c5..9d5c3bd 100644
--- a/init/hive/init.sh
+++ b/init/hive/init.sh
@@ -21,7 +21,7 @@ sed -i '$d' /usr/local/sbin/start.sh
sed -i '$d' /usr/local/sbin/start.sh
sed -i 's|hdfs://localhost:9000|hdfs://hive:9000|g'
/usr/local/hive/conf/hive-site.xml
/bin/bash /usr/local/sbin/start.sh
-hdfs dfs -mkdir /user/iceberg/
-hdfs dfs -mkdir /user/iceberg/warehouse
+hdfs dfs -mkdir -p /user/gravitino
+hdfs dfs -mkdir -p /user/iceberg/warehouse
hdfs dfs -chmod 777 /user/iceberg/warehouse/
tail -f /dev/null
diff --git a/init/jupyter/gravitino-fileset-example.ipynb
b/init/jupyter/gravitino-fileset-example.ipynb
index 6bb78aa..17b7efc 100644
--- a/init/jupyter/gravitino-fileset-example.ipynb
+++ b/init/jupyter/gravitino-fileset-example.ipynb
@@ -23,9 +23,9 @@
"hdfs_client = InsecureClient('http://hive:50070', user='root')\n",
"\n",
"# List HDFS file and directories\n",
- "print(hdfs_client.list('/user/datastrato'))\n",
+ "print(hdfs_client.list('/user/gravitino'))\n",
"\n",
- "# hdfs_client.delete(\"/user/datastrato\")"
+ "hdfs_client.delete(\"/user/gravitino\")"
]
},
{
@@ -124,7 +124,7 @@
"source": [
"# Create schema entity via Gravition client\n",
"schema_name=\"schema\"\n",
- "schema_path=\"/user/datastrato/\"+schema_name\n",
+ "schema_path=\"/user/gravitino/\"+schema_name\n",
"schema_hdfs_path=\"hdfs://hive:9000\"+schema_path\n",
"\n",
"catalog.as_schemas().create_schema(schema_name=schema_name, \n",
@@ -149,7 +149,7 @@
"source": [
"# Create a managed type of Fileset\n",
"managed_fileset_name=\"managed_fileset\"\n",
-
"managed_fileset_path=\"/user/datastrato/\"+schema_name+\"/\"+managed_fileset_name\n",
+
"managed_fileset_path=\"/user/gravitino/\"+schema_name+\"/\"+managed_fileset_name\n",
"managed_fileset_hdfs_path=\"hdfs://hive:9000\"+managed_fileset_path\n",
"\n",
"managed_fileset_ident: NameIdentifier = NameIdentifier.of(schema_name,
managed_fileset_name)\n",
@@ -176,7 +176,7 @@
"outputs": [],
"source": [
"external_fileset_name=\"external_fileset\"\n",
-
"external_fileset_path=\"/user/datastrato/\"+schema_name+\"/\"+external_fileset_name\n",
+
"external_fileset_path=\"/user/gravitino/\"+schema_name+\"/\"+external_fileset_name\n",
"external_fileset_hdfs_path=\"hdfs://hive:9000\"+external_fileset_path\n",
"\n",
"# Create a fileset path in HDFS in advance\n",
diff --git a/init/jupyter/gravitino-spark-trino-example.ipynb
b/init/jupyter/gravitino-spark-trino-example.ipynb
index 0664266..eaf14fe 100644
--- a/init/jupyter/gravitino-spark-trino-example.ipynb
+++ b/init/jupyter/gravitino-spark-trino-example.ipynb
@@ -21,7 +21,7 @@
"spark = SparkSession.builder \\\n",
" .appName(\"PySpark SQL Example\") \\\n",
" .config(\"spark.plugins\",
\"org.apache.gravitino.spark.connector.plugin.GravitinoSparkPlugin\") \\\n",
- " .config(\"spark.jars\",
\"/opt/spark/jars/iceberg-spark-runtime-3.4_2.12-1.5.2.jar,/opt/spark/jars/gravitino-spark-connector-runtime-3.4_2.12-0.6.0-incubating.jar\")
\\\n",
+ " .config(\"spark.jars\",
\"/opt/spark/jars/iceberg-spark-runtime-3.4_2.12-1.5.2.jar,/opt/spark/jars/gravitino-spark-connector-runtime-3.4_2.12-0.6.1-incubating.jar\")
\\\n",
" .config(\"spark.sql.gravitino.uri\", \"http://gravitino:8090\";) \\\n",
" .config(\"spark.sql.gravitino.metalake\", \"metalake_demo\") \\\n",
" .config(\"spark.sql.gravitino.enableIcebergSupport\", \"true\") \\\n",
diff --git a/init/spark/init.sh b/init/spark/init.sh
index e0f9c38..798d73f 100644
--- a/init/spark/init.sh
+++ b/init/spark/init.sh
@@ -18,8 +18,9 @@
#
mkdir -p /opt/spark/conf
cp /tmp/spark/spark-defaults.conf /opt/spark/conf
-wget
https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-3.4_2.12/1.5.2/iceberg-spark-runtime-3.4_2.12-1.5.2.jar
-O /opt/spark/jars/iceberg-spark-runtime-3.4_2.12-1.5.2.jar
-wget
https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-spark-connector-runtime-3.4_2.12/0.6.1-incubating/gravitino-spark-connector-runtime-3.4_2.12-0.6.1-incubating.jar
-O
/opt/spark/jars/gravitino-spark-connector-runtime-3.4_2.12-0.6.1-incubating.jar
-wget
https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar
-O /opt/spark/jars/mysql-connector-java-8.0.27.jar
-sh /tmp/common/init_metalake_catalog.sh
+cp /tmp/spark/packages/iceberg-spark-runtime-3.4_2.12-1.5.2.jar
/opt/spark/jars/iceberg-spark-runtime-3.4_2.12-1.5.2.jar
+cp
/tmp/spark/packages/gravitino-spark-connector-runtime-3.4_2.12-0.6.1-incubating.jar
/opt/spark/jars/gravitino-spark-connector-runtime-3.4_2.12-0.6.1-incubating.jar
+cp /tmp/spark/packages/mysql-connector-java-8.0.27.jar
/opt/spark/jars/mysql-connector-java-8.0.27.jar
+cp /tmp/spark/packages/kyuubi-spark-authz-shaded_2.12-1.9.2.jar
/opt/spark/jars/kyuubi-spark-authz-shaded_2.12-1.9.2.jar
+sh /tmp/common/init_metalake_catalog.sh
tail -f /dev/null
diff --git a/init/spark/spark-dependency.sh b/init/spark/spark-dependency.sh
new file mode 100755
index 0000000..29c1a85
--- /dev/null
+++ b/init/spark/spark-dependency.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+spark_dir="$(dirname "${BASH_SOURCE-$0}")"
+spark_dir="$(
+ cd "${spark_dir}" >/dev/null
+ pwd
+)"
+. "${spark_dir}/../common/common.sh"
+
+# Prepare download packages
+if [[ ! -d "${spark_dir}/packages" ]]; then
+ mkdir -p "${spark_dir}/packages"
+fi
+
+ICEBERG_SPARK_RUNTIME_JAR="https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-3.4_2.12/1.5.2/iceberg-spark-runtime-3.4_2.12-1.5.2.jar";
+ICEBERG_SPARK_RUNTIME_MD5="${ICEBERG_SPARK_RUNTIME_JAR}.md5"
+download_and_verify "${ICEBERG_SPARK_RUNTIME_JAR}"
"${ICEBERG_SPARK_RUNTIME_MD5}" "${spark_dir}"
+
+GRAVITINO_SPARK_CONNECTOR_RUNTIME_JAR="https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-spark-connector-runtime-3.4_2.12/0.6.1-incubating/gravitino-spark-connector-runtime-3.4_2.12-0.6.1-incubating.jar";
+GRAVITINO_SPARK_CONNECTOR_RUNTIME_MD5="${GRAVITINO_SPARK_CONNECTOR_RUNTIME_JAR}.md5"
+download_and_verify "${GRAVITINO_SPARK_CONNECTOR_RUNTIME_JAR}"
"${GRAVITINO_SPARK_CONNECTOR_RUNTIME_MD5}" "${spark_dir}"
+
+MYSQL_CONNECTOR_JAVA_JAR="https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar";
+MYSQL_CONNECTOR_JAVA_MD5="${MYSQL_CONNECTOR_JAVA_JAR}.md5"
+download_and_verify "${MYSQL_CONNECTOR_JAVA_JAR}"
"${MYSQL_CONNECTOR_JAVA_MD5}" "${spark_dir}"
+
+KYUUBI_SPARK_AUTHZ_SHADED_JAR="https://repo1.maven.org/maven2/org/apache/kyuubi/kyuubi-spark-authz-shaded_2.12/1.9.2/kyuubi-spark-authz-shaded_2.12-1.9.2.jar";
+KYUUBI_SPARK_AUTHZ_SHADED_MD5="${KYUUBI_SPARK_AUTHZ_SHADED_JAR}.md5"
+download_and_verify "${KYUUBI_SPARK_AUTHZ_SHADED_JAR}"
"${KYUUBI_SPARK_AUTHZ_SHADED_MD5}" "${spark_dir}"
diff --git a/init/trino/init.sh b/init/trino/init.sh
index 21ba967..922b0d1 100644
--- a/init/trino/init.sh
+++ b/init/trino/init.sh
@@ -17,7 +17,7 @@
# under the License.
#
-sh /tmp/common/init_metalake_catalog.sh
+sh /tmp/common/init_metalake_catalog.sh
/etc/trino/update-trino-conf.sh
nohup /usr/lib/trino/bin/run-trino &
@@ -25,14 +25,13 @@ nohup /usr/lib/trino/bin/run-trino &
counter=0
while [ $counter -le 240 ]; do
counter=$((counter + 1))
- trino_ready=$(trino --execute "SHOW CATALOGS LIKE 'catalog_hive'"| grep
"catalog_hive" | wc -l)
- if [ "$trino_ready" -eq 0 ];
- then
+ trino_ready=$(trino --execute "SHOW CATALOGS LIKE 'catalog_hive'" | grep
"catalog_hive" | wc -l)
+ if [ "$trino_ready" -eq 0 ]; then
echo "Wait for the initialization of services"
- sleep 5;
+ sleep 5
else
echo "Import the data of the Hive warehouse"
- trino < /tmp/trino/init.sql
+ trino </tmp/trino/init.sql
echo "Import ends"
# persist the container
diff --git a/playground.sh b/playground.sh
new file mode 100755
index 0000000..990d5bc
--- /dev/null
+++ b/playground.sh
@@ -0,0 +1,125 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+playground_dir="$(dirname "${BASH_SOURCE-$0}")"
+playground_dir="$(
+ cd "${playground_dir}" >/dev/null
+ pwd
+)"
+
+testDocker() {
+ echo "Testing Docker environment by running hello-world..."
+ docker run --pull always hello-world >/dev/null 2>&1
+ if [ $? -eq 0 ]; then
+ echo "Docker is working correctly!"
+ else
+ echo "There was an issue running the hello-world container. Please check
your Docker installation."
+ exit 1
+ fi
+}
+
+checkCompose() {
+ isExist=$(which docker-compose)
+ if [ $isExist ]; then
+ true # Placeholder, do nothing
+ else
+ echo "ERROR: No docker service environment found. Please install
docker-compose."
+ exit
+ fi
+}
+
+checkPortInUse() {
+ local port=$1
+ if [[ "$(uname)" == "Darwin" ]]; then
+ openPort=$(lsof -i :$port)
+ elif [[ "$(uname)" == "Linux" ]]; then
+ openPort=$(sudo lsof -i :$port)
+ fi
+ if [ -z "${openPort}" ]; then
+ echo "Port $port is ok."
+ else
+ echo "Port $port is in use. Please check it."
+ exit 1
+ fi
+}
+
+start() {
+ echo "Starting the playground..."
+ testDocker
+ checkCompose
+
+ ports=(8090 9001 3307 19000 19083 60070 13306 15342 18080 18888)
+ for port in "${ports[@]}"; do
+ checkPortInUse ${port}
+ done
+
+ cd ${playground_dir}
+ echo "Preparing packages..."
+ ./init/spark/spark-dependency.sh
+ ./init/gravitino/gravitino-dependency.sh
+
+ logSuffix=$(date +%Y%m%d%H%m%s)
+ docker-compose up --detach
+ docker compose logs -f >${playground_dir}/playground-${logSuffix}.log 2>&1 &
+ echo "Check log details: ${playground_dir}/playground-${logSuffix}.log"
+}
+
+status() {
+ docker-compose ps
+}
+
+stop() {
+ echo "Stopping the playground..."
+ docker-compose down
+ if [ $? -eq 0 ]; then
+ echo "Playground stopped!"
+ fi
+}
+
+case "$1" in
+start)
+ if [[ "$2" == "-y" ]]; then
+ input="y"
+ else
+ echo "The playground requires 2 CPU cores, 8 GB of RAM, and 25 GB of disk
storage to operate efficiently."
+ read -r -p "Confirm the requirement is available in your OS [Y/n]:" input
+ fi
+ case $input in
+ [yY][eE][sS] | [yY]) ;;
+ [nN][oO] | [nN])
+ exit 0
+ ;;
+ *)
+ echo "Invalid input!"
+ exit 1
+ ;;
+ esac
+ start
+ ;;
+status)
+ status
+ ;;
+stop)
+ stop
+ ;;
+*)
+ echo "Usage: $0 [start | status | stop]"
+ exit 1
+ ;;
+esac
