vaijosh commented on code in PR #3081: URL: https://github.com/apache/hugegraph/pull/3081#discussion_r3631841310
########## docker/cloud-storage/scripts/test-graph-queries-and-sst.sh: ########## @@ -0,0 +1,917 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +STACK_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +REPO_ROOT="$(cd "${STACK_DIR}/../.." && pwd)" +GENERATED_DIR="${STACK_DIR}/.generated" +ARTIFACTS_DIR="${STACK_DIR}/.artifacts" +COMPOSE_FILE="${GENERATED_DIR}/docker-compose.yml" +FULL_TEST_REPORT="${GENERATED_DIR}/FULL-TEST-REPORT.txt" +TEST_REPORT="${GENERATED_DIR}/test-report.txt" +MINIO_REPORT="${GENERATED_DIR}/minio-verification.txt" +STORE_CLUSTER_LOG="${GENERATED_DIR}/store-cluster.log" +CLI_LOAD_LOG="${GENERATED_DIR}/cli-load.log" +LOAD_DATA_FILE="${GENERATED_DIR}/load-data.tsv" +export SERVER_GRAPHS_DIR="${GENERATED_DIR}/graphs" +export SERVER_GRAPH_CONF="${SERVER_GRAPHS_DIR}/hugegraph.properties" +export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-cloud-storage-test}" +HG_PD_IMAGE="${HG_PD_IMAGE:-hugegraph/pd:cloud-storage-local}" +HG_STORE_IMAGE="${HG_STORE_IMAGE:-hugegraph/store:cloud-storage-local}" +HG_SERVER_IMAGE="${HG_SERVER_IMAGE:-hugegraph/server:1.7.0}" +MINIO_IMAGE="${MINIO_IMAGE:-minio/minio:latest}" +MINIO_MC_IMAGE="${MINIO_MC_IMAGE:-minio/mc:latest}" +MINIO_ROOT_USER="${MINIO_ROOT_USER:-minioadmin}" +MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD:-minioadmin}" +S3_BUCKET_STORE0="${S3_BUCKET_STORE0:-hugegraph-store0}" +S3_BUCKET_STORE1="${S3_BUCKET_STORE1:-hugegraph-store1}" +S3_BUCKET_STORE2="${S3_BUCKET_STORE2:-hugegraph-store2}" +S3_REGION="${S3_REGION:-us-east-1}" +S3_ENDPOINT="${S3_ENDPOINT:-http://minio:9000}"; +GRAPH_API_BASE="${GRAPH_API_BASE:-http://localhost:8080/graphs/hugegraph}"; +STORE_ROCKSDB_CLOUD_ENABLED="${STORE_ROCKSDB_CLOUD_ENABLED:-true}" +STORE_ROCKSDB_CLOUD_SYNC_INTERVAL_SECONDS="${STORE_ROCKSDB_CLOUD_SYNC_INTERVAL_SECONDS:-30}" +KEEP_UP="${KEEP_UP:-true}" +SKIP_SMOKE_TESTS="${SKIP_SMOKE_TESTS:-false}" +SCRIPT_START_TS="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" +RECOVERY_STATUS="NOT_RUN" +DELETE_CLEANUP_STATUS="NOT_RUN" +RECREATE_STATUS="NOT_RUN" +SMOKE_STATUS="NOT_RUN" +INFRA_READY="false" +NETWORK="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --keep-stack) KEEP_UP=true ;; + --skip-smoke-tests|--infra-only) + SKIP_SMOKE_TESTS=true ;; + -h|--help) + cat <<'USAGE' + Usage: test-graph-queries-and-sst.sh [--keep-stack] [--skip-smoke-tests|--infra-only] + + This script runs a full end-to-end cloud storage test suite: + + 1. Recovery Test: + load data -> flush+compact -> verify a consistent + {CURRENT, MANIFEST, OPTIONS, SST} set in MinIO -> + wipe each store's local RocksDB state (raft/ preserved) -> + restart -> confirm data is recovered from cloud (not empty DB). + + 2. DB Deletion Cleanup Test: + create test graph -> load data -> sync to cloud -> + delete graph -> verify cloud storage prefix is cleaned up + (tests onDBDeleted() -> purgeRemotePrefix() behavior). + + --keep-stack Leave the stack running on exit (same as KEEP_UP=true). + --skip-smoke-tests Start infrastructure only; skip data load + validation tests. + --infra-only Alias of --skip-smoke-tests. +USAGE + exit 0 ;; + *) echo "unknown arg: $1 (see --help)" >&2; exit 2 ;; + esac + shift +done +log() { printf "[cloud-storage] %s\n" "$*"; } +need_cmd() { command -v "$1" >/dev/null 2>&1 || { echo "ERROR: $1 not found" >&2; exit 2; }; } + +report_line() { + local file="$1" + shift + printf "%s\n" "$*" >> "$file" +} + +report_event() { + local category="$1" + shift + report_line "$FULL_TEST_REPORT" "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] [$category] $*" +} + +record_phase() { + local phase="$1" + local status="$2" + local details="$3" + report_line "$TEST_REPORT" "$(date -u +"%Y-%m-%dT%H:%M:%SZ")\t${phase}\t${status}\t${details}" + report_event "$phase" "${status} - ${details}" +} + +init_reports() { + mkdir -p "$GENERATED_DIR" + : > "$FULL_TEST_REPORT" + : > "$TEST_REPORT" + : > "$MINIO_REPORT" + : > "$STORE_CLUSTER_LOG" + : > "$CLI_LOAD_LOG" + : > "$LOAD_DATA_FILE" + + report_line "$FULL_TEST_REPORT" "Cloud Storage E2E Test Report" + report_line "$FULL_TEST_REPORT" "started_at=${SCRIPT_START_TS}" + report_line "$FULL_TEST_REPORT" "script=${BASH_SOURCE[0]}" + report_line "$FULL_TEST_REPORT" "compose_project=${COMPOSE_PROJECT_NAME}" + report_line "$FULL_TEST_REPORT" "" + + report_line "$TEST_REPORT" "timestamp_utc phase status details" + report_line "$MINIO_REPORT" "MinIO Verification Report" + report_line "$MINIO_REPORT" "started_at=${SCRIPT_START_TS}" + report_line "$MINIO_REPORT" "" + report_line "$CLI_LOAD_LOG" "CLI/Data Load Report" + report_line "$CLI_LOAD_LOG" "started_at=${SCRIPT_START_TS}" + report_line "$CLI_LOAD_LOG" "" + report_line "$LOAD_DATA_FILE" "name age city" +} + +collect_store_logs() { + if [[ -f "$COMPOSE_FILE" ]]; then + docker compose -f "$COMPOSE_FILE" logs store0 store1 store2 > "$STORE_CLUSTER_LOG" 2>&1 || true + fi +} + +finalize_reports() { + local exit_code="$1" + local end_ts + end_ts="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + + report_line "$FULL_TEST_REPORT" "" + report_line "$FULL_TEST_REPORT" "Summary" + report_line "$FULL_TEST_REPORT" "ended_at=${end_ts}" + report_line "$FULL_TEST_REPORT" "exit_code=${exit_code}" + report_line "$FULL_TEST_REPORT" "infra_ready=${INFRA_READY}" + report_line "$FULL_TEST_REPORT" "smoke_tests=${SMOKE_STATUS}" + report_line "$FULL_TEST_REPORT" "recovery_test=${RECOVERY_STATUS}" + report_line "$FULL_TEST_REPORT" "db_deletion_cleanup_test=${DELETE_CLEANUP_STATUS}" + report_line "$FULL_TEST_REPORT" "db_recreation_no_orphan_test=${RECREATE_STATUS}" + report_line "$FULL_TEST_REPORT" "" + report_line "$FULL_TEST_REPORT" "Generated artifacts:" + report_line "$FULL_TEST_REPORT" "- ${FULL_TEST_REPORT}" + report_line "$FULL_TEST_REPORT" "- ${TEST_REPORT}" + report_line "$FULL_TEST_REPORT" "- ${MINIO_REPORT}" + report_line "$FULL_TEST_REPORT" "- ${STORE_CLUSTER_LOG}" + report_line "$FULL_TEST_REPORT" "- ${CLI_LOAD_LOG}" + report_line "$FULL_TEST_REPORT" "- ${LOAD_DATA_FILE}" +} + +assert_all_reports_present() { + local missing=0 + for report_file in "$FULL_TEST_REPORT" "$TEST_REPORT" "$MINIO_REPORT" "$STORE_CLUSTER_LOG" "$CLI_LOAD_LOG" "$LOAD_DATA_FILE"; do + if [[ ! -f "$report_file" ]]; then + echo "ERROR: expected report file missing: $report_file" >&2 + missing=$((missing + 1)) + fi + done + if (( missing > 0 )); then + echo "ERROR: $missing expected report file(s) are missing; CI contract violated" >&2 + return 1 + fi + return 0 +} + +find_dist_dir() { + local glob="$1" + local d + for d in $glob; do + [[ -d "$d" ]] && { echo "$d"; return 0; } + done + return 1 +} + +find_plugin_jar() { + local f + for f in "${REPO_ROOT}"/hugegraph-store/hg-store-cloud-s3/target/hg-store-cloud-s3-*.jar; do + [[ -f "$f" ]] || continue + case "$f" in + *-sources.jar|*-javadoc.jar|*original*) continue ;; + esac + echo "$f" + return 0 + done + return 1 +} + +prepare_artifacts() { + local pd_src store_src plugin_jar plugin_dep_dir dep_base + + pd_src="$(find_dist_dir "${REPO_ROOT}/hugegraph-pd/apache-hugegraph-pd-*")" || { + echo "ERROR: PD dist not found under ${REPO_ROOT}/hugegraph-pd/apache-hugegraph-pd-*" >&2 + echo "Run: mvn clean package -DskipTests" >&2 + exit 2 + } + store_src="$(find_dist_dir "${REPO_ROOT}/hugegraph-store/apache-hugegraph-store-*")" || { + echo "ERROR: Store dist not found under ${REPO_ROOT}/hugegraph-store/apache-hugegraph-store-*" >&2 + echo "Run: mvn clean package -DskipTests" >&2 + exit 2 + } + plugin_jar="$(find_plugin_jar)" || { + echo "ERROR: S3 plugin jar not found under ${REPO_ROOT}/hugegraph-store/hg-store-cloud-s3/target/" >&2 + echo "Run: mvn clean package -DskipTests" >&2 + exit 2 + } + + log "preparing Docker artifacts in ${ARTIFACTS_DIR}" + rm -rf "${ARTIFACTS_DIR}" + mkdir -p "${ARTIFACTS_DIR}/pd-dist" "${ARTIFACTS_DIR}/store-dist" "${ARTIFACTS_DIR}/plugins" + + cp -R "${pd_src}/." "${ARTIFACTS_DIR}/pd-dist/" + cp -R "${store_src}/." "${ARTIFACTS_DIR}/store-dist/" + cp "${plugin_jar}" "${ARTIFACTS_DIR}/plugins/" + + plugin_dep_dir="${REPO_ROOT}/hugegraph-store/hg-store-cloud-s3/target/dependency" + if [[ -d "${plugin_dep_dir}" ]]; then + # Keep only external plugin deps; internal HugeGraph jars must come from /hugegraph-store/lib. + for dep in "${plugin_dep_dir}"/*.jar; do + [[ -f "${dep}" ]] || continue + dep_base="$(basename "${dep}")" + case "${dep_base}" in + hg-*.jar|hugegraph-*.jar) continue ;; + esac + cp "${dep}" "${ARTIFACTS_DIR}/plugins/" + done + else + log "warning: plugin dependency dir not found at ${plugin_dep_dir}; continuing with plugin jar only" + fi + + # Prevent duplicate SLF4J binding clashes from plugin dependency staging. + rm -f "${ARTIFACTS_DIR}"/plugins/log4j-slf4j-impl-*.jar "${ARTIFACTS_DIR}"/plugins/slf4j-log4j12-*.jar || true +} + +ensure_image() { + local img="$1" + docker image inspect "$img" >/dev/null 2>&1 && return 0 + # Check if it's a local build image (contains "cloud-storage-local") + if [[ "$img" == *"cloud-storage-local"* ]]; then + log "image $img is local-build only (will be built via docker compose build)" + return 0 + fi + log "pulling $img..." + docker pull "$img" >/dev/null || exit 3 +} +ensure_minio_buckets() { + for bucket in "$S3_BUCKET_STORE0" "$S3_BUCKET_STORE1" "$S3_BUCKET_STORE2"; do + docker run --rm --network "$1" --entrypoint /bin/sh "$MINIO_MC_IMAGE" -c "mc alias set local http://minio:9000 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD >/dev/null && mc mb --ignore-existing local/$bucket >/dev/null" + done +} +wait_svc() { + local svc max=120 i=0 + svc="$1" + while [[ $i -lt $max ]]; do + local cid status + cid=$(docker compose -f "$COMPOSE_FILE" ps -q "$svc" 2>/dev/null || true) + [[ -z "$cid" ]] && { sleep 2; i=$((i+1)); continue; } + status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$cid" 2>/dev/null || true) + [[ "$status" == "healthy" || "$status" == "running" ]] && { log "✓ $svc"; return 0; } + sleep 2 + i=$((i+1)) + done + echo "ERROR: $svc timeout" >&2 + return 1 +} +wait_http() { + local url max=120 i=0 + url="$1" + while [[ $i -lt $max ]]; do + [[ $(curl -so /dev/null -w "%{http_code}" "$url" 2>/dev/null) == "200" ]] && { log "✓ $url ready"; return 0; } + sleep 2 + i=$((i+1)) + done + echo "ERROR: $url timeout" >&2 + return 1 +} + +delete_graph_with_confirm() { + local graph_api="$1" + local resp_file status + resp_file="/tmp/hg-delete-$$.json" + status=$(curl -s -o "$resp_file" -w "%{http_code}" -X DELETE "$graph_api" \ + --get --data-urlencode "confirm_message=I'm sure to drop the graph") + if [[ "$status" != 2* ]]; then + echo "ERROR: graph delete failed with HTTP ${status}: ${graph_api}" >&2 + head -80 "$resp_file" >&2 || true + rm -f "$resp_file" || true + return 1 + fi + rm -f "$resp_file" || true +} + +clear_graph_with_confirm() { + local graph_api="$1" + local resp_file status clear_api + resp_file="/tmp/hg-clear-$$.json" + clear_api="${graph_api}/clear" + status=$(curl -s -o "$resp_file" -w "%{http_code}" -X DELETE "$clear_api" \ + --get --data-urlencode "confirm_message=I'm sure to delete all data") + if [[ "$status" != 2* ]]; then + echo "ERROR: graph clear failed with HTTP ${status}: ${clear_api}" >&2 + head -80 "$resp_file" >&2 || true + rm -f "$resp_file" || true + return 1 + fi + rm -f "$resp_file" || true +} + +on_exit() { + local status=$? + local assertion_rc + collect_store_logs + finalize_reports "$status" + assert_all_reports_present + assertion_rc=$? + # Preserve original exit code unless test passed but assertion failed + [[ $status -eq 0 && $assertion_rc -ne 0 ]] && status=$assertion_rc + [[ "$KEEP_UP" == "true" ]] || (docker compose -f "$COMPOSE_FILE" down -v 2>/dev/null || true) +} +trap on_exit EXIT + +# ----------------------------------------------------------------------------- +# Total-loss recovery E2E helpers +# ----------------------------------------------------------------------------- + +# Current vertex count via the server graph API (0 if the query fails). +graph_vertex_count() { + curl -s --compressed "${GRAPH_API_BASE}/graph/vertices" 2>/dev/null \ + | python3 -c "import sys,json; print(len(json.load(sys.stdin).get('vertices',[])))" \ + 2>/dev/null || echo 0 +} + +# Create a minimal schema and insert enough vertices to generate SST files. +load_test_data() { + log "loading test data (schema + 150 vertices)..." + report_event "graph-load" "starting schema + vertex load" + local insert_ok=0 insert_fail=0 + + # Create deterministic input file promised by README. + for i in $(seq 1 150); do + printf "person_%s\t%s\tcity_%s\n" "$i" "$((20 + i % 50))" "$((i % 5))" >> "$LOAD_DATA_FILE" + done + + for pk in '{"name":"name","data_type":"TEXT","cardinality":"SINGLE"}' \ + '{"name":"age","data_type":"INT","cardinality":"SINGLE"}' \ + '{"name":"city","data_type":"TEXT","cardinality":"SINGLE"}'; do + curl -s -o /dev/null -X POST "${GRAPH_API_BASE}/schema/propertykeys" \ + -H 'Content-Type: application/json' -d "$pk" || true + done + curl -s -o /dev/null -X POST "${GRAPH_API_BASE}/schema/vertexlabels" \ + -H 'Content-Type: application/json' \ + -d '{"name":"person","id_strategy":"AUTOMATIC","properties":["name","age","city"]}' || true + + while IFS=$'\t' read -r name age city; do + [[ "$name" == "name" ]] && continue + local payload code + payload="{\"label\":\"person\",\"properties\":{\"name\":\"${name}\",\"age\":${age},\"city\":\"${city}\"}}" + code=$(curl -s -o /dev/null -w "%{http_code}" -X POST "${GRAPH_API_BASE}/graph/vertices" \ + -H 'Content-Type: application/json' -d "$payload" || true) + if [[ "$code" =~ ^2 ]]; then + insert_ok=$((insert_ok + 1)) + if (( insert_ok % 25 == 0 )); then + report_line "$CLI_LOAD_LOG" "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] inserted=${insert_ok} failed=${insert_fail}" + fi + else + insert_fail=$((insert_fail + 1)) + report_line "$CLI_LOAD_LOG" "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] vertex_insert_failed name=${name} http=${code}" + fi + done < "$LOAD_DATA_FILE" + + report_line "$CLI_LOAD_LOG" "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] load_complete inserted=${insert_ok} failed=${insert_fail}" + + if (( insert_ok == 0 )); then + record_phase "graph-load" "FAIL" "no vertices inserted" + echo "ERROR: failed to insert any vertices" >&2 + return 1 + fi + + log " ✓ inserted ${insert_ok} vertices (count now = $(graph_vertex_count))" + record_phase "graph-load" "PASS" "inserted=${insert_ok};failed=${insert_fail};vertex_count=$(graph_vertex_count)" +} + +# Assert every bucket holds a consistent {CURRENT, MANIFEST, OPTIONS, SST} set. +# This deterministic check verifies metadata objects from ordered +# CURRENT/MANIFEST/OPTIONS mirroring ran, so the SSTs are no longer orphans. +verify_metadata_in_minio() { + local out rc + set +e + out=$(docker run --rm --entrypoint /bin/sh --network "$NETWORK" "$MINIO_MC_IMAGE" -c ' + mc alias set local http://minio:9000 '"$MINIO_ROOT_USER $MINIO_ROOT_PASSWORD"' >/dev/null 2>&1 + rc=0 + for b in '"$S3_BUCKET_STORE0 $S3_BUCKET_STORE1 $S3_BUCKET_STORE2"'; do + sst=$(mc find local/$b --name "*.sst" 2>/dev/null | wc -l | tr -d " ") + cur=$(mc find local/$b --name "CURRENT" 2>/dev/null | wc -l | tr -d " ") + man=$(mc find local/$b --name "MANIFEST-*" 2>/dev/null | wc -l | tr -d " ") + opt=$(mc find local/$b --name "OPTIONS-*" 2>/dev/null | wc -l | tr -d " ") + printf "### %s: sst=%s CURRENT=%s MANIFEST=%s OPTIONS=%s\n" "$b" "$sst" "$cur" "$man" "$opt" + if [ "$cur" -lt 1 ] || [ "$man" -lt 1 ] || [ "$sst" -lt 1 ]; then + echo " ✗ recovery metadata/SST set incomplete for $b"; rc=1 + else + echo " ✓ consistent {CURRENT, MANIFEST, OPTIONS, SST} set present" + fi + done + exit $rc + ' 2>&1) + rc=$? + set -e + + printf "%s\n" "$out" | tee -a "$MINIO_REPORT" + return $rc +} + +# Stop a store, wipe its RocksDB state-machine data (db/ + metadata graph) while +# preserving raft/ and snapshot/, then start it again. This models local +# state-machine loss where the node still rejoins its raft group but must +# re-hydrate its RocksDB data from cloud on open (pre-hydration path). +wipe_store_rocksdb_state() { + local idx="$1" + local vol="${COMPOSE_PROJECT_NAME}_hg-store${idx}-data" + docker compose -f "$COMPOSE_FILE" stop "store${idx}" >/dev/null 2>&1 || true + docker run --rm --entrypoint /bin/sh -v "${vol}:/s" "$MINIO_MC_IMAGE" -c ' Review Comment: Addressed -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
