This is an automated email from the ASF dual-hosted git repository. madhan pushed a commit to branch atlas-2.5 in repository https://gitbox.apache.org/repos/asf/atlas.git
commit d65c6ad2c541d1375a295d3211bc82bfc05ff014 Author: Abhishek Kumar <[email protected]> AuthorDate: Wed Jul 9 13:43:17 2025 -0700 ATLAS-5058: Update ci.yml to rely on healthchecks (#391) (cherry picked from commit 2d4b81ba19099c25815f05773efac51c1008fa37) --- .github/workflows/ci.yml | 134 +++++++++++++++++++++ .../atlas-docker/docker-compose.atlas-hadoop.yml | 2 - .../atlas-docker/docker-compose.atlas-hive.yml | 13 +- 3 files changed, 143 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..136d1a54d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,134 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: CI + +on: + push: + pull_request: + branches: [ "master" ] + +env: + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + +jobs: + docker-build: + runs-on: ubuntu-22.04 + timeout-minutes: 180 + steps: + - uses: actions/checkout@v4 + + - name: Cache for maven dependencies + uses: actions/cache@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/atlas + key: maven-repo-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-repo- + + - name: Set up .m2 directory and permissions + run: | + mkdir -p ~/.m2/repository + chmod -R 777 ~/.m2 + chmod -R 777 ./ + + - name: Set up JDK 8 + uses: actions/setup-java@v4 + with: + java-version: '8' + distribution: 'temurin' + + - name: Clean up Docker space + run: docker system prune --all --force --volumes + + - name: Build Atlas - JDK 8 + run: | + cd dev-support/atlas-docker + export DOCKER_BUILDKIT=1 + export COMPOSE_DOCKER_CLI_BUILD=1 + SKIPTESTS=false docker compose -f docker-compose.atlas-base.yml -f docker-compose.atlas-build.yml up + ATLAS_BUILD_CONTAINER=$(docker ps -a -q --filter "name=atlas-build") + EXIT_CODE=$(docker inspect --format '{{.State.ExitCode}}' "$ATLAS_BUILD_CONTAINER") + + # If the exit code is non-zero, fail the workflow + if [ "$EXIT_CODE" -ne 0 ]; then + echo "atlas-build container failed with exit code $EXIT_CODE." + exit 1 + fi + + - name: Upload Code Coverage Results + uses: actions/upload-artifact@v4 + with: + name: coverage + path: dev-support/atlas-docker/dist/coverage/* + + - name: Cache downloaded archives + if: success() + uses: actions/cache@v4 + with: + path: dev-support/atlas-docker/downloads + key: ${{ runner.os }}-atlas-downloads-${{ hashFiles('dev-support/atlas-docker/.env') }} + restore-keys: | + ${{ runner.os }}-atlas-downloads- + + - name: Run download-archives.sh + run: | + cd dev-support/atlas-docker + chmod +x download-archives.sh && ./download-archives.sh + + - name: Bring up containers + run: | + cd dev-support/atlas-docker + export DOCKER_BUILDKIT=1 + export COMPOSE_DOCKER_CLI_BUILD=1 + docker compose -f docker-compose.atlas-base.yml build + docker compose \ + -f docker-compose.atlas.yml \ + -f docker-compose.atlas-hadoop.yml \ + -f docker-compose.atlas-hbase.yml \ + -f docker-compose.atlas-kafka.yml \ + -f docker-compose.atlas-hive.yml up -d --wait + + - name: Check status of containers and remove them + run: | + containers=(atlas atlas-hadoop atlas-hbase atlas-kafka atlas-hive); + flag=true; + for container in "${containers[@]}"; do + if [[ $(docker inspect -f '{{.State.Running}}' $container 2>/dev/null) == "true" ]]; then + echo "Container $container is running!"; + else + flag=false; + echo "Container $container is NOT running!"; + fi + done + + if [[ $flag == true ]]; then + echo "All required containers are up and running"; + docker stop $(docker ps -q) && docker rm $(docker ps -aq); + else + docker stop $(docker ps -q) && docker rm $(docker ps -aq); + exit 1; + fi diff --git a/dev-support/atlas-docker/docker-compose.atlas-hadoop.yml b/dev-support/atlas-docker/docker-compose.atlas-hadoop.yml index 16c1949ba..a6880fc22 100644 --- a/dev-support/atlas-docker/docker-compose.atlas-hadoop.yml +++ b/dev-support/atlas-docker/docker-compose.atlas-hadoop.yml @@ -15,8 +15,6 @@ services: ports: - "9000:9000" - "8088:8088" - depends_on: - - atlas-base healthcheck: test: [ "CMD-SHELL", "su hdfs -c \"/opt/hadoop/bin/hdfs dfsadmin -report | grep -q 'Live datanodes'\"" ] interval: 30s diff --git a/dev-support/atlas-docker/docker-compose.atlas-hive.yml b/dev-support/atlas-docker/docker-compose.atlas-hive.yml index 3acb431a1..9934a9d12 100644 --- a/dev-support/atlas-docker/docker-compose.atlas-hive.yml +++ b/dev-support/atlas-docker/docker-compose.atlas-hive.yml @@ -14,6 +14,12 @@ services: tty: true networks: - atlas + healthcheck: + test: [ "CMD-SHELL", "su hdfs -c \"/opt/hadoop/bin/hdfs dfsadmin -report | grep -q 'Live datanodes'\"" ] + interval: 30s + timeout: 10s + retries: 30 + start_period: 40s ports: - "10000:10000" depends_on: @@ -40,11 +46,10 @@ services: networks: - atlas healthcheck: - test: [ "CMD-SHELL", "su hdfs -c \"/opt/hadoop/bin/hdfs dfsadmin -report | grep -q 'Live datanodes'\"" ] - interval: 30s - timeout: 10s + test: 'su -c "pg_isready -q" postgres' + interval: 10s + timeout: 2s retries: 30 - start_period: 40s networks: atlas:
