This is an automated email from the ASF dual-hosted git repository.
adelapena pushed a commit to branch cassandra-3.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cassandra-3.0 by this push:
new 02048cffea Remove -l / -m / -h designation and have two options: free
or paid tier circle config
02048cffea is described below
commit 02048cffeae4267054541f96741b16b1d0d89a20
Author: Andrés de la Peña <[email protected]>
AuthorDate: Tue Feb 28 11:44:31 2023 +0000
Remove -l / -m / -h designation and have two options: free or paid tier
circle config
patch by Andrés de la Peña; reviewed by Ekaterina Dimitrova and Josh
McKenzie for CASSANDRA-18012
---
.circleci/config-2_1.yml.high_res.patch | 105 -
.circleci/{config.yml.LOWRES => config.yml.FREE} | 0
.circleci/config.yml.HIGHRES | 3103 --------------------
.circleci/{config.yml.MIDRES => config.yml.PAID} | 0
.circleci/{config-2_1.yml => config_template.yml} | 0
...id_res.patch => config_template.yml.PAID.patch} | 0
.circleci/generate.sh | 107 +-
.circleci/readme.md | 98 +-
8 files changed, 93 insertions(+), 3320 deletions(-)
diff --git a/.circleci/config-2_1.yml.high_res.patch
b/.circleci/config-2_1.yml.high_res.patch
deleted file mode 100644
index d62ba2d740..0000000000
--- a/.circleci/config-2_1.yml.high_res.patch
+++ /dev/null
@@ -1,105 +0,0 @@
---- config-2_1.yml 2023-02-07 19:05:10.000000000 -0500
-+++ config-2_1.yml.HIGHRES 2023-02-07 19:25:03.000000000 -0500
-@@ -40,8 +40,8 @@
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
-- CCM_MAX_HEAP_SIZE: 1024M
-- CCM_HEAP_NEWSIZE: 256M
-+ CCM_MAX_HEAP_SIZE: 2048M
-+ CCM_HEAP_NEWSIZE: 512M
-
- # Whether the repeated test iterations should stop on the first failure
by default.
- REPEATED_TESTS_STOP_ON_FAILURE: false
-@@ -127,14 +127,14 @@
- j8_par_executor: &j8_par_executor
- executor:
- name: java8-executor
-- #exec_resource_class: xlarge
-- parallelism: 4
-+ exec_resource_class: xlarge
-+ parallelism: 100
-
- j8_small_par_executor: &j8_small_par_executor
- executor:
- name: java8-executor
-- #exec_resource_class: xlarge
-- parallelism: 1
-+ exec_resource_class: xlarge
-+ parallelism: 2
-
- j8_small_executor: &j8_small_executor
- executor:
-@@ -145,34 +145,38 @@
- j8_medium_par_executor: &j8_medium_par_executor
- executor:
- name: java8-executor
-- #exec_resource_class: xlarge
-- parallelism: 1
-+ exec_resource_class: xlarge
-+ parallelism: 2
-
- j8_seq_executor: &j8_seq_executor
- executor:
- name: java8-executor
-- #exec_resource_class: xlarge
-+ exec_resource_class: xlarge
- parallelism: 1 # sequential, single container tests: no parallelism benefits
-
- j8_repeated_utest_executor: &j8_repeated_utest_executor
- executor:
- name: java8-executor
-- parallelism: 4
-+ exec_resource_class: xlarge
-+ parallelism: 100
-
- j8_repeated_dtest_executor: &j8_repeated_dtest_executor
- executor:
- name: java8-executor
-- parallelism: 4
-+ exec_resource_class: xlarge
-+ parallelism: 100
-
- j8_repeated_upgrade_dtest_executor: &j8_repeated_upgrade_dtest_executor
- executor:
- name: java8-executor
-- parallelism: 4
-+ exec_resource_class: xlarge
-+ parallelism: 100
-
- j8_repeated_jvm_upgrade_dtest_executor:
&j8_repeated_jvm_upgrade_dtest_executor
- executor:
- name: java8-executor
-- parallelism: 4
-+ exec_resource_class: xlarge
-+ parallelism: 100
-
- separate_jobs: &separate_jobs
- jobs:
-@@ -579,7 +583,7 @@
- pytest_extra_args: '--skip-resource-intensive-tests'
-
- j8_dtests_large_vnode:
-- <<: *j8_par_executor
-+ <<: *j8_seq_executor
- steps:
- - attach_workspace:
- at: /home/cassandra
-@@ -593,7 +597,7 @@
- pytest_extra_args: '--use-vnodes --num-tokens=32
--only-resource-intensive-tests --force-resource-intensive-tests'
-
- j8_dtests_large:
-- <<: *j8_par_executor
-+ <<: *j8_seq_executor
- steps:
- - attach_workspace:
- at: /home/cassandra
-@@ -1187,7 +1191,7 @@
- # Put manually specified tests and automatically detected tests
together, removing duplicates
- tests=$(echo <<parameters.tests>> | sed -e "s/<nil>//" | sed -e
"s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
--
-+
- # Prepare the testtag for the target, used by the test macro in
build.xml to group the output files
- target=<<parameters.target>>
- testtag=""
diff --git a/.circleci/config.yml.LOWRES b/.circleci/config.yml.FREE
similarity index 100%
rename from .circleci/config.yml.LOWRES
rename to .circleci/config.yml.FREE
diff --git a/.circleci/config.yml.HIGHRES b/.circleci/config.yml.HIGHRES
deleted file mode 100644
index 6719225b5e..0000000000
--- a/.circleci/config.yml.HIGHRES
+++ /dev/null
@@ -1,3103 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-version: 2
-jobs:
- j8_jvm_upgrade_dtests:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 1
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Determine distributed Tests to Run
- command: |
- # reminder: this code (along with all the steps) is independently
executed on every circle container
- # so the goal here is to get the circleci script to return the tests
*this* container will run
- # which we do via the `circleci` cli tool.
-
- rm -fr ~/cassandra-dtest/upgrade_tests
- echo "***java tests***"
-
- # get all of our unit test filenames
- set -eo pipefail && circleci tests glob
"$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
-
- # split up the unit tests into groups based on the number of
containers we have
- set -eo pipefail && circleci tests split --split-by=timings
--timings-type=filename --index=${CIRCLE_NODE_INDEX}
--total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt >
/tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt |
sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" |
grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
- cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- no_output_timeout: 15m
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Run Unit Tests (testclasslist)
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- test_timeout=$(grep 'name="test.distributed.timeout"' build.xml |
awk -F'"' '{print $4}' || true)
- if [ -z "$test_timeout" ]; then
- test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"'
'{print $4}')
- fi
- ant testclasslist -Dtest.timeout="$test_timeout"
-Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
-Dtest.classlistprefix=distributed -Dno-build-test=true
- no_output_timeout: 15m
- - store_test_results:
- path: /tmp/cassandra/build/test/output/
- - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_compression_repeat:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Repeatedly run new or modifed JUnit tests
- no_output_timeout: 15m
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current
parallel runner.
- count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} %
CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests
together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/
//" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in
build.xml to group the output files
- target=test-compression
- testtag=""
- if [[ $target == "test-compression" ]]; then
- testtag="compression"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class
name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "long-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant test-compression $name_arg $methods_arg
-Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && ((
$exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
- - store_test_results:
- path: /tmp/results/repeated_utests/output
- - store_artifacts:
- path: /tmp/results/repeated_utests/stdout
- destination: stdout
- - store_artifacts:
- path: /tmp/results/repeated_utests/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/results/repeated_utests/logs
- destination: logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests_large_vnode:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 1
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1
$DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be
pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the
docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things
slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest
requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_large_with_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is
independently executed on every circle container\n# so the goal here is to get
the circleci script to return the tests *this* container will run\n# which we
do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource
~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n
export \nfi\n\necho \"***Collected DTests (j8_large_with_vnodes)***\"\nset -eo
pipefail && ./run_dtests.py --use-vnodes -- [...]
- - run:
- name: Run dtests (j8_large_with_vnodes)
- no_output_timeout: 15m
- command: "echo \"cat
/tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\"\ncat
/tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\n\nsource
~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n
export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho
\"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need
the \"set -o pipefail\" here so that the exit code that circleci will actually
use is from pytest an [...]
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_large_with_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_large_with_vnodes_logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_cqlshlib_cython_tests:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 1
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Run cqlshlib Unit Tests
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- export cython="yes"
- time mv ~/cassandra /tmp
- cd /tmp/cassandra/
- ./pylib/cassandra-cqlsh-tests.sh $(pwd)
- no_output_timeout: 15m
- - store_test_results:
- path: /tmp/cassandra/pylib
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests_large_repeat:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1
$DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be
pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the
docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things
slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest
requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Run repeated Python DTests
- no_output_timeout: 15m
- command: |
- if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
- echo "Repeated dtest name hasn't been defined, exiting without
running any test"
- elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
- echo "Repeated dtest count hasn't been defined, exiting without
running any test"
- elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
- echo "Repeated dtest count is lesser or equals than zero, exiting
without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current
parallel runner.
- # Since we are running the same test multiple times there is no
need to use `circleci tests split`.
- count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} %
CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_LARGE_DTESTS} $count times"
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
-
- tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
-
- stop_on_failure_arg=""
- if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
- stop_on_failure_arg="-x"
- fi
-
- vnodes_args=""
- if false; then
- vnodes_args="--use-vnodes --num-tokens=32"
- fi
-
- upgrade_arg=""
- if false; then
- upgrade_arg="--execute-upgrade-tests
--upgrade-target-version-only --upgrade-version-selection all"
- fi
-
- # we need the "set -o pipefail" here so that the exit code that
circleci will actually use is from pytest and not the exit code from tee
- set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args
--count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG
--junit-xml=/tmp/results/dtests/pytest_result.xml -s
--cassandra-dir=/home/cassandra/cassandra --keep-test-dir
--only-resource-intensive-tests --force-resource-intensive-tests $tests_arg |
tee /tmp/dtest/stdout.txt
- fi
- fi
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_long_repeat:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Repeatedly run new or modifed JUnit tests
- no_output_timeout: 15m
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current
parallel runner.
- count=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} %
CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests
together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS_LONG} | sed -e "s/<nil>//" | sed -e
"s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in
build.xml to group the output files
- target=long-testsome
- testtag=""
- if [[ $target == "test-compression" ]]; then
- testtag="compression"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class
name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "long-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant long-testsome $name_arg $methods_arg
-Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && ((
$exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
- - store_test_results:
- path: /tmp/results/repeated_utests/output
- - store_artifacts:
- path: /tmp/results/repeated_utests/stdout
- destination: stdout
- - store_artifacts:
- path: /tmp/results/repeated_utests/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/results/repeated_utests/logs
- destination: logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_unit_tests_repeat:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Repeatedly run new or modifed JUnit tests
- no_output_timeout: 15m
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current
parallel runner.
- count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} %
CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests
together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/
//" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in
build.xml to group the output files
- target=testsome
- testtag=""
- if [[ $target == "test-compression" ]]; then
- testtag="compression"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class
name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "long-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant testsome $name_arg $methods_arg -Dno-build-test=true
| \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && ((
$exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
- - store_test_results:
- path: /tmp/results/repeated_utests/output
- - store_artifacts:
- path: /tmp/results/repeated_utests/stdout
- destination: stdout
- - store_artifacts:
- path: /tmp/results/repeated_utests/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/results/repeated_utests/logs
- destination: logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests_large:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 1
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1
$DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be
pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the
docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things
slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest
requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_large_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is
independently executed on every circle container\n# so the goal here is to get
the circleci script to return the tests *this* container will run\n# which we
do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource
~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n
export \nfi\n\necho \"***Collected DTests (j8_large_without_vnodes)***\"\nset
-eo pipefail && ./run_dtests.py --only-resou [...]
- - run:
- name: Run dtests (j8_large_without_vnodes)
- no_output_timeout: 15m
- command: "echo \"cat
/tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\"\ncat
/tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\n\nsource
~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n
export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho
\"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need
the \"set -o pipefail\" here so that the exit code that circleci will actually
use is from pyt [...]
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_large_without_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_large_without_vnodes_logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_upgrade_dtests_repeat:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1
$DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be
pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the
docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things
slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest
requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Run repeated Python DTests
- no_output_timeout: 15m
- command: |
- if [ "${REPEATED_UPGRADE_DTESTS}" == "<nil>" ]; then
- echo "Repeated dtest name hasn't been defined, exiting without
running any test"
- elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "<nil>" ]; then
- echo "Repeated dtest count hasn't been defined, exiting without
running any test"
- elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then
- echo "Repeated dtest count is lesser or equals than zero, exiting
without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current
parallel runner.
- # Since we are running the same test multiple times there is no
need to use `circleci tests split`.
- count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} %
CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_UPGRADE_DTESTS} $count times"
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
-
- tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g")
-
- stop_on_failure_arg=""
- if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
- stop_on_failure_arg="-x"
- fi
-
- vnodes_args=""
- if false; then
- vnodes_args="--use-vnodes --num-tokens=32"
- fi
-
- upgrade_arg=""
- if true; then
- upgrade_arg="--execute-upgrade-tests
--upgrade-target-version-only --upgrade-version-selection all"
- fi
-
- # we need the "set -o pipefail" here so that the exit code that
circleci will actually use is from pytest and not the exit code from tee
- set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args
--count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG
--junit-xml=/tmp/results/dtests/pytest_result.xml -s
--cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee
/tmp/dtest/stdout.txt
- fi
- fi
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- build:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 1
- steps:
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Clone Cassandra Repository (via git)
- command: |
- git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH
https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git
~/cassandra
- - run:
- name: Build Cassandra
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- # Loop to prevent failure due to maven-ant-tasks not downloading a
jar..
- for x in $(seq 1 3); do
- ${ANT_HOME}/bin/ant clean jar build-test
- RETURN="$?"
- if [ "${RETURN}" -eq "0" ]; then
- break
- fi
- done
- # Exit, if we didn't build successfully
- if [ "${RETURN}" -ne "0" ]; then
- echo "Build failed with exit code: ${RETURN}"
- exit ${RETURN}
- fi
- no_output_timeout: 15m
- - run:
- name: Run eclipse-warnings
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- ant eclipse-warnings
- - persist_to_workspace:
- root: /home/cassandra
- paths:
- - cassandra
- - .m2
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests_vnode_repeat:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1
$DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be
pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the
docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things
slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest
requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Run repeated Python DTests
- no_output_timeout: 15m
- command: |
- if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
- echo "Repeated dtest name hasn't been defined, exiting without
running any test"
- elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
- echo "Repeated dtest count hasn't been defined, exiting without
running any test"
- elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
- echo "Repeated dtest count is lesser or equals than zero, exiting
without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current
parallel runner.
- # Since we are running the same test multiple times there is no
need to use `circleci tests split`.
- count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} %
CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_DTESTS} $count times"
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
-
- tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
-
- stop_on_failure_arg=""
- if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
- stop_on_failure_arg="-x"
- fi
-
- vnodes_args=""
- if true; then
- vnodes_args="--use-vnodes --num-tokens=32"
- fi
-
- upgrade_arg=""
- if false; then
- upgrade_arg="--execute-upgrade-tests
--upgrade-target-version-only --upgrade-version-selection all"
- fi
-
- # we need the "set -o pipefail" here so that the exit code that
circleci will actually use is from pytest and not the exit code from tee
- set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args
--count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG
--junit-xml=/tmp/results/dtests/pytest_result.xml -s
--cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee
/tmp/dtest/stdout.txt
- fi
- fi
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_upgrade_dtests:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1
$DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be
pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the
docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things
slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest
requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_upgradetests_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is
independently executed on every circle container\n# so the goal here is to get
the circleci script to return the tests *this* container will run\n# which we
do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource
~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n
export \nfi\n\necho \"***Collected DTests
(j8_upgradetests_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py
--exe [...]
- - run:
- name: Run dtests (j8_upgradetests_without_vnodes)
- no_output_timeout: 15m
- command: "echo \"cat
/tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat
/tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource
~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n
export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho
\"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need
the \"set -o pipefail\" here so that the exit code that circleci will actually
u [...]
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_upgradetests_without_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_upgradetests_without_vnodes_logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_jvm_upgrade_dtests_repeat:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Repeatedly run new or modifed JUnit tests
- no_output_timeout: 15m
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current
parallel runner.
- count=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} %
CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests
together, removing duplicates
- tests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e "s/<nil>//" |
sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in
build.xml to group the output files
- target=test-jvm-dtest-some
- testtag=""
- if [[ $target == "test-compression" ]]; then
- testtag="compression"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class
name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "long-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant test-jvm-dtest-some $name_arg $methods_arg
-Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && ((
$exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
- - store_test_results:
- path: /tmp/results/repeated_utests/output
- - store_artifacts:
- path: /tmp/results/repeated_utests/stdout
- destination: stdout
- - store_artifacts:
- path: /tmp/results/repeated_utests/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/results/repeated_utests/logs
- destination: logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_unit_tests:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Determine unit Tests to Run
- command: |
- # reminder: this code (along with all the steps) is independently
executed on every circle container
- # so the goal here is to get the circleci script to return the tests
*this* container will run
- # which we do via the `circleci` cli tool.
-
- rm -fr ~/cassandra-dtest/upgrade_tests
- echo "***java tests***"
-
- # get all of our unit test filenames
- set -eo pipefail && circleci tests glob
"$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
-
- # split up the unit tests into groups based on the number of
containers we have
- set -eo pipefail && circleci tests split --split-by=timings
--timings-type=filename --index=${CIRCLE_NODE_INDEX}
--total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt >
/tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt |
sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" >
/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
- cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- no_output_timeout: 15m
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Run Unit Tests (testclasslist)
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"'
'{print $4}' || true)
- if [ -z "$test_timeout" ]; then
- test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"'
'{print $4}')
- fi
- ant testclasslist -Dtest.timeout="$test_timeout"
-Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
-Dtest.classlistprefix=unit -Dno-build-test=true
- no_output_timeout: 15m
- - store_test_results:
- path: /tmp/cassandra/build/test/output/
- - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1
$DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be
pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the
docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things
slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest
requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is
independently executed on every circle container\n# so the goal here is to get
the circleci script to return the tests *this* container will run\n# which we
do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource
~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n
export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo
pipefail && ./run_dtests.py --skip-resource-in [...]
- - run:
- name: Run dtests (j8_without_vnodes)
- no_output_timeout: 15m
- command: "echo \"cat
/tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat
/tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource
~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n
export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho
\"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need
the \"set -o pipefail\" here so that the exit code that circleci will actually
use is from pytest and not [...]
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_without_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_without_vnodes_logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests_vnode:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1
$DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be
pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the
docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things
slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest
requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_with_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is
independently executed on every circle container\n# so the goal here is to get
the circleci script to return the tests *this* container will run\n# which we
do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource
~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n
export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo
pipefail && ./run_dtests.py --use-vnodes --skip-r [...]
- - run:
- name: Run dtests (j8_with_vnodes)
- no_output_timeout: 15m
- command: "echo \"cat
/tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat
/tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource
~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n
export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho
\"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need
the \"set -o pipefail\" here so that the exit code that circleci will actually
use is from pytest and not the ex [...]
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_with_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_with_vnodes_logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_jvm_dtests_repeat:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Repeatedly run new or modifed JUnit tests
- no_output_timeout: 15m
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current
parallel runner.
- count=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} %
CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests
together, removing duplicates
- tests=$(echo ${REPEATED_JVM_DTESTS} | sed -e "s/<nil>//" | sed -e
"s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in
build.xml to group the output files
- target=test-jvm-dtest-some
- testtag=""
- if [[ $target == "test-compression" ]]; then
- testtag="compression"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class
name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "long-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant test-jvm-dtest-some $name_arg $methods_arg
-Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && ((
$exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
- - store_test_results:
- path: /tmp/results/repeated_utests/output
- - store_artifacts:
- path: /tmp/results/repeated_utests/stdout
- destination: stdout
- - store_artifacts:
- path: /tmp/results/repeated_utests/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/results/repeated_utests/logs
- destination: logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_repeated_ant_test:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Run repeated JUnit test
- no_output_timeout: 15m
- command: |
- if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then
- echo "Repeated utest class name hasn't been defined, exiting
without running any test"
- elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then
- echo "Repeated utest count hasn't been defined, exiting without
running any test"
- elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then
- echo "Repeated utest count is lesser or equals than zero, exiting
without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current
parallel runner.
- # Since we are running the same test multiple times there is no
need to use `circleci tests split`.
- count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} %
CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_ANT_TEST_TARGET}
${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS}
${REPEATED_ANT_TEST_COUNT} times"
-
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- target=${REPEATED_ANT_TEST_TARGET}
- class_path=${REPEATED_ANT_TEST_CLASS}
- class_name="${class_path##*.}"
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class
name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "long-test" ]]; then
- name="-Dtest.name=$class_name"
- else
- name="-Dtest.name=$class_path"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then
- methods=""
- else
- methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}"
- fi
-
- # Run the test target as many times as requested collecting the
exit code,
- # stopping the iteration only if stop_on_failure is set.
- exit_code="$?"
- for i in $(seq -w 1 $count); do
-
- echo "Running test iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && ant $target $name $methods
-Dno-build-test=true | tee stdout.txt ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utest/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt
$dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt
-
- # move the XML output files
- source=build/test/output
- dest=/tmp/results/repeated_utest/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs
- dest=/tmp/results/repeated_utest/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && ((
$exit_code > 0 )); then
- break
- fi
- done
-
- (exit ${exit_code})
- fi
- fi
- - store_test_results:
- path: /tmp/results/repeated_utest/output
- - store_artifacts:
- path: /tmp/results/repeated_utest/stdout
- destination: stdout
- - store_artifacts:
- path: /tmp/results/repeated_utest/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/results/repeated_utest/logs
- destination: logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests_repeat:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1
$DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be
pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the
docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things
slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest
requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Run repeated Python DTests
- no_output_timeout: 15m
- command: |
- if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
- echo "Repeated dtest name hasn't been defined, exiting without
running any test"
- elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
- echo "Repeated dtest count hasn't been defined, exiting without
running any test"
- elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
- echo "Repeated dtest count is lesser or equals than zero, exiting
without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current
parallel runner.
- # Since we are running the same test multiple times there is no
need to use `circleci tests split`.
- count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} %
CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_DTESTS} $count times"
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
-
- tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
-
- stop_on_failure_arg=""
- if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
- stop_on_failure_arg="-x"
- fi
-
- vnodes_args=""
- if false; then
- vnodes_args="--use-vnodes --num-tokens=32"
- fi
-
- upgrade_arg=""
- if false; then
- upgrade_arg="--execute-upgrade-tests
--upgrade-target-version-only --upgrade-version-selection all"
- fi
-
- # we need the "set -o pipefail" here so that the exit code that
circleci will actually use is from pytest and not the exit code from tee
- set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args
--count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG
--junit-xml=/tmp/results/dtests/pytest_result.xml -s
--cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee
/tmp/dtest/stdout.txt
- fi
- fi
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_jvm_dtests:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 1
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Determine distributed Tests to Run
- command: |
- # reminder: this code (along with all the steps) is independently
executed on every circle container
- # so the goal here is to get the circleci script to return the tests
*this* container will run
- # which we do via the `circleci` cli tool.
-
- rm -fr ~/cassandra-dtest/upgrade_tests
- echo "***java tests***"
-
- # get all of our unit test filenames
- set -eo pipefail && circleci tests glob
"$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
-
- # split up the unit tests into groups based on the number of
containers we have
- set -eo pipefail && circleci tests split --split-by=timings
--timings-type=filename --index=${CIRCLE_NODE_INDEX}
--total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt >
/tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt |
sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" |
grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
- cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- no_output_timeout: 15m
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Run Unit Tests (testclasslist)
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- test_timeout=$(grep 'name="test.distributed.timeout"' build.xml |
awk -F'"' '{print $4}' || true)
- if [ -z "$test_timeout" ]; then
- test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"'
'{print $4}')
- fi
- ant testclasslist -Dtest.timeout="$test_timeout"
-Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
-Dtest.classlistprefix=distributed -Dno-build-test=true
- no_output_timeout: 15m
- - store_test_results:
- path: /tmp/cassandra/build/test/output/
- - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_long:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 1
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Run Unit Tests (long-test)
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- ant long-test -Dno-build-test=true
- no_output_timeout: 15m
- - store_test_results:
- path: /tmp/cassandra/build/test/output/
- - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_cqlshlib_tests:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 1
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Run cqlshlib Unit Tests
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra/
- ./pylib/cassandra-cqlsh-tests.sh $(pwd)
- no_output_timeout: 15m
- - store_test_results:
- path: /tmp/cassandra/pylib
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests_large_vnode_repeat:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1
$DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be
pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the
docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things
slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest
requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Run repeated Python DTests
- no_output_timeout: 15m
- command: |
- if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
- echo "Repeated dtest name hasn't been defined, exiting without
running any test"
- elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
- echo "Repeated dtest count hasn't been defined, exiting without
running any test"
- elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
- echo "Repeated dtest count is lesser or equals than zero, exiting
without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current
parallel runner.
- # Since we are running the same test multiple times there is no
need to use `circleci tests split`.
- count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} %
CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_LARGE_DTESTS} $count times"
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
-
- tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
-
- stop_on_failure_arg=""
- if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
- stop_on_failure_arg="-x"
- fi
-
- vnodes_args=""
- if true; then
- vnodes_args="--use-vnodes --num-tokens=32"
- fi
-
- upgrade_arg=""
- if false; then
- upgrade_arg="--execute-upgrade-tests
--upgrade-target-version-only --upgrade-version-selection all"
- fi
-
- # we need the "set -o pipefail" here so that the exit code that
circleci will actually use is from pytest and not the exit code from tee
- set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args
--count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG
--junit-xml=/tmp/results/dtests/pytest_result.xml -s
--cassandra-dir=/home/cassandra/cassandra --keep-test-dir
--only-resource-intensive-tests --force-resource-intensive-tests $tests_arg |
tee /tmp/dtest/stdout.txt
- fi
- fi
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_compression:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Determine unit Tests to Run
- command: |
- # reminder: this code (along with all the steps) is independently
executed on every circle container
- # so the goal here is to get the circleci script to return the tests
*this* container will run
- # which we do via the `circleci` cli tool.
-
- rm -fr ~/cassandra-dtest/upgrade_tests
- echo "***java tests***"
-
- # get all of our unit test filenames
- set -eo pipefail && circleci tests glob
"$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
-
- # split up the unit tests into groups based on the number of
containers we have
- set -eo pipefail && circleci tests split --split-by=timings
--timings-type=filename --index=${CIRCLE_NODE_INDEX}
--total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt >
/tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt |
sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" >
/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
- cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- no_output_timeout: 15m
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Run Unit Tests (testclasslist-compression)
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"'
'{print $4}' || true)
- if [ -z "$test_timeout" ]; then
- test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"'
'{print $4}')
- fi
- ant testclasslist-compression -Dtest.timeout="$test_timeout"
-Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
-Dtest.classlistprefix=unit -Dno-build-test=true
- no_output_timeout: 15m
- - store_test_results:
- path: /tmp/cassandra/build/test/output/
- - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtest_jars_build:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 1
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Build Cassandra DTest jars
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- mkdir ~/dtest_jars
- git remote add apache https://github.com/apache/cassandra.git
- for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do
- # check out the correct cassandra version:
- git remote set-branches --add apache '$branch'
- git fetch --depth 1 apache $branch
- git checkout $branch
- git clean -fd
- # Loop to prevent failure due to maven-ant-tasks not downloading a
jar..
- for x in $(seq 1 3); do
- ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar
dtest-jar
- RETURN="$?"
- if [ "${RETURN}" -eq "0" ]; then
- cp build/dtest*.jar ~/dtest_jars
- break
- fi
- done
- # Exit, if we didn't build successfully
- if [ "${RETURN}" -ne "0" ]; then
- echo "Build failed with exit code: ${RETURN}"
- exit ${RETURN}
- fi
- done
- # and build the dtest-jar for the branch under test
- ${ANT_HOME}/bin/ant realclean
- git checkout origin/$CIRCLE_BRANCH
- git clean -fd
- for x in $(seq 1 3); do
- ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar
- RETURN="$?"
- if [ "${RETURN}" -eq "0" ]; then
- cp build/dtest*.jar ~/dtest_jars
- break
- fi
- done
- # Exit, if we didn't build successfully
- if [ "${RETURN}" -ne "0" ]; then
- echo "Build failed with exit code: ${RETURN}"
- exit ${RETURN}
- fi
- ls -l ~/dtest_jars
- no_output_timeout: 15m
- - persist_to_workspace:
- root: /home/cassandra
- paths:
- - dtest_jars
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_LARGE_DTESTS: null
- - REPEATED_LARGE_DTESTS_COUNT: 100
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
-workflows:
- version: 2
- separate_tests:
- jobs:
- - start_build:
- type: approval
- - build:
- requires:
- - start_build
- - start_j8_unit_tests:
- type: approval
- - j8_unit_tests:
- requires:
- - start_j8_unit_tests
- - build
- - start_j8_jvm_dtests:
- type: approval
- - j8_jvm_dtests:
- requires:
- - start_j8_jvm_dtests
- - build
- - start_j8_cqlshlib_tests:
- type: approval
- - j8_cqlshlib_tests:
- requires:
- - start_j8_cqlshlib_tests
- - build
- - start_j8_cqlshlib_cython_tests:
- type: approval
- - j8_cqlshlib_cython_tests:
- requires:
- - start_j8_cqlshlib_cython_tests
- - build
- - start_utests_long:
- type: approval
- - utests_long:
- requires:
- - start_utests_long
- - build
- - start_utests_compression:
- type: approval
- - utests_compression:
- requires:
- - start_utests_compression
- - build
- - start_j8_dtest_jars_build:
- type: approval
- - j8_dtest_jars_build:
- requires:
- - start_j8_dtest_jars_build
- - build
- - start_jvm_upgrade_dtests:
- type: approval
- - j8_jvm_upgrade_dtests:
- requires:
- - start_jvm_upgrade_dtests
- - j8_dtest_jars_build
- - start_j8_dtests:
- type: approval
- - j8_dtests:
- requires:
- - start_j8_dtests
- - build
- - start_j8_dtests_vnode:
- type: approval
- - j8_dtests_vnode:
- requires:
- - start_j8_dtests_vnode
- - build
- - start_j8_dtests_large:
- type: approval
- - j8_dtests_large:
- requires:
- - start_j8_dtests_large
- - build
- - start_j8_dtests_large_vnode:
- type: approval
- - j8_dtests_large_vnode:
- requires:
- - start_j8_dtests_large_vnode
- - build
- - start_j8_upgrade_dtests:
- type: approval
- - j8_upgrade_dtests:
- requires:
- - start_j8_upgrade_dtests
- - build
- pre-commit_tests:
- jobs:
- - start_pre-commit_tests:
- type: approval
- - build:
- requires:
- - start_pre-commit_tests
- - j8_unit_tests:
- requires:
- - build
- - j8_jvm_dtests:
- requires:
- - build
- - j8_cqlshlib_tests:
- requires:
- - build
- - j8_cqlshlib_cython_tests:
- requires:
- - build
- - start_utests_long:
- type: approval
- - utests_long:
- requires:
- - start_utests_long
- - build
- - start_utests_compression:
- type: approval
- - utests_compression:
- requires:
- - start_utests_compression
- - build
- - start_jvm_upgrade_dtests:
- type: approval
- - j8_dtest_jars_build:
- requires:
- - start_jvm_upgrade_dtests
- - build
- - j8_jvm_upgrade_dtests:
- requires:
- - j8_dtest_jars_build
- - j8_dtests:
- requires:
- - build
- - j8_dtests_vnode:
- requires:
- - build
- - start_j8_dtests_large:
- type: approval
- - j8_dtests_large:
- requires:
- - start_j8_dtests_large
- - build
- - j8_dtests_large_vnode:
- requires:
- - start_j8_dtests_large
- - build
- - start_upgrade_dtests:
- type: approval
- - j8_upgrade_dtests:
- requires:
- - start_upgrade_dtests
- - build
diff --git a/.circleci/config.yml.MIDRES b/.circleci/config.yml.PAID
similarity index 100%
rename from .circleci/config.yml.MIDRES
rename to .circleci/config.yml.PAID
diff --git a/.circleci/config-2_1.yml b/.circleci/config_template.yml
similarity index 100%
rename from .circleci/config-2_1.yml
rename to .circleci/config_template.yml
diff --git a/.circleci/config-2_1.yml.mid_res.patch
b/.circleci/config_template.yml.PAID.patch
similarity index 100%
rename from .circleci/config-2_1.yml.mid_res.patch
rename to .circleci/config_template.yml.PAID.patch
diff --git a/.circleci/generate.sh b/.circleci/generate.sh
index 6308da477b..ee62a9b9da 100755
--- a/.circleci/generate.sh
+++ b/.circleci/generate.sh
@@ -30,13 +30,12 @@ die ()
print_help()
{
- echo "Usage: $0 [-l|-m|-h|-f|-e]"
- echo " -a Generate the default config.yml using low resources and the
three templates"
- echo " (config.yml.LOWRES, config.yml.MIDRES and config.yml.HIGHRES).
Use this for"
- echo " permanent changes in config-2_1.yml that will be committed to
the main repo."
- echo " -l Generate config.yml using low resources"
- echo " -m Generate config.yml using mid resources"
- echo " -h Generate config.yml using high resources"
+ echo "Usage: $0 [-f|-p|-a|-e|-i]"
+ echo " -a Generate the config.yml, config.yml.FREE and config.yml.PAID
expanded configuration"
+ echo " files from the main config_template.yml reusable configuration
file."
+ echo " Use this for permanent changes in config that will be committed
to the main repo."
+ echo " -f Generate config.yml for tests compatible with the CircleCI free
tier resources"
+ echo " -p Generate config.yml for tests compatible with the CircleCI paid
tier resources"
echo " -e <key=value> Environment variables to be used in the generated
config.yml, e.g.:"
echo " -e DTEST_BRANCH=CASSANDRA-8272"
echo " -e
DTEST_REPO=https://github.com/adelapena/cassandra-dtest.git";
@@ -60,28 +59,25 @@ print_help()
echo " -e
REPEATED_ANT_TEST_METHODS=testCompoundPartitionKey,testStaticTable"
echo " -e REPEATED_ANT_TEST_COUNT=500"
echo " For the complete list of environment variables,
please check the"
- echo " list of examples in config-2_1.yml and/or the
documentation."
+ echo " list of examples in config_template.yml and/or the
documentation."
echo " If you want to specify multiple environment
variables simply add"
- echo " multiple -e options. The flags -l/-m/-h should be
used when using -e."
- echo " -f Stop checking that the environment variables are known"
+ echo " multiple -e options. The flags -f/-p should be used
when using -e."
+ echo " -i Ignore unknown environment variables"
}
all=false
-lowres=false
-midres=false
-highres=false
+free=false
+paid=false
env_vars=""
has_env_vars=false
check_env_vars=true
-while getopts "e:almhf" opt; do
+while getopts "e:afpi" opt; do
case $opt in
a ) all=true
;;
- l ) lowres=true
+ f ) free=true
;;
- m ) midres=true
- ;;
- h ) highres=true
+ p ) paid=true
;;
e ) if (!($has_env_vars)); then
env_vars="$OPTARG"
@@ -90,7 +86,7 @@ while getopts "e:almhf" opt; do
fi
has_env_vars=true
;;
- f ) check_env_vars=false
+ i ) check_env_vars=false
;;
\?) die "Invalid option: -$OPTARG"
;;
@@ -131,52 +127,40 @@ if $has_env_vars && $check_env_vars; then
done
fi
-if $lowres; then
- ($all || $midres || $highres) && die "Cannot use option -l with options -a,
-m or -h"
- echo "Generating new config.yml file with low resources from config-2_1.yml"
- circleci config process $BASEDIR/config-2_1.yml >
$BASEDIR/config.yml.LOWRES.tmp
- cat $BASEDIR/license.yml $BASEDIR/config.yml.LOWRES.tmp > $BASEDIR/config.yml
- rm $BASEDIR/config.yml.LOWRES.tmp
-
-elif $midres; then
- ($all || $lowres || $highres) && die "Cannot use option -m with options -a,
-l or -h"
- echo "Generating new config.yml file with middle resources from
config-2_1.yml"
- patch -o $BASEDIR/config-2_1.yml.MIDRES $BASEDIR/config-2_1.yml
$BASEDIR/config-2_1.yml.mid_res.patch
- circleci config process $BASEDIR/config-2_1.yml.MIDRES >
$BASEDIR/config.yml.MIDRES.tmp
- cat $BASEDIR/license.yml $BASEDIR/config.yml.MIDRES.tmp > $BASEDIR/config.yml
- rm $BASEDIR/config-2_1.yml.MIDRES $BASEDIR/config.yml.MIDRES.tmp
+if $free; then
+ ($all || $paid) && die "Cannot use option -f with options -a or -p"
+ echo "Generating new config.yml file for free tier from config_template.yml"
+ circleci config process $BASEDIR/config_template.yml >
$BASEDIR/config.yml.FREE.tmp
+ cat $BASEDIR/license.yml $BASEDIR/config.yml.FREE.tmp > $BASEDIR/config.yml
+ rm $BASEDIR/config.yml.FREE.tmp
-elif $highres; then
- ($all || $lowres || $midres) && die "Cannot use option -h with options -a,
-l or -m"
- echo "Generating new config.yml file with high resources from config-2_1.yml"
- patch -o $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config-2_1.yml
$BASEDIR/config-2_1.yml.high_res.patch
- circleci config process $BASEDIR/config-2_1.yml.HIGHRES >
$BASEDIR/config.yml.HIGHRES.tmp
- cat $BASEDIR/license.yml $BASEDIR/config.yml.HIGHRES.tmp >
$BASEDIR/config.yml
- rm $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config.yml.HIGHRES.tmp
+elif $paid; then
+ ($all || $free) && die "Cannot use option -p with options -a or -f"
+ echo "Generating new config.yml file for paid tier from config_template.yml"
+ patch -o $BASEDIR/config_template.yml.PAID $BASEDIR/config_template.yml
$BASEDIR/config_template.yml.PAID.patch
+ circleci config process $BASEDIR/config_template.yml.PAID >
$BASEDIR/config.yml.PAID.tmp
+ cat $BASEDIR/license.yml $BASEDIR/config.yml.PAID.tmp > $BASEDIR/config.yml
+ rm $BASEDIR/config_template.yml.PAID $BASEDIR/config.yml.PAID.tmp
elif $all; then
- ($lowres || $midres || $highres || $has_env_vars) && die "Cannot use option
-a with options -l, -m, -h or -e"
- echo "Generating new config.yml file with low resources and
LOWRES/MIDRES/HIGHRES templates from config-2_1.yml"
-
- # setup lowres
- circleci config process $BASEDIR/config-2_1.yml >
$BASEDIR/config.yml.LOWRES.tmp
- cat $BASEDIR/license.yml $BASEDIR/config.yml.LOWRES.tmp >
$BASEDIR/config.yml.LOWRES
- rm $BASEDIR/config.yml.LOWRES.tmp
+ ($free || $paid || $has_env_vars) && die "Cannot use option -a with options
-f, -p or -e"
+ echo "Generating new default config.yml file for free tier and FREE/PAID
templates from config_template.yml."
+ echo "Make sure you commit the newly generated config.yml, config.yml.FREE
and config.yml.PAID files"
+ echo "after running this command if you want them to persist."
- # setup midres
- patch -o $BASEDIR/config-2_1.yml.MIDRES $BASEDIR/config-2_1.yml
$BASEDIR/config-2_1.yml.mid_res.patch
- circleci config process $BASEDIR/config-2_1.yml.MIDRES >
$BASEDIR/config.yml.MIDRES.tmp
- cat $BASEDIR/license.yml $BASEDIR/config.yml.MIDRES.tmp >
$BASEDIR/config.yml.MIDRES
- rm $BASEDIR/config-2_1.yml.MIDRES $BASEDIR/config.yml.MIDRES.tmp
+ # setup config for free tier
+ circleci config process $BASEDIR/config_template.yml >
$BASEDIR/config.yml.FREE.tmp
+ cat $BASEDIR/license.yml $BASEDIR/config.yml.FREE.tmp >
$BASEDIR/config.yml.FREE
+ rm $BASEDIR/config.yml.FREE.tmp
- # setup highres
- patch -o $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config-2_1.yml
$BASEDIR/config-2_1.yml.high_res.patch
- circleci config process $BASEDIR/config-2_1.yml.HIGHRES >
$BASEDIR/config.yml.HIGHRES.tmp
- cat $BASEDIR/license.yml $BASEDIR/config.yml.HIGHRES.tmp >
$BASEDIR/config.yml.HIGHRES
- rm $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config.yml.HIGHRES.tmp
+ # setup config for paid tier
+ patch -o $BASEDIR/config_template.yml.PAID $BASEDIR/config_template.yml
$BASEDIR/config_template.yml.PAID.patch
+ circleci config process $BASEDIR/config_template.yml.PAID >
$BASEDIR/config.yml.PAID.tmp
+ cat $BASEDIR/license.yml $BASEDIR/config.yml.PAID.tmp >
$BASEDIR/config.yml.PAID
+ rm $BASEDIR/config_template.yml.PAID $BASEDIR/config.yml.PAID.tmp
- # copy lower into config.yml to make sure this gets updated
- cp $BASEDIR/config.yml.LOWRES $BASEDIR/config.yml
+ # copy free tier into config.yml to make sure this gets updated
+ cp $BASEDIR/config.yml.FREE $BASEDIR/config.yml
elif (!($has_env_vars)); then
print_help
@@ -290,7 +274,6 @@ delete_repeated_jobs()
delete_repeated_jobs "config.yml"
if $all; then
- delete_repeated_jobs "config.yml.LOWRES"
- delete_repeated_jobs "config.yml.MIDRES"
- delete_repeated_jobs "config.yml.HIGHRES"
+ delete_repeated_jobs "config.yml.FREE"
+ delete_repeated_jobs "config.yml.PAID"
fi
diff --git a/.circleci/readme.md b/.circleci/readme.md
index 9eb37678f8..e65be5158a 100644
--- a/.circleci/readme.md
+++ b/.circleci/readme.md
@@ -22,42 +22,47 @@
This directory contains the configuration for CircleCI continous integration
platform.
The file `config.yml` is the configuration file that is read by CircleCI. This
file is
-automatically generated by the `generate.sh` script from the `config-2_1.yml`
file.
+automatically generated by the `generate.sh` script from the
`config_template.yml` file.
The provided `config.yml` file uses low resources so users of the CircleCI
free tier can
-use it. Additionally, there are three versions of this file using different
resources so
+use it. Additionally, there are two versions of this file using different
resources so
users who have access to premium CircleCI resources can use larger instances
and more
-parallelism. These files are `config.yml.LOWRES`, `config.yml.MIDRES` and
`config.yml.HIGHRES`.
-The default `config.yml` file is just a copy of `config.yml.LOWRES`.
+parallelism. These files are `config.yml.FREE` and `config.yml.PAID`.
+The default `config.yml` file is just a copy of `config.yml.FREE`.
## Switching to higher resource settings
-This directory contains generated files for low, middle and high resource
settings.
-Switch between them by copying the correct file to `config.yml` and committing
the result:
+This directory contains generated files for free and paid resource settings.
+Switch between them by copying the correct file to `config.yml` and committing
the result.
+For example, for using higher resources benefiting from CircleCI's paid tier:
-`cp .circleci/config.yml.HIGHRES .circleci/config.yml`
+`cp .circleci/config.yml.PAID .circleci/config.yml`
-Alternatively, you can run the `generate.sh` script with the flags
`-l`/`-m`/`-h`
-to regenerate the `config.yml` file from `config-2_1.yml` using
LOWRES/MIDRES/HIGHRES.
-This script validates and applies any changes to the `config-2_1.yml` file,
and it
+And for using lower resources comaptible with CircleCI's free tier:
+
+`cp .circleci/config.yml.FREE .circleci/config.yml`
+
+Alternatively, you can run the `generate.sh` script with the flags `-f`/`-p`
+to regenerate the `config.yml` file from `config_template.yml` using free or
paid resources.
+This script validates and applies any changes to the `config_template.yml`
file, and it
requires the [CircleCI CLI](https://circleci.com/docs/2.0/local-cli/#install)
to be
installed.
## Setting environment variables
-Both `config-2_1.yml` and `config.yml` files contain a set of environment
variables
+Both `config_template.yml` and `config.yml` files contain a set of environment
variables
defining things like what dtest repo and branch to use, what tests could be
repeatedly
run, etc.
These environment variables can be directly edited in the `config.yml` file,
although if
you do this you should take into account that the entire set of env vars is
repeated on
-every job.
+every job.
-A probably better approach is editing them in `config-2_1.yml` and then
regenerate the
+A probably better approach is editing them in `config_template.yml` and then
regenerate the
`config.yml` file using the `generate.sh` script. You can also directly pass
environment
variable values to the `generate.sh` script with the `-e` flag. For example,
to set the
dtest repo and branch with MIDRES config you can run:
```
-generate.sh -m \
+generate.sh -p \
-e DTEST_REPO=https://github.com/adelapena/cassandra-dtest.git \
-e DTEST_BRANCH=CASSANDRA-8272
@@ -69,7 +74,7 @@ The script will then create jobs to run each of these new or
modified tests for
number of times, to verify that they are stable. You can use environment
variables to
specify the number of iterations of each type of test:
```
-generate.sh -m \
+generate.sh -p \
-e REPEATED_UTESTS_COUNT=500 \
-e REPEATED_UTESTS_LONG_COUNT=100 \
-e REPEATED_SIMULATOR_DTESTS_COUNT=500 \
@@ -82,12 +87,12 @@ generate.sh -m \
```
You can also specify whether the iteration should fail on the first test
failure:
```
-generate.sh -m -e REPEATED_TESTS_STOP_ON_FAILURE=false
+generate.sh -p -e REPEATED_TESTS_STOP_ON_FAILURE=false
```
In addition to the automatically detected tests, it's also possible to provide
lists of
specific tests to be repeated:
```
-generate.sh -m \
+generate.sh -p \
-e
REPEATED_UTESTS=org.apache.cassandra.cql3.ViewTest,org.apache.cassandra.db.CellTest
\
-e
REPEATED_UTESTS_LONG=org.apache.cassandra.io.sstable.CQLSSTableWriterLongTest#testWideRow
\
-e
REPEATED_DTESTS=cql_test.py,consistency_test.py::TestAvailability::test_simple_strategy
\
@@ -99,7 +104,7 @@ generate.sh -m \
For particular Ant test targets that are not included in the regular test
suites, you can
use the `run_repeated_utest` job:
```
-generate.sh -m \
+generate.sh -p \
-e REPEATED_ANT_TEST_TARGET=test-cdc \
-e REPEATED_ANT_TEST_CLASS=org.apache.cassandra.cql3.ViewTest \
-e REPEATED_ANT_TEST_METHODS=testCompoundPartitionKey,testStaticTable \
@@ -107,7 +112,7 @@ generate.sh -m \
```
Putting all together, you can have runs as complex as:
```
-generate.sh -m \
+generate.sh -p \
-e REPEATED_TESTS_STOP_ON_FAILURE=true \
-e
REPEATED_UTESTS=org.apache.cassandra.cql3.ViewTest,org.apache.cassandra.db.CellTest
\
-e REPEATED_UTESTS_COUNT=500 \
@@ -135,45 +140,38 @@ default medium config and Large containers.
## Updating the config
For configuration changes meant to be permanent in the Apache repo you should
never edit
-the `config.yml` file manually. Instead, you should edit the `config-2_1.yml`
file and then
-regenerate the `config.yml`, `config.yml.LOWRES`, `config.yml.MIDRES` and
`config.yml.HIGHRES`
+the `config.yml` file manually. Instead, you should edit the
`config_template.yml` file and then
+regenerate the `config.yml`, `config.yml.FREE` and `config.yml.PAID`
files by runnining the `generate.sh` script with `-a` flag. For using this
script you
need to install the [CircleCI
CLI](https://circleci.com/docs/2.0/local-cli/#install).
As for temporal changes done while working in a patch, such as pointing to you
dtest repo or
-running a test repeatedly, you can either directly edit `config.yml` or edit
`config-2_1.yml`
-and then regenerate `config.yml` with the `generate.sh` script using a
`-l`/`-m`/`-h` flag.
+running a test repeatedly, you can either directly edit `config.yml` or edit
`config_template.yml`
+and then regenerate `config.yml` with the `generate.sh` script using a
`-f`/`-p` flag.
When this flag is used only the `config.yml` will be generated.
Please note that any previous swapping or edition of the generated files will
be overriden
-by running `generate.sh` with `-a` argument, returning `config.yml` to the
default LOWRES. So if
-you previously swapped your `config.yml` to MIDRES or HIGHRES you would need
to either swap it
-again or use the `-l`/`-m`/`-h` script flags.
+by running `generate.sh` with `-a` argument, returning `config.yml` to the
default FREE. So if
+you previously swapped your `config.yml` to MIDRES you would need to either
swap it
+again or use the `-f`/`-p` script flags.
Read below for details how to generate the files manually without the
`generate.sh` script:
-1. make your edits to config-2_1.yml - let it stay at lowres settings
-2. generate a valid LOWRES file:
- `circleci config process config-2_1.yml > config.yml.LOWRES`
-3. add the Apache license header to the newly created LOWRES file:
- `cat license.yml config.yml.LOWRES > config.yml.LOWRES.new && mv
config.yml.LOWRES.new config.yml.LOWRES`
-4. then apply the highres patch to config-2_1.yml;
- `patch -o config-2_1.yml.HIGHRES config-2_1.yml
config-2_1.yml.high_res.patch`
- (this creates a new file `config-2_1.yml.HIGHRES` instead of in-place
patching
- config-2_1.yml)
- Note that if the patch no longer applies to `config-2_1.yml` a new patch
file
- is needed, do this by manually making `config-2_1.yml` high resource and
create
+1. make your edits to config_template.yml - let it stay at free tier settings
+2. generate a valid FREE file:
+ `circleci config process config_template.yml > config.yml.FREE`
+3. add the Apache license header to the newly created FREE file:
+ `cat license.yml config.yml.FREE > config.yml.FREE.new && mv
config.yml.FREE.new config.yml.FREE`
+4. then apply the PAID patch to config_template.yml;
+ `patch -o config_template.yml.PAID config_template.yml
config_template.yml.PAID.patch`
+ (this creates a new file `config_template.yml.PAID` instead of in-place
patching
+ config_template.yml).
+ Note that if the patch no longer applies to `config_template.yml` a new
patch file
+ is needed, do this by manually creating and editing
`config_template.yml.PAID`, and create
the patch file based on the diff (don't commit it though).
-5. generate the HIGHRES file:
- `circleci config process config-2_1.yml.HIGHRES > config.yml.HIGHRES`
-6. remove the temporary patched HIGHRES file: `rm config-2_1.yml.HIGHRES`
-7. add the Apache license header to the newly created HIGHRES file:
- `cat license.yml config.yml.HIGHRES > config.yml.HIGHRES.new && mv
config.yml.HIGHRES.new config.yml.HIGHRES`
-8. repeat the last steps to generate the MIDRES file:
- ```
- patch -o config-2_1.yml.MIDRES config-2_1.yml config-2_1.yml.mid_res.patch
- circleci config process config-2_1.yml.MIDRES > config.yml.MIDRES
- rm config-2_1.yml.MIDRES
- cat license.yml config.yml.MIDRES > config.yml.MIDRES.new && mv
config.yml.MIDRES.new config.yml.MIDRES
- ```
-9. finally, remember to update the config.yml
+5. generate the PAID file:
+ `circleci config process config_template.yml.PAID > config.yml.PAID`
+6. remove the temporary patched PAID file: `rm config_template.yml.PAID`
+7. add the Apache license header to the newly created PAID file:
+ `cat license.yml config.yml.PAID > config.yml.PAID.new && mv
config.yml.PAID.new config.yml.PAID`
+8. finally, remember to update the config.yml
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
