This is an automated email from the ASF dual-hosted git repository. adelapena pushed a commit to branch cassandra-4.0 in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 72f3b7901879aba902f92b45215a25ee130ff057 Merge: f646a7c 7e95c92 Author: Andrés de la Peña <[email protected]> AuthorDate: Tue Oct 19 11:42:56 2021 +0100 Merge branch 'cassandra-3.11' into cassandra-4.0 .circleci/config-2_1.yml | 42 ++++++++++++--- .circleci/config-2_1.yml.high_res.patch | 48 ++++++++++++++++- .circleci/config-2_1.yml.mid_res.patch | 91 ++++++++++++++++++++++++--------- .circleci/config.yml.MIDRES | 8 +-- 4 files changed, 153 insertions(+), 36 deletions(-) diff --cc .circleci/config-2_1.yml index 034720f,2650830..9e823af --- a/.circleci/config-2_1.yml +++ b/.circleci/config-2_1.yml @@@ -128,31 -121,33 +128,61 @@@ j8_seq_executor: &j8_seq_executo #exec_resource_class: xlarge parallelism: 1 # sequential, single container tests: no parallelism benefits +j11_par_executor: &j11_par_executor + executor: + name: java11-executor + #exec_resource_class: xlarge + parallelism: 4 + +j11_small_par_executor: &j11_small_par_executor + executor: + name: java11-executor + #exec_resource_class: xlarge + parallelism: 1 + +j11_small_executor: &j11_small_executor + executor: + name: java11-executor + #exec_resource_class: medium + parallelism: 1 + + j8_repeated_utest_executor: &j8_repeated_utest_executor + executor: + name: java8-executor + parallelism: 4 + + j8_repeated_dtest_executor: &j8_repeated_dtest_executor + executor: + name: java8-executor + parallelism: 4 + + j8_repeated_upgrade_dtest_executor: &j8_repeated_upgrade_dtest_executor + executor: + name: java8-executor + parallelism: 4 + + j8_repeated_jvm_upgrade_dtest_executor: &j8_repeated_jvm_upgrade_dtest_executor + executor: + name: java8-executor + parallelism: 4 + -with_dtests_jobs: &with_dtest_jobs ++j11_repeated_utest_executor: &j11_repeated_utest_executor ++ executor: ++ name: java11-executor ++ parallelism: 4 ++ ++j11_repeated_dtest_executor: &j11_repeated_dtest_executor ++ executor: ++ name: java11-executor ++ parallelism: 4 ++ +j8_with_dtests_jobs: &j8_with_dtests_jobs jobs: - - start_build: + - start_j8_build: type: approval - - build: + - j8_build: requires: - - start_build + - start_j8_build # Java 8 unit tests - start_j8_unit_tests: type: approval @@@ -908,205 -471,16 +938,205 @@@ jobs - create_venv - create_dtest_containers: file_tag: j8_upgradetests_without_vnodes - run_dtests_extra_args: '--execute-upgrade-tests' - extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' - tests_filter_pattern: '^upgrade_tests' + run_dtests_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all' - run_dtests: file_tag: j8_upgradetests_without_vnodes - extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' - pytest_extra_args: '--execute-upgrade-tests' + pytest_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all' + + j8_cqlsh-dtests-py2-with-vnodes: + <<: *j8_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv + - create_dtest_containers: + file_tag: j8_with_vnodes + run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'" + - run_dtests: + file_tag: j8_with_vnodes + pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7' + + j8_cqlsh-dtests-py3-with-vnodes: + <<: *j8_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv + - create_dtest_containers: + file_tag: j8_with_vnodes + run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'" + - run_dtests: + file_tag: j8_with_vnodes + pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' + + j8_cqlsh-dtests-py38-with-vnodes: + <<: *j8_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv: + python_version: '3.8' + - create_dtest_containers: + file_tag: j8_with_vnodes + run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'" + python_version: '3.8' + - run_dtests: + file_tag: j8_with_vnodes + pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8' + python_version: '3.8' + + j8_cqlsh-dtests-py2-no-vnodes: + <<: *j8_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv + - create_dtest_containers: + file_tag: j8_without_vnodes + run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'" + - run_dtests: + file_tag: j8_without_vnodes + pytest_extra_args: '--skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7' + + j8_cqlsh-dtests-py3-no-vnodes: + <<: *j8_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv + - create_dtest_containers: + file_tag: j8_without_vnodes + run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'" + - run_dtests: + file_tag: j8_without_vnodes + pytest_extra_args: '--skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' + + j8_cqlsh-dtests-py38-no-vnodes: + <<: *j8_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv: + python_version: '3.8' + - create_dtest_containers: + file_tag: j8_without_vnodes + run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'" + python_version: '3.8' + - run_dtests: + file_tag: j8_without_vnodes + pytest_extra_args: '--skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8' + python_version: '3.8' + + j11_cqlsh-dtests-py2-with-vnodes: + <<: *j11_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv + - create_dtest_containers: + file_tag: j11_with_vnodes + run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'" + - run_dtests: + file_tag: j11_with_vnodes + pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7' + + j11_cqlsh-dtests-py3-with-vnodes: + <<: *j11_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv + - create_dtest_containers: + file_tag: j11_with_vnodes + run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'" + - run_dtests: + file_tag: j11_with_vnodes + pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' + + j11_cqlsh-dtests-py38-with-vnodes: + <<: *j11_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv: + python_version: '3.8' + - create_dtest_containers: + file_tag: j11_with_vnodes + run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'" + python_version: '3.8' + - run_dtests: + file_tag: j11_with_vnodes + pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8' + python_version: '3.8' + + j11_cqlsh-dtests-py2-no-vnodes: + <<: *j11_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv + - create_dtest_containers: + file_tag: j11_without_vnodes + run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'" + - run_dtests: + file_tag: j11_without_vnodes + pytest_extra_args: '--skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7' + + j11_cqlsh-dtests-py3-no-vnodes: + <<: *j11_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv + - create_dtest_containers: + file_tag: j11_without_vnodes + run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'" + - run_dtests: + file_tag: j11_without_vnodes + pytest_extra_args: '--skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' + + j11_cqlsh-dtests-py38-no-vnodes: + <<: *j11_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv: + python_version: '3.8' + - create_dtest_containers: + file_tag: j11_without_vnodes + run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'" + python_version: '3.8' + - run_dtests: + file_tag: j11_without_vnodes + pytest_extra_args: '--skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8' + python_version: '3.8' j8_repeated_utest: - <<: *j8_par_executor + <<: *j8_repeated_utest_executor steps: - attach_workspace: at: /home/cassandra @@@ -1118,21 -492,8 +1148,21 @@@ count: ${REPEATED_UTEST_COUNT} stop_on_failure: ${REPEATED_UTEST_STOP_ON_FAILURE} + j11_repeated_utest: - <<: *j11_par_executor ++ <<: *j11_repeated_utest_executor + steps: + - attach_workspace: + at: /home/cassandra + - log_environment + - run_repeated_utest: + target: ${REPEATED_UTEST_TARGET} + class: ${REPEATED_UTEST_CLASS} + methods: ${REPEATED_UTEST_METHODS} + count: ${REPEATED_UTEST_COUNT} + stop_on_failure: ${REPEATED_UTEST_STOP_ON_FAILURE} + j8_repeated_dtest: - <<: *j8_par_executor + <<: *j8_repeated_dtest_executor steps: - attach_workspace: at: /home/cassandra @@@ -1145,23 -506,8 +1175,23 @@@ count: ${REPEATED_DTEST_COUNT} stop_on_failure: ${REPEATED_DTEST_STOP_ON_FAILURE} + j11_repeated_dtest: - <<: *j11_par_executor ++ <<: *j11_repeated_dtest_executor + steps: + - attach_workspace: + at: /home/cassandra + - log_environment + - clone_dtest + - create_venv + - run_repeated_dtest: + tests: ${REPEATED_DTEST_NAME} + vnodes: ${REPEATED_DTEST_VNODES} + upgrade: "false" + count: ${REPEATED_DTEST_COUNT} + stop_on_failure: ${REPEATED_DTEST_STOP_ON_FAILURE} + repeated_jvm_upgrade_dtest: - <<: *j8_par_executor + <<: *j8_repeated_jvm_upgrade_dtest_executor steps: - attach_workspace: at: /home/cassandra diff --cc .circleci/config-2_1.yml.high_res.patch index 1a0ba53,ffe37e2..45c54b2 --- a/.circleci/config-2_1.yml.high_res.patch +++ b/.circleci/config-2_1.yml.high_res.patch @@@ -1,4 -1,17 +1,6 @@@ - @@ -19,14 +19,14 @@ default_env_vars: &default_env_vars ---- .circleci/config-2_1.yml 2021-10-18 11:17:08.000000000 +0100 -+++ .circleci/config-2_1.yml.HIGHRES 2021-10-18 11:22:50.000000000 +0100 -@@ -40,8 +40,8 @@ - CASSANDRA_SKIP_SYNC: true - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - DTEST_BRANCH: trunk -- CCM_MAX_HEAP_SIZE: 1024M -- CCM_HEAP_NEWSIZE: 256M -+ CCM_MAX_HEAP_SIZE: 2048M -+ CCM_HEAP_NEWSIZE: 512M - - # The Ant test target to run, for example: - # REPEATED_UTEST_TARGET: testsome -@@ -100,46 +100,50 @@ ++--- .circleci/config-2_1.yml 2021-10-18 11:48:35.000000000 +0100 +++++ .circleci/config-2_1.yml.HIGHRES 2021-10-18 12:03:47.000000000 +0100 ++@@ -101,14 +101,14 @@ j8_par_executor: &j8_par_executor executor: name: java8-executor @@@ -13,11 -26,8 +15,11 @@@ - #exec_resource_class: xlarge - parallelism: 1 + exec_resource_class: xlarge -+ parallelism: 2 ++ parallelism: 5 + j8_small_executor: &j8_small_executor + executor: - @@ -37,32 +37,32 @@ j8_small_executor: &j8_small_executor ++@@ -119,62 +119,68 @@ j8_medium_par_executor: &j8_medium_par_executor executor: name: java8-executor @@@ -33,30 -43,33 +35,72 @@@ + exec_resource_class: xlarge parallelism: 1 # sequential, single container tests: no parallelism benefits + j11_par_executor: &j11_par_executor + executor: + name: java11-executor +- #exec_resource_class: xlarge +- parallelism: 4 ++ exec_resource_class: xlarge ++ parallelism: 100 + +-j11_small_par_executor: &j11_small_par_executor ++j11_small_executor: &j11_small_executor + executor: + name: java11-executor +- #exec_resource_class: xlarge ++ exec_resource_class: medium + parallelism: 1 + +-j11_small_executor: &j11_small_executor ++j11_small_par_executor: &j11_small_par_executor + executor: + name: java11-executor +- #exec_resource_class: medium +- parallelism: 1 ++ exec_resource_class: xlarge ++ parallelism: 2 + + j8_repeated_utest_executor: &j8_repeated_utest_executor + executor: + name: java8-executor + - parallelism: 4 + + exec_resource_class: xlarge + + parallelism: 100 + + j8_repeated_dtest_executor: &j8_repeated_dtest_executor + executor: + name: java8-executor + - parallelism: 4 + + exec_resource_class: xlarge + + parallelism: 100 + + j8_repeated_upgrade_dtest_executor: &j8_repeated_upgrade_dtest_executor + executor: + name: java8-executor + - parallelism: 4 + + exec_resource_class: xlarge + + parallelism: 100 + + j8_repeated_jvm_upgrade_dtest_executor: &j8_repeated_jvm_upgrade_dtest_executor + executor: + name: java8-executor + - parallelism: 4 + + exec_resource_class: xlarge + + parallelism: 100 + - with_dtests_jobs: &with_dtest_jobs ++ j11_repeated_utest_executor: &j11_repeated_utest_executor ++ executor: ++ name: java11-executor ++- parallelism: 4 +++ exec_resource_class: xlarge +++ parallelism: 100 ++ ++ j11_repeated_dtest_executor: &j11_repeated_dtest_executor ++ executor: ++ name: java11-executor ++- parallelism: 4 +++ exec_resource_class: xlarge +++ parallelism: 100 ++ + j8_with_dtests_jobs: &j8_with_dtests_jobs jobs: diff --cc .circleci/config-2_1.yml.mid_res.patch index 0c360d9,bf1f3f7..27df9cc --- a/.circleci/config-2_1.yml.mid_res.patch +++ b/.circleci/config-2_1.yml.mid_res.patch @@@ -1,8 -1,9 +1,10 @@@ - @@ -19,14 +19,14 @@ default_env_vars: &default_env_vars ---- .circleci/config-2_1.yml 2021-10-18 11:17:08.000000000 +0100 -+++ .circleci/config-2_1.yml.MIDRES 2021-10-18 11:20:45.000000000 +0100 -@@ -101,45 +101,65 @@ ++--- .circleci/config-2_1.yml 2021-10-18 11:48:35.000000000 +0100 +++++ .circleci/config-2_1.yml.MIDRES 2021-10-18 12:02:44.000000000 +0100 ++@@ -101,14 +101,14 @@ + j8_par_executor: &j8_par_executor executor: name: java8-executor - #exec_resource_class: xlarge +- #exec_resource_class: xlarge - parallelism: 4 + exec_resource_class: medium + parallelism: 25 @@@ -15,9 -16,6 +17,9 @@@ + exec_resource_class: large + parallelism: 10 + j8_small_executor: &j8_small_executor + executor: - @@ -37,20 +37,32 @@ j8_small_executor: &j8_small_executor ++@@ -119,20 +119,32 @@ j8_medium_par_executor: &j8_medium_par_executor executor: name: java8-executor @@@ -45,31 -43,38 +47,72 @@@ + exec_resource_class: medium parallelism: 1 # sequential, single container tests: no parallelism benefits + j11_par_executor: &j11_par_executor + executor: + name: java11-executor +- #exec_resource_class: xlarge +- parallelism: 4 ++ exec_resource_class: medium ++ parallelism: 25 + + j11_small_par_executor: &j11_small_par_executor + executor: - @@ -64,6 +76,12 @@ j11_small_executor: &j11_small_executor ++@@ -146,35 +158,47 @@ + #exec_resource_class: medium + parallelism: 1 + ++j11_large_par_executor: &j11_large_par_executor ++ executor: ++ name: java11-executor ++ exec_resource_class: large ++ parallelism: 50 ++ + j8_repeated_utest_executor: &j8_repeated_utest_executor + executor: + name: java8-executor + - parallelism: 4 + + exec_resource_class: medium + + parallelism: 25 + + j8_repeated_dtest_executor: &j8_repeated_dtest_executor + executor: + name: java8-executor + - parallelism: 4 + + exec_resource_class: large + + parallelism: 25 + + j8_repeated_upgrade_dtest_executor: &j8_repeated_upgrade_dtest_executor + executor: + name: java8-executor + - parallelism: 4 + + exec_resource_class: xlarge + + parallelism: 25 + + j8_repeated_jvm_upgrade_dtest_executor: &j8_repeated_jvm_upgrade_dtest_executor + executor: + name: java8-executor + - parallelism: 4 + + exec_resource_class: large + + parallelism: 25 + - with_dtests_jobs: &with_dtest_jobs ++ j11_repeated_utest_executor: &j11_repeated_utest_executor ++ executor: ++ name: java11-executor ++- parallelism: 4 +++ exec_resource_class: medium +++ parallelism: 25 ++ ++ j11_repeated_dtest_executor: &j11_repeated_dtest_executor ++ executor: ++ name: java11-executor ++- parallelism: 4 +++ exec_resource_class: large +++ parallelism: 25 ++ + j8_with_dtests_jobs: &j8_with_dtests_jobs jobs: - - j8_build - @@ -456,7 +474,7 @@ jobs: - target: fqltool-test -@@ -435,7 +455,7 @@ - target: stress-test ++@@ -872,7 +896,7 @@ ++ target: testclasslist-system-keyspace-directory j8_dtests-with-vnodes: - <<: *j8_par_executor @@@ -77,17 -82,8 +120,17 @@@ steps: - attach_workspace: at: /home/cassandra - @@ -470,7 +488,7 @@ jobs: -@@ -449,7 +469,7 @@ - pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests' ++@@ -886,7 +910,7 @@ + pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests' + + j11_dtests-with-vnodes: +- <<: *j11_par_executor ++ <<: *j11_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra - @@ -485,7 +503,7 @@ jobs: ++@@ -901,7 +925,7 @@ + pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests' j8_dtests-no-vnodes: - <<: *j8_par_executor @@@ -95,134 -91,12 +138,134 @@@ steps: - attach_workspace: at: /home/cassandra - @@ -499,7 +517,7 @@ jobs: -@@ -463,7 +483,7 @@ ++@@ -915,7 +939,7 @@ pytest_extra_args: '--skip-resource-intensive-tests' + j11_dtests-no-vnodes: +- <<: *j11_par_executor ++ <<: *j11_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra - @@ -514,7 +532,7 @@ jobs: ++@@ -930,7 +954,7 @@ + pytest_extra_args: '--skip-resource-intensive-tests' + j8_upgradetests-no-vnodes: - <<: *j8_par_executor + <<: *j8_very_large_par_executor steps: - attach_workspace: at: /home/cassandra - @@ -531,7 +549,7 @@ jobs: - pytest_extra_args: '--execute-upgrade-tests' ++@@ -944,7 +968,7 @@ ++ pytest_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all' + + j8_cqlsh-dtests-py2-with-vnodes: +- <<: *j8_par_executor ++ <<: *j8_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra - @@ -546,7 +564,7 @@ jobs: ++@@ -959,7 +983,7 @@ + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7' + + j8_cqlsh-dtests-py3-with-vnodes: +- <<: *j8_par_executor ++ <<: *j8_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra - @@ -561,7 +579,7 @@ jobs: ++@@ -974,7 +998,7 @@ + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' + + j8_cqlsh-dtests-py38-with-vnodes: +- <<: *j8_par_executor ++ <<: *j8_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra - @@ -579,7 +597,7 @@ jobs: ++@@ -992,7 +1016,7 @@ + python_version: '3.8' + + j8_cqlsh-dtests-py2-no-vnodes: +- <<: *j8_par_executor ++ <<: *j8_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra - @@ -594,7 +612,7 @@ jobs: ++@@ -1007,7 +1031,7 @@ + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7' + + j8_cqlsh-dtests-py3-no-vnodes: +- <<: *j8_par_executor ++ <<: *j8_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra - @@ -609,7 +627,7 @@ jobs: ++@@ -1022,7 +1046,7 @@ + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' + + j8_cqlsh-dtests-py38-no-vnodes: +- <<: *j8_par_executor ++ <<: *j8_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra - @@ -627,7 +645,7 @@ jobs: ++@@ -1040,7 +1064,7 @@ + python_version: '3.8' + + j11_cqlsh-dtests-py2-with-vnodes: +- <<: *j11_par_executor ++ <<: *j11_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra - @@ -642,7 +660,7 @@ jobs: ++@@ -1055,7 +1079,7 @@ + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7' + + j11_cqlsh-dtests-py3-with-vnodes: +- <<: *j11_par_executor ++ <<: *j11_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra - @@ -657,7 +675,7 @@ jobs: ++@@ -1070,7 +1094,7 @@ + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' + + j11_cqlsh-dtests-py38-with-vnodes: +- <<: *j11_par_executor ++ <<: *j11_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra - @@ -675,7 +693,7 @@ jobs: ++@@ -1088,7 +1112,7 @@ + python_version: '3.8' + + j11_cqlsh-dtests-py2-no-vnodes: +- <<: *j11_par_executor ++ <<: *j11_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra - @@ -690,7 +708,7 @@ jobs: ++@@ -1103,7 +1127,7 @@ + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7' + + j11_cqlsh-dtests-py3-no-vnodes: +- <<: *j11_par_executor ++ <<: *j11_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra - @@ -705,7 +723,7 @@ jobs: ++@@ -1118,7 +1142,7 @@ + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' + + j11_cqlsh-dtests-py38-no-vnodes: +- <<: *j11_par_executor ++ <<: *j11_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra - @@ -1063,3 +1081,4 @@ commands: ++@@ -1760,3 +1784,4 @@ + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_logs ++ diff --cc .circleci/config.yml.MIDRES index f89c09e,5bfb7f6..142bc88 --- a/.circleci/config.yml.MIDRES +++ b/.circleci/config.yml.MIDRES @@@ -269,2307 -268,8 +269,2307 @@@ jobs path: /tmp/results/repeated_utest/logs destination: logs environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh-dtests-py2-with-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip- [...] + - run: + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then + export CQLSH_PYTHON=/usr/bin/python2.7 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j11_unit_tests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Determine unit Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) + if [ -z "$test_timeout" ]; then + test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') + fi + ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + repeated_upgrade_dtest: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium ++ resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Run repeated Python dtest + no_output_timeout: 15m + command: | + if [ "${REPEATED_UPGRADE_DTEST_NAME}" == "<nil>" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_UPGRADE_DTEST_COUNT}" == "<nil>" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_UPGRADE_DTEST_COUNT}" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_UPGRADE_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_UPGRADE_DTEST_NAME} $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + stop_on_failure_arg="" + if ${REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE}; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if false; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + upgrade_arg="" + if true; then + upgrade_arg="--execute-upgrade-tests" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_UPGRADE_DTEST_NAME} | tee /tmp/dtest/stdout.txt + fi + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh-dtests-py38-no-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.8/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...] + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt + + source ~/env3.8/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then + export CQLSH_PYTHON=/usr/bin/python3.8 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j11_cqlsh-dtests-py3-with-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip [...] + - run: + name: Run dtests (j11_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then + export CQLSH_PYTHON=/usr/bin/python3.6 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_cqlsh-dtests-py3-no-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource- [...] + - run: + name: Run dtests (j11_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then + export CQLSH_PYTHON=/usr/bin/python3.6 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_cqlsh-dtests-py38-with-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.8/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip [...] + - run: + name: Run dtests (j11_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt + + source ~/env3.8/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then + export CQLSH_PYTHON=/usr/bin/python3.8 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j8_cqlsh-dtests-py3-with-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip- [...] + - run: + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then + export CQLSH_PYTHON=/usr/bin/python3.6 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh-dtests-py2-no-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...] + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then + export CQLSH_PYTHON=/usr/bin/python2.7 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j11_repeated_dtest: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 - resource_class: medium ++ resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Run repeated Python dtest + no_output_timeout: 15m + command: | + if [ "${REPEATED_DTEST_NAME}" == "<nil>" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTEST_COUNT}" == "<nil>" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTEST_COUNT}" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_DTEST_NAME} $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + stop_on_failure_arg="" + if ${REPEATED_DTEST_STOP_ON_FAILURE}; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if ${REPEATED_DTEST_VNODES}; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + upgrade_arg="" + if false; then + upgrade_arg="--execute-upgrade-tests" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_DTEST_NAME} | tee /tmp/dtest/stdout.txt + fi + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j8_repeated_dtest: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium ++ resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Run repeated Python dtest + no_output_timeout: 15m + command: | + if [ "${REPEATED_DTEST_NAME}" == "<nil>" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTEST_COUNT}" == "<nil>" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTEST_COUNT}" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_DTEST_NAME} $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + stop_on_failure_arg="" + if ${REPEATED_DTEST_STOP_ON_FAILURE}; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if ${REPEATED_DTEST_VNODES}; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + upgrade_arg="" + if false; then + upgrade_arg="--execute-upgrade-tests" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_DTEST_NAME} | tee /tmp/dtest/stdout.txt + fi + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j11_cqlsh-dtests-py2-with-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip [...] + - run: + name: Run dtests (j11_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then + export CQLSH_PYTHON=/usr/bin/python2.7 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_dtests-with-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip [...] + - run: + name: Run dtests (j11_with_vnodes) + no_output_timeout: 15m + command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the [...] + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j8_dtests-no-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...] + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not [...] + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_upgradetests-no-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_upgradetests_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_upgradetests_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --ex [...] + - run: + name: Run dtests (j8_upgradetests_without_vnodes) + no_output_timeout: 15m + command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually [...] + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_upgradetests_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_upgradetests_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + utests_stress: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Run Unit Tests (stress-test) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + ant stress-test + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_unit_tests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Determine unit Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) + if [ -z "$test_timeout" ]; then + test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') + fi + ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j11_jvm_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Determine distributed Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) + if [ -z "$test_timeout" ]; then + test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') + fi + ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_build: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Clone Cassandra Repository (via git) + command: | + git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra + - run: + name: Build Cassandra + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean realclean jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + no_output_timeout: 15m + - run: + name: Run eclipse-warnings + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + ant eclipse-warnings + - persist_to_workspace: + root: /home/cassandra + paths: + - cassandra + - .m2 + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_cqlsh-dtests-py2-no-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource- [...] + - run: + name: Run dtests (j11_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then + export CQLSH_PYTHON=/usr/bin/python2.7 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_cqlshlib_tests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Run cqlshlib Unit Tests + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra/pylib + ./cassandra-cqlsh-tests.sh .. + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/pylib + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_repeated_utest: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run repeated JUnit test + no_output_timeout: 15m + command: | + if [ "${REPEATED_UTEST_CLASS}" == "<nil>" ]; then + echo "Repeated utest class name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_UTEST_COUNT}" == "<nil>" ]; then + echo "Repeated utest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_UTEST_COUNT}" -le 0 ]; then + echo "Repeated utest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_UTEST_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UTEST_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_UTEST_TARGET} ${REPEATED_UTEST_CLASS} ${REPEATED_UTEST_METHODS} ${REPEATED_UTEST_COUNT} times" + + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + target=${REPEATED_UTEST_TARGET} + class_path=${REPEATED_UTEST_CLASS} + class_name="${class_path##*.}" + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" ]]; then + name="-Dtest.name=$class_name" + else + name="-Dtest.name=$class_path" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ "${REPEATED_UTEST_METHODS}" == "<nil>" ]; then + methods="" + else + methods="-Dtest.methods=${REPEATED_UTEST_METHODS}" + fi + + # Run the test target as many times as requested collecting the exit code, + # stopping the iteration only if stop_on_failure is set. + exit_code="$?" + for i in $(seq -w 1 $count); do + + echo "Running test iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utest/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${REPEATED_UTEST_TARGET}-${REPEATED_UTEST_CLASS}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utest/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utest/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_UTEST_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + + (exit ${exit_code}) + fi + fi + - store_test_results: + path: /tmp/results/repeated_utest/output + - store_artifacts: + path: /tmp/results/repeated_utest/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utest/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utest/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j8_dtests-with-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip- [...] + - run: + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the e [...] + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - LANG: en_US.UTF-8 - KEEP_TEST_DIR: true - DEFAULT_DIR: /home/cassandra/cassandra-dtest --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
