This is an automated email from the ASF dual-hosted git repository. michaelsembwever pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 8b7a4e2cc0d5da6c4075a4ea55f74e828a720d73 Merge: 4ab34e1c1c 7d9fa6a154 Author: mck <[email protected]> AuthorDate: Sat Sep 5 19:58:53 2026 +0200 Merge branch 'cassandra-6.0' into trunk * cassandra-6.0: CI: Take the report merge off the test fan-out, stop the agent pod churn, report split balance .build/ci/cell_balance.py | 328 +++++++++++++++++++++++++++++++++++ .build/ci/generate-ci-summary.sh | 6 +- .build/run-ci.d/run-ci-test.py | 13 +- .jenkins/Jenkinsfile | 242 +++++++++++++++++++------- .jenkins/k8s/README.md | 12 +- .jenkins/k8s/jenkins-deployment.yaml | 155 ++++++++++++++--- 6 files changed, 664 insertions(+), 92 deletions(-) diff --cc .build/ci/generate-ci-summary.sh index 3828fa2c1e,3828fa2c1e..3767bc43f0 --- a/.build/ci/generate-ci-summary.sh +++ b/.build/ci/generate-ci-summary.sh @@@ -68,7 -68,7 +68,11 @@@ cat >${DIST_DIR}/ci_summary.html <<EO ... EOL ++# Unguarded, and this script is `sh -e`: a summary the parser could not write is a failure. ${CASSANDRA_DIR}/.build/ci/ci_parser.py --mute --input ${DIST_DIR}/test/output/ --output ${DIST_DIR}/ci_summary.html --exit $? ++# How evenly each target's splits divided, and which came near their cell deadline. ++${CASSANDRA_DIR}/.build/ci/cell_balance.py --input ${DIST_DIR}/test --output ${DIST_DIR}/ci_summary.html || echo "failed cell_balance.py" ++ ++exit 0 diff --cc .build/run-ci.d/run-ci-test.py index 98fc6b754a,957aa574d7..203a3ea5a7 --- a/.build/run-ci.d/run-ci-test.py +++ b/.build/run-ci.d/run-ci-test.py @@@ -296,8 -295,8 +296,11 @@@ class TestCIPipeline(unittest.TestCase) maxSize is the only in-cluster record of what a pool can hold, and a pool at zero nodes has no nodes to count, so the check reads it from here. """ ++ # Two groups per size, each pair summing to that size's instanceCap in jenkins-deployment.yaml. Raise ++ # these whenever a cap is raised, or the committed values stop passing their own check. groups = [(f"eks-amd64-{size}-ondemand-{n}-{n}cfd1c1", 0, maximum) -- for size, maximum in (("large", 80), ("medium", 65), ("small", 25)) for n in (2, 3)] ++ for size, maximum in (("large", 153), ("medium", 75), ("small", 10), ("report", 2)) ++ for n in (2, 3)] groups.append(("eks-jenkins-controller-0-2acd8787", 1, 1)) return yaml.safe_dump({"nodeGroups": [ @@@ -341,9 -340,9 +344,9 @@@ self.assertEqual(0, self.capacity_check(self.deployed_values(), nodes=self.LARGE_NODE)) def test_check_agent_capacity_blocks_a_cap_above_the_pool(self): -- # 200 against the 160 nodes two large groups can hold: 40 agents could never be scheduled, which is ++ # 400 against the 306 nodes two large groups can hold: 94 agents could never be scheduled, which is # not idle but a churn loop, and is what preceded the 2026-08-11 controller stall -- over = self.deployed_values("large", instanceCap=200, instanceCapStr="200") ++ over = self.deployed_values("large", instanceCap=400, instanceCapStr="400") self.assertEqual(1, self.capacity_check(over)) # a cluster whose ceilings cannot be read leaves it unchecked rather than blocking a valid deploy self.assertEqual(0, self.capacity_check(over, autoscaler=False)) @@@ -352,14 -351,14 +355,14 @@@ # the live cluster nests a group's maximum under its health condition, and the check also takes it # from the group. Reading the wrong key costs nothing visible: the ceilings come out empty and # every cap passes unchecked, so the shapes are pinned here rather than in a deploy -- over = self.deployed_values("large", instanceCap=200, instanceCapStr="200") ++ over = self.deployed_values("large", instanceCap=400, instanceCapStr="400") for nested in (True, False): self.assertEqual(1, self.capacity_check(over, nested=nested)) self.assertEqual(0, self.capacity_check(self.deployed_values(), nested=nested)) def test_check_agent_capacity_blocks_contradictory_config(self): # the plugin takes the cap from either key, so a disagreement resolves to whichever applies last -- self.assertEqual(1, self.capacity_check(self.deployed_values("large", instanceCapStr="200"))) ++ self.assertEqual(1, self.capacity_check(self.deployed_values("large", instanceCapStr="400"))) # a nodeSelector the live nodes contradict strands every agent of that size typo = self.deployed_values("large", nodeSelector="cassandra.jenkins.agent.large=ture") self.assertEqual(1, self.capacity_check(typo, nodes=self.LARGE_NODE)) diff --cc .jenkins/Jenkinsfile index 4ca6a4443e,4ca6a4443e..4b4b4c6857 --- a/.jenkins/Jenkinsfile +++ b/.jenkins/Jenkinsfile @@@ -35,6 -35,6 +35,8 @@@ // - cassandra-small + cassandra-${arch}-small : 1 cpu, 1GB ram (alias for above but for any arch) // - cassandra-medium + cassandra-${arch}-medium : 3 cpu, 5GB ram // - cassandra-large + cassandra-${arch}-large : 7 cpu, 16GB ram ++// - cassandra-report + cassandra-${arch}-report : 2 cpu, 9GB ram ++// // // Performance targets required a `cassandra-${arch}-large-dedicated` labelled nodes. // @@@ -61,6 -61,6 +63,28 @@@ import groovy.transform.Fiel @Field List<String> archsSupported = ["amd64", "arm64"] @Field List<String> pythonsSupported = ["3.8", "3.11", "3.12", "3.13"] @Field String pythonDefault = "3.8" ++ ++// Shell defining cpus(), to prepend to any `sh` script fanning out with `xargs -P` or `xz -T` ++@Field String cpusShell = ''' ++cpus() { ++ cpus_n= ++ # The v2 line of /proc/self/cgroup is "0::<path>", this cgroup relative to the host's root. ++ cpus_own="$(sed -n 's/^0:://p' /proc/self/cgroup 2>/dev/null | head -1)" ++ for cpus_file in "/sys/fs/cgroup${cpus_own}/cpu.max" /sys/fs/cgroup/cpu.max ; do ++ cpus_quota= cpus_period= ++ read -r cpus_quota cpus_period 2>/dev/null < "${cpus_file}" || continue ++ # "max <period>" is a cgroup with no cpu limit, where nproc is right. ++ case "${cpus_quota}${cpus_period}" in ""|*[!0-9]*) continue ;; esac ++ [ "${cpus_period:-0}" -gt 0 ] || continue ++ cpus_n=$(( cpus_quota / cpus_period )) ++ break ++ done ++ [ -n "${cpus_n}" ] || cpus_n="$(nproc 2>/dev/null || echo 1)" ++ [ "${cpus_n}" -ge 1 ] 2>/dev/null || cpus_n=1 ++ echo "${cpus_n}" ++} ++''' ++ /** CONSTANTS end **********************************/ pipeline { @@@ -199,7 -199,7 +223,7 @@@ def tasks() 'test-latest': [splits: 20], 'test-compression': [splits: 20], 'stress-test': [splits: 1, size: 'small'], -- 'test-burn': [splits: 4], ++ 'test-burn': [splits: 5], 'long-test': [splits: 4], 'test-oa': [splits: 20], 'test-system-keyspace-directory': [splits: 20], @@@ -432,52 -432,52 +456,44 @@@ def test(command, cell) fetchDockerImages(['ubuntu-test']) def cell_suffix = "_jdk${cell.jdk}_python_${cell.python}_${cell.cython}_${cell.arch}_${cell.split}_${splits}" def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz" -- def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail -- script_vars = "${script_vars} python_version=\'${cell.python}\'" -- script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'" -- if ("cqlsh-test" == cell.step) { -- script_vars = "${script_vars} cython=\'${cell.cython}\'" -- } -- script_vars = fetchDTestsSource(command, script_vars) -- timeout(time: command.timeout_hours, unit: 'HOURS') { // best throughput with each cell at ~10 minutes -- def timer = System.currentTimeMillis() -- try { -- buildJVMDTestJars(cell, script_vars, logfile) -- script_vars = "${script_vars} docker_timeout_hours=\"${command.timeout_hours}\"" -- def status = sh label: "RUNNING TESTS ${cell.step}...", script: filterConsoleOutput("${script_vars} .build/docker/run-tests.sh -a ${cell.step} -c '${cell.split}/${splits}' -j ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )"), returnStatus: true -- dir("build") { -- archiveArtifacts artifacts: "${logfile}", fingerprint: true -- } -- if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") } -- } catch (exc) { -- if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) { -- def descriptions = [] -- for (def cause in exc.getCauses()) { -- echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}" -- if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption') || cause.getClass().getName().contains('ParallelStep$FailFastCause')) { -- throw exc // user abort or fail-fast — do not retry ++ def script_vars = testScriptVars(command, cell) ++ def timer = System.currentTimeMillis() ++ def outcome = "ok" ++ try { ++ timeout(time: command.timeout_hours, unit: 'HOURS') { // best throughput with each cell at ~10 minutes ++ try { ++ buildJVMDTestJars(cell, script_vars, logfile) ++ script_vars = "${script_vars} docker_timeout_hours=\"${command.timeout_hours}\"" ++ def status = sh label: "RUNNING TESTS ${cell.step}...", script: filterConsoleOutput("${script_vars} .build/docker/run-tests.sh -a ${cell.step} -c '${cell.split}/${splits}' -j ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )"), returnStatus: true ++ dir("build") { ++ archiveArtifacts artifacts: "${logfile}", fingerprint: true ++ } ++ if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") } ++ } catch (exc) { ++ outcome = "failed" ++ if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) { ++ def descriptions = [] ++ for (def cause in exc.getCauses()) { ++ echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}" ++ if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption') || cause.getClass().getName().contains('ParallelStep$FailFastCause')) { ++ throw exc // user abort or fail-fast — do not retry ++ } ++ if (cause.getClass().getName().contains('TimeoutStepExecution')) { outcome = "timeout" } ++ descriptions.add(cause.getShortDescription()) } -- descriptions.add(cause.getShortDescription()) ++ error("Retryable interruption: ${descriptions.join(', ')}") } -- error("Retryable interruption: ${descriptions.join(', ')}") ++ throw exc ++ } finally { ++ def duration = System.currentTimeMillis() - timer ++ def formattedTime = String.format("%tT.%tL", duration, duration) ++ echo "Time ${cell.step}${cell_suffix}: ${formattedTime}" } -- throw exc -- } finally { -- def duration = System.currentTimeMillis() - timer -- def formattedTime = String.format("%tT.%tL", duration, duration) -- echo "Time ${cell.step}${cell_suffix}: ${formattedTime}" -- } -- } -- dir("build") { -- organiseTestResultFiles(cell, cell_suffix) -- if (!cell.step.startsWith("microbench")) { -- junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']] } -- debugOomKiller() -- compressTestResultFiles() -- archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json", fingerprint: true -- copyToNightlies("${logfile},test/logs/**,test/**/jmh-result.json", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_")) ++ } finally { ++ recordCellTime(cell, cell_suffix, splits, command.timeout_hours, System.currentTimeMillis() - timer, outcome) } ++ processResults(cell, cell_suffix, logfile, splits) } finally { cleanAgent(cell.step) } @@@ -506,6 -506,6 +522,19 @@@ def fetchDTestsSource(command, script_v return script_vars } ++// The prefix every test cell's `sh` script starts with: bash with pipefail, which the tee needs, and the ++// variables run-tests.sh reads. fetchDTestsSource is here because it both checks out the python dtests and ++// names their directory in the prefix. ++def testScriptVars(command, cell) { ++ def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail ++ script_vars = "${script_vars} python_version=\'${cell.python}\'" ++ script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'" ++ if ("cqlsh-test" == cell.step) { ++ script_vars = "${script_vars} cython=\'${cell.cython}\'" ++ } ++ return fetchDTestsSource(command, script_vars) ++} ++ def buildJVMDTestJars(cell, script_vars, logfile) { if (cell.step.startsWith("jvm-dtest-upgrade")) { try { @@@ -632,6 -632,6 +661,78 @@@ def _stash(cell) stash name: "${cell.arch}_${cell.jdk}" } ++// One line per cell, for the split balance table .build/ci/cell_balance.py puts in ci_summary.html. ++// ++// The duration is the cell's own, not the sum of its tests: the cell deadline covers the setup before the ++// first test too. ++// ++// outcome is ok, timeout, or failed. A timed-out cell's duration is its deadline and so a lower bound, and a ++// failed cell's may be short because the failure ended it. ++def recordCellTime(cell, cell_suffix, splits, timeout_hours, duration_ms, outcome) { ++ try { ++ dir("build") { ++ writeCellTime(cell, cell_suffix, splits, timeout_hours, duration_ms, outcome) ++ archiveArtifacts artifacts: "test/cell-times/**", fingerprint: true ++ } ++ } catch (hudson.AbortException | IOException exc) { ++ echo "no cell time recorded for ${cell.step}${cell_suffix}: ${exc}" ++ } ++} ++ ++def processResults(cell, cell_suffix, logfile, splits) { ++ dir("build") { ++ organiseTestResultFiles(cell, cell_suffix) ++ if (!cell.step.startsWith("microbench")) { ++ junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml" ++ } ++ debugOomKiller() ++ compressTestResultFiles() ++ archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json", fingerprint: true ++ copyToNightlies("${logfile},test/logs/**,test/**/jmh-result.json", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_")) ++ } ++} ++ ++// The .tsv and the .suites of one cell, written in build/ ++def writeCellTime(cell, cell_suffix, splits, timeout_hours, duration_ms, outcome) { ++ // Both files are written before organiseTestResultFiles: here test/output/ holds only this cell's results, ++ // and one step later they are merged with the split gone from every path, so no report can say which cell ++ // ran which test again. ++ // ++ // Aggregated by a testcase's classname. The cell's whole test time is recorded too, so ++ // cell_balance.py can subtract it from the duration and state the setup time spent exactly. ++ sh label: "recording cell time...", script: """ ++ mkdir -p test/cell-times ++ suites='test/cell-times/${cell.step}${cell_suffix}.suites' ++ ++ # <seconds>\\t<classname>, longest first, with the total on stderr. grep -o rather than a line-oriented ++ # read, because pytest puts many testcase elements on one line. A parser would be tidier; this runs ++ # once per cell, of which a build has over a thousand. ++ find test/output -type f -name '*.xml' -print0 2>/dev/null \\ ++ | xargs -0 -r grep -ho '<testcase [^>]*>' 2>/dev/null \\ ++ | awk ' ++ { ++ cls = ""; secs = "" ++ # time=" is anchored on a space because name=" is a substring of classname=". Anchoring both ++ # costs nothing and records the trap. ++ if (match(\$0, /classname="[^"]*"/)) { cls = substr(\$0, RSTART + 11, RLENGTH - 12) } ++ if (match(\$0, /[ \\t]time="[^"]*"/)) { secs = substr(\$0, RSTART + 7, RLENGTH - 8) } ++ if (cls != "" && secs != "") { total[cls] += secs; grand += secs } ++ } ++ END { ++ for (c in total) { printf "%.3f\\t%s\\n", total[c], c } ++ printf "%.3f\\n", grand > "/dev/stderr" ++ }' 2>test/cell-times/grand \\ ++ | sort -rn | head -12 > "\${suites}" || true ++ test_seconds="\$(cat test/cell-times/grand 2>/dev/null || echo 0)" ++ rm -f test/cell-times/grand ++ ++ printf '%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n' \\ ++ '${cell.step}' '${cell.arch}' '${cell.jdk}' '${cell.split}' '${splits}' \\ ++ '${timeout_hours}' '${duration_ms}' "\${test_seconds}" '${outcome}' \\ ++ > 'test/cell-times/${cell.step}${cell_suffix}.tsv' ++ """ ++} ++ def organiseTestResultFiles(cell, cell_suffix) { sh label: "organise test result files...", script: """ mkdir -p test/output/${cell.step} @@@ -650,9 -650,9 +751,11 @@@ def debugOomKiller() } def compressTestResultFiles() { -- sh label: "compress test result files...", script: """ ++ sh label: "compress test result files...", script: """${cpusShell} { set +x; } 2>/dev/null -- find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f ++ # -n64, not -n1: one xz per file is a fork and an exec for a few kilobytes, and 64 per process still ++ # leaves more batches than parallel slots. cpus(), not nproc: see cpusShell. ++ find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n64 -P"\$(cpus)" xz -f echo "\$(find test/output -type f -name "*.xml.xz" | wc -l) test result files compressed" """ } @@@ -662,7 -662,7 +765,7 @@@ ///////////////////////////////////////// def generateTestReports() { -- node("cassandra-medium") { ++ node("cassandra-report") { cleanAgent("generateTestReports") checkout changelog: false, scm: scmGit(branches: [[name: params.branch]], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true)], userRemoteConfigs: [[url: params.repository]]) def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_generateTestReports.log.xz" @@@ -673,35 -673,35 +776,52 @@@ // copyArtifacts takes >4hrs, hack with manual download sh label: "manual download (instead of copyArtifacts)...", script: """${script_vars} ( mkdir -p build/test -- wget -q ${BUILD_URL}/artifact/test/output/*zip*/output.zip -- unzip -x -d build/test -q output.zip ) ${teeSuffix} ++ wget -q ${BUILD_URL}/artifact/test/output/*zip*/output.zip ++ unzip -x -d build/test -q output.zip ++ ( wget -q ${BUILD_URL}/artifact/test/cell-times/*zip*/cell-times.zip && unzip -x -d build/test -q cell-times.zip ) || echo "no cell-times artefact to download" ++ ) ${teeSuffix} """ } else { -- copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true ++ copyArtifacts filter: 'test/cell-times/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true } // merge and summarise test reports if (fileExists('build/test/output') && sh(script: 'test -n "$(find build/test/output -type f -name "*.xml.xz" -print -quit)"', returnStatus: true) == 0) { // merge splits for each target's test report, other axes are kept separate -- // TODO parallelised for loop // TODO results_details.tar.xz needs to include all logs for failed tests -- sh label: "merging splits test reports...", script: """${script_vars} ( ++ sh label: "merging splits test reports...", script: """${script_vars}${cpusShell} ( echo "test result files to decompress"; find build/test/output -type f -name "*.xml.xz" | wc -l -- find build/test/output -type f -name "*.xml.xz" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f --decompress -- -- for target in \$(ls build/test/output/) ; do -- if test -d build/test/output/\${target} ; then -- mkdir -p build/test/reports/\${target} -- echo "Report for \${target} (\$(find build/test/output/\${target} -name '*.xml' | wc -l) test files)" -- CASSANDRA_DOCKER_ANT_OPTS="-Dbuild.test.output.dir=build/test/output/\${target} -Dbuild.test.report.dir=build/test/reports/\${target}" ++ find build/test/output -type f -name "*.xml.xz" -print0 | xargs -0 -r -n64 -P"\$(cpus)" xz -f --decompress ++ ++ # One ant junitreport per test target, up to three at a time: cpus() is this jnlp container's limit, so the report template's 2 cpu runs two. ++ report_jobs=\$(cpus) ; [ "\${report_jobs}" -le 3 ] || report_jobs=3 ++ report_target() { ++ target="\$1" ++ test -d "build/test/output/\${target}" || return 0 ++ mkdir -p "build/test/reports/\${target}" ++ # Held and printed as one block per target: concurrent runs otherwise interleave line by line ++ report_log="\$(mktemp)" ++ { ++ echo "Report for \${target} (\$(find "build/test/output/\${target}" -name '*.xml' | wc -l) test files)" ++ # -Xmx2g per concurrent ant jvm; unset, each takes a quarter of dind's limit. Exported inside ++ # the function, which xargs runs as its own process, so targets cannot see each other's value. ++ CASSANDRA_DOCKER_ANT_OPTS="-Xmx2g -Dbuild.test.output.dir=build/test/output/\${target} -Dbuild.test.report.dir=build/test/reports/\${target}" export CASSANDRA_DOCKER_ANT_OPTS .build/docker/_docker_run.sh debian-build.docker ci/generate-test-report.sh -- fi -- done ++ } > "\${report_log}" 2>&1 ++ report_status=\$? ++ cat "\${report_log}" ; rm -f "\${report_log}" ++ return \${report_status} ++ } ++ export -f report_target ++ ls build/test/output/ | xargs -r -n1 -P"\${report_jobs}" bash -xc 'report_target "\$0"' .build/docker/_docker_run.sh debian-build.docker ci/generate-ci-summary.sh || echo "failed generate-ci-summary.sh" tar -cf build/results_details.tar -C build/test/ reports -- xz -8f build/results_details.tar ) ${teeSuffix} ++ # -T with a count: xz's own -T0 reads nproc and would thread for the node, not this container. ++ # --memlimit-compress caps the total: -8 takes ~700MB per thread, and against a limit xz drops ++ # threads (then the preset) instead of growing until the kernel's OOM killer takes it. ++ xz -8f -T"\$(cpus)" --memlimit-compress=2g build/results_details.tar ) ${teeSuffix} """ dir('build/') { @@@ -778,4 -778,4 +898,4 @@@ def emailContent() ------------------------------------------------------------------------------- For complete test report and logs see https://nightlies.apache.org/cassandra/${JOB_NAME}/${BUILD_NUMBER}/ ''' --} ++} diff --cc .jenkins/k8s/README.md index 14ac7175fd,14ac7175fd..b8d631d759 --- a/.jenkins/k8s/README.md +++ b/.jenkins/k8s/README.md @@@ -24,15 -24,15 +24,21 @@@ ZONE="us-central1-c gcloud container clusters create ${CLUSTER_NAME} --machine-type e2-standard-8 --disk-type=pd-ssd --num-nodes 1 --node-labels=cassandra.jenkins.controller=true --autoscaling-profile optimize-utilization --zone ${ZONE} # small resource nodes --gcloud container node-pools create agents-small --cluster ${CLUSTER_NAME} --machine-type e2-highcpu-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=50 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.small=true --zone ${ZONE} ++gcloud container node-pools create agents-small --cluster ${CLUSTER_NAME} --machine-type e2-highcpu-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=20 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.small=true --zone ${ZONE} ++ ++# report resource nodes, for the generateTestReports stage (the agent-dind-report podTemplate) ++# a standard machine, not highcpu: that template's dind container has a 9G memory limit ++gcloud container node-pools create agents-report --cluster ${CLUSTER_NAME} --machine-type n2-standard-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=4 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.report=true --zone ${ZONE} # medium resource nodes # preference (by cost): n2-highcpu-8, c3-highcpu-8, n4-highcpu-8, n1-highcpu-16 --gcloud container node-pools create agents-medium --cluster ${CLUSTER_NAME} --machine-type n2-highcpu-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=100 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.medium=true --zone ${ZONE} ++gcloud container node-pools create agents-medium --cluster ${CLUSTER_NAME} --machine-type n2-highcpu-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=150 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.medium=true --zone ${ZONE} # large resource nodes --gcloud container node-pools create agents-large --cluster ${CLUSTER_NAME} --machine-type n2-standard-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=160 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.large=true --zone ${ZONE} ++gcloud container node-pools create agents-large --cluster ${CLUSTER_NAME} --machine-type n2-standard-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=306 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.large=true --zone ${ZONE} ++# Each --max-nodes above is that size's agent.podTemplates.*.instanceCap in jenkins-deployment.yaml, one agent per node. ++# Raise the pool first: `.build/run-ci --only-setup` blocks a deploy whose instanceCap exceeds the nodes its pool can hold. # For each sized resource nodes, pick any machine type that fits, those listed above should work and be the most cost-effective, but this can change region to region # See https://github.com/apache/cassandra/blob/cassandra-6.0/.jenkins/Jenkinsfile#L35-L38 # and agent.podTemplates.*.resourceLimitCpu and agent.podTemplates.*.resourceLimitMemory (adding gke/eks requirements) in https://github.com/apache/cassandra/blob/cassandra-6.0/.jenkins/k8s/jenkins-deployment.yaml diff --cc .jenkins/k8s/jenkins-deployment.yaml index 6e59340a13,6e59340a13..37e257da19 --- a/.jenkins/k8s/jenkins-deployment.yaml +++ b/.jenkins/k8s/jenkins-deployment.yaml @@@ -170,11 -170,11 +170,16 @@@ controller enabled: true agent: disableDefaultAgent: true -- maxRequestsPerHostStr: "3200" -- containerCap: 300 ++ # Concurrent requests the plugin's client may hold to one host, the API server being its only host. The ++ # plugin's default of 32 is for a cloud of a few agents. ++ maxRequestsPerHostStr: "5120" ++ # The cap across every podTemplate below, and equal to the sum of their four instanceCaps, so on a cluster ++ # whose pools can each reach their own cap this never binds. Deliberately: a lower shared cap does not ++ # divide the shortfall evenly, it hands it to whichever pool holds its agents longest, which is large. ++ containerCap: 480 node-selector: cassandra.jenkins.agent: true -- waitForPodSec: "180" ++ waitForPodSec: "900" # Reap agent pods the controller no longer tracks. A restart of the controller JVM clears its in-memory # agent registry, and because agent pods are bare pods with no ownerReference nothing else deletes them. # Preferred over a pod activeDeadlineSeconds, which cannot distinguish an orphan from a long (6h) build. @@@ -206,13 -206,13 +211,15 @@@ nodeSelector: 'cassandra.jenkins.agent.small=true' # 0 = no pod lifetime cap: idleMinutes reuse means a pod's age is unrelated to any one build's timeout. Orphans are reaped by agent.garbageCollection instead. activeDeadlineSeconds: '0' -- idleMinutes: 1 -- # should match the small pool's 50 nodes (README's --max-nodes), i.e. one agent per node. must not be higher than max nodes possible. -- instanceCap: 50 -- instanceCapStr: "50" ++ idleMinutes: 5 ++ # This pool's ceiling, not the cluster's; agent.containerCap above is that, and is the one to lower ++ # when the account cannot hold this many nodes. Must not exceed the small pool's own maximum ++ # (README's --max-nodes), one agent per node. No other template selects that pool. ++ instanceCap: 20 ++ instanceCapStr: "20" nodeUsageMode: "NORMAL" showRawYaml: 'true' -- slaveConnectTimeout: '30' ++ slaveConnectTimeout: '600' yamlMergeStrategy: override containers: - name: jnlp @@@ -308,19 -308,19 +315,126 @@@ volumeMounts: - name: docker-storage mountPath: /var/lib/docker ++ agent-dind-report: | ++ - name: agent-dind-report ++ label: agent-dind cassandra-report cassandra-amd64-report ++ nodeSelector: 'cassandra.jenkins.agent.report=true' ++ activeDeadlineSeconds: '0' ++ idleMinutes: 5 ++ instanceCap: 4 ++ instanceCapStr: "4" ++ nodeUsageMode: "NORMAL" ++ showRawYaml: 'true' ++ slaveConnectTimeout: '600' ++ yamlMergeStrategy: override ++ containers: ++ - name: jnlp ++ # https://github.com/jenkinsci/kubernetes-plugin#pipeline-support ++ alwaysPullImage: true ++ envVars: ++ - envVar: ++ key: DOCKER_TLS_CERTDIR ++ value: /certs/client/ ++ - envVar: ++ key: DOCKER_CERT_PATH ++ value: /certs/client/ ++ - envVar: ++ key: DOCKER_TLS_VERIFY ++ value: 'true' ++ - envVar: ++ key: DOCKER_HOST ++ value: tcp://localhost:2376 ++ - envVar: ++ key: JENKINS_JAVA_OPTS ++ value: '-Dorg.jenkinsci.plugins.durabletask.BourneShellScript.USE_BINARY_WRAPPER=true' ++ image: apache.jfrog.io/cassan-docker/apache/cassandra-jenkins-k8s ++ livenessProbe: ++ failureThreshold: '0' ++ initialDelaySeconds: '0' ++ periodSeconds: '0' ++ successThreshold: '0' ++ timeoutSeconds: '0' ++ privileged: 'true' ++ resourceRequestCpu: 1 ++ resourceLimitCpu: 2 ++ resourceRequestMemory: 1G ++ resourceLimitMemory: 2400M ++ # the workspace emptyDir: the downloaded output.zip, the decompressed test xml, and the tar ++ resourceRequestEphemeralStorage: 10Gi ++ resourceLimitEphemeralStorage: 20Gi ++ ttyEnabled: 'true' ++ workingDir: /home/jenkins/agent ++ - name: dind ++ alwaysPullImage: 'false' ++ envVars: ++ - envVar: ++ key: DOCKER_TLS_CERTDIR ++ value: /certs ++ - envVar: ++ key: "DOCKER_IPTABLES_LEGACY" ++ value: "1" ++ image: docker:dind ++ args: "--default-address-pool base=192.168.96.0/20,size=24" # overwrite docker subnet in case of overlapping ++ livenessProbe: ++ failureThreshold: '0' ++ initialDelaySeconds: '0' ++ periodSeconds: '0' ++ successThreshold: '0' ++ timeoutSeconds: '0' ++ privileged: 'true' ++ resourceRequestCpu: 2 ++ resourceLimitCpu: 5 ++ resourceRequestMemory: 3400M ++ # 9G against the medium template's 5G: generateTestReports now runs several ant junitreport ++ # jvms at once, each in a container this daemon holds. Moves with the Jenkinsfile's ++ # report_jobs and -Xmx. ++ resourceLimitMemory: 9G ++ # docker's images and containers, in the docker-storage emptyDir ++ resourceRequestEphemeralStorage: 40Gi ++ resourceLimitEphemeralStorage: 60Gi ++ ttyEnabled: 'true' ++ workingDir: /home/jenkins/agent ++ volumes: ++ # /var/lib/docker is not here but in `yaml:` below, the only place it can carry a sizeLimit ++ - emptyDirVolume: ++ memory: 'false' ++ mountPath: /certs ++ # limit one agent pod per node for simpler operations (like orphan cleanup) ++ yaml: | ++ spec: ++ affinity: ++ podAntiAffinity: ++ requiredDuringSchedulingIgnoredDuringExecution: ++ - labelSelector: ++ matchExpressions: ++ - key: jenkins/cassius-jenkins-agent ++ operator: In ++ values: ++ - "true" ++ topologyKey: kubernetes.io/hostname ++ # docker's storage, named and mounted here so that the sizeLimit survives the plugin's merge. ++ # 60Gi bounds the images and their containers alone, of the pod's 80Gi ++ # fetchDockerImages in Jenkinsfile warns as the node fills as our image sizes grow. ++ volumes: ++ - name: docker-storage ++ emptyDir: ++ sizeLimit: 60Gi ++ containers: ++ - name: dind ++ volumeMounts: ++ - name: docker-storage ++ mountPath: /var/lib/docker agent-dind-medium: | - name: agent-dind-medium label: agent-dind cassandra-medium cassandra-amd64-medium nodeSelector: 'cassandra.jenkins.agent.medium=true' -- # 0 = no pod lifetime cap: idleMinutes reuse means a pod's age is unrelated to any one build's timeout. Orphans are reaped by agent.garbageCollection instead. activeDeadlineSeconds: '0' -- idleMinutes: 1 -- # should match the medium pools 100 nodes (README's --max-nodes), i.e. one agent per node. must not be higher than max nodes possible. -- instanceCap: 100 -- instanceCapStr: "100" ++ idleMinutes: 5 ++ instanceCap: 150 ++ instanceCapStr: "150" nodeUsageMode: "NORMAL" showRawYaml: 'true' -- slaveConnectTimeout: '30' ++ slaveConnectTimeout: '600' yamlMergeStrategy: override containers: - name: jnlp @@@ -420,17 -420,17 +534,14 @@@ - name: agent-dind-large label: agent-dind cassandra-large cassandra-amd64-large cassandra-amd64-large-dedicated nodeSelector: 'cassandra.jenkins.agent.large=true' -- # 0 = no pod lifetime cap. A microbench cell runs up to 6h (timeout_hours in the Jenkinsfile) -- # and idleMinutes reuse extends a pod's age past any one build, so age cannot stand in for -- # health here. Orphans are reaped by agent.garbageCollection instead. activeDeadlineSeconds: '0' -- idleMinutes: 1 -- # should match the large pools 160 nodes (README's --max-nodes), i.e. one agent per node. must not be higher than max nodes possible. -- instanceCap: 160 -- instanceCapStr: "160" ++ idleMinutes: 5 ++ instanceCap: 306 ++ instanceCapStr: "306" nodeUsageMode: "NORMAL" showRawYaml: 'true' -- slaveConnectTimeout: '30' ++ # Raised from 30 seconds. See the small template above. ++ slaveConnectTimeout: '600' yamlMergeStrategy: override containers: - name: jnlp --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
