https://github.com/DavidSpickett updated
https://github.com/llvm/llvm-project/pull/113447
>From ba99bed96bd69db7c86d43a2119255569872f41f Mon Sep 17 00:00:00 2001
From: David Spickett <david.spick...@linaro.org>
Date: Mon, 21 Oct 2024 12:34:17 +0000
Subject: [PATCH 1/8] [ci] Write test results to unique file names
In this patch I'm using a new lit option so that the pipeline
writes many results files, one for each time lit is run:
```
--use-unique-output-file-name
When enabled, lit will add a unique element to the output file name, before
the extension. For example "results.xml" will become "results.<something>.xml".
The
"<something>" is not ordered in any way and is chosen so that existing files
are not overwritten. [Default: Off]
```
(I added this to lit recently)
Now if I run the Linux build:
$ bash ./.ci/monolithic-linux.sh "clang;lldb;lld" "check-lldb-shell check-lld"
"libcxx;libcxxabi" "check-libcxx check-libcxxabi"
I get multiple test result files. In my case some tests fail so runtimes aren't
checked, but all projects are
so there is 1 file for lldb and one for lld:
$ ls build/*.xml
build/test-results.klc82utf.xml build/test-results.majylh73.xml
This change just collects the XML files as artifacts. Once I know that's
working, I can setup a test reporting plugin to build a summary from them.
---
.ci/generate-buildkite-pipeline-premerge | 4 ++--
.ci/monolithic-linux.sh | 13 +++++++++----
.ci/monolithic-windows.sh | 2 +-
3 files changed, 12 insertions(+), 7 deletions(-)
diff --git a/.ci/generate-buildkite-pipeline-premerge
b/.ci/generate-buildkite-pipeline-premerge
index 7676ff716c4185..e52133751f09b1 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -272,7 +272,7 @@ if [[ "${linux_projects}" != "" ]]; then
artifact_paths:
- 'artifacts/**/*'
- '*_result.json'
- - 'build/test-results.xml'
+ - 'build/test-results*.xml'
agents: ${LINUX_AGENTS}
retry:
automatic:
@@ -295,7 +295,7 @@ if [[ "${windows_projects}" != "" ]]; then
artifact_paths:
- 'artifacts/**/*'
- '*_result.json'
- - 'build/test-results.xml'
+ - 'build/test-results*.xml'
agents: ${WINDOWS_AGENTS}
retry:
automatic:
diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index b78dc59432b65c..17ea51c08fafd3 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -37,6 +37,8 @@ trap show-stats EXIT
projects="${1}"
targets="${2}"
+lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml
--use-unique-output-file-name --timeout=1200 --time-tests"
+
echo "--- cmake"
pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt
pip install -q -r "${MONOREPO_ROOT}"/lldb/test/requirements.txt
@@ -47,7 +49,7 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
-D LLVM_ENABLE_ASSERTIONS=ON \
-D LLVM_BUILD_EXAMPLES=ON \
-D COMPILER_RT_BUILD_LIBFUZZER=OFF \
- -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml
--timeout=1200 --time-tests" \
+ -D LLVM_LIT_ARGS="${lit_args}" \
-D LLVM_ENABLE_LLD=ON \
-D CMAKE_CXX_FLAGS=-gmlt \
-D LLVM_CCACHE_BUILD=ON \
@@ -87,7 +89,8 @@ if [[ "${runtimes}" != "" ]]; then
-D CMAKE_BUILD_TYPE=RelWithDebInfo \
-D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
-D LIBCXX_TEST_PARAMS="std=c++03" \
- -D LIBCXXABI_TEST_PARAMS="std=c++03"
+ -D LIBCXXABI_TEST_PARAMS="std=c++03" \
+ -D LLVM_LIT_ARGS="${lit_args}"
echo "--- ninja runtimes C++03"
@@ -104,7 +107,8 @@ if [[ "${runtimes}" != "" ]]; then
-D CMAKE_BUILD_TYPE=RelWithDebInfo \
-D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
-D LIBCXX_TEST_PARAMS="std=c++26" \
- -D LIBCXXABI_TEST_PARAMS="std=c++26"
+ -D LIBCXXABI_TEST_PARAMS="std=c++26" \
+ -D LLVM_LIT_ARGS="${lit_args}"
echo "--- ninja runtimes C++26"
@@ -121,7 +125,8 @@ if [[ "${runtimes}" != "" ]]; then
-D CMAKE_BUILD_TYPE=RelWithDebInfo \
-D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
-D LIBCXX_TEST_PARAMS="enable_modules=clang" \
- -D LIBCXXABI_TEST_PARAMS="enable_modules=clang"
+ -D LIBCXXABI_TEST_PARAMS="enable_modules=clang" \
+ -D LLVM_LIT_ARGS="${lit_args}"
echo "--- ninja runtimes clang modules"
diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh
index 91e719c52d4363..9ec44c22442d06 100755
--- a/.ci/monolithic-windows.sh
+++ b/.ci/monolithic-windows.sh
@@ -53,7 +53,7 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
-D LLVM_ENABLE_ASSERTIONS=ON \
-D LLVM_BUILD_EXAMPLES=ON \
-D COMPILER_RT_BUILD_LIBFUZZER=OFF \
- -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml
--timeout=1200 --time-tests" \
+ -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml
--use-unique-output-file-name --timeout=1200 --time-tests" \
-D COMPILER_RT_BUILD_ORC=OFF \
-D CMAKE_C_COMPILER_LAUNCHER=sccache \
-D CMAKE_CXX_COMPILER_LAUNCHER=sccache \
>From c2d541afc37226a09eb89cf7b6e71836889940fd Mon Sep 17 00:00:00 2001
From: David Spickett <david.spick...@linaro.org>
Date: Wed, 23 Oct 2024 11:39:15 +0100
Subject: [PATCH 2/8] WIP
---
.ci/generate_test_report.py | 251 ++++++++++++++++++++++++++++++++++++
1 file changed, 251 insertions(+)
create mode 100644 .ci/generate_test_report.py
diff --git a/.ci/generate_test_report.py b/.ci/generate_test_report.py
new file mode 100644
index 00000000000000..672b131dbbd311
--- /dev/null
+++ b/.ci/generate_test_report.py
@@ -0,0 +1,251 @@
+# Script to parse many JUnit XML result files and produce a combined report
+# that CI jobs can show.
+#
+# To run the unittests:
+# python3 -m unittest discover -p generate_test_report.py
+
+import argparse
+import unittest
+from io import StringIO
+from junitparser import JUnitXml, Failure
+from textwrap import dedent
+
+
+def junit_from_xml(xml):
+ return JUnitXml.fromfile(StringIO(xml))
+
+
+class TestReports(unittest.TestCase):
+ def test_title(self):
+ self.assertEqual(_generate_report("Foo", []), "# Foo")
+
+ def test_report_single_file_single_testsuite(self):
+ self.assertEqual(
+ _generate_report(
+ "Foo",
+ [
+ junit_from_xml(
+ dedent(
+ """\
+ <?xml version="1.0" encoding="UTF-8"?>
+ <testsuites time="8.89">
+ <testsuite name="Bar" tests="4" failures="2" skipped="1"
time="410.63">
+ <testcase classname="Bar/test_1" name="test_1" time="0.02"/>
+ <testcase classname="Bar/test_2" name="test_2" time="0.02">
+ <skipped message="Reason"/>
+ </testcase>
+ <testcase classname="Bar/test_3" name="test_3" time="0.02">
+ <failure><![CDATA[Output goes here]]></failure>
+ </testcase>
+ <testcase classname="Bar/test_4" name="test_4" time="0.02">
+ <failure><![CDATA[Other output goes here]]></failure>
+ </testcase>
+ </testsuite>
+ </testsuites>"""
+ )
+ )
+ ],
+ ),
+ dedent(
+ """\
+ # Foo
+
+ 1 tests passed
+ 1 tests skipped
+ 2 tests failed
+
+ ## Failed tests
+ (click to see output)
+
+ ### Bar
+ <details>
+ <summary>Bar/test_3/test_3</summary>
+
+ ```
+ Output goes here
+ ```
+ </details>
+ <details>
+ <summary>Bar/test_4/test_4</summary>
+
+ ```
+ Other output goes here
+ ```
+ </details>"""
+ ),
+ )
+
+ MULTI_SUITE_OUTPUT = dedent(
+ """\
+ # ABC and DEF
+
+ 1 tests passed
+ 1 tests skipped
+ 2 tests failed
+
+ ## Failed tests
+ (click to see output)
+
+ ### ABC
+ <details>
+ <summary>ABC/test_2/test_2</summary>
+
+ ```
+ ABC/test_2 output goes here
+ ```
+ </details>
+
+ ### DEF
+ <details>
+ <summary>DEF/test_2/test_2</summary>
+
+ ```
+ DEF/test_2 output goes here
+ ```
+ </details>"""
+ )
+
+ def test_report_single_file_multiple_testsuites(self):
+ self.assertEqual(
+ _generate_report(
+ "ABC and DEF",
+ [
+ junit_from_xml(
+ dedent(
+ """\
+ <?xml version="1.0" encoding="UTF-8"?>
+ <testsuites time="8.89">
+ <testsuite name="ABC" tests="2" failures="1" skipped="0"
time="410.63">
+ <testcase classname="ABC/test_1" name="test_1" time="0.02"/>
+ <testcase classname="ABC/test_2" name="test_2" time="0.02">
+ <failure><![CDATA[ABC/test_2 output goes here]]></failure>
+ </testcase>
+ </testsuite>
+ <testsuite name="DEF" tests="2" failures="1" skipped="1"
time="410.63">
+ <testcase classname="DEF/test_1" name="test_1" time="0.02">
+ <skipped message="reason"/>
+ </testcase>
+ <testcase classname="DEF/test_2" name="test_2" time="0.02">
+ <failure><![CDATA[DEF/test_2 output goes here]]></failure>
+ </testcase>
+ </testsuite>
+ </testsuites>"""
+ )
+ )
+ ],
+ ),
+ self.MULTI_SUITE_OUTPUT,
+ )
+
+ def test_report_multiple_files_multiple_testsuites(self):
+ self.assertEqual(
+ _generate_report(
+ "ABC and DEF",
+ [
+ junit_from_xml(
+ dedent(
+ """\
+ <?xml version="1.0" encoding="UTF-8"?>
+ <testsuites time="8.89">
+ <testsuite name="ABC" tests="2" failures="1" skipped="0"
time="410.63">
+ <testcase classname="ABC/test_1" name="test_1" time="0.02"/>
+ <testcase classname="ABC/test_2" name="test_2" time="0.02">
+ <failure><![CDATA[ABC/test_2 output goes here]]></failure>
+ </testcase>
+ </testsuite>
+ </testsuites>"""
+ )
+ ),
+ junit_from_xml(
+ dedent(
+ """\
+ <?xml version="1.0" encoding="UTF-8"?>
+ <testsuites time="8.89">
+ <testsuite name="DEF" tests="2" failures="1" skipped="1"
time="410.63">
+ <testcase classname="DEF/test_1" name="test_1" time="0.02">
+ <skipped message="reason"/>
+ </testcase>
+ <testcase classname="DEF/test_2" name="test_2" time="0.02">
+ <failure><![CDATA[DEF/test_2 output goes here]]></failure>
+ </testcase>
+ </testsuite>
+ </testsuites>"""
+ )
+ ),
+ ],
+ ),
+ self.MULTI_SUITE_OUTPUT,
+ )
+
+
+def _generate_report(title, junit_objects):
+ report = [f"# {title}"]
+
+ if junit_objects:
+ report.append("")
+ failures = {}
+
+ tests_run = 0
+ tests_skipped = 0
+ tests_failed = 0
+
+ for results in junit_objects:
+ for testsuite in results:
+ tests_run += testsuite.tests
+ tests_skipped += testsuite.skipped
+ tests_failed += testsuite.failures
+
+ for test in testsuite:
+ if (
+ not test.is_passed
+ and test.result
+ and isinstance(test.result[0], Failure)
+ ):
+ if failures.get(testsuite.name) is None:
+ failures[testsuite.name] = []
+ failures[testsuite.name].append(
+ (test.classname + "/" + test.name,
test.result[0].text)
+ )
+
+ tests_passed = tests_run - tests_skipped - tests_failed
+ report.extend(
+ [
+ f"{tests_passed} tests passed",
+ f"{tests_skipped} tests skipped",
+ f"{tests_failed} tests failed",
+ ]
+ )
+
+ if failures:
+ report.extend(["", "## Failed tests", "(click to see output)"])
+ for testsuite_name, failures in failures.items():
+ report.extend(["", f"### {testsuite_name}"])
+ for name, output in failures:
+ report.extend(
+ [
+ "<details>",
+ f"<summary>{name}</summary>",
+ "",
+ "```",
+ output,
+ "```",
+ "</details>",
+ ]
+ )
+
+ return "\n".join(report)
+
+
+def generate_report(title, junit_files):
+ return _generate_report(title, [JUnitXml.fromfile(p) for p in junit_files])
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "title", help="Title of the test report, without Markdown formatting."
+ )
+ parser.add_argument("junit_files", help="Paths to JUnit report files.",
nargs="*")
+ args = parser.parse_args()
+
+ print(generate_report(args.title, args.junit_files))
>From dc897ccd87ce96aca24f116c46f25e1a3bfc0d1d Mon Sep 17 00:00:00 2001
From: David Spickett <david.spick...@linaro.org>
Date: Wed, 23 Oct 2024 12:30:07 +0100
Subject: [PATCH 3/8] Add deliberate test failures
---
clang/test/PCH/arc.m | 2 +-
llvm/test/MC/AArch64/adr.s | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/clang/test/PCH/arc.m b/clang/test/PCH/arc.m
index e4ad71a469b956..ec7a4a2249ec23 100644
--- a/clang/test/PCH/arc.m
+++ b/clang/test/PCH/arc.m
@@ -14,5 +14,5 @@
array0 a0;
array1 a1;
-// CHECK-ERR1: Objective-C automated reference counting was enabled in AST
file '{{.*}}' but is currently disabled
+// CHECK-ERR1: Objective-C automated reference counting was enabled in AST
file '{{.*}}' but is currently disabled and this will fail the test
// CHECK-ERR2: Objective-C automated reference counting was disabled in AST
file '{{.*}}' but is currently enabled
diff --git a/llvm/test/MC/AArch64/adr.s b/llvm/test/MC/AArch64/adr.s
index c2ddc8fcb38b56..ef9491209a01c9 100644
--- a/llvm/test/MC/AArch64/adr.s
+++ b/llvm/test/MC/AArch64/adr.s
@@ -19,7 +19,7 @@
adr x5, (0xffffffff000f1000 - 0xffffffff00000000 + Symbol)
adr x6, Symbol + (0xffffffff000f1000 - 0xffffffff00000000)
-// CHECK-NEXT: adrp x0, 0x0
+// CHECK-NEXT: adrp x0, 0x0 -- will fail the test!!!
// CHECK-NEXT: R_AARCH64_ADR_PREL_PG_HI21 Symbol
// CHECK-NEXT: adrp x2, 0x0
// CHECK-NEXT: R_AARCH64_ADR_PREL_PG_HI21 Symbol
>From 0c8176c4f595b174db68fedc2b2ae110813554ce Mon Sep 17 00:00:00 2001
From: David Spickett <david.spick...@linaro.org>
Date: Wed, 23 Oct 2024 12:36:31 +0100
Subject: [PATCH 4/8] hook this up to the linux build
---
.ci/generate-buildkite-pipeline-premerge | 48 ++++++++++++------------
.ci/monolithic-linux.sh | 8 +++-
2 files changed, 30 insertions(+), 26 deletions(-)
diff --git a/.ci/generate-buildkite-pipeline-premerge
b/.ci/generate-buildkite-pipeline-premerge
index e52133751f09b1..06defbb8d037f5 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -289,27 +289,27 @@ if [[ "${linux_projects}" != "" ]]; then
EOF
fi
-if [[ "${windows_projects}" != "" ]]; then
- cat <<EOF
-- label: ':windows: Windows x64'
- artifact_paths:
- - 'artifacts/**/*'
- - '*_result.json'
- - 'build/test-results*.xml'
- agents: ${WINDOWS_AGENTS}
- retry:
- automatic:
- - exit_status: -1 # Agent was lost
- limit: 2
- - exit_status: 255 # Forced agent shutdown
- limit: 2
- timeout_in_minutes: 150
- env:
- CC: 'cl'
- CXX: 'cl'
- LD: 'link'
- commands:
- - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'
- - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')"
"$(echo ${windows_check_targets})"'
-EOF
-fi
+# if [[ "${windows_projects}" != "" ]]; then
+# cat <<EOF
+# - label: ':windows: Windows x64'
+# artifact_paths:
+# - 'artifacts/**/*'
+# - '*_result.json'
+# - 'build/test-results*.xml'
+# agents: ${WINDOWS_AGENTS}
+# retry:
+# automatic:
+# - exit_status: -1 # Agent was lost
+# limit: 2
+# - exit_status: 255 # Forced agent shutdown
+# limit: 2
+# timeout_in_minutes: 150
+# env:
+# CC: 'cl'
+# CXX: 'cl'
+# LD: 'link'
+# commands:
+# - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64
-host_arch=amd64'
+# - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' '
';')" "$(echo ${windows_check_targets})"'
+# EOF
+# fi
diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index 17ea51c08fafd3..965039b28e6af4 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -28,11 +28,14 @@ if [[ -n "${CLEAR_CACHE:-}" ]]; then
ccache --clear
fi
-function show-stats {
+function at-exit {
+ python3 "${MONOREPO_ROOT}"/.ci/generate_test_report.py ":linux: Linux x64
Test Results" "${BUILD_DIR}"/test-results*.xml | \
+ buildkite-agent annotate --context "linux-x64-test-results"
+
mkdir -p artifacts
ccache --print-stats > artifacts/ccache_stats.txt
}
-trap show-stats EXIT
+trap at-exit EXIT
projects="${1}"
targets="${2}"
@@ -42,6 +45,7 @@ lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml
--use-unique-outpu
echo "--- cmake"
pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt
pip install -q -r "${MONOREPO_ROOT}"/lldb/test/requirements.txt
+pip install -q junitparser==3.2.0
cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
-D LLVM_ENABLE_PROJECTS="${projects}" \
-G Ninja \
>From 5a42871170f8a8a504381c4d0984a67e32e5a87f Mon Sep 17 00:00:00 2001
From: David Spickett <david.spick...@linaro.org>
Date: Wed, 23 Oct 2024 13:28:17 +0100
Subject: [PATCH 5/8] improve test formatting
---
.ci/generate_test_report.py | 172 +++++++++++++++++++++++-------------
.ci/monolithic-linux.sh | 2 +
2 files changed, 114 insertions(+), 60 deletions(-)
diff --git a/.ci/generate_test_report.py b/.ci/generate_test_report.py
index 672b131dbbd311..3d5ce5f08f9dfa 100644
--- a/.ci/generate_test_report.py
+++ b/.ci/generate_test_report.py
@@ -16,8 +16,54 @@ def junit_from_xml(xml):
class TestReports(unittest.TestCase):
- def test_title(self):
- self.assertEqual(_generate_report("Foo", []), "# Foo")
+ def test_title_only(self):
+ self.assertEqual(_generate_report("Foo", []), "")
+
+ def test_no_tests_in_testsuite(self):
+ self.assertEqual(
+ _generate_report(
+ "Foo",
+ [
+ junit_from_xml(
+ dedent(
+ """\
+ <?xml version="1.0" encoding="UTF-8"?>
+ <testsuites time="0.00">
+ <testsuite name="Empty" tests="0" failures="0" skipped="0"
time="0.00">
+ </testsuite>
+ </testsuites>"""
+ )
+ )
+ ],
+ ),
+ "",
+ )
+
+ def test_no_failures(self):
+ self.assertEqual(
+ _generate_report(
+ "Foo",
+ [
+ junit_from_xml(
+ dedent(
+ """\
+ <?xml version="1.0" encoding="UTF-8"?>
+ <testsuites time="0.00">
+ <testsuite name="Passed" tests="1" failures="0" skipped="0"
time="0.00">
+ <testcase classname="Bar/test_1" name="test_1" time="0.00"/>
+ </testsuite>
+ </testsuites>"""
+ )
+ )
+ ],
+ ),
+ dedent(
+ """\
+ # Foo
+
+ * 1 test passed"""
+ ),
+ )
def test_report_single_file_single_testsuite(self):
self.assertEqual(
@@ -50,9 +96,9 @@ def test_report_single_file_single_testsuite(self):
"""\
# Foo
- 1 tests passed
- 1 tests skipped
- 2 tests failed
+ * 1 test passed
+ * 1 test skipped
+ * 2 tests failed
## Failed tests
(click to see output)
@@ -79,9 +125,9 @@ def test_report_single_file_single_testsuite(self):
"""\
# ABC and DEF
- 1 tests passed
- 1 tests skipped
- 2 tests failed
+ * 1 test passed
+ * 1 test skipped
+ * 2 tests failed
## Failed tests
(click to see output)
@@ -179,60 +225,66 @@ def test_report_multiple_files_multiple_testsuites(self):
def _generate_report(title, junit_objects):
- report = [f"# {title}"]
-
- if junit_objects:
- report.append("")
- failures = {}
-
- tests_run = 0
- tests_skipped = 0
- tests_failed = 0
-
- for results in junit_objects:
- for testsuite in results:
- tests_run += testsuite.tests
- tests_skipped += testsuite.skipped
- tests_failed += testsuite.failures
-
- for test in testsuite:
- if (
- not test.is_passed
- and test.result
- and isinstance(test.result[0], Failure)
- ):
- if failures.get(testsuite.name) is None:
- failures[testsuite.name] = []
- failures[testsuite.name].append(
- (test.classname + "/" + test.name,
test.result[0].text)
- )
-
- tests_passed = tests_run - tests_skipped - tests_failed
- report.extend(
- [
- f"{tests_passed} tests passed",
- f"{tests_skipped} tests skipped",
- f"{tests_failed} tests failed",
- ]
- )
-
- if failures:
- report.extend(["", "## Failed tests", "(click to see output)"])
- for testsuite_name, failures in failures.items():
- report.extend(["", f"### {testsuite_name}"])
- for name, output in failures:
- report.extend(
- [
- "<details>",
- f"<summary>{name}</summary>",
- "",
- "```",
- output,
- "```",
- "</details>",
- ]
+ if not junit_objects:
+ return ""
+
+ failures = {}
+ tests_run = 0
+ tests_skipped = 0
+ tests_failed = 0
+
+ for results in junit_objects:
+ for testsuite in results:
+ tests_run += testsuite.tests
+ tests_skipped += testsuite.skipped
+ tests_failed += testsuite.failures
+
+ for test in testsuite:
+ if (
+ not test.is_passed
+ and test.result
+ and isinstance(test.result[0], Failure)
+ ):
+ if failures.get(testsuite.name) is None:
+ failures[testsuite.name] = []
+ failures[testsuite.name].append(
+ (test.classname + "/" + test.name, test.result[0].text)
)
+ if not tests_run:
+ return ""
+
+ report = [f"# {title}", ""]
+
+ tests_passed = tests_run - tests_skipped - tests_failed
+
+ def plural(num_tests):
+ return "test" if num_tests == 1 else "tests"
+
+ if tests_passed:
+ report.append(f"* {tests_passed} {plural(tests_passed)} passed")
+ if tests_skipped:
+ report.append(f"* {tests_skipped} {plural(tests_skipped)} skipped")
+ if tests_failed:
+ report.append(f"* {tests_failed} {plural(tests_failed)} failed")
+
+ if failures:
+ report.extend(["", "## Failed tests", "(click to see output)"])
+ for testsuite_name, failures in failures.items():
+ report.extend(["", f"### {testsuite_name}"])
+ for name, output in failures:
+ report.extend(
+ [
+ "<details>",
+ f"<summary>{name}</summary>",
+ "",
+ "```",
+ output,
+ "```",
+ "</details>",
+ ]
+ )
+
return "\n".join(report)
diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index 965039b28e6af4..c4bbc02e44b79a 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -29,6 +29,8 @@ if [[ -n "${CLEAR_CACHE:-}" ]]; then
fi
function at-exit {
+ # TODO: don't annotate if no test results
+ # TODO: use styles for the annotatinon success or failure
python3 "${MONOREPO_ROOT}"/.ci/generate_test_report.py ":linux: Linux x64
Test Results" "${BUILD_DIR}"/test-results*.xml | \
buildkite-agent annotate --context "linux-x64-test-results"
>From 8cf82ac5b7150d0d6e8f049897390b94eba616e0 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spick...@linaro.org>
Date: Wed, 23 Oct 2024 13:34:22 +0100
Subject: [PATCH 6/8] Also choose a style
---
.ci/generate_test_report.py | 27 +++++++++++++++------------
1 file changed, 15 insertions(+), 12 deletions(-)
diff --git a/.ci/generate_test_report.py b/.ci/generate_test_report.py
index 3d5ce5f08f9dfa..c2329138d51179 100644
--- a/.ci/generate_test_report.py
+++ b/.ci/generate_test_report.py
@@ -17,7 +17,7 @@ def junit_from_xml(xml):
class TestReports(unittest.TestCase):
def test_title_only(self):
- self.assertEqual(_generate_report("Foo", []), "")
+ self.assertEqual(_generate_report("Foo", []), ('', None))
def test_no_tests_in_testsuite(self):
self.assertEqual(
@@ -36,7 +36,7 @@ def test_no_tests_in_testsuite(self):
)
],
),
- "",
+ ('', None),
)
def test_no_failures(self):
@@ -57,12 +57,12 @@ def test_no_failures(self):
)
],
),
- dedent(
+ (dedent(
"""\
# Foo
* 1 test passed"""
- ),
+ ), 'success'),
)
def test_report_single_file_single_testsuite(self):
@@ -92,7 +92,7 @@ def test_report_single_file_single_testsuite(self):
)
],
),
- dedent(
+ (dedent(
"""\
# Foo
@@ -118,10 +118,10 @@ def test_report_single_file_single_testsuite(self):
Other output goes here
```
</details>"""
- ),
+ ), 'error'),
)
- MULTI_SUITE_OUTPUT = dedent(
+ MULTI_SUITE_OUTPUT = (dedent(
"""\
# ABC and DEF
@@ -149,7 +149,7 @@ def test_report_single_file_single_testsuite(self):
DEF/test_2 output goes here
```
</details>"""
- )
+ ), 'error')
def test_report_single_file_multiple_testsuites(self):
self.assertEqual(
@@ -225,8 +225,10 @@ def test_report_multiple_files_multiple_testsuites(self):
def _generate_report(title, junit_objects):
+ style = None
+
if not junit_objects:
- return ""
+ return ('', style)
failures = {}
tests_run = 0
@@ -252,8 +254,9 @@ def _generate_report(title, junit_objects):
)
if not tests_run:
- return ""
-
+ return ('', style)
+
+ style = 'error' if tests_failed else 'success'
report = [f"# {title}", ""]
tests_passed = tests_run - tests_skipped - tests_failed
@@ -285,7 +288,7 @@ def plural(num_tests):
]
)
- return "\n".join(report)
+ return "\n".join(report), style
def generate_report(title, junit_files):
>From 67fa72ac9c3d6cab061e32b5a51309b8f108bf32 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spick...@linaro.org>
Date: Wed, 23 Oct 2024 13:38:02 +0100
Subject: [PATCH 7/8] call the agent from the script
---
.ci/generate_test_report.py | 44 +++++++++++++++++++++++--------------
.ci/monolithic-linux.sh | 6 ++---
2 files changed, 30 insertions(+), 20 deletions(-)
diff --git a/.ci/generate_test_report.py b/.ci/generate_test_report.py
index c2329138d51179..4b637fea3129db 100644
--- a/.ci/generate_test_report.py
+++ b/.ci/generate_test_report.py
@@ -9,6 +9,7 @@
from io import StringIO
from junitparser import JUnitXml, Failure
from textwrap import dedent
+from subprocess import check_call
def junit_from_xml(xml):
@@ -17,7 +18,7 @@ def junit_from_xml(xml):
class TestReports(unittest.TestCase):
def test_title_only(self):
- self.assertEqual(_generate_report("Foo", []), ('', None))
+ self.assertEqual(_generate_report("Foo", []), ("", None))
def test_no_tests_in_testsuite(self):
self.assertEqual(
@@ -36,7 +37,7 @@ def test_no_tests_in_testsuite(self):
)
],
),
- ('', None),
+ ("", None),
)
def test_no_failures(self):
@@ -57,12 +58,15 @@ def test_no_failures(self):
)
],
),
- (dedent(
- """\
+ (
+ dedent(
+ """\
# Foo
* 1 test passed"""
- ), 'success'),
+ ),
+ "success",
+ ),
)
def test_report_single_file_single_testsuite(self):
@@ -92,8 +96,9 @@ def test_report_single_file_single_testsuite(self):
)
],
),
- (dedent(
- """\
+ (
+ dedent(
+ """\
# Foo
* 1 test passed
@@ -118,11 +123,14 @@ def test_report_single_file_single_testsuite(self):
Other output goes here
```
</details>"""
- ), 'error'),
+ ),
+ "error",
+ ),
)
- MULTI_SUITE_OUTPUT = (dedent(
- """\
+ MULTI_SUITE_OUTPUT = (
+ dedent(
+ """\
# ABC and DEF
* 1 test passed
@@ -149,7 +157,9 @@ def test_report_single_file_single_testsuite(self):
DEF/test_2 output goes here
```
</details>"""
- ), 'error')
+ ),
+ "error",
+ )
def test_report_single_file_multiple_testsuites(self):
self.assertEqual(
@@ -228,7 +238,7 @@ def _generate_report(title, junit_objects):
style = None
if not junit_objects:
- return ('', style)
+ return ("", style)
failures = {}
tests_run = 0
@@ -254,9 +264,9 @@ def _generate_report(title, junit_objects):
)
if not tests_run:
- return ('', style)
-
- style = 'error' if tests_failed else 'success'
+ return ("", style)
+
+ style = "error" if tests_failed else "success"
report = [f"# {title}", ""]
tests_passed = tests_run - tests_skipped - tests_failed
@@ -300,7 +310,9 @@ def generate_report(title, junit_files):
parser.add_argument(
"title", help="Title of the test report, without Markdown formatting."
)
+ parser.add_argument("context", help="Annotation context to write to.")
parser.add_argument("junit_files", help="Paths to JUnit report files.",
nargs="*")
args = parser.parse_args()
- print(generate_report(args.title, args.junit_files))
+ report, style = generate_report(args.title, args.junit_files)
+ check_call(["buildkite-agent", "--context", args.context, "--style",
style, report])
diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index c4bbc02e44b79a..4c4c1c756ba98f 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -29,10 +29,8 @@ if [[ -n "${CLEAR_CACHE:-}" ]]; then
fi
function at-exit {
- # TODO: don't annotate if no test results
- # TODO: use styles for the annotatinon success or failure
- python3 "${MONOREPO_ROOT}"/.ci/generate_test_report.py ":linux: Linux x64
Test Results" "${BUILD_DIR}"/test-results*.xml | \
- buildkite-agent annotate --context "linux-x64-test-results"
+ python3 "${MONOREPO_ROOT}"/.ci/generate_test_report.py ":linux: Linux x64
Test Results" \
+ "linux-x64-test-results" "${BUILD_DIR}"/test-results*.xml
mkdir -p artifacts
ccache --print-stats > artifacts/ccache_stats.txt
>From 0a0de55c533b90d3c72ffa0cacda8a0982a7848a Mon Sep 17 00:00:00 2001
From: David Spickett <david.spick...@linaro.org>
Date: Wed, 23 Oct 2024 14:06:44 +0100
Subject: [PATCH 8/8] fix buildkite command
---
.ci/generate_test_report.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.ci/generate_test_report.py b/.ci/generate_test_report.py
index 4b637fea3129db..16f5ab3f6c3548 100644
--- a/.ci/generate_test_report.py
+++ b/.ci/generate_test_report.py
@@ -315,4 +315,4 @@ def generate_report(title, junit_files):
args = parser.parse_args()
report, style = generate_report(args.title, args.junit_files)
- check_call(["buildkite-agent", "--context", args.context, "--style",
style, report])
+ check_call(["buildkite-agent", "annotate", "--context", args.context,
"--style", style, report])
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
