In r15-2354-g4d1f71d49e396c I added the ability to use Python to write
tests of SARIF output via a new "run-sarif-pytest" based
on "run-gcov-pytest", with a sarif.py support script in
testsuite/gcc.dg/sarif-output.
This followup patch:
(a) removes the limitation of such tests needing to be in
testsuite/gcc.dg/sarif-output by moving sarif.py to testsuite/lib
and adding logic to add that directory to PYTHONPATH when invoking
pytest.
(b) uses this to replace fragile regexp-based tests in
gcc.dg/plugin/diagnostic-test-paths-multithreaded-sarif.c with
Python logic that verifies the structure within the generated JSON,
and to add test coverage for SARIF output relating to GCC plugins.
Successfully bootstrapped & regrtested on x86_64-pc-linux-gnu.
Pushed to trunk as r15-3198-gaa3b950291119a.
gcc/ChangeLog:
* diagnostic-format-sarif.cc: Add comments noting that we don't
yet capture any diagnostic_metadata::rules associated with a
diagnostic.
gcc/testsuite/ChangeLog:
* gcc.dg/plugin/diagnostic-test-metadata-sarif.c: New test,
based on diagnostic-test-metadata.c.
* gcc.dg/plugin/diagnostic-test-metadata-sarif.py: New script.
* gcc.dg/plugin/diagnostic-test-paths-multithreaded-sarif.c:
Replace scan-sarif-file directives with run-sarif-pytest, to
run...
* gcc.dg/plugin/diagnostic-test-paths-multithreaded-sarif.py:
...this new test.
* gcc.dg/plugin/plugin.exp (plugin_test_list): Add
diagnostic-test-metadata-sarif.c.
* gcc.dg/sarif-output/sarif.py: Move to...
* lib/sarif.py: ...here.
* lib/scansarif.exp (run-sarif-pytest): Prepend "lib" to
PYTHONPATH before running python scripts.
Signed-off-by: David Malcolm <dmalc...@redhat.com>
---
gcc/diagnostic-format-sarif.cc | 6 +-
.../plugin/diagnostic-test-metadata-sarif.c | 17 +++
.../plugin/diagnostic-test-metadata-sarif.py | 55 +++++++++
...iagnostic-test-paths-multithreaded-sarif.c | 17 +--
...agnostic-test-paths-multithreaded-sarif.py | 109 ++++++++++++++++++
gcc/testsuite/gcc.dg/plugin/plugin.exp | 4 +-
.../{gcc.dg/sarif-output => lib}/sarif.py | 0
gcc/testsuite/lib/scansarif.exp | 16 +++
8 files changed, 208 insertions(+), 16 deletions(-)
create mode 100644 gcc/testsuite/gcc.dg/plugin/diagnostic-test-metadata-sarif.c
create mode 100644
gcc/testsuite/gcc.dg/plugin/diagnostic-test-metadata-sarif.py
create mode 100644
gcc/testsuite/gcc.dg/plugin/diagnostic-test-paths-multithreaded-sarif.py
rename gcc/testsuite/{gcc.dg/sarif-output => lib}/sarif.py (100%)
diff --git a/gcc/diagnostic-format-sarif.cc b/gcc/diagnostic-format-sarif.cc
index 963a185f6ced..1d99c904ff0c 100644
--- a/gcc/diagnostic-format-sarif.cc
+++ b/gcc/diagnostic-format-sarif.cc
@@ -580,7 +580,9 @@ public:
(SARIF v2.1.0 section 3.27.13).
- doesn't capture -Werror cleanly
- doesn't capture inlining information (can SARIF handle this?)
- - doesn't capture macro expansion information (can SARIF handle this?). */
+ - doesn't capture macro expansion information (can SARIF handle this?).
+ - doesn't capture any diagnostic_metadata::rules associated with
+ a diagnostic. */
class sarif_builder
{
@@ -1522,6 +1524,8 @@ sarif_builder::make_result_object (diagnostic_context
&context,
}
diagnostic.metadata->maybe_add_sarif_properties (*result_obj);
+
+ /* We don't yet support diagnostic_metadata::rule. */
}
/* "level" property (SARIF v2.1.0 section 3.27.10). */
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-metadata-sarif.c
b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-metadata-sarif.c
new file mode 100644
index 000000000000..246a8429090d
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-metadata-sarif.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-options "-fdiagnostics-format=sarif-file" } */
+
+extern char *gets (char *s);
+
+void test_cwe (void)
+{
+ char buf[1024];
+ gets (buf);
+}
+
+/* Verify that some JSON was written to a file with the expected name. */
+/* { dg-final { verify-sarif-file } } */
+
+/* Use a Python script to verify various properties about the generated
+ .sarif file:
+ { dg-final { run-sarif-pytest diagnostic-test-metadata-sarif.c
"diagnostic-test-metadata-sarif.py" } } */
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-metadata-sarif.py
b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-metadata-sarif.py
new file mode 100644
index 000000000000..959e6f2e9942
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-metadata-sarif.py
@@ -0,0 +1,55 @@
+# We expect a warning with this textual form:
+#
+# . PATH/diagnostic-test-metadata-sarif.c: In function 'test_cwe':
+# . PATH/diagnostic-test-metadata-sarif.c:8:3: warning: never use 'gets'
[CWE-242] [STR34-C]
+
+from sarif import *
+
+import pytest
+
+@pytest.fixture(scope='function', autouse=True)
+def sarif():
+ return sarif_from_env()
+
+def test_basics(sarif):
+ schema = sarif['$schema']
+ assert schema ==
"https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json";
+
+ version = sarif['version']
+ assert version == "2.1.0"
+
+def test_plugin_metadata(sarif):
+ runs = sarif['runs']
+ assert len(runs) == 1
+
+ run = runs[0]
+ tool = run['tool']
+
+ # We expect an extension, for the plugin.
+ extensions = tool['extensions']
+ assert len(extensions) == 1
+
+ extension = extensions[0]
+ assert extension['name'] == 'diagnostic_plugin_test_metadata'
+ assert 'diagnostic_plugin_test_metadata' in extension['fullName']
+
+ # TODO: ideally there would be a rule for [STR34-C] within
+ # the extension for the plugin with
+ # 'id': 'STR34-C',
+ # 'helpUri': 'https://example.com/'
+
+def test_result(sarif):
+ runs = sarif['runs']
+ run = runs[0]
+ results = run['results']
+
+ assert len(results) == 1
+
+ result = results[0]
+ assert result['level'] == 'warning'
+ assert result['message']['text'] == "never use 'gets'"
+
+ assert len(result['taxa']) == 1
+ taxon = result['taxa'][0]
+ assert taxon['id'] == '242'
+ assert taxon['toolComponent']['name'] == 'cwe'
diff --git
a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-paths-multithreaded-sarif.c
b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-paths-multithreaded-sarif.c
index 727d1bb6469b..f0f31d01fef9 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-paths-multithreaded-sarif.c
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-paths-multithreaded-sarif.c
@@ -19,17 +19,6 @@ void bar ()
/* Verify that some JSON was written to a file with the expected name. */
/* { dg-final { verify-sarif-file } } */
-/* We expect various properties.
- The indentation here reflects the expected hierarchy, though these tests
- don't check for that, merely the string fragments we expect.
-
- { dg-final { scan-sarif-file {"version": "2.1.0"} } }
- { dg-final { scan-sarif-file {"text": "deadlock due to inconsistent lock
acquisition order"} } }
- { dg-final { scan-sarif-file {"id": "Thread 1"} } }
- { dg-final { scan-sarif-file {"executionOrder": 1} } }
- { dg-final { scan-sarif-file {"executionOrder": 2} } }
- { dg-final { scan-sarif-file {"executionOrder": 5} } }
- { dg-final { scan-sarif-file {"id": "Thread 2"} } }
- { dg-final { scan-sarif-file {"executionOrder": 3} } }
- { dg-final { scan-sarif-file {"executionOrder": 4} } }
- { dg-final { scan-sarif-file {"executionOrder": 6} } } */
+/* Use a Python script to verify various properties about the generated
+ .sarif file:
+ { dg-final { run-sarif-pytest diagnostic-test-paths-multithreaded-sarif.c
"diagnostic-test-paths-multithreaded-sarif.py" } } */
diff --git
a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-paths-multithreaded-sarif.py
b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-paths-multithreaded-sarif.py
new file mode 100644
index 000000000000..cff78aa8ac8e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-paths-multithreaded-sarif.py
@@ -0,0 +1,109 @@
+# We expect a warning with this textual form:
+#
+# . warning: deadlock due to inconsistent lock acquisition order
+# . 17 | acquire_lock_a (); /* { dg-warning "deadlock due to inconsistent
lock acquisition order" } */
+# . | ^~~~~~~~~~~~~~~~~
+# . Thread: 'Thread 1'
+# . 'foo': event 1
+# . |
+# . | 9 | {
+# . | | ^
+# . | | |
+# . | | (1) entering 'foo'
+# . |
+# . +--> 'foo': event 2
+# . |
+# . | 10 | acquire_lock_a ();
+# . | | ^~~~~~~~~~~~~~~~~
+# . | | |
+# . | | (2) lock a is now held by thread 1
+# . |
+# .
+# . Thread: 'Thread 2'
+# . 'bar': event 3
+# . |
+# . | 15 | {
+# . | | ^
+# . | | |
+# . | | (3) entering 'bar'
+# . |
+# . +--> 'bar': event 4
+# . |
+# . | 16 | acquire_lock_b ();
+# . | | ^~~~~~~~~~~~~~~~~
+# . | | |
+# . | | (4) lock b is now held by thread 2
+# . |
+
+from sarif import *
+
+import pytest
+
+@pytest.fixture(scope='function', autouse=True)
+def sarif():
+ return sarif_from_env()
+
+def test_basics(sarif):
+ schema = sarif['$schema']
+ assert schema ==
"https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json";
+
+ version = sarif['version']
+ assert version == "2.1.0"
+
+def test_execution_successful(sarif):
+ runs = sarif['runs']
+ run = runs[0]
+
+ invocations = run['invocations']
+ assert len(invocations) == 1
+ invocation = invocations[0]
+
+ # We expect a mere 'warning' to allow executionSuccessful be true
+ assert invocation['executionSuccessful'] == True
+
+def test_result(sarif):
+ runs = sarif['runs']
+ run = runs[0]
+ results = run['results']
+
+ assert len(results) == 1
+
+ result = results[0]
+ assert result['level'] == 'warning'
+ assert result['message']['text'] == "deadlock due to inconsistent lock
acquisition order"
+
+ code_flows = result['codeFlows']
+ assert len(code_flows) == 1
+
+ code_flow = code_flows[0]
+
+ thread_flows = code_flow['threadFlows']
+ assert len(thread_flows) == 2
+
+ tf0 = thread_flows[0]
+ assert tf0['id'] == 'Thread 1'
+
+ tf1 = thread_flows[1]
+ assert tf1['id'] == 'Thread 2'
+
+ assert len(tf0['locations']) == 3
+ assert tf0['locations'][0]['executionOrder'] == 1
+ assert tf0['locations'][0]['location']['message']['text'] \
+ == "entering 'foo'"
+ assert tf0['locations'][1]['executionOrder'] == 2
+ assert tf0['locations'][1]['location']['message']['text'] \
+ == "lock a is now held by thread 1"
+ assert tf0['locations'][2]['executionOrder'] == 5
+ assert tf0['locations'][2]['location']['message']['text'] \
+ == "deadlocked due to waiting for lock b in thread 1..."
+
+ assert len(tf1['locations']) == 3
+ assert tf1['locations'][0]['executionOrder'] == 3
+ assert tf1['locations'][0]['location']['message']['text'] \
+ == "entering 'bar'"
+ assert tf1['locations'][1]['executionOrder'] == 4
+ assert tf1['locations'][1]['location']['message']['text'] \
+ == "lock b is now held by thread 2"
+ assert tf1['locations'][2]['executionOrder'] == 6
+ assert tf1['locations'][2]['location']['message']['text'] \
+ == "...whilst waiting for lock a in thread 2"
diff --git a/gcc/testsuite/gcc.dg/plugin/plugin.exp
b/gcc/testsuite/gcc.dg/plugin/plugin.exp
index 933f9a5850bc..2c2d919eddf0 100644
--- a/gcc/testsuite/gcc.dg/plugin/plugin.exp
+++ b/gcc/testsuite/gcc.dg/plugin/plugin.exp
@@ -101,7 +101,9 @@ set plugin_test_list [list \
diagnostic-test-inlining-2.c \
diagnostic-test-inlining-3.c \
diagnostic-test-inlining-4.c } \
- { diagnostic_plugin_test_metadata.c diagnostic-test-metadata.c } \
+ { diagnostic_plugin_test_metadata.c \
+ diagnostic-test-metadata.c \
+ diagnostic-test-metadata-sarif.c } \
{ diagnostic_plugin_test_paths.c \
diagnostic-test-paths-1.c \
diagnostic-test-paths-2.c \
diff --git a/gcc/testsuite/gcc.dg/sarif-output/sarif.py
b/gcc/testsuite/lib/sarif.py
similarity index 100%
rename from gcc/testsuite/gcc.dg/sarif-output/sarif.py
rename to gcc/testsuite/lib/sarif.py
diff --git a/gcc/testsuite/lib/scansarif.exp b/gcc/testsuite/lib/scansarif.exp
index e08f80c9ce18..ea6a88340b5e 100644
--- a/gcc/testsuite/lib/scansarif.exp
+++ b/gcc/testsuite/lib/scansarif.exp
@@ -135,8 +135,24 @@ proc run-sarif-pytest { args } {
}
setenv SARIF_PATH $testcase
+ set libdir "${srcdir}/lib"
+
+ # Set/prepend libdir to PYTHONPATH
+ if [info exists ::env(PYTHONPATH)] {
+ set old_PYTHONPATH $::env(PYTHONPATH)
+ setenv PYTHONPATH "${libdir}:${old_PYTHONPATH}"
+ } else {
+ setenv PYTHONPATH "${libdir}"
+ }
+
+ verbose "PYTHONPATH=[getenv PYTHONPATH]" 2
+
spawn -noecho python3 -m pytest --color=no -rap -s --tb=no
$srcdir/$subdir/$pytest_script
+ if [info exists old_PYTHONPATH] {
+ setenv PYTHONPATH ${old_PYTHONPATH}
+ }
+
set prefix "\[^\r\n\]*"
expect {
-re "FAILED($prefix)\[^\r\n\]+\r\n" {
--
2.26.3
