https://gcc.gnu.org/g:68c7747dd29016d51e6f04e7fe1d8543423f093f
commit r15-2292-g68c7747dd29016d51e6f04e7fe1d8543423f093f
Author: David Malcolm <dmalc...@redhat.com>
Date: Wed Jul 24 18:07:56 2024 -0400
diagnostics: add selftests for SARIF output
The existing DejaGnu-based tests for our SARIF output used regexes
to verify the JSON at the string level, which lets us test for
the presence of properties, but doesn't check the overall structure.
This patch uses the selftest framework to verify the structure of
the tree of JSON values for a log containing one diagnostic.
No functional change intended.
gcc/ChangeLog:
* diagnostic-format-sarif.cc (sarif_builder::flush_to_object):
New, using code moved from...
(sarif_builder::end_group): ...here.
(class selftest::test_sarif_diagnostic_context): New.
(selftest::test_simple_log): New.
(selftest::diagnostic_format_sarif_cc_tests): Call it.
* json.h (json::object::is_empty): New.
* selftest-diagnostic.cc (test_diagnostic_context::report): New.
* selftest-diagnostic.h (test_diagnostic_context::report): New
decl.
* selftest-json.cc (selftest::assert_json_string_eq): New.
(selftest::expect_json_object_with_string_property): New.
(selftest::assert_json_string_property_eq): New.
* selftest-json.h (selftest::assert_json_string_eq): New decl.
(ASSERT_JSON_STRING_EQ): New macro.
(selftest::expect_json_object_with_string_property): New decl.
(EXPECT_JSON_OBJECT_WITH_STRING_PROPERTY): New macro.
Signed-off-by: David Malcolm <dmalc...@redhat.com>
Diff:
---
gcc/diagnostic-format-sarif.cc | 185 +++++++++++++++++++++++++++++++++++++++--
gcc/json.h | 2 +
gcc/selftest-diagnostic.cc | 14 ++++
gcc/selftest-diagnostic.h | 10 +++
gcc/selftest-json.cc | 34 +++++++-
gcc/selftest-json.h | 27 ++++++
6 files changed, 264 insertions(+), 8 deletions(-)
diff --git a/gcc/diagnostic-format-sarif.cc b/gcc/diagnostic-format-sarif.cc
index afb29eab5839..816f3210036e 100644
--- a/gcc/diagnostic-format-sarif.cc
+++ b/gcc/diagnostic-format-sarif.cc
@@ -380,6 +380,7 @@ public:
const diagnostic_diagram &diagram);
void end_group ();
+ std::unique_ptr<sarif_log> flush_to_object ();
void flush_to_file (FILE *outf);
std::unique_ptr<json::array>
@@ -860,6 +861,20 @@ sarif_builder::end_group ()
m_cur_group_result = nullptr;
}
+/* Create a top-level object, and add it to all the results
+ (and other entities) we've seen so far, moving ownership
+ to the object. */
+
+std::unique_ptr<sarif_log>
+sarif_builder::flush_to_object ()
+{
+ m_invocation_obj->prepare_to_flush (m_context);
+ std::unique_ptr<sarif_log> top
+ = make_top_level_object (std::move (m_invocation_obj),
+ std::move (m_results_array));
+ return top;
+}
+
/* Create a top-level object, and add it to all the results
(and other entities) we've seen so far.
@@ -868,12 +883,8 @@ sarif_builder::end_group ()
void
sarif_builder::flush_to_file (FILE *outf)
{
- m_invocation_obj->prepare_to_flush (m_context);
- std::unique_ptr<sarif_log> top
- = make_top_level_object (std::move (m_invocation_obj),
- std::move (m_results_array));
+ std::unique_ptr<sarif_log> top = flush_to_object ();
top->dump (outf, m_formatted);
- m_invocation_obj = nullptr;
fprintf (outf, "\n");
}
@@ -2434,6 +2445,54 @@ diagnostic_output_format_init_sarif_stream
(diagnostic_context &context,
namespace selftest {
+/* A subclass of sarif_output_format for writing selftests.
+ The JSON output is cached internally, rather than written
+ out to a file. */
+
+class test_sarif_diagnostic_context : public test_diagnostic_context
+{
+public:
+ test_sarif_diagnostic_context ()
+ {
+ diagnostic_output_format_init_sarif (*this);
+
+ m_format = new buffered_output_format (*this,
+ "MAIN_INPUT_FILENAME",
+ true);
+ set_output_format (m_format); // give ownership;
+ }
+
+ std::unique_ptr<sarif_log> flush_to_object ()
+ {
+ return m_format->flush_to_object ();
+ }
+
+private:
+ class buffered_output_format : public sarif_output_format
+ {
+ public:
+ buffered_output_format (diagnostic_context &context,
+ const char *main_input_filename_,
+ bool formatted)
+ : sarif_output_format (context, main_input_filename_, formatted)
+ {
+ }
+ bool machine_readable_stderr_p () const final override
+ {
+ return false;
+ }
+ std::unique_ptr<sarif_log> flush_to_object ()
+ {
+ return m_builder.flush_to_object ();
+ }
+ };
+
+ buffered_output_format *m_format; // borrowed
+};
+
+/* Test making a sarif_location for a complex rich_location
+ with labels and escape-on-output. */
+
static void
test_make_location_object (const line_table_case &case_)
{
@@ -2550,12 +2609,128 @@ test_make_location_object (const line_table_case
&case_)
}
}
+/* Test of reporting a diagnostic to a diagnostic_context and
+ examining the generated sarif_log.
+ Verify various basic properties. */
+
+static void
+test_simple_log ()
+{
+ test_sarif_diagnostic_context dc;
+
+ rich_location richloc (line_table, UNKNOWN_LOCATION);
+ dc.report (DK_ERROR, richloc, nullptr, 0, "this is a test: %i", 42);
+
+ auto log_ptr = dc.flush_to_object ();
+
+ // 3.13 sarifLog:
+ auto log = log_ptr.get ();
+ ASSERT_JSON_STRING_PROPERTY_EQ (log, "$schema", SARIF_SCHEMA); // 3.13.3
+ ASSERT_JSON_STRING_PROPERTY_EQ (log, "version", SARIF_VERSION); // 3.13.2
+
+ auto runs = EXPECT_JSON_OBJECT_WITH_ARRAY_PROPERTY (log, "runs"); // 3.13.4
+ ASSERT_EQ (runs->size (), 1);
+
+ // 3.14 "run" object:
+ auto run = (*runs)[0];
+
+ {
+ // 3.14.6:
+ auto tool = EXPECT_JSON_OBJECT_WITH_OBJECT_PROPERTY (run, "tool");
+
+ EXPECT_JSON_OBJECT_WITH_OBJECT_PROPERTY (tool, "driver"); // 3.18.2
+ }
+
+ {
+ // 3.14.11
+ auto invocations
+ = EXPECT_JSON_OBJECT_WITH_ARRAY_PROPERTY (run, "invocations");
+ ASSERT_EQ (invocations->size (), 1);
+
+ {
+ // 3.20 "invocation" object:
+ auto invocation = (*invocations)[0];
+
+ // 3.20.3 arguments property
+
+ // 3.20.7 startTimeUtc property
+ EXPECT_JSON_OBJECT_WITH_STRING_PROPERTY (invocation, "startTimeUtc");
+
+ // 3.20.8 endTimeUtc property
+ EXPECT_JSON_OBJECT_WITH_STRING_PROPERTY (invocation, "endTimeUtc");
+
+ // 3.20.19 workingDirectory property
+ {
+ auto wd_obj
+ = EXPECT_JSON_OBJECT_WITH_OBJECT_PROPERTY (invocation,
+ "workingDirectory");
+ EXPECT_JSON_OBJECT_WITH_STRING_PROPERTY (wd_obj, "uri");
+ }
+
+ // 3.20.21 toolExecutionNotifications property
+ auto notifications
+ = EXPECT_JSON_OBJECT_WITH_ARRAY_PROPERTY
+ (invocation, "toolExecutionNotifications");
+ ASSERT_EQ (notifications->size (), 0);
+ }
+ }
+
+ {
+ // 3.14.15:
+ auto artifacts = EXPECT_JSON_OBJECT_WITH_ARRAY_PROPERTY (run, "artifacts");
+ ASSERT_EQ (artifacts->size (), 1);
+
+ {
+ // 3.24 "artifact" object:
+ auto artifact = (*artifacts)[0];
+
+ // 3.24.2:
+ auto location
+ = EXPECT_JSON_OBJECT_WITH_OBJECT_PROPERTY (artifact, "location");
+ ASSERT_JSON_STRING_PROPERTY_EQ (location, "uri", "MAIN_INPUT_FILENAME");
+
+ // 3.24.6:
+ auto roles = EXPECT_JSON_OBJECT_WITH_ARRAY_PROPERTY (artifact, "roles");
+ ASSERT_EQ (roles->size (), 1);
+ {
+ auto role = (*roles)[0];
+ ASSERT_JSON_STRING_EQ (role, "analysisTarget");
+ }
+ }
+ }
+
+ {
+ // 3.14.23:
+ auto results = EXPECT_JSON_OBJECT_WITH_ARRAY_PROPERTY (run, "results");
+ ASSERT_EQ (results->size (), 1);
+
+ {
+ // 3.27 "result" object:
+ auto result = (*results)[0];
+ ASSERT_JSON_STRING_PROPERTY_EQ (result, "ruleId", "error");
+ ASSERT_JSON_STRING_PROPERTY_EQ (result, "level", "error"); // 3.27.10
+
+ {
+ // 3.27.11:
+ auto message
+ = EXPECT_JSON_OBJECT_WITH_OBJECT_PROPERTY (result, "message");
+ ASSERT_JSON_STRING_PROPERTY_EQ (message, "text",
+ "this is a test: 42");
+ }
+
+ // 3.27.12:
+ EXPECT_JSON_OBJECT_WITH_ARRAY_PROPERTY (result, "locations");
+ }
+ }
+}
+
/* Run all of the selftests within this file. */
void
diagnostic_format_sarif_cc_tests ()
{
for_each_line_table_case (test_make_location_object);
+ test_simple_log ();
}
} // namespace selftest
diff --git a/gcc/json.h b/gcc/json.h
index 96721edf5365..21f71fe1c4ab 100644
--- a/gcc/json.h
+++ b/gcc/json.h
@@ -109,6 +109,8 @@ class object : public value
enum kind get_kind () const final override { return JSON_OBJECT; }
void print (pretty_printer *pp, bool formatted) const final override;
+ bool is_empty () const { return m_map.is_empty (); }
+
void set (const char *key, value *v);
/* Set the property KEY of this object, requiring V
diff --git a/gcc/selftest-diagnostic.cc b/gcc/selftest-diagnostic.cc
index 2684216a49fe..c9e9d7094bd3 100644
--- a/gcc/selftest-diagnostic.cc
+++ b/gcc/selftest-diagnostic.cc
@@ -60,6 +60,20 @@ test_diagnostic_context::start_span_cb (diagnostic_context
*context,
default_diagnostic_start_span_fn (context, exploc);
}
+bool
+test_diagnostic_context::report (diagnostic_t kind,
+ rich_location &richloc,
+ const diagnostic_metadata *metadata,
+ int option,
+ const char * fmt, ...)
+{
+ va_list ap;
+ va_start (ap, fmt);
+ bool result = diagnostic_impl (&richloc, metadata, option, fmt, &ap, kind);
+ va_end (ap);
+ return result;
+}
+
} // namespace selftest
#endif /* #if CHECKING_P */
diff --git a/gcc/selftest-diagnostic.h b/gcc/selftest-diagnostic.h
index 72a65fdb977f..f899443c4961 100644
--- a/gcc/selftest-diagnostic.h
+++ b/gcc/selftest-diagnostic.h
@@ -40,6 +40,16 @@ class test_diagnostic_context : public diagnostic_context
real filename (to avoid printing the names of tempfiles). */
static void
start_span_cb (diagnostic_context *context, expanded_location exploc);
+
+ /* Report a diagnostic to this context. For a selftest, this
+ should only be called on a context that uses a non-standard formatter
+ that e.g. gathers the results in memory, rather than emits to stderr. */
+ bool
+ report (diagnostic_t kind,
+ rich_location &richloc,
+ const diagnostic_metadata *metadata,
+ int option,
+ const char * fmt, ...) ATTRIBUTE_GCC_DIAG(6,7);
};
} // namespace selftest
diff --git a/gcc/selftest-json.cc b/gcc/selftest-json.cc
index 271e9b441120..4f52a87538ae 100644
--- a/gcc/selftest-json.cc
+++ b/gcc/selftest-json.cc
@@ -33,6 +33,20 @@ along with GCC; see the file COPYING3. If not see
namespace selftest {
+/* Assert that VALUE is a non-null json::string
+ equalling EXPECTED_VALUE.
+ Use LOC for any failures. */
+
+void
+assert_json_string_eq (const location &loc,
+ const json::value *value,
+ const char *expected_value)
+{
+ ASSERT_EQ_AT (loc, value->get_kind (), json::JSON_STRING);
+ const json::string *str = static_cast<const json::string *> (value);
+ ASSERT_STREQ_AT (loc, expected_value, str->get_string ());
+}
+
/* Assert that VALUE is a non-null json::object,
returning it as such, failing at LOC if this isn't the case. */
@@ -112,6 +126,22 @@ expect_json_object_with_array_property (const location
&loc,
return static_cast<const json::array *> (property_value);
}
+/* Assert that VALUE is a non-null json::object that has property
+ PROPERTY_NAME, and that the property value is a non-null JSON string.
+ Return the value of the property as a json::string.
+ Use LOC for any failures. */
+
+const json::string *
+expect_json_object_with_string_property (const location &loc,
+ const json::value *value,
+ const char *property_name)
+{
+ const json::value *property_value
+ = expect_json_object_with_property (loc, value, property_name);
+ ASSERT_EQ_AT (loc, property_value->get_kind (), json::JSON_STRING);
+ return static_cast<const json::string *> (property_value);
+}
+
/* Assert that VALUE is a non-null json::object that has property
PROPERTY_NAME, and that the value of that property is a non-null
JSON string equalling EXPECTED_VALUE.
@@ -125,9 +155,7 @@ assert_json_string_property_eq (const location &loc,
{
const json::value *property_value
= expect_json_object_with_property (loc, value, property_name);
- ASSERT_EQ_AT (loc, property_value->get_kind (), json::JSON_STRING);
- const json::string *str = static_cast<const json::string *> (property_value);
- ASSERT_STREQ_AT (loc, expected_value, str->get_string ());
+ assert_json_string_eq (loc, property_value, expected_value);
}
} // namespace selftest
diff --git a/gcc/selftest-json.h b/gcc/selftest-json.h
index 23b4d18951ca..80527d702813 100644
--- a/gcc/selftest-json.h
+++ b/gcc/selftest-json.h
@@ -30,6 +30,19 @@ along with GCC; see the file COPYING3. If not see
namespace selftest {
+/* Assert that VALUE is a non-null json::string
+ equalling EXPECTED_VALUE.
+ Use LOC for any failures. */
+
+void
+assert_json_string_eq (const location &loc,
+ const json::value *value,
+ const char *expected_value);
+#define ASSERT_JSON_STRING_EQ(JSON_VALUE, EXPECTED_VALUE) \
+ assert_json_string_eq ((SELFTEST_LOCATION), \
+ (JSON_VALUE), \
+ (EXPECTED_VALUE))
+
/* Assert that VALUE is a non-null json::object,
returning it as such, failing at LOC if this isn't the case. */
@@ -91,6 +104,20 @@ expect_json_object_with_array_property (const location &loc,
(JSON_VALUE), \
(PROPERTY_NAME))
+/* Assert that VALUE is a non-null json::object that has property
+ PROPERTY_NAME, and that the property value is a non-null JSON string.
+ Return the value of the property as a json::string.
+ Use LOC for any failures. */
+
+const json::string *
+expect_json_object_with_string_property (const location &loc,
+ const json::value *value,
+ const char *property_name);
+#define EXPECT_JSON_OBJECT_WITH_STRING_PROPERTY(JSON_VALUE, PROPERTY_NAME) \
+ expect_json_object_with_string_property ((SELFTEST_LOCATION),
\
+ (JSON_VALUE), \
+ (PROPERTY_NAME))
+
/* Assert that VALUE is a non-null json::object that has property
PROPERTY_NAME, and that the value of that property is a non-null
JSON string equalling EXPECTED_VALUE.
