Adds support for the AArch64 fmv priority syntax.
This allows users to override the default function ordering.
For example:
```c
int bar [[gnu::target_version("default")]] (int){
return 1;
}
int bar [[gnu::target_version("dotprod;priority=2")]] (int) {
return 2;
}
int bar [[gnu::target_version("sve;priority=1")]] (int) {
return 3;
}
```
gcc/ChangeLog:
* config/aarch64/aarch64.cc (aarch64_parse_fmv_features): Add parsing
for priority arguments.
(aarch64_process_target_version_attr): Update call to
aarch64_parse_fmv_features.
(get_feature_mask_for_version): Update call to
aarch64_parse_fmv_features.
(aarch64_compare_version_priority): Add logic to order by priority if
present.
(aarch64_functions_b_resolvable_from_a): Update call to
aarch64_parse_fmv_features.
(aarch64_mangle_decl_assembler_name): Update call to
aarch64_parse_fmv_features.
(dispatch_function_versions): Add logic to sort by priority.
(aarch64_same_function_versions): Add diagnostic if invalid use of
priority syntax.
(aarch64_merge_decl_attributes): Add logic to make suer priority
arguments are preserved.
(aarch64_check_target_clone_version): Update call to
aarch64_parse_fmv_features.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/fmv_priority3.c: New test.
* gcc.target/aarch64/fmv_priority_error1.c: New test.
* gcc.target/aarch64/fmv_priority_error2.c: New test.
---
gcc/config/aarch64/aarch64.cc | 143 ++++++++++++++----
.../gcc.target/aarch64/fmv_priority3.c | 26 ++++
.../gcc.target/aarch64/fmv_priority_error1.c | 11 ++
.../gcc.target/aarch64/fmv_priority_error2.c | 8 +
4 files changed, 160 insertions(+), 28 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/aarch64/fmv_priority3.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/fmv_priority_error1.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/fmv_priority_error2.c
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index 6f6dea67e0d..1da67b882d0 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -20401,25 +20401,29 @@ static aarch64_fmv_feature_datum
aarch64_fmv_feature_data[] = {
static enum aarch_parse_opt_result
aarch64_parse_fmv_features (string_slice str, aarch64_feature_flags *isa_flags,
aarch64_fmv_feature_mask *feature_mask,
+ unsigned int *priority,
std::string *invalid_extension)
{
if (feature_mask)
*feature_mask = 0ULL;
+ if (priority)
+ *priority = 0;
+
+ str = str.strip ();
if (str == "default")
return AARCH_PARSE_OK;
- gcc_assert (str.is_valid ());
-
- while (str.is_valid ())
+ /* Parse the architecture part of the string. */
+ string_slice arch_string = string_slice::tokenize (&str, ";");
+ while (arch_string.is_valid ())
{
string_slice ext;
- ext = string_slice::tokenize (&str, "+");
-
- gcc_assert (ext.is_valid ());
+ ext = string_slice::tokenize (&arch_string, "+");
+ ext = ext.strip ();
- if (!ext.is_valid () || ext.empty ())
+ if (ext.empty ())
return AARCH_PARSE_MISSING_ARG;
int num_features = ARRAY_SIZE (aarch64_fmv_feature_data);
@@ -20456,6 +20460,59 @@ aarch64_parse_fmv_features (string_slice str,
aarch64_feature_flags *isa_flags,
}
}
+ /* Parse any extra arguments. */
+ unsigned int priority_res = 0;
+ while (str.is_valid ())
+ {
+ string_slice argument = string_slice::tokenize (&str, ";").strip ();
+ /* Save the whole argument for diagnostics. */
+ string_slice arg = argument;
+ string_slice name = string_slice::tokenize (&argument, "=").strip ();
+ if (!argument.is_valid () || argument.empty ())
+ {
+ *invalid_extension = std::string (arg.begin (), arg.size ());
+ return AARCH_PARSE_INVALID_FEATURE;
+ }
+
+ /* priority=N argument (only one is allowed). */
+ if (name == "priority" && priority_res == 0)
+ {
+ /* Priority values can only be between 1 and 255, so any greater than
+ 3 digits long are invalid. */
+ if (argument.size () > 3)
+ {
+ *invalid_extension = std::string (arg.begin (), arg.size ());
+ return AARCH_PARSE_INVALID_FEATURE;
+ }
+
+ /* Parse the string value. */
+ for (char c : argument)
+ {
+ if (!ISDIGIT (c))
+ {
+ priority_res = 0;
+ break;
+ }
+ priority_res = 10 * priority_res + c - '0';
+ }
+
+ /* Check the entire string parsed, and that the value is in range. */
+ if (priority_res < 1 || priority_res > 255)
+ {
+ *invalid_extension = std::string (arg.begin (), arg.size ());
+ return AARCH_PARSE_INVALID_FEATURE;
+ }
+ if (priority)
+ *priority = priority_res;
+ }
+ else
+ {
+ /* Unrecognised argument. */
+ *invalid_extension = std::string (arg.begin (), arg.size ());
+ return AARCH_PARSE_INVALID_FEATURE;
+ }
+ }
+
return AARCH_PARSE_OK;
}
@@ -20487,7 +20544,7 @@ aarch64_process_target_version_attr (tree args)
auto isa_flags = aarch64_asm_isa_flags;
std::string invalid_extension;
- parse_res = aarch64_parse_fmv_features (str, &isa_flags, NULL,
+ parse_res = aarch64_parse_fmv_features (str, &isa_flags, NULL, NULL,
&invalid_extension);
if (parse_res == AARCH_PARSE_OK)
@@ -20576,7 +20633,7 @@ aarch64_option_valid_version_attribute_p (tree fndecl,
tree, tree args, int)
add or remove redundant feature requirements. */
static aarch64_fmv_feature_mask
-get_feature_mask_for_version (tree decl)
+get_feature_mask_for_version (tree decl, unsigned int *priority)
{
tree version_attr = lookup_attribute ("target_version",
DECL_ATTRIBUTES (decl));
@@ -20590,7 +20647,7 @@ get_feature_mask_for_version (tree decl)
aarch64_fmv_feature_mask feature_mask;
parse_res = aarch64_parse_fmv_features (version_string, NULL,
- &feature_mask, NULL);
+ &feature_mask, priority, NULL);
/* We should have detected any errors before getting here. */
gcc_assert (parse_res == AARCH_PARSE_OK);
@@ -20626,9 +20683,13 @@ compare_feature_masks (aarch64_fmv_feature_mask mask1,
int
aarch64_compare_version_priority (tree decl1, tree decl2)
{
- auto mask1 = get_feature_mask_for_version (decl1);
- auto mask2 = get_feature_mask_for_version (decl2);
+ unsigned int priority_1 = 0;
+ unsigned int priority_2 = 0;
+ auto mask1 = get_feature_mask_for_version (decl1, &priority_1);
+ auto mask2 = get_feature_mask_for_version (decl2, &priority_2);
+ if (priority_1 != priority_2)
+ return priority_1 > priority_2 ? 1 : -1;
return compare_feature_masks (mask1, mask2);
}
@@ -20645,9 +20706,9 @@ aarch64_functions_b_resolvable_from_a (tree decl_a,
tree decl_b, tree baseline)
auto a_version = get_target_version (decl_a);
auto b_version = get_target_version (decl_b);
if (a_version.is_valid ())
- aarch64_parse_fmv_features (a_version, &isa_a, NULL, NULL);
+ aarch64_parse_fmv_features (a_version, &isa_a, NULL, NULL, NULL);
if (b_version.is_valid ())
- aarch64_parse_fmv_features (b_version, &isa_b, NULL, NULL);
+ aarch64_parse_fmv_features (b_version, &isa_b, NULL, NULL, NULL);
/* Are there any bits of b that arent in a. */
if (isa_b & (~isa_a))
@@ -20725,7 +20786,7 @@ aarch64_mangle_decl_assembler_name (tree decl, tree id)
else if (DECL_FUNCTION_VERSIONED (decl))
{
aarch64_fmv_feature_mask feature_mask
- = get_feature_mask_for_version (decl);
+ = get_feature_mask_for_version (decl, NULL);
if (feature_mask == 0ULL)
return clone_identifier (id, "default");
@@ -21028,7 +21089,7 @@ dispatch_function_versions (tree dispatch_decl,
FOR_EACH_VEC_ELT_REVERSE ((*fndecls), i, version_decl)
{
aarch64_fmv_feature_mask feature_mask
- = get_feature_mask_for_version (version_decl);
+ = get_feature_mask_for_version (version_decl, NULL);
*empty_bb = add_condition_to_bb (dispatch_decl, version_decl,
feature_mask, mask_var, *empty_bb);
}
@@ -21151,23 +21212,35 @@ aarch64_get_function_versions_dispatcher (void *decl)
return dispatch_decl;
}
-/* This function returns true if STR1 and STR2 are version strings for the same
- function. */
+/* This function returns true if OLD_STR and NEW_STR are version strings for
the
+ same function. */
bool
-aarch64_same_function_versions (string_slice str1, string_slice str2)
+aarch64_same_function_versions (string_slice old_str, string_slice new_str)
{
enum aarch_parse_opt_result parse_res;
- aarch64_fmv_feature_mask feature_mask1;
- aarch64_fmv_feature_mask feature_mask2;
- parse_res = aarch64_parse_fmv_features (str1, NULL,
- &feature_mask1, NULL);
+ aarch64_fmv_feature_mask old_feature_mask;
+ aarch64_fmv_feature_mask new_feature_mask;
+ unsigned int old_priority;
+ unsigned int new_priority;
+ parse_res = aarch64_parse_fmv_features (old_str, NULL, &old_feature_mask,
+ &old_priority, NULL);
gcc_assert (parse_res == AARCH_PARSE_OK);
- parse_res = aarch64_parse_fmv_features (str2, NULL,
- &feature_mask2, NULL);
+ parse_res = aarch64_parse_fmv_features (new_str, NULL, &new_feature_mask,
+ &new_priority, NULL);
gcc_assert (parse_res == AARCH_PARSE_OK);
- return feature_mask1 == feature_mask2;
+ if (old_feature_mask != new_feature_mask)
+ return false;
+
+ /* Accept the case where the old version had a defined priority value but the
+ new version does not, as we infer the new version inherits the same
+ priority. */
+ if (old_priority != new_priority && (new_priority != 0))
+ error ("inconsistent function multi-version priority values "
+ "(was previously %d)", old_priority);
+
+ return true;
}
/* Implement TARGET_FUNCTION_ATTRIBUTE_INLINABLE_P. Use an opt-out
@@ -30274,6 +30347,20 @@ aarch64_merge_decl_attributes (tree olddecl, tree
newdecl)
aarch64_check_arm_new_against_type (TREE_VALUE (new_new), newdecl);
}
+ /* For target_version and target_clones, make sure we take the old version
+ as priority syntax cannot be added addetively. */
+ tree old_target_version = lookup_attribute ("target_version", old_attrs);
+ tree new_target_version = lookup_attribute ("target_version", new_attrs);
+
+ if (old_target_version && new_target_version)
+ TREE_VALUE (new_target_version) = TREE_VALUE (old_target_version);
+
+ tree old_target_clones = lookup_attribute ("target_clones", old_attrs);
+ tree new_target_clones = lookup_attribute ("target_clones", new_attrs);
+
+ if (old_target_clones && new_target_clones)
+ TREE_VALUE (new_target_clones) = TREE_VALUE (old_target_clones);
+
return merge_attributes (old_attrs, new_attrs);
}
@@ -31947,8 +32034,8 @@ aarch64_check_target_clone_version (string_slice str,
location_t *loc)
auto isa_flags = aarch64_asm_isa_flags;
std::string invalid_extension;
- parse_res
- = aarch64_parse_fmv_features (str, &isa_flags, NULL, &invalid_extension);
+ parse_res = aarch64_parse_fmv_features (str, &isa_flags, NULL, NULL,
+ &invalid_extension);
if (loc == NULL)
return parse_res == AARCH_PARSE_OK;
diff --git a/gcc/testsuite/gcc.target/aarch64/fmv_priority3.c
b/gcc/testsuite/gcc.target/aarch64/fmv_priority3.c
new file mode 100644
index 00000000000..535e99ab6a0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/fmv_priority3.c
@@ -0,0 +1,26 @@
+/* { dg-do compile } */
+/* { dg-require-ifunc "" } */
+/* { dg-options "-O0 -fdump-ipa-targetclone1-details" } */
+
+int foo [[gnu::target_version("dotprod;priority=1")]] (void);
+int foo [[gnu::target_version("default")]] (void) { return 1; }
+int foo [[gnu::target_version("dotprod;priority=1")]] (void) { return 2; }
+int foo [[gnu::target_version("sve")]] (void) { return 3; }
+
+// Priority1 dotprod > sve
+/* { dg-final { scan-ipa-dump-times "Version order for
foo/\[0-9\]+:\\nfoo\.default/\[0-9\]+\\nfoo\._Msve/\[0-9\]+\\nfoo\._Mdotprod/\[0-9\]+\\n"
1 "targetclone1" } } */
+
+int bar [[gnu::target_version("dotprod;priority=2")]] (void);
+int bar [[gnu::target_version("default")]] (void) { return 1; }
+int bar [[gnu::target_version("dotprod")]] (void) { return 2; }
+int bar [[gnu::target_version("sve;priority=1")]] (void) { return 3; }
+
+// Priority2 dotprod > Priority1 sve
+/* { dg-final { scan-ipa-dump-times "Version order for
bar/\[0-9\]+:\\nbar\.default/\[0-9\]+\\nbar\._Msve/\[0-9\]+\\nbar\._Mdotprod/\[0-9\]+\\n"
1 "targetclone1" } } */
+
+int baz [[gnu::target_version("default")]] (void) { return 1; }
+int baz [[gnu::target_version("dotprod;priority=1")]] (void) { return 2; }
+int baz [[gnu::target_version("sve;priority=1")]] (void) { return 3; }
+
+// Priority1 sve > Priority1 dotprod
+/* { dg-final { scan-ipa-dump-times "Version order for
baz/\[0-9\]+:\\nbaz\.default/\[0-9\]+\\nbaz\._Mdotprod/\[0-9\]+\\nbaz\._Msve/\[0-9\]+\\n"
1 "targetclone1" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/fmv_priority_error1.c
b/gcc/testsuite/gcc.target/aarch64/fmv_priority_error1.c
new file mode 100644
index 00000000000..0a6dd1f4045
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/fmv_priority_error1.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-require-ifunc } */
+/* { dg-options "-std=c23 -pedantic-errors" } */
+
+// Value was 2, now is 1
+int foo [[gnu::target_version("dotprod;priority=2")]] (void);
+int foo [[gnu::target_version("dotprod;priority=1")]] (void) { return 2; } /*
{ dg-error "inconsistent function multi-version priority values \\\(was
previously 2\\\)" } */
+
+// Value was 0, now is 2
+int bar [[gnu::target_version("dotprod")]] (void);
+int bar [[gnu::target_version("dotprod;priority=2")]] (void) { return 2; } /*
{ dg-error "inconsistent function multi-version priority values \\\(was
previously 0\\\)" } */
diff --git a/gcc/testsuite/gcc.target/aarch64/fmv_priority_error2.c
b/gcc/testsuite/gcc.target/aarch64/fmv_priority_error2.c
new file mode 100644
index 00000000000..0f212ff6aae
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/fmv_priority_error2.c
@@ -0,0 +1,8 @@
+/* { dg-do compile } */
+/* { dg-require-ifunc } */
+/* { dg-options "-std=c23 -pedantic-errors" } */
+
+int foo [[gnu::target_version("dotprod;priority=0")]] (void); /* { dg-error
"invalid feature modifier .priority=0. of value .dotprod;priority=0. in
.target_version. attribute" } */
+int foo [[gnu::target_version("dotprod;priority=-1")]] (void); /* { dg-error
"invalid feature modifier .priority=-1. of value .dotprod;priority=-1. in
.target_version. attribute" } */
+int foo [[gnu::target_version("dotprod;priority=256")]] (void); /* { dg-error
"invalid feature modifier .priority=256. of value .dotprod;priority=256. in
.target_version. attribute" } */
+int foo [[gnu::target_version("priority=4;dotprod")]] (void); /* { dg-error
"invalid feature modifier .priority=4. of value .priority=4;dotprod. in
.target_version. attribute" } */
--
2.34.1
