Hi Tobias,
Thanks for your thorough review. Please find attached a revised patch.
On 09/10/2024 14:55, Tobias Burnus wrote:
Paul-Antoine Arras wrote:
This patch adds middle-end support for the `dispatch` construct and the
`adjust_args` clause. The heavy lifting is done in
`gimplify_omp_dispatch` and
`gimplify_call_expr` respectively. For `adjust_args`, this mostly
consists in
emitting a call to `gomp_get_mapped_ptr` for the adequate device.
omp_get_… not gomp_get_…
Fixed.
For dispatch, the following steps are performed:
* Handle the device clause, if any: set the default-device ICV at the
top of the
dispatch region and restore its previous value at the end.
* Handle novariants and nocontext clauses, if any. Evaluate compile-time
constants and select a variant, if possible. Otherwise, emit code to
handle all
possible cases at run time.
* If depend clauses are present, add a taskwait construct before the
dispatch
region and move them there.
The latter is not done here – but already in the front ends, i.e.
OMP_TASK are handled in part 3 (C), 4 (C++) and 6 (Fortran) of this series.
Forgot to move that during a previous iteration. Fixed now.
...
--- a/gcc/gimple.cc
+++ b/gcc/gimple.cc
...
+/* Build a GIMPLE_OMP_DISPATCH statement.
+
+ BODY is the target function call to be dispatched.
+ CLAUSES are any of the OMP dispatch construct's clauses: ... */
Looks as if you planned to add something here. How about:
s/: ..././ ?
Right, fixed.
@@ -4067,23 +4069,125 @@ gimplify_call_expr (tree *expr_p, gimple_seq
*pre_p, bool want_value)
+ if (flag_openmp && EXPR_P (CALL_EXPR_FN (*expr_p))
+ && DECL_P (TREE_OPERAND (CALL_EXPR_FN (*expr_p), 0))
+ && (adjust_args_list = lookup_attribute (
+ "omp declare variant variant adjust_args",
+ DECL_ATTRIBUTES (
+ TREE_OPERAND (CALL_EXPR_FN (*expr_p), 0))))
+ != NULL_TREE
+ && gimplify_omp_ctxp != NULL
+ && gimplify_omp_ctxp->code == OMP_DISPATCH
+ && !gimplify_omp_ctxp->in_call_args)
+ {
!= should be under 'a' of 'adjust (remove one space)
clang-format is consistently doing it wrong... I have to be careful and
fix it manually.
And I wonder whether it is a bit more readable and a tiny bit faster if
you move the gimplify_omp_ctx checks directly after flag_openmp
and only if successfull ('&&') check for the attributes.
Agreed!
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IS_DEVICE_PTR)
+ {
+ tree decl1 = DECL_NAME (OMP_CLAUSE_DECL (c));
+ tree decl2 = tree_strip_nop_conversions (*arg_p);
+ if (TREE_CODE (decl2) == ADDR_EXPR)
+ decl2 = TREE_OPERAND (decl2, 0);
+ gcc_assert (TREE_CODE (decl2) == VAR_DECL
+ || TREE_CODE (decl2) == PARM_DECL);
The first one can be 'VAR_P (decl2)'. I keep wondering whether there
Changed.
can be cases where that's not true (e.g. VAR_DECL) or some indirect ref.
I don't remember running into such case. I think this is here just for
extra safety.
For Fortran, I could imagine that array descriptors make problems, e.g.
subroutine f(x)
integer, pointer :: x(:)
where 'x->data' is the device pointer and not 'x'.
(TODO: something to check + possibly to revisite when handling the
Fortran part; for now (including C/C++), I think we can leave it as is.)
Or something with reference types (→ C++, Fortran), albeit that's more
for need_device_addr / has_device_addr, which is not yet implemeted.
+ bool need_device_ptr = false;
+ for (tree arg
+ = TREE_PURPOSE (TREE_VALUE (adjust_args_list));
+ arg != NULL; arg = TREE_CHAIN (arg))
+ {
...
+ }
+
+ if (need_device_ptr && !is_device_ptr)
Actually, the is_device_ptr loop is only needed when need_device_ptr
(or, later, need_device_addr) is true; I wonder whether it should be
swapped and is_device_ptr only be checked conditionally?
Good point! Changed as suggested.
+ *arg_p = (TREE_CODE (*arg_p) == NOP_EXPR)
+ ? TREE_OPERAND (*arg_p, 0)
+ : *arg_p;
Use tree_strip_nop_conversions or STRIP_NOPS ? However, it is not clear
why it is needed here ...
Correct, it is not needed here. Removed.
+ gimplify_arg (arg_p, pre_p, loc);
+ gimplify_arg (&device_num, pre_p, loc);
+ call = gimple_build_call (fn, 2, *arg_p, device_num);
+ tree mapped_arg
+ = create_tmp_var (gimple_call_return_type (call));
+ gimple_call_set_lhs (call, mapped_arg);
+ gimplify_seq_add_stmt (pre_p, call);
+
+ *arg_p = mapped_arg;
This line causes the following to attempt to fail:
+ // Mark mapped argument as device pointer to ensure
+ // idempotency in gimplification
+ gcc_assert (gimplify_omp_ctxp->code == OMP_DISPATCH);
+ tree c = build_omp_clause (input_location,
+ OMP_CLAUSE_IS_DEVICE_PTR);
+ OMP_CLAUSE_DECL (c) = *arg_p;
and, to continue the previous topic, I wonder whether it would be
sufficient to add the STRIP_NOPS on the RHS.
* * *
Testcase (could be a candidate for a scan-tree-dump-times test):
void foobar(int *x, int *y);
#pragma omp declare variant(foobar) match(construct={dispatch})
adjust_args(need_device_ptr: x,y)
void bar(int *x, int *y);
void sub(int *y, int *z)
{
#pragma omp dispatch //is_device_ptr(y)
bar(y, y);
}
yields:
_3 = __builtin_omp_get_default_device ();
_6 = __builtin_omp_get_mapped_ptr (y_4(D), _3);
_8 = __builtin_omp_get_mapped_ptr (y_4(D), _3);
foobar (_8, _6);
The quoted piece of code is aimed at avoiding code duplication in case
gimplify_call_expr is called multiple times. It is not meant to
factorise argument handling.
+/* Callback for walk_tree to find a IFN_GOMP_DISPATCH. */
+
+static tree
+find_ifn_gomp_dispatch (tree *tp, int *, void *modify)
Typo: "... an IFN_..." (a -> an)
Fixed.
+/* Try to evaluate a novariants clause. Return 1 if true, 0 if false
or absent,
+ * -1 if run-time evaluation is needed. */
+
+int
+omp_has_novariants (void)
+{
+ for (struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp; ctx;
+ ctx = ctx->outer_context)
+ {
+ if (ctx->code == OMP_DISPATCH && !ctx->in_call_args)
Shouldn't dispatch not always be the innermost OpenMP context?
Likewise for nocontext.
Correct. Removed the loop.
+ while (TREE_CODE (base_call_expr) == FLOAT_EXPR
+ || TREE_CODE (base_call_expr) == CONVERT_EXPR
+ || TREE_CODE (base_call_expr) == COMPLEX_EXPR
+ || TREE_CODE (base_call_expr) == INDIRECT_REF)
+ base_call_expr = TREE_OPERAND (base_call_expr, 0);
Is there a reason that NOP_EXPR is not in that list? Especially as you
use STRIP_NOPS in the next line?
There might have been a reason in a previous revision, but not anymore.
Corrected.
+ tree base_fndecl = get_callee_fndecl (STRIP_NOPS (base_call_expr));
+ if (base_fndecl != NULL_TREE)
...
+ if (base_fndecl != variant_fndecl
+ && (omp_has_novariants () == -1 || omp_has_nocontext () == -1))
+ {
+ tree novariants_clause = NULL_TREE, nocontext_clause = NULL_TREE,
+ novariants_cond = NULL_TREE, nocontext_cond = NULL_TREE;
+ for (tree c = OMP_DISPATCH_CLAUSES (expr); c; c = TREE_CHAIN (c))
+ {
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NOVARIANTS)
&& !integer_zerop (OMP_CLAUSE_NOVARIANTS_EXPR (c)) ?
Yup.
+ {
+ gcc_assert (novariants_cond == NULL_TREE);
+ novariants_clause = c;
+ novariants_cond = OMP_CLAUSE_NOVARIANTS_EXPR (c);
+ }
+ else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NOCONTEXT)
&& !integer_zerop (OMP_CLAUSE_NOCONTEXT_EXPR (c)) ?
Just to avoid some extra gimplification / optimization work later on?
Sure.
+ {
+ gcc_assert (nocontext_cond == NULL_TREE);
+ nocontext_clause = c;
+ nocontext_cond = OMP_CLAUSE_NOCONTEXT_EXPR (c);
+ }
+ }
+ gcc_assert (novariants_cond != NULL_TREE
+ || nocontext_cond != NULL_TREE);
+
+ enum gimplify_status ret
+ = gimplify_expr (&novariants_cond, &body, NULL, is_gimple_val,
+ fb_rvalue);
+ if (ret == GS_ERROR || ret == GS_UNHANDLED)
+ return ret;
+ ret = gimplify_expr (&nocontext_cond, &body, NULL, is_gimple_val,
+ fb_rvalue);
+ if (ret == GS_ERROR || ret == GS_UNHANDLED)
+ return ret;
+
+ tree base_label = create_artificial_label (UNKNOWN_LOCATION);
+ tree variant1_label = create_artificial_label (UNKNOWN_LOCATION);
+ tree cond_label = create_artificial_label (UNKNOWN_LOCATION);
+ tree variant2_label = create_artificial_label (UNKNOWN_LOCATION);
+ tree end_label = create_artificial_label (UNKNOWN_LOCATION);
+
+ if (novariants_cond != NULL_TREE)
+ {
+ gcond *novariants_cond_stmt
+ = gimple_build_cond_from_tree (novariants_cond, base_label,
+ cond_label);
+ gimplify_seq_add_stmt (&body, novariants_cond_stmt);
+
+ gimplify_seq_add_stmt (&body, gimple_build_label
(base_label));
+ tree base_call_expr2 = copy_node (base_call_expr);
+ if (TREE_CODE (dispatch_body) == MODIFY_EXPR)
+ {
+ base_call_expr2 = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst,
+ base_call_expr2);
+ }
+ OMP_CLAUSE_NOVARIANTS_EXPR (novariants_clause)
+ = boolean_true_node;
+ gimplify_and_add (base_call_expr2, &body);
+ gimplify_seq_add_stmt (&body, gimple_build_goto (end_label));
+
+ OMP_CLAUSE_NOVARIANTS_EXPR (novariants_clause)
+ = boolean_false_node;
+ }
+
+ gimplify_seq_add_stmt (&body, gimple_build_label (cond_label));
You can move the cond_label into the 'if' - and do likewise with the
first two label creations - those are not used outside of the 'if'.
(except for end_label, of course)
Likewise for:
+ if (nocontext_cond != NULL_TREE)
+ {
...
+ }
+
+ gimplify_seq_add_stmt (&body, gimple_build_label
(variant2_label));
Those.
Moved labels to inner scope as suggested.
+ tree variant_call_expr = copy_node (base_call_expr);
Is there a reason to copy the node? I understand that you copy them for
the variant calls - but I think one of the nodes can be used without
copying (e.g. this one), or do I miss something?
Indeed, the last one can be used directly. Done.
+ if (TREE_CODE (dispatch_body) == MODIFY_EXPR)
+ {
+ variant_call_expr
+ = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, variant_call_expr);
+ }
+ gimplify_and_add (variant_call_expr, &body);
+ gimplify_seq_add_stmt (&body, gimple_build_goto (end_label));
+ gimplify_seq_add_stmt (&body, gimple_build_label (end_label));
+ }
I think the last goto is superflous as it just jumps to the next line.
Absolutely right. Removed.
+ else
+ gimplify_and_add (OMP_DISPATCH_BODY (expr), &body);
+ }
+ else
+ gimplify_and_add (OMP_DISPATCH_BODY (expr), &body);
+
+ // Restore default-device-var ICV
+ if (device)
+ {
+ tree fn = builtin_decl_explicit (BUILT_IN_OMP_SET_DEFAULT_DEVICE);
+ gcall *call = gimple_build_call (fn, 1, saved_device_icv);
+ gimplify_seq_add_stmt (&body, call);
+ }
+
+ // Wrap dispatch body into a bind
+ gimple *bind = gimple_build_bind (NULL_TREE, body, NULL_TREE);
+ pop_gimplify_context (bind);
+
+ // Manually tear down context created by gimplify_scan_omp_clauses
to avoid a
+ // call to gimplify_adjust_omp_clauses
+ gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
+ if (ctx != NULL)
+ {
+ gcc_assert (ctx->code == OMP_DISPATCH);
+ gimplify_omp_ctxp = ctx->outer_context;
+ delete_omp_context (ctx);
+ }
+
+ // Remove nowait as it has no effect on dispatch (OpenMP 5.2), and
depend and
+ // device clauses as they have already been handled
"Remove ..., device as it has been handled above, and depend as the
front end handled it by inserting taskwait."
Done.
Or something like that - to avoid searching for OMP_TASK/OMP_CLAUSE_DEPEND.
--- a/gcc/omp-general.cc
+++ b/gcc/omp-general.cc
@@ -1049,7 +1049,7 @@ omp_construct_traits_to_codes (tree ctx, int
nconstructs,
/* Order must match the OMP_TRAIT_CONSTRUCT_* enumerators in
enum omp_ts_code. */
static enum tree_code code_map[]
- = { OMP_TARGET, OMP_TEAMS, OMP_PARALLEL, OMP_FOR, OMP_SIMD };
+ = { OMP_TARGET, OMP_TEAMS, OMP_PARALLEL, OMP_FOR, OMP_SIMD,
OMP_DISPATCH };
for (tree ts = ctx; ts; ts = TREE_CHAIN (ts), i--)
{
@@ -1142,6 +1142,7 @@ const char *omp_tss_map[] =
"target_device",
"implementation",
"user",
+ "need_device_ptr",
NULL
};
This one looks bogus - and is also effectively unused.
Leftover from a previous iteration. Removed.
On 17/10/2024 12:37, Tobias Burnus wrote:
Minor follow-up comments:
Paul-Antoine Arras wrote:
This patch adds middle-end support for the `dispatch` construct and the
`adjust_args` clause. The heavy lifting is done in `gimplify_omp_dispatch` and
`gimplify_call_expr` respectively. For `adjust_args`, this mostly consists in
emitting a call to `gomp_get_mapped_ptr` for the adequate device.
...
@@ -4067,23 +4069,125 @@ gimplify_call_expr
...
+ // Mark mapped argument as device pointer to ensure
+ // idempotency in gimplification
+ gcc_assert (gimplify_omp_ctxp->code == OMP_DISPATCH);
Just an observation: This assert looks as if it really should never
become true if one reads the code flow. I think it is therefore a
candidate for: gcc_checking_assert
While gcc_assert is enabled by --enable-checking=assert_checking, which
is on by default, gcc_checking_assert is enabled by …=checking, which is
enabled by default only when gcc/DEV-PHASE == experimental.
Thus, in releases this saves a few bytes (code + string) and µs execution time
...
Replaced with gcc_checking_assert.
* * *
+/* Gimplify an OMP_DISPATCH construct. */
+
+static enum gimplify_status
+gimplify_omp_dispatch (tree *expr_p, gimple_seq *pre_p)
+{
...
+ // If device clause, adjust ICV
+ tree device
+ = omp_find_clause (OMP_DISPATCH_CLAUSES (expr), OMP_CLAUSE_DEVICE);
+ tree saved_device_icv;
+ if (device)
+ {
I think you should do:
if (device
&& (TREE_CODE (TREE_VALUE (device)) != INTEGER_CST
|| !wi::eq_p (device, -1 /* omp_initial_device */ )))
Done.
--- a/gcc/omp-builtins.def
+++ b/gcc/omp-builtins.def
@@ -80,6 +80,12 @@ DEF_GOMP_BUILTIN (BUILT_IN_OMP_GET_TEAM_NUM,
"omp_get_team_num",
BT_FN_INT, ATTR_CONST_NOTHROW_LEAF_LIST)
DEF_GOMP_BUILTIN (BUILT_IN_OMP_GET_NUM_TEAMS, "omp_get_num_teams",
BT_FN_INT, ATTR_CONST_NOTHROW_LEAF_LIST)
+DEF_GOMP_BUILTIN (BUILT_IN_OMP_GET_MAPPED_PTR, "omp_get_mapped_ptr",
+ BT_FN_PTR_CONST_PTR_INT, ATTR_NOTHROW_LEAF_LIST)
That's 'void *omp_get_mapped_ptr (const void *, int)'.
When using the function (the builtin), the pointer address of the first
argument escapes for the compiler, but we know it doesn't. Thus, I think
we want to use 'fn attr' here
→ builtin_fnspec in builtins.cc and attr-fnspec.h for the syntax.
Added ". R " for BUILT_IN_OMP_GET_MAPPED_PTR in builtins.cc.
Thanks,
--
PA
commit 0f4e2830fd4232bcaddd4d8131c573d5626c21da
Author: Paul-Antoine Arras <par...@baylibre.com>
Date: Fri May 24 15:53:45 2024 +0200
OpenMP: middle-end support for dispatch + adjust_args
This patch adds middle-end support for the `dispatch` construct and the
`adjust_args` clause. The heavy lifting is done in `gimplify_omp_dispatch` and
`gimplify_call_expr` respectively. For `adjust_args`, this mostly consists in
emitting a call to `omp_get_mapped_ptr` for the adequate device.
For dispatch, the following steps are performed:
* Handle the device clause, if any: set the default-device ICV at the top of the
dispatch region and restore its previous value at the end.
* Handle novariants and nocontext clauses, if any. Evaluate compile-time
constants and select a variant, if possible. Otherwise, emit code to handle all
possible cases at run time.
gcc/ChangeLog:
* gimple-low.cc (lower_stmt): Handle GIMPLE_OMP_DISPATCH.
* gimple-pretty-print.cc (dump_gimple_omp_dispatch): New function.
(pp_gimple_stmt_1): Handle GIMPLE_OMP_DISPATCH.
* gimple-walk.cc (walk_gimple_stmt): Likewise.
* gimple.cc (gimple_build_omp_dispatch): New function.
(gimple_copy): Handle GIMPLE_OMP_DISPATCH.
* gimple.def (GIMPLE_OMP_DISPATCH): Define.
* gimple.h (gimple_build_omp_dispatch): Declare.
(gimple_has_substatements): Handle GIMPLE_OMP_DISPATCH.
(gimple_omp_dispatch_clauses): New function.
(gimple_omp_dispatch_clauses_ptr): Likewise.
(gimple_omp_dispatch_set_clauses): Likewise.
(gimple_return_set_retval): Handle GIMPLE_OMP_DISPATCH.
* gimplify.cc (enum omp_region_type): Add ORT_DISPATCH.
(gimplify_call_expr): Handle need_device_ptr arguments.
(is_gimple_stmt): Handle OMP_DISPATCH.
(gimplify_scan_omp_clauses): Handle OMP_CLAUSE_DEVICE in a dispatch
construct. Handle OMP_CLAUSE_NOVARIANTS and OMP_CLAUSE_NOCONTEXT.
(omp_construct_selector_matches): Handle OMP_DISPATCH with nocontext
clause.
(omp_has_novariants): New function.
(omp_has_nocontext): Likewise.
(find_ifn_gomp_dispatch): New function.
(gimplify_omp_dispatch): Likewise.
(gimplify_expr): Handle OMP_DISPATCH.
* gimplify.h (omp_has_novariants): Declare.
* internal-fn.cc (expand_GOMP_DISPATCH): New function.
* internal-fn.def (GOMP_DISPATCH): Define.
* omp-builtins.def (BUILT_IN_OMP_GET_MAPPED_PTR): Define.
(BUILT_IN_OMP_GET_DEFAULT_DEVICE): Define.
(BUILT_IN_OMP_SET_DEFAULT_DEVICE): Define.
* omp-general.cc (omp_construct_traits_to_codes): Add OMP_DISPATCH.
(struct omp_ts_info): Add dispatch.
(omp_resolve_declare_variant): Handle novariants. Adjust
DECL_ASSEMBLER_NAME.
* omp-low.cc (scan_omp_1_stmt): Handle GIMPLE_OMP_DISPATCH.
(lower_omp_dispatch): New function.
(lower_omp_1): Call it.
* tree-inline.cc (remap_gimple_stmt): Handle GIMPLE_OMP_DISPATCH.
(estimate_num_insns): Handle GIMPLE_OMP_DISPATCH.
diff --git gcc/builtins.cc gcc/builtins.cc
index 37c7c98e5c7..c7a8310bb3f 100644
--- gcc/builtins.cc
+++ gcc/builtins.cc
@@ -12580,6 +12580,8 @@ builtin_fnspec (tree callee)
by its first argument. */
case BUILT_IN_POSIX_MEMALIGN:
return ".cOt";
+ case BUILT_IN_OMP_GET_MAPPED_PTR:
+ return ". R ";
default:
return "";
diff --git gcc/gimple-low.cc gcc/gimple-low.cc
index e0371988705..712a1ebf776 100644
--- gcc/gimple-low.cc
+++ gcc/gimple-low.cc
@@ -746,6 +746,7 @@ lower_stmt (gimple_stmt_iterator *gsi, struct lower_data *data)
case GIMPLE_EH_MUST_NOT_THROW:
case GIMPLE_OMP_FOR:
case GIMPLE_OMP_SCOPE:
+ case GIMPLE_OMP_DISPATCH:
case GIMPLE_OMP_SECTIONS:
case GIMPLE_OMP_SECTIONS_SWITCH:
case GIMPLE_OMP_SECTION:
diff --git gcc/gimple-pretty-print.cc gcc/gimple-pretty-print.cc
index 01d7c9f6eeb..7a45e8ec843 100644
--- gcc/gimple-pretty-print.cc
+++ gcc/gimple-pretty-print.cc
@@ -1726,6 +1726,35 @@ dump_gimple_omp_scope (pretty_printer *pp, const gimple *gs,
}
}
+/* Dump a GIMPLE_OMP_DISPATCH tuple on the pretty_printer BUFFER. */
+
+static void
+dump_gimple_omp_dispatch (pretty_printer *buffer, const gimple *gs, int spc,
+ dump_flags_t flags)
+{
+ if (flags & TDF_RAW)
+ {
+ dump_gimple_fmt (buffer, spc, flags, "%G <%+BODY <%S>%nCLAUSES <", gs,
+ gimple_omp_body (gs));
+ dump_omp_clauses (buffer, gimple_omp_dispatch_clauses (gs), spc, flags);
+ dump_gimple_fmt (buffer, spc, flags, " >");
+ }
+ else
+ {
+ pp_string (buffer, "#pragma omp dispatch");
+ dump_omp_clauses (buffer, gimple_omp_dispatch_clauses (gs), spc, flags);
+ if (!gimple_seq_empty_p (gimple_omp_body (gs)))
+ {
+ newline_and_indent (buffer, spc + 2);
+ pp_left_brace (buffer);
+ pp_newline (buffer);
+ dump_gimple_seq (buffer, gimple_omp_body (gs), spc + 4, flags);
+ newline_and_indent (buffer, spc + 2);
+ pp_right_brace (buffer);
+ }
+ }
+}
+
/* Dump a GIMPLE_OMP_TARGET tuple on the pretty_printer PP. */
static void
@@ -2805,6 +2834,10 @@ pp_gimple_stmt_1 (pretty_printer *pp, const gimple *gs, int spc,
dump_gimple_omp_scope (pp, gs, spc, flags);
break;
+ case GIMPLE_OMP_DISPATCH:
+ dump_gimple_omp_dispatch(pp, gs, spc, flags);
+ break;
+
case GIMPLE_OMP_MASTER:
case GIMPLE_OMP_SECTION:
case GIMPLE_OMP_STRUCTURED_BLOCK:
diff --git gcc/gimple-walk.cc gcc/gimple-walk.cc
index 9f768ca20fd..1122713a98b 100644
--- gcc/gimple-walk.cc
+++ gcc/gimple-walk.cc
@@ -707,6 +707,7 @@ walk_gimple_stmt (gimple_stmt_iterator *gsi, walk_stmt_fn callback_stmt,
case GIMPLE_OMP_PARALLEL:
case GIMPLE_OMP_TASK:
case GIMPLE_OMP_SCOPE:
+ case GIMPLE_OMP_DISPATCH:
case GIMPLE_OMP_SECTIONS:
case GIMPLE_OMP_SINGLE:
case GIMPLE_OMP_TARGET:
diff --git gcc/gimple.cc gcc/gimple.cc
index 6e28cf291e1..8fb9b41c991 100644
--- gcc/gimple.cc
+++ gcc/gimple.cc
@@ -1235,6 +1235,21 @@ gimple_build_omp_scope (gimple_seq body, tree clauses)
return p;
}
+/* Build a GIMPLE_OMP_DISPATCH statement.
+
+ BODY is the target function call to be dispatched.
+ CLAUSES are any of the OMP dispatch construct's clauses. */
+
+gimple *
+gimple_build_omp_dispatch (gimple_seq body, tree clauses)
+{
+ gimple *p = gimple_alloc (GIMPLE_OMP_DISPATCH, 0);
+ gimple_omp_dispatch_set_clauses (p, clauses);
+ if (body)
+ gimple_omp_set_body (p, body);
+
+ return p;
+}
/* Build a GIMPLE_OMP_TARGET statement.
@@ -2148,6 +2163,11 @@ gimple_copy (gimple *stmt)
gimple_omp_scope_set_clauses (copy, t);
goto copy_omp_body;
+ case GIMPLE_OMP_DISPATCH:
+ t = unshare_expr (gimple_omp_dispatch_clauses (stmt));
+ gimple_omp_dispatch_set_clauses (copy, t);
+ goto copy_omp_body;
+
case GIMPLE_OMP_TARGET:
{
gomp_target *omp_target_stmt = as_a <gomp_target *> (stmt);
diff --git gcc/gimple.def gcc/gimple.def
index fbcd727f945..21c7405875d 100644
--- gcc/gimple.def
+++ gcc/gimple.def
@@ -350,6 +350,11 @@ DEFGSCODE(GIMPLE_OMP_SCAN, "gimple_omp_scan", GSS_OMP_SINGLE_LAYOUT)
CLAUSES is an OMP_CLAUSE chain holding the associated clauses. */
DEFGSCODE(GIMPLE_OMP_SCOPE, "gimple_omp_scope", GSS_OMP_SINGLE_LAYOUT)
+/* GIMPLE_OMP_DISPATCH <BODY, CLAUSES> represents #pragma omp dispatch
+ BODY is the target function call to be dispatched.
+ CLAUSES is an OMP_CLAUSE chain holding the associated clauses. */
+DEFGSCODE(GIMPLE_OMP_DISPATCH, "gimple_omp_dispatch", GSS_OMP_SINGLE_LAYOUT)
+
/* OMP_SECTION <BODY> represents #pragma omp section.
BODY is the sequence of statements in the section body. */
DEFGSCODE(GIMPLE_OMP_SECTION, "gimple_omp_section", GSS_OMP)
diff --git gcc/gimple.h gcc/gimple.h
index 4a6e0e97d1e..b6967e63de2 100644
--- gcc/gimple.h
+++ gcc/gimple.h
@@ -742,7 +742,7 @@ struct GTY((tag("GSS_OMP_CONTINUE")))
};
/* GIMPLE_OMP_SINGLE, GIMPLE_OMP_ORDERED, GIMPLE_OMP_TASKGROUP,
- GIMPLE_OMP_SCAN, GIMPLE_OMP_MASKED, GIMPLE_OMP_SCOPE. */
+ GIMPLE_OMP_SCAN, GIMPLE_OMP_MASKED, GIMPLE_OMP_SCOPE, GIMPLE_OMP_DISPATCH. */
struct GTY((tag("GSS_OMP_SINGLE_LAYOUT")))
gimple_statement_omp_single_layout : public gimple_statement_omp
@@ -1591,6 +1591,7 @@ gomp_task *gimple_build_omp_task (gimple_seq, tree, tree, tree, tree,
gimple *gimple_build_omp_section (gimple_seq);
gimple *gimple_build_omp_structured_block (gimple_seq);
gimple *gimple_build_omp_scope (gimple_seq, tree);
+gimple *gimple_build_omp_dispatch (gimple_seq, tree);
gimple *gimple_build_omp_master (gimple_seq);
gimple *gimple_build_omp_masked (gimple_seq, tree);
gimple *gimple_build_omp_taskgroup (gimple_seq, tree);
@@ -1882,6 +1883,7 @@ gimple_has_substatements (gimple *g)
case GIMPLE_OMP_PARALLEL:
case GIMPLE_OMP_TASK:
case GIMPLE_OMP_SCOPE:
+ case GIMPLE_OMP_DISPATCH:
case GIMPLE_OMP_SECTIONS:
case GIMPLE_OMP_SINGLE:
case GIMPLE_OMP_TARGET:
@@ -5434,6 +5436,34 @@ gimple_omp_scope_set_clauses (gimple *gs, tree clauses)
= clauses;
}
+/* Return the clauses associated with OMP_DISPATCH statement GS. */
+
+inline tree
+gimple_omp_dispatch_clauses (const gimple *gs)
+{
+ GIMPLE_CHECK (gs, GIMPLE_OMP_DISPATCH);
+ return static_cast<const gimple_statement_omp_single_layout *> (gs)->clauses;
+}
+
+/* Return a pointer to the clauses associated with OMP dispatch statement
+ GS. */
+
+inline tree *
+gimple_omp_dispatch_clauses_ptr (gimple *gs)
+{
+ GIMPLE_CHECK (gs, GIMPLE_OMP_DISPATCH);
+ return &static_cast<gimple_statement_omp_single_layout *> (gs)->clauses;
+}
+
+/* Set CLAUSES to be the clauses associated with OMP dispatch statement
+ GS. */
+
+inline void
+gimple_omp_dispatch_set_clauses (gimple *gs, tree clauses)
+{
+ GIMPLE_CHECK (gs, GIMPLE_OMP_DISPATCH);
+ static_cast<gimple_statement_omp_single_layout *> (gs)->clauses = clauses;
+}
/* Return the kind of the OMP_FOR statemement G. */
@@ -6768,6 +6798,7 @@ gimple_return_set_retval (greturn *gs, tree retval)
case GIMPLE_OMP_TARGET: \
case GIMPLE_OMP_TEAMS: \
case GIMPLE_OMP_SCOPE: \
+ case GIMPLE_OMP_DISPATCH: \
case GIMPLE_OMP_SECTION: \
case GIMPLE_OMP_STRUCTURED_BLOCK: \
case GIMPLE_OMP_MASTER: \
diff --git gcc/gimplify.cc gcc/gimplify.cc
index 3f602469d57..c86928bb1fc 100644
--- gcc/gimplify.cc
+++ gcc/gimplify.cc
@@ -161,7 +161,8 @@ enum omp_region_type
{
ORT_WORKSHARE = 0x00,
ORT_TASKGROUP = 0x01,
- ORT_SIMD = 0x04,
+ ORT_DISPATCH = 0x02,
+ ORT_SIMD = 0x04,
ORT_PARALLEL = 0x08,
ORT_COMBINED_PARALLEL = ORT_PARALLEL | 1,
@@ -258,6 +259,7 @@ struct gimplify_omp_ctx
bool order_concurrent;
bool has_depend;
bool in_for_exprs;
+ bool in_call_args;
int defaultmap[5];
};
@@ -4071,23 +4073,136 @@ gimplify_call_expr (tree *expr_p, gimple_seq *pre_p, bool want_value)
/* Gimplify the function arguments. */
if (nargs > 0)
{
+ tree device_num = NULL_TREE;
for (i = (PUSH_ARGS_REVERSED ? nargs - 1 : 0);
- PUSH_ARGS_REVERSED ? i >= 0 : i < nargs;
- PUSH_ARGS_REVERSED ? i-- : i++)
- {
- enum gimplify_status t;
+ PUSH_ARGS_REVERSED ? i >= 0 : i < nargs;
+ PUSH_ARGS_REVERSED ? i-- : i++)
+ {
+ enum gimplify_status t;
- /* Avoid gimplifying the second argument to va_start, which needs to
- be the plain PARM_DECL. */
- if ((i != 1) || !builtin_va_start_p)
- {
- t = gimplify_arg (&CALL_EXPR_ARG (*expr_p, i), pre_p,
- EXPR_LOCATION (*expr_p), ! returns_twice);
+ /* Avoid gimplifying the second argument to va_start, which needs to
+ be the plain PARM_DECL. */
+ if ((i != 1) || !builtin_va_start_p)
+ {
+ tree *arg_p = &CALL_EXPR_ARG (*expr_p, i);
+ tree adjust_args_list;
+ if (flag_openmp && gimplify_omp_ctxp != NULL
+ && gimplify_omp_ctxp->code == OMP_DISPATCH
+ && !gimplify_omp_ctxp->in_call_args
+ && EXPR_P (CALL_EXPR_FN (*expr_p))
+ && DECL_P (TREE_OPERAND (CALL_EXPR_FN (*expr_p), 0))
+ && (adjust_args_list = lookup_attribute (
+ "omp declare variant variant adjust_args",
+ DECL_ATTRIBUTES (
+ TREE_OPERAND (CALL_EXPR_FN (*expr_p), 0))))
+ != NULL_TREE)
+ {
+ tree param
+ = DECL_ARGUMENTS (TREE_OPERAND (CALL_EXPR_FN (*expr_p), 0));
- if (t == GS_ERROR)
- ret = GS_ERROR;
- }
- }
+ if (param != NULL_TREE)
+ {
+ for (int param_idx = 0; param_idx < i; param_idx++)
+ param = TREE_CHAIN (param);
+
+ bool need_device_ptr = false;
+ for (tree arg
+ = TREE_PURPOSE (TREE_VALUE (adjust_args_list));
+ arg != NULL; arg = TREE_CHAIN (arg))
+ {
+ if (TREE_VALUE (arg)
+ && TREE_CODE (TREE_VALUE (arg)) == INTEGER_CST
+ && wi::eq_p (i, wi::to_wide (TREE_VALUE (arg))))
+ {
+ need_device_ptr = true;
+ break;
+ }
+ }
+
+ if (need_device_ptr)
+ {
+ bool is_device_ptr = false;
+ for (tree c = gimplify_omp_ctxp->clauses; c;
+ c = TREE_CHAIN (c))
+ {
+ if (OMP_CLAUSE_CODE (c)
+ == OMP_CLAUSE_IS_DEVICE_PTR)
+ {
+ tree decl1 = DECL_NAME (OMP_CLAUSE_DECL (c));
+ tree decl2
+ = tree_strip_nop_conversions (*arg_p);
+ if (TREE_CODE (decl2) == ADDR_EXPR)
+ decl2 = TREE_OPERAND (decl2, 0);
+ if (VAR_P (decl2)
+ || TREE_CODE (decl2) == PARM_DECL)
+ {
+ decl2 = DECL_NAME (decl2);
+ if (decl1 == decl2)
+ is_device_ptr = true;
+ }
+ }
+ else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEVICE)
+ device_num = OMP_CLAUSE_OPERAND (c, 0);
+ }
+
+ if (!is_device_ptr)
+ {
+ if (device_num == NULL_TREE)
+ {
+ // device_num = omp_get_default_device ()
+ tree fn = builtin_decl_explicit (
+ BUILT_IN_OMP_GET_DEFAULT_DEVICE);
+ gcall *call = gimple_build_call (fn, 0);
+ device_num = create_tmp_var (
+ gimple_call_return_type (call));
+ gimple_call_set_lhs (call, device_num);
+ gimplify_seq_add_stmt (pre_p, call);
+ }
+
+ // mapped_arg = omp_get_mapped_ptr (arg,
+ // device_num)
+ tree fn = builtin_decl_explicit (
+ BUILT_IN_OMP_GET_MAPPED_PTR);
+ gimplify_arg (arg_p, pre_p, loc);
+ gimplify_arg (&device_num, pre_p, loc);
+ call
+ = gimple_build_call (fn, 2, *arg_p, device_num);
+ tree mapped_arg = create_tmp_var (
+ gimple_call_return_type (call));
+ gimple_call_set_lhs (call, mapped_arg);
+ gimplify_seq_add_stmt (pre_p, call);
+
+ *arg_p = mapped_arg;
+
+ // gimplify_call_expr might be called several
+ // times on the same call, which would result in
+ // duplicated calls to omp_get_default_device and
+ // omp_get_mapped_ptr. To prevent that, we mark
+ // already mapped arguments as device pointers.
+ gcc_checking_assert (gimplify_omp_ctxp->code
+ == OMP_DISPATCH);
+ tree c
+ = build_omp_clause (input_location,
+ OMP_CLAUSE_IS_DEVICE_PTR);
+ OMP_CLAUSE_DECL (c) = *arg_p;
+ OMP_CLAUSE_CHAIN (c) = gimplify_omp_ctxp->clauses;
+ gimplify_omp_ctxp->clauses = c;
+ }
+ }
+ }
+ }
+
+ if (gimplify_omp_ctxp && gimplify_omp_ctxp->code == OMP_DISPATCH)
+ gimplify_omp_ctxp->in_call_args = true;
+ t = gimplify_arg (arg_p, pre_p, EXPR_LOCATION (*expr_p),
+ !returns_twice);
+ if (gimplify_omp_ctxp && gimplify_omp_ctxp->code == OMP_DISPATCH)
+ gimplify_omp_ctxp->in_call_args = false;
+
+ if (t == GS_ERROR)
+ ret = GS_ERROR;
+ }
+ }
}
/* Gimplify the static chain. */
@@ -6341,6 +6456,7 @@ is_gimple_stmt (tree t)
case OACC_LOOP:
case OMP_SCAN:
case OMP_SCOPE:
+ case OMP_DISPATCH:
case OMP_SECTIONS:
case OMP_SECTION:
case OMP_STRUCTURED_BLOCK:
@@ -13161,6 +13277,21 @@ gimplify_scan_omp_clauses (tree *list_p, gimple_seq *pre_p,
break;
}
}
+ else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEVICE
+ && code == OMP_DISPATCH)
+ {
+ bool saved_into_ssa = gimplify_ctxp->into_ssa;
+ gimplify_ctxp->into_ssa = false;
+ if (gimplify_expr (&OMP_CLAUSE_DEVICE_ID (c), pre_p, NULL,
+ is_gimple_val, fb_rvalue)
+ == GS_ERROR)
+ remove = true;
+ else if (DECL_P (OMP_CLAUSE_DEVICE_ID (c)))
+ omp_add_variable (ctx, OMP_CLAUSE_DEVICE_ID (c),
+ GOVD_SHARED | GOVD_SEEN);
+ gimplify_ctxp->into_ssa = saved_into_ssa;
+ break;
+ }
/* Fall through. */
case OMP_CLAUSE_PRIORITY:
@@ -13401,6 +13532,14 @@ gimplify_scan_omp_clauses (tree *list_p, gimple_seq *pre_p,
}
break;
+ case OMP_CLAUSE_NOVARIANTS:
+ OMP_CLAUSE_NOVARIANTS_EXPR (c)
+ = gimple_boolify (OMP_CLAUSE_NOVARIANTS_EXPR (c));
+ break;
+ case OMP_CLAUSE_NOCONTEXT:
+ OMP_CLAUSE_NOCONTEXT_EXPR (c)
+ = gimple_boolify (OMP_CLAUSE_NOCONTEXT_EXPR (c));
+ break;
case OMP_CLAUSE_NOHOST:
default:
gcc_unreachable ();
@@ -14728,6 +14867,54 @@ gimplify_adjust_omp_clauses (gimple_seq *pre_p, gimple_seq body, tree *list_p,
delete_omp_context (ctx);
}
+/* Try to evaluate a novariants clause. Return 1 if true, 0 if false or absent,
+ * -1 if run-time evaluation is needed. */
+
+int
+omp_has_novariants (void)
+{
+ struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
+ if (ctx != NULL && ctx->code == OMP_DISPATCH && !ctx->in_call_args)
+ {
+ tree c = omp_find_clause (ctx->clauses, OMP_CLAUSE_NOVARIANTS);
+ if (c != NULL_TREE)
+ {
+ if (integer_nonzerop (OMP_CLAUSE_NOVARIANTS_EXPR (c)))
+ return 1;
+ else if (integer_zerop (OMP_CLAUSE_NOVARIANTS_EXPR (c)))
+ return 0;
+ else
+ return -1;
+ }
+ return 0;
+ }
+ return 0;
+}
+
+/* Try to evaluate a nocontext clause. Return 1 if true, 0 if false or absent,
+ * -1 if run-time evaluation is needed. */
+
+static int
+omp_has_nocontext (void)
+{
+ struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
+ if (ctx != NULL && ctx->code == OMP_DISPATCH)
+ {
+ tree c = omp_find_clause (ctx->clauses, OMP_CLAUSE_NOCONTEXT);
+ if (c != NULL_TREE)
+ {
+ if (integer_nonzerop (OMP_CLAUSE_NOCONTEXT_EXPR (c)))
+ return 1;
+ else if (integer_zerop (OMP_CLAUSE_NOCONTEXT_EXPR (c)))
+ return 0;
+ else
+ return -1;
+ }
+ return 0;
+ }
+ return 0;
+}
+
/* Return 0 if CONSTRUCTS selectors don't match the OpenMP context,
-1 if unknown yet (simd is involved, won't be known until vectorization)
and 1 if they do. If SCORES is non-NULL, it should point to an array
@@ -14755,9 +14942,9 @@ omp_construct_selector_matches (enum tree_code *constructs, int nconstructs,
== ORT_TARGET && ctx->code == OMP_TARGET)
|| ((ctx->region_type & ORT_TEAMS) && ctx->code == OMP_TEAMS)
|| (ctx->region_type == ORT_WORKSHARE && ctx->code == OMP_FOR)
- || (ctx->region_type == ORT_SIMD
- && ctx->code == OMP_SIMD
- && !omp_find_clause (ctx->clauses, OMP_CLAUSE_BIND)))
+ || (ctx->region_type == ORT_SIMD && ctx->code == OMP_SIMD
+ && !omp_find_clause (ctx->clauses, OMP_CLAUSE_BIND))
+ || (ctx->code == OMP_DISPATCH && omp_has_nocontext () != 1))
{
++cnt;
if (scores)
@@ -17868,6 +18055,272 @@ gimplify_omp_ordered (tree expr, gimple_seq body)
return gimple_build_omp_ordered (body, OMP_ORDERED_CLAUSES (expr));
}
+/* Callback for walk_tree to find an IFN_GOMP_DISPATCH. */
+
+static tree
+find_ifn_gomp_dispatch (tree *tp, int *, void *modify)
+{
+ tree t = *tp;
+
+ if (TREE_CODE (t) == CALL_EXPR && CALL_EXPR_IFN (t) == IFN_GOMP_DISPATCH)
+ {
+ *tp = CALL_EXPR_ARG (t, 0);
+ return *(tree *) modify ? *(tree *) modify : *tp;
+ }
+
+ if (TREE_CODE (t) == MODIFY_EXPR)
+ *(tree *) modify = *tp;
+
+ return NULL_TREE;
+}
+
+/* Gimplify an OMP_DISPATCH construct. */
+
+static enum gimplify_status
+gimplify_omp_dispatch (tree *expr_p, gimple_seq *pre_p)
+{
+ tree expr = *expr_p;
+ gimple_seq body = NULL;
+
+ gimplify_scan_omp_clauses (&OMP_DISPATCH_CLAUSES (expr), pre_p, ORT_DISPATCH,
+ OMP_DISPATCH);
+ push_gimplify_context ();
+
+ // If device clause, adjust ICV
+ tree device
+ = omp_find_clause (OMP_DISPATCH_CLAUSES (expr), OMP_CLAUSE_DEVICE);
+ tree saved_device_icv = NULL_TREE;
+ if (device
+ && (TREE_CODE (OMP_CLAUSE_DEVICE_ID (device)) != INTEGER_CST
+ || !wi::eq_p (wi::to_wide (OMP_CLAUSE_DEVICE_ID (device)),
+ -1 /* omp_initial_device */)))
+ {
+ // Save current default-device-var ICV
+ saved_device_icv = create_tmp_var (integer_type_node);
+ tree fn = builtin_decl_explicit (BUILT_IN_OMP_GET_DEFAULT_DEVICE);
+ gcall *call = gimple_build_call (fn, 0);
+ gimple_call_set_lhs (call, saved_device_icv);
+ gimplify_seq_add_stmt (&body, call);
+
+ // Set default device
+ fn = builtin_decl_explicit (BUILT_IN_OMP_SET_DEFAULT_DEVICE);
+ call = gimple_build_call (fn, 1, OMP_CLAUSE_DEVICE_ID (device));
+ gimplify_seq_add_stmt (&body, call);
+ }
+
+ // If the novariants and nocontext clauses are not compile-time constants,
+ // we need to generate code for all possible cases:
+ // if (novariants) // implies nocontext
+ // base()
+ // else if (nocontext)
+ // variant1()
+ // else
+ // variant2()
+ tree *dispatch_body_p = &OMP_DISPATCH_BODY (expr);
+ if (TREE_CODE (*dispatch_body_p) == BIND_EXPR)
+ dispatch_body_p = &BIND_EXPR_BODY (*dispatch_body_p);
+ tree dispatch_body = *dispatch_body_p;
+
+ // Look for IFN_GOMP_DISPATCH and extract the base function call
+ tree base_call_expr = NULL_TREE;
+ if (TREE_CODE (dispatch_body) == STATEMENT_LIST)
+ for (tree_stmt_iterator tsi = tsi_start (dispatch_body); !tsi_end_p (tsi);
+ tsi_next (&tsi))
+ {
+ tree modify = NULL_TREE;
+ tree stmt = tsi_stmt (tsi);
+ base_call_expr
+ = walk_tree (&stmt, find_ifn_gomp_dispatch, &modify, NULL);
+ if (base_call_expr != NULL_TREE)
+ {
+ tsi_link_before (&tsi, base_call_expr, TSI_CONTINUE_LINKING);
+ tsi_next (&tsi);
+ tsi_delink (&tsi);
+ break;
+ }
+ }
+ else
+ {
+ tree modify = NULL_TREE;
+ base_call_expr
+ = walk_tree (dispatch_body_p, find_ifn_gomp_dispatch, &modify, NULL);
+ }
+ gcc_assert (base_call_expr != NULL_TREE);
+
+ tree dst = NULL_TREE;
+ if (TREE_CODE (base_call_expr) == MODIFY_EXPR)
+ {
+ dst = TREE_OPERAND (base_call_expr, 0);
+ base_call_expr = TREE_OPERAND (base_call_expr, 1);
+ }
+ while (TREE_CODE (base_call_expr) == FLOAT_EXPR
+ || TREE_CODE (base_call_expr) == CONVERT_EXPR
+ || TREE_CODE (base_call_expr) == COMPLEX_EXPR
+ || TREE_CODE (base_call_expr) == INDIRECT_REF
+ || TREE_CODE (base_call_expr) == NOP_EXPR)
+ base_call_expr = TREE_OPERAND (base_call_expr, 0);
+
+ tree base_fndecl = get_callee_fndecl (base_call_expr);
+ if (base_fndecl != NULL_TREE)
+ {
+ if (DECL_VIRTUAL_P (base_fndecl))
+ {
+ error_at (
+ EXPR_LOCATION (base_call_expr),
+ "%qD is a virtual function but only a direct call is allowed "
+ "in a dispatch construct",
+ DECL_NAME (base_fndecl));
+ }
+
+ tree variant_fndecl = omp_resolve_declare_variant (base_fndecl);
+ if (base_fndecl != variant_fndecl
+ && (omp_has_novariants () == -1 || omp_has_nocontext () == -1))
+ {
+ tree novariants_clause = NULL_TREE, nocontext_clause = NULL_TREE,
+ novariants_cond = NULL_TREE, nocontext_cond = NULL_TREE;
+ for (tree c = OMP_DISPATCH_CLAUSES (expr); c; c = TREE_CHAIN (c))
+ {
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NOVARIANTS
+ && !integer_zerop (OMP_CLAUSE_NOVARIANTS_EXPR (c)))
+ {
+ gcc_assert (novariants_cond == NULL_TREE);
+ novariants_clause = c;
+ novariants_cond = OMP_CLAUSE_NOVARIANTS_EXPR (c);
+ }
+ else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NOCONTEXT
+ && !integer_zerop (OMP_CLAUSE_NOCONTEXT_EXPR (c)))
+ {
+ gcc_assert (nocontext_cond == NULL_TREE);
+ nocontext_clause = c;
+ nocontext_cond = OMP_CLAUSE_NOCONTEXT_EXPR (c);
+ }
+ }
+ gcc_assert (novariants_cond != NULL_TREE
+ || nocontext_cond != NULL_TREE);
+
+ enum gimplify_status ret
+ = gimplify_expr (&novariants_cond, &body, NULL, is_gimple_val,
+ fb_rvalue);
+ if (ret == GS_ERROR || ret == GS_UNHANDLED)
+ return ret;
+ ret = gimplify_expr (&nocontext_cond, &body, NULL, is_gimple_val,
+ fb_rvalue);
+ if (ret == GS_ERROR || ret == GS_UNHANDLED)
+ return ret;
+
+ tree end_label = create_artificial_label (UNKNOWN_LOCATION);
+
+ if (novariants_cond != NULL_TREE)
+ {
+ tree base_label = create_artificial_label (UNKNOWN_LOCATION);
+ tree cond_label = create_artificial_label (UNKNOWN_LOCATION);
+ gcond *novariants_cond_stmt
+ = gimple_build_cond_from_tree (novariants_cond, base_label,
+ cond_label);
+ gimplify_seq_add_stmt (&body, novariants_cond_stmt);
+
+ gimplify_seq_add_stmt (&body, gimple_build_label (base_label));
+ tree base_call_expr2 = copy_node (base_call_expr);
+ if (TREE_CODE (dispatch_body) == MODIFY_EXPR)
+ {
+ base_call_expr2 = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst,
+ base_call_expr2);
+ }
+ OMP_CLAUSE_NOVARIANTS_EXPR (novariants_clause)
+ = boolean_true_node;
+ gimplify_and_add (base_call_expr2, &body);
+ gimplify_seq_add_stmt (&body, gimple_build_goto (end_label));
+
+ OMP_CLAUSE_NOVARIANTS_EXPR (novariants_clause)
+ = boolean_false_node;
+ gimplify_seq_add_stmt (&body, gimple_build_label (cond_label));
+ }
+
+ if (nocontext_cond != NULL_TREE)
+ {
+ tree variant1_label = create_artificial_label (UNKNOWN_LOCATION);
+ tree variant2_label = create_artificial_label (UNKNOWN_LOCATION);
+ gcond *nocontext_cond_stmt
+ = gimple_build_cond_from_tree (nocontext_cond, variant1_label,
+ variant2_label);
+ gimplify_seq_add_stmt (&body, nocontext_cond_stmt);
+
+ gimplify_seq_add_stmt (&body,
+ gimple_build_label (variant1_label));
+ tree variant_call_expr = copy_node (base_call_expr);
+ if (TREE_CODE (dispatch_body) == MODIFY_EXPR)
+ {
+ variant_call_expr = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst,
+ variant_call_expr);
+ }
+ OMP_CLAUSE_NOCONTEXT_EXPR (nocontext_clause) = boolean_true_node;
+ gimplify_and_add (variant_call_expr, &body);
+ gimplify_seq_add_stmt (&body, gimple_build_goto (end_label));
+ OMP_CLAUSE_NOCONTEXT_EXPR (nocontext_clause) = boolean_false_node;
+ gimplify_seq_add_stmt (&body,
+ gimple_build_label (variant2_label));
+ }
+
+ tree variant_call_expr = base_call_expr;
+ if (TREE_CODE (dispatch_body) == MODIFY_EXPR)
+ {
+ variant_call_expr
+ = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, variant_call_expr);
+ }
+ gimplify_and_add (variant_call_expr, &body);
+ gimplify_seq_add_stmt (&body, gimple_build_label (end_label));
+ }
+ else
+ gimplify_and_add (OMP_DISPATCH_BODY (expr), &body);
+ }
+ else
+ gimplify_and_add (OMP_DISPATCH_BODY (expr), &body);
+
+ // Restore default-device-var ICV
+ if (saved_device_icv != NULL_TREE)
+ {
+ tree fn = builtin_decl_explicit (BUILT_IN_OMP_SET_DEFAULT_DEVICE);
+ gcall *call = gimple_build_call (fn, 1, saved_device_icv);
+ gimplify_seq_add_stmt (&body, call);
+ }
+
+ // Wrap dispatch body into a bind
+ gimple *bind = gimple_build_bind (NULL_TREE, body, NULL_TREE);
+ pop_gimplify_context (bind);
+
+ // Manually tear down context created by gimplify_scan_omp_clauses to avoid a
+ // call to gimplify_adjust_omp_clauses
+ gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
+ if (ctx != NULL)
+ {
+ gcc_assert (ctx->code == OMP_DISPATCH);
+ gimplify_omp_ctxp = ctx->outer_context;
+ delete_omp_context (ctx);
+ }
+
+ // Remove nowait as it has no effect on dispatch (OpenMP 5.2), device as it
+ // has been handled above, and depend as the front end handled it by inserting
+ // taskwait.
+ tree *dispatch_clauses_ptr = &OMP_DISPATCH_CLAUSES (expr);
+ for (tree c = *dispatch_clauses_ptr; c; c = *dispatch_clauses_ptr)
+ {
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NOWAIT
+ || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
+ || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEVICE)
+ {
+ *dispatch_clauses_ptr = OMP_CLAUSE_CHAIN (c);
+ break;
+ }
+ else
+ dispatch_clauses_ptr = &OMP_CLAUSE_CHAIN (c);
+ }
+
+ gimple *stmt = gimple_build_omp_dispatch (bind, OMP_DISPATCH_CLAUSES (expr));
+ gimplify_seq_add_stmt (pre_p, stmt);
+ *expr_p = NULL_TREE;
+ return GS_ALL_DONE;
+}
+
/* Convert the GENERIC expression tree *EXPR_P to GIMPLE. If the
expression produces a value to be used as an operand inside a GIMPLE
statement, the value will be stored back in *EXPR_P. This value will
@@ -18806,6 +19259,10 @@ gimplify_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p,
ret = gimplify_omp_atomic (expr_p, pre_p);
break;
+ case OMP_DISPATCH:
+ ret = gimplify_omp_dispatch (expr_p, pre_p);
+ break;
+
case TRANSACTION_EXPR:
ret = gimplify_transaction (expr_p, pre_p);
break;
@@ -19131,7 +19588,8 @@ gimplify_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p,
&& code != OMP_SECTION
&& code != OMP_STRUCTURED_BLOCK
&& code != OMP_SINGLE
- && code != OMP_SCOPE);
+ && code != OMP_SCOPE
+ && code != OMP_DISPATCH);
}
#endif
diff --git gcc/gimplify.h gcc/gimplify.h
index ac3cc8eb552..2e912677022 100644
--- gcc/gimplify.h
+++ gcc/gimplify.h
@@ -77,6 +77,7 @@ extern enum gimplify_status gimplify_expr (tree *, gimple_seq *, gimple_seq *,
bool (*) (tree), fallback_t);
int omp_construct_selector_matches (enum tree_code *, int, int *);
+int omp_has_novariants (void);
extern void gimplify_type_sizes (tree, gimple_seq *);
extern void gimplify_one_sizepos (tree *, gimple_seq *);
diff --git gcc/internal-fn.cc gcc/internal-fn.cc
index d89a04fe412..1aac6f154b6 100644
--- gcc/internal-fn.cc
+++ gcc/internal-fn.cc
@@ -662,6 +662,14 @@ expand_GOMP_SIMD_ORDERED_END (internal_fn, gcall *)
gcc_unreachable ();
}
+/* This should get expanded in gimplify_omp_dispatch. */
+
+static void
+expand_GOMP_DISPATCH (internal_fn, gcall *)
+{
+ gcc_unreachable ();
+}
+
/* This should get expanded in the sanopt pass. */
static void
diff --git gcc/internal-fn.def gcc/internal-fn.def
index 23b4ab02b30..b7c08659e96 100644
--- gcc/internal-fn.def
+++ gcc/internal-fn.def
@@ -469,6 +469,7 @@ DEF_INTERNAL_FN (GOMP_SIMD_VF, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (GOMP_SIMD_LAST_LANE, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (GOMP_SIMD_ORDERED_START, ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (GOMP_SIMD_ORDERED_END, ECF_LEAF | ECF_NOTHROW, NULL)
+DEF_INTERNAL_FN (GOMP_DISPATCH, ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (LOOP_VECTORIZED, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (LOOP_DIST_ALIAS, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (ANNOTATE, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
diff --git gcc/omp-builtins.def gcc/omp-builtins.def
index 7b49ef1c0e5..c70e077063b 100644
--- gcc/omp-builtins.def
+++ gcc/omp-builtins.def
@@ -80,6 +80,12 @@ DEF_GOMP_BUILTIN (BUILT_IN_OMP_GET_TEAM_NUM, "omp_get_team_num",
BT_FN_INT, ATTR_CONST_NOTHROW_LEAF_LIST)
DEF_GOMP_BUILTIN (BUILT_IN_OMP_GET_NUM_TEAMS, "omp_get_num_teams",
BT_FN_INT, ATTR_CONST_NOTHROW_LEAF_LIST)
+DEF_GOMP_BUILTIN (BUILT_IN_OMP_GET_MAPPED_PTR, "omp_get_mapped_ptr",
+ BT_FN_PTR_CONST_PTR_INT, ATTR_NOTHROW_LEAF_LIST)
+DEF_GOMP_BUILTIN (BUILT_IN_OMP_GET_DEFAULT_DEVICE, "omp_get_default_device",
+ BT_FN_INT, ATTR_NOTHROW_LEAF_LIST)
+DEF_GOMP_BUILTIN (BUILT_IN_OMP_SET_DEFAULT_DEVICE, "omp_set_default_device",
+ BT_FN_INT, ATTR_NOTHROW_LEAF_LIST)
DEF_GOMP_BUILTIN (BUILT_IN_GOMP_ATOMIC_START, "GOMP_atomic_start",
BT_FN_VOID, ATTR_NOTHROW_LEAF_LIST)
diff --git gcc/omp-general.cc gcc/omp-general.cc
index f4c5f577047..52f93a574f4 100644
--- gcc/omp-general.cc
+++ gcc/omp-general.cc
@@ -1049,7 +1049,7 @@ omp_construct_traits_to_codes (tree ctx, int nconstructs,
/* Order must match the OMP_TRAIT_CONSTRUCT_* enumerators in
enum omp_ts_code. */
static enum tree_code code_map[]
- = { OMP_TARGET, OMP_TEAMS, OMP_PARALLEL, OMP_FOR, OMP_SIMD };
+ = { OMP_TARGET, OMP_TEAMS, OMP_PARALLEL, OMP_FOR, OMP_SIMD, OMP_DISPATCH };
for (tree ts = ctx; ts; ts = TREE_CHAIN (ts), i--)
{
@@ -1253,10 +1253,14 @@ struct omp_ts_info omp_ts_map[] =
OMP_TRAIT_PROPERTY_CLAUSE_LIST, false,
NULL
},
+ { "dispatch",
+ (1 << OMP_TRAIT_SET_CONSTRUCT),
+ OMP_TRAIT_PROPERTY_NONE, false,
+ NULL
+ },
{ NULL, 0, OMP_TRAIT_PROPERTY_NONE, false, NULL } /* OMP_TRAIT_LAST */
};
-
/* Return a name from PROP, a property in selectors accepting
name lists. */
@@ -2579,6 +2583,9 @@ omp_resolve_declare_variant (tree base)
if (cfun && (cfun->curr_properties & PROP_gimple_any) != 0)
return omp_resolve_late_declare_variant (base);
+ if (omp_has_novariants () == 1)
+ return base;
+
auto_vec <tree, 16> variants;
auto_vec <bool, 16> defer;
bool any_deferred = false;
@@ -2725,6 +2732,8 @@ omp_resolve_declare_variant (tree base)
(*slot)->variants = entry.variants;
tree alt = build_decl (DECL_SOURCE_LOCATION (base), FUNCTION_DECL,
DECL_NAME (base), TREE_TYPE (base));
+ if (DECL_ASSEMBLER_NAME_SET_P (base))
+ SET_DECL_ASSEMBLER_NAME (alt, DECL_ASSEMBLER_NAME (base));
DECL_ARTIFICIAL (alt) = 1;
DECL_IGNORED_P (alt) = 1;
TREE_STATIC (alt) = 1;
diff --git gcc/omp-low.cc gcc/omp-low.cc
index da2051b0279..54e91605775 100644
--- gcc/omp-low.cc
+++ gcc/omp-low.cc
@@ -4205,6 +4205,11 @@ scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
scan_omp (gimple_omp_body_ptr (stmt), ctx);
break;
+ case GIMPLE_OMP_DISPATCH:
+ ctx = new_omp_context (stmt, ctx);
+ scan_omp (gimple_omp_body_ptr (stmt), ctx);
+ break;
+
case GIMPLE_OMP_SECTIONS:
scan_omp_sections (as_a <gomp_sections *> (stmt), ctx);
break;
@@ -8946,6 +8951,31 @@ lower_omp_scope (gimple_stmt_iterator *gsi_p, omp_context *ctx)
if (BLOCK_VARS (block))
TREE_USED (block) = 1;
}
+
+/* Lower code for an OMP dispatch directive. */
+
+static void
+lower_omp_dispatch (gimple_stmt_iterator *gsi_p, omp_context *ctx)
+{
+ tree block;
+ gimple *stmt = gsi_stmt (*gsi_p);
+ gbind *bind;
+
+ push_gimplify_context ();
+
+ block = make_node (BLOCK);
+ bind = gimple_build_bind (NULL, NULL, block);
+ gsi_replace (gsi_p, bind, true);
+
+ lower_omp (gimple_omp_body_ptr (stmt), ctx);
+ gimple_bind_set_body (bind, maybe_catch_exception (gimple_omp_body (stmt)));
+
+ pop_gimplify_context (bind);
+
+ gimple_bind_append_vars (bind, ctx->block_vars);
+ BLOCK_VARS (block) = ctx->block_vars;
+}
+
/* Expand code for an OpenMP master or masked directive. */
static void
@@ -14577,6 +14607,11 @@ lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx)
gcc_assert (ctx);
lower_omp_scope (gsi_p, ctx);
break;
+ case GIMPLE_OMP_DISPATCH:
+ ctx = maybe_lookup_ctx (stmt);
+ gcc_assert (ctx);
+ lower_omp_dispatch (gsi_p, ctx);
+ break;
case GIMPLE_OMP_SINGLE:
ctx = maybe_lookup_ctx (stmt);
gcc_assert (ctx);
diff --git gcc/testsuite/c-c++-common/gomp/dispatch-10.c gcc/testsuite/c-c++-common/gomp/dispatch-10.c
new file mode 100644
index 00000000000..7daecd73463
--- /dev/null
+++ gcc/testsuite/c-c++-common/gomp/dispatch-10.c
@@ -0,0 +1,27 @@
+/* { dg-do compile } */
+
+int *f();
+
+struct t {
+ int *a, *b;
+};
+
+void construct(int *x, int *y);
+void noconstruct(int *x, int *y);
+
+#pragma omp declare variant(construct) match(construct={dispatch}) adjust_args(need_device_ptr: x,y)
+#pragma omp declare variant(noconstruct) match(implementation={vendor(gnu)})
+void bar(int *x, int *y);
+
+int nocontext, novariant;
+
+void sub(struct t *s, void *y)
+{
+ bar( f(), s->b);
+ #pragma omp dispatch device(0) is_device_ptr(s)
+ bar( f(), s->b);
+
+ bar( (int *) y, s->b);
+ #pragma omp dispatch device(0) is_device_ptr(y)
+ bar( (int *) y, s->b);
+}
diff --git gcc/tree-inline.cc gcc/tree-inline.cc
index 037fd1e946a..3c9591ae79c 100644
--- gcc/tree-inline.cc
+++ gcc/tree-inline.cc
@@ -1679,6 +1679,12 @@ remap_gimple_stmt (gimple *stmt, copy_body_data *id)
(s1, gimple_omp_scope_clauses (stmt));
break;
+ case GIMPLE_OMP_DISPATCH:
+ s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
+ copy = gimple_build_omp_dispatch (s1,
+ gimple_omp_dispatch_clauses (stmt));
+ break;
+
case GIMPLE_OMP_TASKGROUP:
s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
copy = gimple_build_omp_taskgroup
@@ -4609,6 +4615,7 @@ estimate_num_insns (gimple *stmt, eni_weights *weights)
case GIMPLE_OMP_MASTER:
case GIMPLE_OMP_MASKED:
case GIMPLE_OMP_SCOPE:
+ case GIMPLE_OMP_DISPATCH:
case GIMPLE_OMP_TASKGROUP:
case GIMPLE_OMP_ORDERED:
case GIMPLE_OMP_SCAN:
