llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-ir
Author: Madhur Amilkanthwar (madhur13490)
<details>
<summary>Changes</summary>
Replace the two-operand boolean form
!{!"llvm.loop.vectorize.scalable.enable", i1 0/1} with a single-operand
enable/disable pair:
!{!"llvm.loop.vectorize.scalable.enable"}
!{!"llvm.loop.vectorize.scalable.disable"}
The Verifier rejects the two-operand form, AutoUpgrade rewrites old bitcode,
and the readers and producers in LLVM, Clang and MLIR are updated.
RFC:
https://discourse.llvm.org/t/rfc-enforce-single-operand-format-for-all-enable-metadata-nodes/90571
---
Patch is 62.47 KiB, truncated to 20.00 KiB below, full version:
https://github.com/llvm/llvm-project/pull/215742.diff
82 Files Affected:
- (modified) clang/lib/CodeGen/CGLoopInfo.cpp (+4-5)
- (modified) clang/test/CodeGenCXX/pragma-loop-pr27643.cpp (+1-1)
- (modified) clang/test/CodeGenCXX/pragma-loop-predicate.cpp (+1-1)
- (modified) clang/test/CodeGenCXX/pragma-loop.cpp (+2-2)
- (modified) clang/test/CodeGenCXX/pragma-unroll.cpp (+1-1)
- (modified) llvm/docs/LangRef.md (+4-7)
- (modified) llvm/include/llvm/IR/AutoUpgrade.h (+3-1)
- (modified) llvm/include/llvm/Transforms/Utils/LoopUtils.h (+4-2)
- (modified) llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
(+8-8)
- (modified) llvm/lib/Transforms/Utils/LoopUtils.cpp (+5-3)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
(+16-15)
- (added) llvm/test/Bitcode/upgrade-loop-vectorize-scalable-enable.ll (+37)
- (added) llvm/test/Bitcode/upgrade-loop-vectorize-scalable-enable.ll.bc ()
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/bounded-load.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll (+1-1)
- (modified)
llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll (+1-1)
- (modified)
llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/masked-op-cost.ll
(+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/masked_ldst_sme.ll
(+1-1)
- (modified)
llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll (+1-1)
- (modified)
llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll
(+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/scalable-alloca.ll
(+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll (+1-1)
- (modified)
llvm/test/Transforms/LoopVectorize/AArch64/scalable-predicate-instruction.ll
(+1-1)
- (modified)
llvm/test/Transforms/LoopVectorize/AArch64/scalable-reduction-inloop-cond.ll
(+1-1)
- (modified)
llvm/test/Transforms/LoopVectorize/AArch64/scalable-reductions-tf.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/scalable-reductions.ll
(+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
(+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/scalable-vf-hint.ll
(+8-8)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sme-vectorize.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-basic-vec.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll
(+1-1)
- (modified)
llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll
(+1-1)
- (modified)
llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll (+1-1)
- (modified)
llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll
(+1-1)
- (modified)
llvm/test/Transforms/LoopVectorize/AArch64/sve-extract-last-veclane.ll (+1-1)
- (modified)
llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter-cost.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll
(+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-illegal-type.ll
(+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll
(+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-loads.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-store.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-invalid-costs.ll
(+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-large-strides.ll
(+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-masked-loadstore.ll
(+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-select-cmp.ll
(+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-strict-fadd-cost.ll
(+1-1)
- (modified)
llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll (+1-1)
- (modified)
llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-reductions.ll
(+1-1)
- (modified)
llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding.ll
(+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-type-conv.ll (+1-1)
- (modified)
llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse-mask4.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
(+1-1)
- (modified)
llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-extractvalue.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-styles.ll
(+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll
(+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll
(+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/illegal-type.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
(+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/scalable-vf-hint.ll
(+1-1)
- (modified)
llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-gather-scatter.ll (+1-1)
- (modified)
llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-uniform-store.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/X86/cost-model.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/X86/iv-live-outs.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/cast-costs.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/metadata-width.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/no_array_bounds_scalable.ll
(+1-1)
- (modified)
llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization-scalable.ll
(+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/outer_loop_hcfg_construction.ll
(+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/scalable-inductions.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/scalable-trunc-min-bitwidth.ll
(+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/scalable-vf-hint.ll (+1-1)
- (added) llvm/test/Verifier/llvm.loop.vectorize.scalable.ll (+41)
- (modified) mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp (+2-1)
- (modified) mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp (+3-2)
- (modified) mlir/test/Target/LLVMIR/Import/metadata-loop.ll (+1-1)
- (modified) mlir/test/Target/LLVMIR/loop-metadata.mlir (+1-1)
``````````diff
diff --git a/clang/lib/CodeGen/CGLoopInfo.cpp b/clang/lib/CodeGen/CGLoopInfo.cpp
index 23eff005f7004..e4e9c56fbfc77 100644
--- a/clang/lib/CodeGen/CGLoopInfo.cpp
+++ b/clang/lib/CodeGen/CGLoopInfo.cpp
@@ -245,11 +245,10 @@ clang::CodeGen::LoopInfo::createLoopVectorizeMetadata(
if (Attrs.VectorizeScalable != LoopAttributes::Unspecified) {
bool IsScalable = Attrs.VectorizeScalable == LoopAttributes::Enable;
- Metadata *Vals[] = {
- MDString::get(Ctx, "llvm.loop.vectorize.scalable.enable"),
- ConstantAsMetadata::get(
- ConstantInt::get(llvm::Type::getInt1Ty(Ctx), IsScalable))};
- Args.push_back(MDNode::get(Ctx, Vals));
+ Args.push_back(MDNode::get(
+ Ctx, {MDString::get(
+ Ctx, IsScalable ? "llvm.loop.vectorize.scalable.enable"
+ : "llvm.loop.vectorize.scalable.disable")}));
}
// Setting interleave.count
diff --git a/clang/test/CodeGenCXX/pragma-loop-pr27643.cpp
b/clang/test/CodeGenCXX/pragma-loop-pr27643.cpp
index afab64f4c6867..b0ce9c38e2637 100644
--- a/clang/test/CodeGenCXX/pragma-loop-pr27643.cpp
+++ b/clang/test/CodeGenCXX/pragma-loop-pr27643.cpp
@@ -42,7 +42,7 @@ void loop4(int *List, int Length) {
// CHECK: ![[LOOP1]] = distinct !{![[LOOP1]], [[MP:![0-9]+]],
![[VEC_WIDTH_1:.*]], ![[FIXED_WIDTH:.*]], ![[VEC_ENABLE:.*]]}
// CHECK: ![[VEC_WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1}
-// CHECK: ![[FIXED_WIDTH]] = !{!"llvm.loop.vectorize.scalable.enable", i1
false}
+// CHECK: ![[FIXED_WIDTH]] = !{!"llvm.loop.vectorize.scalable.disable"}
// CHECK: ![[VEC_ENABLE]] = !{!"llvm.loop.vectorize.enable"}
// CHECK: ![[LOOP2]] = distinct !{![[LOOP2]], [[MP]], ![[VEC_WIDTH_2:.*]],
![[FIXED_WIDTH:.*]], ![[VEC_ENABLE]]}
diff --git a/clang/test/CodeGenCXX/pragma-loop-predicate.cpp
b/clang/test/CodeGenCXX/pragma-loop-predicate.cpp
index cb7dfdb810215..bfacddc207f26 100644
--- a/clang/test/CodeGenCXX/pragma-loop-predicate.cpp
+++ b/clang/test/CodeGenCXX/pragma-loop-predicate.cpp
@@ -119,7 +119,7 @@ void test9(int *List, int Length) {
// CHECK-NEXT: ![[LOOP5]] = distinct !{![[LOOP5]], [[MP]], [[GEN6]], [[GEN10]]}
// CHECK-NEXT: ![[LOOP6]] = distinct !{![[LOOP6]], [[MP]], [[GEN8]],
[[GEN10]], [[GEN11:![0-9]+]]}
-// CHECK-NEXT: [[GEN11]] = !{!"llvm.loop.vectorize.scalable.enable", i1 false}
+// CHECK-NEXT: [[GEN11]] = !{!"llvm.loop.vectorize.scalable.disable"}
// CHECK-NEXT: ![[LOOP7]] = distinct !{![[LOOP7]], [[MP]], [[GEN8]],
[[GEN12:![0-9]+]], [[GEN11]], [[GEN3]]}
// CHECK-NEXT: [[GEN12]] = !{!"llvm.loop.vectorize.width", i32 4}
diff --git a/clang/test/CodeGenCXX/pragma-loop.cpp
b/clang/test/CodeGenCXX/pragma-loop.cpp
index 68ab7b51ea833..75961aa86dcad 100644
--- a/clang/test/CodeGenCXX/pragma-loop.cpp
+++ b/clang/test/CodeGenCXX/pragma-loop.cpp
@@ -257,8 +257,8 @@ void for_test_interleave_vectorize_disable_unroll(int
*List, int Length) {
// CHECK-DAG: ![[INTERLEAVE_16:[0-9]+]] = !{!"llvm.loop.interleave.count", i32
16}
// CHECK-DAG: ![[VECTORIZE_ENABLE:[0-9]+]] = !{!"llvm.loop.vectorize.enable"}
-// CHECK-DAG: ![[FIXED_VEC:[0-9]+]] =
!{!"llvm.loop.vectorize.scalable.enable", i1 false}
-// CHECK-DAG: ![[SCALABLE_VEC:[0-9]+]] =
!{!"llvm.loop.vectorize.scalable.enable", i1 true}
+// CHECK-DAG: ![[FIXED_VEC:[0-9]+]] =
!{!"llvm.loop.vectorize.scalable.disable"}
+// CHECK-DAG: ![[SCALABLE_VEC:[0-9]+]] =
!{!"llvm.loop.vectorize.scalable.enable"}
// CHECK-DAG: ![[WIDTH_1:[0-9]+]] = !{!"llvm.loop.vectorize.width", i32 1}
// CHECK-DAG: ![[WIDTH_2:[0-9]+]] = !{!"llvm.loop.vectorize.width", i32 2}
// CHECK-DAG: ![[WIDTH_5:[0-9]+]] = !{!"llvm.loop.vectorize.width", i32 5}
diff --git a/clang/test/CodeGenCXX/pragma-unroll.cpp
b/clang/test/CodeGenCXX/pragma-unroll.cpp
index c89b58907701e..504be03bebb82 100644
--- a/clang/test/CodeGenCXX/pragma-unroll.cpp
+++ b/clang/test/CodeGenCXX/pragma-unroll.cpp
@@ -177,7 +177,7 @@ template void for_vectorize_value_dependent<1>(int *List,
int Length);
// CHECK-DAG: ![[UNROLL_8:[0-9]+]] = !{!"llvm.loop.unroll.count", i32 8}
//
// OTHER-DAG: ![[VEC_1:[0-9]+]] = !{!"llvm.loop.vectorize.width", i32 1}
-// OTHER-DAG: ![[VEC_FIXED:[0-9]+]] =
!{!"llvm.loop.vectorize.scalable.enable", i1 false}
+// OTHER-DAG: ![[VEC_FIXED:[0-9]+]] =
!{!"llvm.loop.vectorize.scalable.disable"}
//
// CHECK-DAG: ![[LOOP_1]] = distinct !{![[LOOP_1]], ![[MP]],
![[UNROLL_ENABLE]]}
// CHECK-DAG: ![[LOOP_2]] = distinct !{![[LOOP_2]], ![[MP]],
![[UNROLL_DISABLE]]}
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index 3a616e8a29fcf..2ecd0f9adfc05 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -8088,18 +8088,15 @@ main loop. Each node has a single operand containing
the name string:
Additionally, enabling predication implicitly enables vectorization.
-#### '`llvm.loop.vectorize.scalable.enable`' Metadata
+#### '`llvm.loop.vectorize.scalable.enable`' and
'`llvm.loop.vectorize.scalable.disable`' Metadata
This metadata selectively enables or disables scalable vectorization for the
loop, and only has any effect if vectorization for the loop is already enabled.
-The first operand is the string `llvm.loop.vectorize.scalable.enable`
-and the second operand is a bit. If the bit operand value is 1 scalable
-vectorization is enabled, whereas a value of 0 reverts to the default fixed
-width vectorization:
+Each node has a single operand containing the name string:
```llvm
-!0 = !{!"llvm.loop.vectorize.scalable.enable", i1 0}
-!1 = !{!"llvm.loop.vectorize.scalable.enable", i1 1}
+!0 = !{!"llvm.loop.vectorize.scalable.enable"}
+!1 = !{!"llvm.loop.vectorize.scalable.disable"}
```
#### '`llvm.loop.vectorize.width`' Metadata
diff --git a/llvm/include/llvm/IR/AutoUpgrade.h
b/llvm/include/llvm/IR/AutoUpgrade.h
index 1f1b3c4c27815..53da20d63a7f1 100644
--- a/llvm/include/llvm/IR/AutoUpgrade.h
+++ b/llvm/include/llvm/IR/AutoUpgrade.h
@@ -121,7 +121,9 @@ namespace llvm {
{"llvm.loop.distribute.enable", "llvm.loop.distribute.disable"},
{"llvm.loop.vectorize.enable", "llvm.loop.vectorize.disable"},
{"llvm.loop.vectorize.predicate.enable",
- "llvm.loop.vectorize.predicate.disable"}};
+ "llvm.loop.vectorize.predicate.disable"},
+ {"llvm.loop.vectorize.scalable.enable",
+ "llvm.loop.vectorize.scalable.disable"}};
/// Return the replacement tags for the enable tag \p Name, or nullptr.
inline const BooleanLoopTags *findBooleanLoopTags(StringRef Name) {
diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index 0bf2d866b72bf..f7c02589e9d1a 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -239,8 +239,10 @@ LLVM_ABI SmallVector<Instruction *, 8>
findDefsUsedOutsideOfLoop(Loop *L);
/// Find a combination of metadata ("llvm.loop.vectorize.width" and
/// "llvm.loop.vectorize.scalable.enable") for a loop and use it to construct a
-/// ElementCount. If the metadata "llvm.loop.vectorize.width" cannot be found
-/// then std::nullopt is returned.
+/// ElementCount. If scalable.enable is present the count is scalable; if
+/// scalable.disable is present or the tag is absent, it is fixed-width. If the
+/// metadata "llvm.loop.vectorize.width" cannot be found then std::nullopt is
+/// returned.
LLVM_ABI std::optional<ElementCount>
getOptionalElementCountLoopAttribute(const Loop *TheLoop);
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 502fec920ece9..7b8b27c6541e1 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -58,7 +58,7 @@ class Type;
/// for example 'force', means a decision has been made. So, we need to be
/// careful NOT to add them if the user hasn't specifically asked so.
class LoopVectorizeHints {
- enum HintKind { HK_WIDTH, HK_INTERLEAVE, HK_ISVECTORIZED, HK_SCALABLE };
+ enum HintKind { HK_WIDTH, HK_INTERLEAVE, HK_ISVECTORIZED };
/// Hint - associates name and validation with the hint value.
struct Hint {
@@ -88,8 +88,9 @@ class LoopVectorizeHints {
/// Vector Predicate; one of ForceKind, carried as a plain value like Force.
unsigned Predicate;
- /// Says whether we should use fixed width or scalable vectorization.
- Hint Scalable;
+ /// Scalable vs fixed-width preference; one of ScalableForceKind. Carried as
+ /// a plain value because the enable/disable pair has no operand to validate.
+ unsigned Scalable;
/// Return the loop metadata prefix.
static StringRef Prefix() { return "llvm.loop."; }
@@ -134,9 +135,8 @@ class LoopVectorizeHints {
ElementCount getWidth() const {
return ElementCount::get(
- Width.Value,
- (ScalableForceKind)Scalable.Value == SK_PreferScalable ||
- (ScalableForceKind)Scalable.Value == SK_AlwaysScalable);
+ Width.Value, (ScalableForceKind)Scalable == SK_PreferScalable ||
+ (ScalableForceKind)Scalable == SK_AlwaysScalable);
}
unsigned getInterleave() const {
@@ -159,13 +159,13 @@ class LoopVectorizeHints {
/// \return true if scalable vectorization has been explicitly disabled.
bool isScalableVectorizationDisabled() const {
- return (ScalableForceKind)Scalable.Value == SK_FixedWidthOnly;
+ return (ScalableForceKind)Scalable == SK_FixedWidthOnly;
}
/// \return true if scalable vectorization is always preferred over
/// fixed-length when feasible, regardless of cost.
bool isScalableVectorizationAlwaysPreferred() const {
- return (ScalableForceKind)Scalable.Value == SK_AlwaysScalable;
+ return (ScalableForceKind)Scalable == SK_AlwaysScalable;
}
/// When enabling loop hints are provided we allow the vectorizer to change
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp
b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index f5dcdb4ace162..7c77acc9d748c 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -276,9 +276,11 @@ llvm::getOptionalElementCountLoopAttribute(const Loop
*TheLoop) {
getOptionalIntLoopAttribute(TheLoop, "llvm.loop.vectorize.width");
if (Width) {
- std::optional<int> IsScalable = getOptionalIntLoopAttribute(
- TheLoop, "llvm.loop.vectorize.scalable.enable");
- return ElementCount::get(*Width, IsScalable.value_or(false));
+ // Presence of the scalable.enable unit node means a scalable ElementCount;
+ // disable or absence both mean fixed-width.
+ bool IsScalable =
+ getBooleanLoopAttribute(TheLoop,
"llvm.loop.vectorize.scalable.enable");
+ return ElementCount::get(*Width, IsScalable);
}
return std::nullopt;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 14e9def164b74..23c0337900565 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -91,7 +91,6 @@ bool LoopVectorizeHints::Hint::validate(unsigned Val) {
case HK_INTERLEAVE:
return isPowerOf2_32(Val) && Val <= MaxInterleaveFactor;
case HK_ISVECTORIZED:
- case HK_SCALABLE:
return (Val == 0 || Val == 1);
}
return false;
@@ -105,9 +104,7 @@ LoopVectorizeHints::LoopVectorizeHints(const Loop *L,
VectorizerParams::VectorizationFactor.getKnownMinValue(),
HK_WIDTH),
Interleave("interleave.count", InterleaveOnlyWhenForced, HK_INTERLEAVE),
Force(FK_Undefined), IsVectorized("isvectorized", 0, HK_ISVECTORIZED),
- Predicate(FK_Undefined),
- Scalable("vectorize.scalable.enable", SK_Unspecified, HK_SCALABLE),
- TheLoop(L), ORE(ORE) {
+ Predicate(FK_Undefined), Scalable(SK_Unspecified), TheLoop(L), ORE(ORE) {
// Populate values with existing loop metadata.
getHintsFromMetadata();
@@ -121,31 +118,31 @@ LoopVectorizeHints::LoopVectorizeHints(const Loop *L,
// - Target default
// - Metadata width
// - Force option (always overrides)
- if ((LoopVectorizeHints::ScalableForceKind)Scalable.Value == SK_Unspecified)
{
+ if ((LoopVectorizeHints::ScalableForceKind)Scalable == SK_Unspecified) {
if (TTI)
- Scalable.Value = TTI->enableScalableVectorization() ? SK_PreferScalable
- : SK_FixedWidthOnly;
+ Scalable = TTI->enableScalableVectorization() ? SK_PreferScalable
+ : SK_FixedWidthOnly;
if (Width.Value)
// If the width is set, but the metadata says nothing about the scalable
// property, then assume it concerns only a fixed-width UserVF.
// If width is not set, the flag takes precedence.
- Scalable.Value = SK_FixedWidthOnly;
+ Scalable = SK_FixedWidthOnly;
}
// If the flag is set to force any use of scalable vectors, override the loop
// hints.
if (ForceScalableVectorization.getValue() !=
LoopVectorizeHints::SK_Unspecified)
- Scalable.Value = ForceScalableVectorization.getValue();
+ Scalable = ForceScalableVectorization.getValue();
// If force-vector-width is scalable, force scalable vectorization.
if (VectorizerParams::VectorizationFactor.isScalable())
- Scalable.Value = SK_AlwaysScalable;
+ Scalable = SK_AlwaysScalable;
// Scalable vectorization is disabled if no preference is specified.
- if ((LoopVectorizeHints::ScalableForceKind)Scalable.Value == SK_Unspecified)
- Scalable.Value = SK_FixedWidthOnly;
+ if ((LoopVectorizeHints::ScalableForceKind)Scalable == SK_Unspecified)
+ Scalable = SK_FixedWidthOnly;
if (IsVectorized.Value != 1)
// If the vectorization width and interleaving count are both 1 then
@@ -293,6 +290,10 @@ void LoopVectorizeHints::getHintsFromMetadata() {
Predicate = FK_Enabled;
else if (Name == "llvm.loop.vectorize.predicate.disable")
Predicate = FK_Disabled;
+ else if (Name == "llvm.loop.vectorize.scalable.enable")
+ Scalable = SK_PreferScalable;
+ else if (Name == "llvm.loop.vectorize.scalable.disable")
+ Scalable = SK_FixedWidthOnly;
continue;
}
if (Args.size() == 1)
@@ -309,9 +310,9 @@ void LoopVectorizeHints::setHint(StringRef Name, Metadata
*Arg) {
return;
unsigned Val = C->getZExtValue();
- // Force and Predicate are omitted: they are only spelled as single-operand
- // enable/disable nodes, which never reach setHint().
- Hint *Hints[] = {&Width, &Interleave, &IsVectorized, &Scalable};
+ // Force, Predicate, and Scalable are omitted: they are only spelled as
+ // single-operand enable/disable nodes, which never reach setHint().
+ Hint *Hints[] = {&Width, &Interleave, &IsVectorized};
for (auto *H : Hints) {
if (Name == H->Name) {
if (H->validate(Val))
diff --git a/llvm/test/Bitcode/upgrade-loop-vectorize-scalable-enable.ll
b/llvm/test/Bitcode/upgrade-loop-vectorize-scalable-enable.ll
new file mode 100644
index 0000000000000..0087bb414031c
--- /dev/null
+++ b/llvm/test/Bitcode/upgrade-loop-vectorize-scalable-enable.ll
@@ -0,0 +1,37 @@
+; Test that older bitcode carrying the two-operand form of
+; "llvm.loop.vectorize.scalable.enable" is auto-upgraded to the
+; single-operand enable/disable pair on load.
+;
+; RUN: llvm-dis < %s.bc | FileCheck %s
+; RUN: verify-uselistorder < %s.bc
+
+define void @enable_true() {
+entry:
+ br label %body
+body:
+ br i1 0, label %body, label %exit, !llvm.loop !0
+exit:
+ ret void
+}
+
+define void @enable_false() {
+entry:
+ br label %body
+body:
+ br i1 0, label %body, label %exit, !llvm.loop !2
+exit:
+ ret void
+}
+
+; i1 true -> single-operand enable.
+; i1 false -> disable.
+; CHECK: !{!"llvm.loop.vectorize.scalable.enable"}
+; CHECK: !{!"llvm.loop.vectorize.scalable.disable"}
+; The old two-operand nodes must be gone from the module.
+; CHECK-NOT: llvm.loop.vectorize.scalable.enable", i1
+; CHECK-NOT: llvm.loop.vectorize.scalable.disable", i1
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
+!2 = distinct !{!2, !3}
+!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 false}
diff --git a/llvm/test/Bitcode/upgrade-loop-vectorize-scalable-enable.ll.bc
b/llvm/test/Bitcode/upgrade-loop-vectorize-scalable-enable.ll.bc
new file mode 100644
index 0000000000000..1922eb1c2aec9
Binary files /dev/null and
b/llvm/test/Bitcode/upgrade-loop-vectorize-scalable-enable.ll.bc differ
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/bounded-load.ll
b/llvm/test/Transforms/LoopVectorize/AArch64/bounded-load.ll
index 7fee6fd8fdda2..f0903c0d9bf69 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/bounded-load.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/bounded-load.ll
@@ -515,7 +515,7 @@ attributes #0 = { vscale_range(1,16)
"target-features"="+sve" }
!1 = !{!"llvm.loop.vectorize.width", i32 4}
!3 = distinct !{!3, !4, !5}
!4 = !{!"llvm.loop.vectorize.width", i32 4}
-!5 = !{!"llvm.loop.interleave.count", i32 8}!2 =
!{!"llvm.loop.vectorize.scalable.enable", i1 true}
+!5 = !{!"llvm.loop.interleave.count", i32 8}!2 =
!{!"llvm.loop.vectorize.scalable.enable"}
; Loop combining a bounded (i % 4) load with a reverse-stride load.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
b/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
index 2143cf45fc1bd..ad59b558fd850 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
@@ -323,5 +323,5 @@ exit:
!0 = distinct !{!0, !1, !2, !3}
!1 = !{!"llvm.loop.vectorize.width", i32 2}
-!2 = !{!"llvm.loop.vectorize.scalable.enable", i1 false}
+!2 = !{!"llvm.loop.vectorize.scalable.disable"}
!3 = !{!"llvm.loop.vectorize.enable"}
diff --git
a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index 5822b91d54226..d51aa872aaf7f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -1340,5 +1340,5 @@ attributes #3 = { "target-cpu"="neoverse-v2" }
!0 = distinct !{!0, !1, !2, !3}
!1 = !{!"llvm.loop.vectorize.width", i32 8}
-!2 = !{!"llvm.loop.vectorize.scalable.enable", i1 false}
+!2 = !{!"llvm.loop.vectorize.scalable.disable"}
!3 = !{!"llvm.loop.vectorize.enable"}
diff --git
a/llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence.ll
b/llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence.ll
index b141e11e107e1..10ffc34389b4e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence.ll
@@ -103,4 +103,4 @@ for.end:
attributes #0 = { vscale_range(1, 16) }
!0 = distinct !{!0, !1}
-!1 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
+!1 = !{!"llvm.loop.vectorize.scalable.enable"}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-op-cost.ll
b/llvm/test/Transforms/LoopVectorize/AArch64/masked-op-cost.ll
index f49b18c80f528..9e4ff7b36e28f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-op-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-op-cost.ll
@@ -88,7 +88,7 @@ attributes #0 = { "target-features"="+neon,+sve" }
!0 = distinct !{!0, !1, !2, !3, !4}
!1 = !{!"llvm.loop.mustprogress"}
!2 = !{!"llvm.loop.vectorize.width", i32 4}
-!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
+!3 = !{!"llvm.loop.vectorize.scalable.enable"}
!4 = !{!"llvm.loop.vectorize.enable"}
!5 = distinct !{!5, !6}
-!6 = !{!"llvm.loop.vectorize.scalable.enable", i1 false}
+!6 = !{!"llvm.loop.vectorize.scalable.disable"}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked_ldst_sme.ll
b/llvm/test/Transforms/LoopVectorize/AArch64/masked_ldst_sme.ll
index 127057c58db2d..082912c13c409 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked_ldst_sme.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked_ldst_sme.ll
@@ -162,7 +162,7 @@ attributes #0 = { uwtable vscale_range(1,16)
"aarch64_pstate_sm_body" "target-fe
!0 = distinct !{!0, !1, !2, !3, !4}
!1 = !{!"llvm.loop.mustprogress"}
!2 = !{!"llvm.loop.vectorize.width", i32 16}
-!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
+!3 = !{!"...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/215742
_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
