Meinersbur created this revision.
Meinersbur added reviewers: hfinkel, aaron.ballman, hsaito, dmgreen, anemet,
rjmccall, Anastasia, pekka.jaaskelainen, meheff, tyler.nowicki.
Herald added a subscriber: zzheng.
Herald added a project: clang.
Before this patch, CGLoop would dump all transformations for a loop into a
single LoopID without encoding any order in which to apply them. rL348944
<https://reviews.llvm.org/rL348944> added the possibility to encode a
transformation order using followup-attributes.
When a loop has more than one transformation, use the follow-up attribute
define the order in which they are applied. The emitted order is the defacto
order as defined by the current LLVM pass pipeline, which is:
1. LoopFullUnrollPass
2. LoopDistributePass
3. LoopVectorizePass
4. LoopUnrollAndJamPass
5. LoopUnrollPass
6. MachinePipeliner
This patch should therefore not change the assembly output, assuming that all
explicit transformations can be applied, and no implicit transformations
in-between. In the former case, WarnMissedTransformationsPass should emit a
warning (except for MachinePipeliner which is not implemented yet). The latter
could be avoided by adding 'llvm.loop.disable_nonforced' attributes.
Because LoopUnrollAndJamPass processes a loop nest, generation of the MDNode is
delayed to after the inner loop metadata have been processed. A temporary
LoopID is therefore used to annotate instructions and RAUW'ed by the actual
LoopID later.
Repository:
rC Clang
https://reviews.llvm.org/D57978
Files:
lib/CodeGen/CGLoopInfo.cpp
lib/CodeGen/CGLoopInfo.h
test/CodeGenCXX/pragma-loop-safety-imperfectly_nested.cpp
test/CodeGenCXX/pragma-loop-safety-nested.cpp
test/CodeGenCXX/pragma-loop-safety-outer.cpp
test/CodeGenCXX/pragma-loop-safety.cpp
test/CodeGenCXX/pragma-loop.cpp
test/CodeGenCXX/pragma-unroll-and-jam.cpp
test/OpenMP/simd_metadata.c
Index: test/OpenMP/simd_metadata.c
===================================================================
--- test/OpenMP/simd_metadata.c
+++ test/OpenMP/simd_metadata.c
@@ -147,16 +147,16 @@
// CHECK: [[LOOP_H1_HEADER:![0-9]+]] = distinct !{[[LOOP_H1_HEADER]], [[LOOP_WIDTH_8:![0-9]+]], [[LOOP_VEC_ENABLE]]}
// CHECK: [[LOOP_WIDTH_8]] = !{!"llvm.loop.vectorize.width", i32 8}
// CHECK: ![[ACCESS_GROUP_7]] = distinct !{}
-// CHECK: [[LOOP_H1_HEADER:![0-9]+]] = distinct !{[[LOOP_H1_HEADER]], [[LOOP_WIDTH_8]], [[LOOP_VEC_ENABLE]], ![[PARALLEL_ACCESSES_9:[0-9]+]]}
+// CHECK: [[LOOP_H1_HEADER:![0-9]+]] = distinct !{[[LOOP_H1_HEADER]], ![[PARALLEL_ACCESSES_9:[0-9]+]], [[LOOP_WIDTH_8]], [[LOOP_VEC_ENABLE]]}
// CHECK: ![[PARALLEL_ACCESSES_9]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_7]]}
//
// Metadata for h2:
// CHECK: ![[ACCESS_GROUP_10]] = distinct !{}
-// CHECK: [[LOOP_H2_HEADER]] = distinct !{[[LOOP_H2_HEADER]], [[LOOP_VEC_ENABLE]], ![[PARALLEL_ACCESSES_12:[0-9]+]]}
+// CHECK: [[LOOP_H2_HEADER]] = distinct !{[[LOOP_H2_HEADER]], ![[PARALLEL_ACCESSES_12:[0-9]+]], [[LOOP_VEC_ENABLE]]}
// CHECK: ![[PARALLEL_ACCESSES_12]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_10]]}
//
// Metadata for h3:
// CHECK: ![[ACCESS_GROUP_13]] = distinct !{}
-// CHECK: [[LOOP_H3_HEADER]] = distinct !{[[LOOP_H3_HEADER]], [[LOOP_VEC_ENABLE]], ![[PARALLEL_ACCESSES_15:[0-9]+]]}
+// CHECK: [[LOOP_H3_HEADER]] = distinct !{[[LOOP_H3_HEADER]], ![[PARALLEL_ACCESSES_15:[0-9]+]], [[LOOP_VEC_ENABLE]]}
// CHECK: ![[PARALLEL_ACCESSES_15]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_13]]}
//
Index: test/CodeGenCXX/pragma-unroll-and-jam.cpp
===================================================================
--- test/CodeGenCXX/pragma-unroll-and-jam.cpp
+++ test/CodeGenCXX/pragma-unroll-and-jam.cpp
@@ -51,5 +51,5 @@
// CHECK: ![[UNJ_4]] = !{!"llvm.loop.unroll_and_jam.count", i32 4}
// CHECK: ![[LOOP_3]] = distinct !{![[LOOP_3]], ![[UNJ_DISABLE:.*]]}
// CHECK: ![[UNJ_DISABLE]] = !{!"llvm.loop.unroll_and_jam.disable"}
-// CHECK: ![[LOOP_7]] = distinct !{![[LOOP_7]], ![[UNROLL_4:.*]], ![[UNJ_DISABLE:.*]]}
+// CHECK: ![[LOOP_7]] = distinct !{![[LOOP_7]], ![[UNJ_DISABLE:.*]], ![[UNROLL_4:.*]]}
// CHECK: ![[UNROLL_4]] = !{!"llvm.loop.unroll.count", i32 4}
Index: test/CodeGenCXX/pragma-loop.cpp
===================================================================
--- test/CodeGenCXX/pragma-loop.cpp
+++ test/CodeGenCXX/pragma-loop.cpp
@@ -158,37 +158,60 @@
for_template_constant_expression_test<double, 2, 4, 8>(List, Length);
}
-// CHECK: ![[LOOP_1]] = distinct !{![[LOOP_1]], ![[WIDTH_4:.*]], ![[INTERLEAVE_4:.*]], ![[INTENABLE_1:.*]], ![[UNROLL_FULL:.*]], ![[DISTRIBUTE_ENABLE:.*]]}
-// CHECK: ![[WIDTH_4]] = !{!"llvm.loop.vectorize.width", i32 4}
-// CHECK: ![[INTERLEAVE_4]] = !{!"llvm.loop.interleave.count", i32 4}
-// CHECK: ![[INTENABLE_1]] = !{!"llvm.loop.vectorize.enable", i1 true}
+// CHECK: ![[LOOP_1]] = distinct !{![[LOOP_1]], ![[UNROLL_FULL:.*]]}
// CHECK: ![[UNROLL_FULL]] = !{!"llvm.loop.unroll.full"}
-// CHECK: ![[DISTRIBUTE_ENABLE]] = !{!"llvm.loop.distribute.enable", i1 true}
-// CHECK: ![[LOOP_2]] = distinct !{![[LOOP_2:.*]], ![[WIDTH_8:.*]], ![[INTERLEAVE_4:.*]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]]}
-// CHECK: ![[WIDTH_8]] = !{!"llvm.loop.vectorize.width", i32 8}
+
+// CHECK: ![[LOOP_2]] = distinct !{![[LOOP_2]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_8:.*]], ![[INTERLEAVE_4:.*]]}
// CHECK: ![[UNROLL_DISABLE]] = !{!"llvm.loop.unroll.disable"}
// CHECK: ![[DISTRIBUTE_DISABLE]] = !{!"llvm.loop.distribute.enable", i1 false}
-// CHECK: ![[LOOP_3]] = distinct !{![[LOOP_3]], ![[INTERLEAVE_4:.*]], ![[UNROLL_8:.*]], ![[INTENABLE_1:.*]]}
+// CHECK: ![[WIDTH_8]] = !{!"llvm.loop.vectorize.width", i32 8}
+// CHECK: ![[INTERLEAVE_4]] = !{!"llvm.loop.interleave.count", i32 4}
+
+// CHECK: ![[LOOP_3]] = distinct !{![[LOOP_3]], ![[INTERLEAVE_4:.*]], ![[INTENABLE_1:.*]], ![[FOLLOWUP_VECTOR_3:.*]]}
+// CHECK: ![[INTENABLE_1]] = !{!"llvm.loop.vectorize.enable", i1 true}
+// CHECK: ![[FOLLOWUP_VECTOR_3]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_3:.*]]}
+// CHECK: ![[AFTER_VECTOR_3]] = distinct !{![[AFTER_VECTOR_3]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]}
+// CHECK: ![[ISVECTORIZED]] = !{!"llvm.loop.isvectorized"}
// CHECK: ![[UNROLL_8]] = !{!"llvm.loop.unroll.count", i32 8}
+
// CHECK: ![[LOOP_4]] = distinct !{![[LOOP_4]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]]}
// CHECK: ![[WIDTH_2]] = !{!"llvm.loop.vectorize.width", i32 2}
// CHECK: ![[INTERLEAVE_2]] = !{!"llvm.loop.interleave.count", i32 2}
-// CHECK: ![[LOOP_5]] = distinct !{![[LOOP_5]], ![[WIDTH_1:.*]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]]}
+
+// CHECK: ![[LOOP_5]] = distinct !{![[LOOP_5]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_1:.*]]}
// CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1}
-// CHECK: ![[LOOP_6]] = distinct !{![[LOOP_6]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]], ![[UNROLL_8:.*]]}
+
+// CHECK: ![[LOOP_6]] = distinct !{![[LOOP_6]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]], ![[FOLLOWUP_VECTOR_6:.*]]}
+// CHECK: ![[FOLLOWUP_VECTOR_6]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_6:.*]]}
+// CHECK: ![[AFTER_VECTOR_6]] = distinct !{![[AFTER_VECTOR_6]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]}
+
// CHECK: ![[LOOP_7]] = distinct !{![[LOOP_7]], ![[WIDTH_5:.*]]}
// CHECK: ![[WIDTH_5]] = !{!"llvm.loop.vectorize.width", i32 5}
+
// CHECK: ![[LOOP_8]] = distinct !{![[LOOP_8]], ![[WIDTH_5:.*]]}
-// CHECK: ![[LOOP_9]] = distinct !{![[LOOP_9]], ![[WIDTH_8:.*]], ![[INTERLEAVE_8:.*]], ![[UNROLL_8:.*]]}
-// CHECK: ![[INTERLEAVE_8]] = !{!"llvm.loop.interleave.count", i32 8}
-// CHECK: ![[LOOP_10]] = distinct !{![[LOOP_10]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]], ![[UNROLL_8:.*]]}
-// CHECK: ![[LOOP_11]] = distinct !{![[LOOP_11]], ![[WIDTH_2:.*]], ![[INTERLEAVE_4:.*]], ![[UNROLL_8:.*]]}
-// CHECK: ![[LOOP_12]] = distinct !{![[LOOP_12]], ![[WIDTH_6:.*]], ![[INTERLEAVE_10:.*]], ![[UNROLL_24:.*]]}
-// CHECK: ![[WIDTH_6]] = !{!"llvm.loop.vectorize.width", i32 6}
-// CHECK: ![[INTERLEAVE_10]] = !{!"llvm.loop.interleave.count", i32 10}
+
+// CHECK: ![[LOOP_9]] = distinct !{![[LOOP_9]], ![[WIDTH_8:.*]], ![[INTERLEAVE_8:.*]], ![[FOLLOWUP_VECTOR_9:.*]]}
+// CHECK: ![[FOLLOWUP_VECTOR_9]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_9:.*]]}
+// CHECK: ![[AFTER_VECTOR_9]] = distinct !{![[AFTER_VECTOR_9]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]}
+
+// CHECK: ![[LOOP_10]] = distinct !{![[LOOP_10]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]], ![[FOLLOWUP_VECTOR_10:.*]]}
+// CHECK: ![[FOLLOWUP_VECTOR_10]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_10:.*]]}
+// CHECK: ![[AFTER_VECTOR_10]] = distinct !{![[AFTER_VECTOR_10]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]}
+
+// CHECK: ![[LOOP_11]] = distinct !{![[LOOP_11]], ![[WIDTH_2:.*]], ![[INTERLEAVE_4:.*]], ![[FOLLOWUP_VECTOR_11:.*]]}
+// CHECK: ![[FOLLOWUP_VECTOR_11]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_11:.*]]}
+// CHECK: ![[AFTER_VECTOR_11]] = distinct !{![[AFTER_VECTOR_11]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]}
+
+// CHECK: ![[LOOP_12]] = distinct !{![[LOOP_12]], ![[WIDTH_6:.*]], ![[INTERLEAVE_10:.*]], ![[FOLLOWUP_VECTOR_12:.*]]}
+// CHECK: ![[FOLLOWUP_VECTOR_12]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_12:.*]]}
+// CHECK: ![[AFTER_VECTOR_12]] = distinct !{![[AFTER_VECTOR_12]], ![[ISVECTORIZED:.*]], ![[UNROLL_24:.*]]}
// CHECK: ![[UNROLL_24]] = !{!"llvm.loop.unroll.count", i32 24}
-// CHECK: ![[LOOP_13]] = distinct !{![[LOOP_13]], ![[WIDTH_8:.*]], ![[INTERLEAVE_16:.*]], ![[UNROLL_32:.*]]}
+
+// CHECK: ![[LOOP_13]] = distinct !{![[LOOP_13]], ![[WIDTH_8:.*]], ![[INTERLEAVE_16:.*]], ![[FOLLOWUP_VECTOR_13:.*]]}
// CHECK: ![[INTERLEAVE_16]] = !{!"llvm.loop.interleave.count", i32 16}
+// CHECK: ![[FOLLOWUP_VECTOR_13]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_13:.*]]}
+// CHECK: ![[AFTER_VECTOR_13]] = distinct !{![[AFTER_VECTOR_13]], ![[ISVECTORIZED:.*]], ![[UNROLL_32:.*]]}
// CHECK: ![[UNROLL_32]] = !{!"llvm.loop.unroll.count", i32 32}
+
// CHECK: ![[LOOP_14]] = distinct !{![[LOOP_14]], ![[WIDTH_10:.*]]}
// CHECK: ![[WIDTH_10]] = !{!"llvm.loop.vectorize.width", i32 10}
Index: test/CodeGenCXX/pragma-loop-safety.cpp
===================================================================
--- test/CodeGenCXX/pragma-loop-safety.cpp
+++ test/CodeGenCXX/pragma-loop-safety.cpp
@@ -47,12 +47,12 @@
}
// CHECK: ![[ACCESS_GROUP_2]] = distinct !{}
-// CHECK: ![[LOOP1_HINTS]] = distinct !{![[LOOP1_HINTS]], ![[INTERLEAVE_1:[0-9]+]], ![[INTENABLE_1:[0-9]+]], ![[UNROLL_DISABLE:[0-9]+]], ![[PARALLEL_ACCESSES_7:[0-9]+]]}
+// CHECK: ![[LOOP1_HINTS]] = distinct !{![[LOOP1_HINTS]], ![[PARALLEL_ACCESSES_7:[0-9]+]], ![[UNROLL_DISABLE:[0-9]+]], ![[INTERLEAVE_1:[0-9]+]], ![[INTENABLE_1:[0-9]+]]}
+// CHECK: ![[PARALLEL_ACCESSES_7]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]}
+// CHECK: ![[UNROLL_DISABLE]] = !{!"llvm.loop.unroll.disable"}
// CHECK: ![[INTERLEAVE_1]] = !{!"llvm.loop.interleave.count", i32 1}
// CHCCK: ![[INTENABLE_1]] = !{!"llvm.loop.vectorize.enable", i1 true}
-// CHECK: ![[UNROLL_DISABLE]] = !{!"llvm.loop.unroll.disable"}
-// CHECK: ![[PARALLEL_ACCESSES_7]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]}
// CHECK: ![[ACCESS_GROUP_8]] = distinct !{}
-// CHECK: ![[LOOP2_HINTS]] = distinct !{![[LOOP2_HINTS]], ![[WIDTH_1:[0-9]+]], ![[INTENABLE_1]], ![[UNROLL_DISABLE]], ![[PARALLEL_ACCESSES_11:[0-9]+]]}
-// CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1}
+// CHECK: ![[LOOP2_HINTS]] = distinct !{![[LOOP2_HINTS]], ![[PARALLEL_ACCESSES_11:[0-9]+]], ![[UNROLL_DISABLE]], ![[WIDTH_1:[0-9]+]], ![[INTENABLE_1]]}
// CHECK: ![[PARALLEL_ACCESSES_11]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_8]]}
+// CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1}
Index: test/CodeGenCXX/pragma-loop-safety-outer.cpp
===================================================================
--- test/CodeGenCXX/pragma-loop-safety-outer.cpp
+++ test/CodeGenCXX/pragma-loop-safety-outer.cpp
@@ -18,5 +18,5 @@
// CHECK: ![[ACCESS_GROUP_2]] = distinct !{}
// CHECK: ![[INNER_LOOPID]] = distinct !{![[INNER_LOOPID]],
-// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], {{.*}} ![[PARALLEL_ACCESSES_9:[0-9]+]]}
+// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], ![[PARALLEL_ACCESSES_9:[0-9]+]]
// CHECK: ![[PARALLEL_ACCESSES_9]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]}
Index: test/CodeGenCXX/pragma-loop-safety-nested.cpp
===================================================================
--- test/CodeGenCXX/pragma-loop-safety-nested.cpp
+++ test/CodeGenCXX/pragma-loop-safety-nested.cpp
@@ -21,7 +21,7 @@
// CHECK: ![[ACCESS_GROUP_2]] = distinct !{}
// CHECK: ![[ACCESS_GROUP_LIST_3]] = !{![[ACCESS_GROUP_2]], ![[ACCESS_GROUP_4:[0-9]+]]}
// CHECK: ![[ACCESS_GROUP_4]] = distinct !{}
-// CHECK: ![[INNER_LOOPID]] = distinct !{![[INNER_LOOPID]], {{.*}} ![[PARALLEL_ACCESSES_8:[0-9]+]]}
+// CHECK: ![[INNER_LOOPID]] = distinct !{![[INNER_LOOPID]], ![[PARALLEL_ACCESSES_8:[0-9]+]]
// CHECK: ![[PARALLEL_ACCESSES_8]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_4]]}
-// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], {{.*}} ![[PARALLEL_ACCESSES_10:[0-9]+]]}
+// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], ![[PARALLEL_ACCESSES_10:[0-9]+]]
// CHECK: ![[PARALLEL_ACCESSES_10]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]}
Index: test/CodeGenCXX/pragma-loop-safety-imperfectly_nested.cpp
===================================================================
--- test/CodeGenCXX/pragma-loop-safety-imperfectly_nested.cpp
+++ test/CodeGenCXX/pragma-loop-safety-imperfectly_nested.cpp
@@ -26,7 +26,7 @@
// CHECK: ![[ACCESS_GROUP_2]] = distinct !{}
// CHECK: ![[ACCESS_GROUP_LIST_3:[0-9]+]] = !{![[ACCESS_GROUP_2]], ![[ACCESS_GROUP_4:[0-9]+]]}
// CHECK: ![[ACCESS_GROUP_4]] = distinct !{}
-// CHECK: ![[INNER_LOOPID]] = distinct !{![[INNER_LOOPID]], {{.*}} ![[PARALLEL_ACCESSES_8:[0-9]+]]}
+// CHECK: ![[INNER_LOOPID]] = distinct !{![[INNER_LOOPID]], ![[PARALLEL_ACCESSES_8:[0-9]+]]
// CHECK: ![[PARALLEL_ACCESSES_8]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_4]]}
-// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], {{.*}} ![[PARALLEL_ACCESSES_10:[0-9]+]]}
+// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], ![[PARALLEL_ACCESSES_10:[0-9]+]]
// CHECK: ![[PARALLEL_ACCESSES_10]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]}
Index: lib/CodeGen/CGLoopInfo.h
===================================================================
--- lib/CodeGen/CGLoopInfo.h
+++ lib/CodeGen/CGLoopInfo.h
@@ -78,10 +78,11 @@
public:
/// Construct a new LoopInfo for the loop with entry Header.
LoopInfo(llvm::BasicBlock *Header, const LoopAttributes &Attrs,
- const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc);
+ const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
+ LoopInfo *Parent);
/// Get the loop id metadata for this loop.
- llvm::MDNode *getLoopID() const { return LoopID; }
+ llvm::MDNode *getLoopID() const { return TempLoopID.get(); }
/// Get the header block of this loop.
llvm::BasicBlock *getHeader() const { return Header; }
@@ -92,15 +93,87 @@
/// Return this loop's access group or nullptr if it does not have one.
llvm::MDNode *getAccessGroup() const { return AccGroup; }
+ /// Create the loop's metadata. Must be called after its nested loops have
+ /// been processed.
+ void finish();
+
private:
/// Loop ID metadata.
- llvm::MDNode *LoopID;
+ llvm::TempMDTuple TempLoopID;
/// Header block of this loop.
llvm::BasicBlock *Header;
/// The attributes for this loop.
LoopAttributes Attrs;
/// The access group for memory accesses parallel to this loop.
llvm::MDNode *AccGroup = nullptr;
+ /// Start location of this loop.
+ llvm::DebugLoc StartLoc;
+ /// End location of this loop.
+ llvm::DebugLoc EndLoc;
+ /// The next outer loop, or nullptr if this is the outermost loop.
+ LoopInfo *Parent;
+ /// If this loop has unroll-and-jam metadata, this can be set by the inner
+ /// loop's LoopInfo to set the llvm.loop.unroll_and_jam.followup_inner
+ /// metadata.
+ llvm::MDNode *UnrollAndJamInnerFollowup = nullptr;
+
+ /// Create a LoopID without any transformations.
+ llvm::MDNode *
+ createLoopPropertiesMetadata(llvm::ArrayRef<llvm::Metadata *> LoopProperties);
+
+ /// Create a LoopID for transformations.
+ ///
+ /// The methods call each other in case multiple transformations are applied
+ /// to a loop. The transformation first to be applied will use LoopID of the
+ /// next transformation in its followup attribute.
+ ///
+ /// @param Attrs The loop's transfomations.
+ /// @param LoopProperties Non-transformation properties such as debug
+ /// location, parallel accesses and disabled
+ /// transformations. These are added to the returned
+ /// LoopID.
+ /// @param HasUserTransforms [out] Set to true if the returned MDNode encoodes
+ /// at least one transformation.
+ ///
+ /// @return A LoopID (metadata node) that can be used for the llvm.loop
+ /// annotation or followup-attribute.
+ /// @{
+ llvm::MDNode *
+ createPipeliningMetadata(const LoopAttributes &Attrs,
+ llvm::ArrayRef<llvm::Metadata *> LoopProperties,
+ bool &HasUserTransforms);
+ llvm::MDNode *
+ createPartialUnrollMetadata(const LoopAttributes &Attrs,
+ llvm::ArrayRef<llvm::Metadata *> LoopProperties,
+ bool &HasUserTransforms);
+ llvm::MDNode *
+ createUnrollAndJamMetadata(const LoopAttributes &Attrs,
+ llvm::ArrayRef<llvm::Metadata *> LoopProperties,
+ bool &HasUserTransforms);
+ llvm::MDNode *
+ createLoopVectorizeMetadata(const LoopAttributes &Attrs,
+ llvm::ArrayRef<llvm::Metadata *> LoopProperties,
+ bool &HasUserTransforms);
+ llvm::MDNode *
+ createLoopDistributeMetadata(const LoopAttributes &Attrs,
+ llvm::ArrayRef<llvm::Metadata *> LoopProperties,
+ bool &HasUserTransforms);
+ llvm::MDNode *
+ createFullUnrollMetadata(const LoopAttributes &Attrs,
+ llvm::ArrayRef<llvm::Metadata *> LoopProperties,
+ bool &HasUserTransforms);
+ /// @}
+
+ /// Create a LoopID for this loop.
+ ///
+ /// @param Attrs This loop's attributes and transfomations.
+ /// @param HasUserTransforms [out] Set to true if the returned MDNode encoodes
+ /// at least one transformation.
+ ///
+ /// @return A LoopID (metadata node) that can be used for the llvm.loop
+ /// annotation.
+ llvm::MDNode *createMetadata(const LoopAttributes &Attrs,
+ bool &HasUserTransforms);
};
/// A stack of loop information corresponding to loop nesting levels.
Index: lib/CodeGen/CGLoopInfo.cpp
===================================================================
--- lib/CodeGen/CGLoopInfo.cpp
+++ lib/CodeGen/CGLoopInfo.cpp
@@ -18,138 +18,394 @@
using namespace clang::CodeGen;
using namespace llvm;
-static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs,
- const llvm::DebugLoc &StartLoc,
- const llvm::DebugLoc &EndLoc, MDNode *&AccGroup) {
+MDNode *
+LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
+ LLVMContext &Ctx = Header->getContext();
+ SmallVector<Metadata *, 4> NewLoopProperties;
+ TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
+ NewLoopProperties.push_back(TempNode.get());
+ NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
+
+ MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
+ LoopID->replaceOperandWith(0, LoopID);
+ return LoopID;
+}
- if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
- Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
- Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
- Attrs.PipelineInitiationInterval == 0 &&
- Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
- Attrs.UnrollEnable == LoopAttributes::Unspecified &&
- Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
- Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
- !EndLoc)
- return nullptr;
+MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
+ ArrayRef<Metadata *> LoopProperties,
+ bool &HasUserTransforms) {
+ LLVMContext &Ctx = Header->getContext();
+
+ Optional<bool> Enabled;
+ if (Attrs.PipelineDisabled)
+ Enabled = false;
+ else if (Attrs.PipelineInitiationInterval != 0)
+ Enabled = true;
+
+ if (Enabled != true) {
+ SmallVector<Metadata *, 4> NewLoopProperties;
+ if (Enabled == false) {
+ NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
+ NewLoopProperties.push_back(
+ MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
+ ConstantAsMetadata::get(ConstantInt::get(
+ llvm::Type::getInt1Ty(Ctx), 1))}));
+ LoopProperties = NewLoopProperties;
+ }
+ return createLoopPropertiesMetadata(LoopProperties);
+ }
SmallVector<Metadata *, 4> Args;
- // Reserve operand 0 for loop id self reference.
- auto TempNode = MDNode::getTemporary(Ctx, None);
+ TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
Args.push_back(TempNode.get());
+ Args.append(LoopProperties.begin(), LoopProperties.end());
- // If we have a valid start debug location for the loop, add it.
- if (StartLoc) {
- Args.push_back(StartLoc.getAsMDNode());
-
- // If we also have a valid end debug location for the loop, add it.
- if (EndLoc)
- Args.push_back(EndLoc.getAsMDNode());
- }
-
- // Setting vectorize.width
- if (Attrs.VectorizeWidth > 0) {
- Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"),
- ConstantAsMetadata::get(ConstantInt::get(
- Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))};
+ if (Attrs.PipelineInitiationInterval > 0) {
+ Metadata *Vals[] = {
+ MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
+ ConstantAsMetadata::get(ConstantInt::get(
+ llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
Args.push_back(MDNode::get(Ctx, Vals));
}
- // Setting interleave.count
- if (Attrs.InterleaveCount > 0) {
- Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"),
- ConstantAsMetadata::get(ConstantInt::get(
- Type::getInt32Ty(Ctx), Attrs.InterleaveCount))};
- Args.push_back(MDNode::get(Ctx, Vals));
+ // No follow-up: This is the last transformation.
+
+ MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
+ LoopID->replaceOperandWith(0, LoopID);
+ HasUserTransforms = true;
+ return LoopID;
+}
+
+MDNode *
+LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
+ ArrayRef<Metadata *> LoopProperties,
+ bool &HasUserTransforms) {
+ LLVMContext &Ctx = Header->getContext();
+
+ Optional<bool> Enabled;
+ if (Attrs.UnrollEnable == LoopAttributes::Disable)
+ Enabled = false;
+ else if (Attrs.UnrollEnable == LoopAttributes::Full)
+ Enabled = None;
+ else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
+ Attrs.UnrollCount != 0)
+ Enabled = true;
+
+ if (Enabled != true) {
+ // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
+ // if unrolling is disabled.
+ return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
}
+ SmallVector<Metadata *, 4> FollowupLoopProperties;
+
+ // Apply all loop properties to the unrolled loop.
+ FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
+
+ // Don't unroll an already unrolled loop.
+ FollowupLoopProperties.push_back(
+ MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
+
+ bool FollowupHasTransforms = false;
+ MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
+ FollowupHasTransforms);
+
+ SmallVector<Metadata *, 4> Args;
+ TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
+ Args.push_back(TempNode.get());
+ Args.append(LoopProperties.begin(), LoopProperties.end());
+
// Setting unroll.count
if (Attrs.UnrollCount > 0) {
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
ConstantAsMetadata::get(ConstantInt::get(
- Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
+ llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
Args.push_back(MDNode::get(Ctx, Vals));
}
- // Setting unroll_and_jam.count
- if (Attrs.UnrollAndJamCount > 0) {
- Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
- ConstantAsMetadata::get(ConstantInt::get(
- Type::getInt32Ty(Ctx), Attrs.UnrollAndJamCount))};
+ // Setting unroll.full or unroll.disable
+ if (Attrs.UnrollEnable == LoopAttributes::Enable) {
+ Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
Args.push_back(MDNode::get(Ctx, Vals));
}
- // Setting vectorize.enable
- if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
- Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
- ConstantAsMetadata::get(ConstantInt::get(
- Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable ==
- LoopAttributes::Enable)))};
- Args.push_back(MDNode::get(Ctx, Vals));
- }
+ if (FollowupHasTransforms)
+ Args.push_back(MDNode::get(
+ Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
- // Setting unroll.full or unroll.disable
- if (Attrs.UnrollEnable != LoopAttributes::Unspecified) {
- std::string Name;
- if (Attrs.UnrollEnable == LoopAttributes::Enable)
- Name = "llvm.loop.unroll.enable";
- else if (Attrs.UnrollEnable == LoopAttributes::Full)
- Name = "llvm.loop.unroll.full";
- else
- Name = "llvm.loop.unroll.disable";
- Metadata *Vals[] = {MDString::get(Ctx, Name)};
- Args.push_back(MDNode::get(Ctx, Vals));
+ MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
+ LoopID->replaceOperandWith(0, LoopID);
+ HasUserTransforms = true;
+ return LoopID;
+}
+
+MDNode *
+LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
+ ArrayRef<Metadata *> LoopProperties,
+ bool &HasUserTransforms) {
+ LLVMContext &Ctx = Header->getContext();
+
+ Optional<bool> Enabled;
+ if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
+ Enabled = false;
+ else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
+ Attrs.UnrollAndJamCount != 0)
+ Enabled = true;
+
+ if (Enabled != true) {
+ SmallVector<Metadata *, 4> NewLoopProperties;
+ if (Enabled == false) {
+ NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
+ NewLoopProperties.push_back(MDNode::get(
+ Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
+ LoopProperties = NewLoopProperties;
+ }
+ return createPartialUnrollMetadata(Attrs, LoopProperties,
+ HasUserTransforms);
}
- // Setting unroll_and_jam.full or unroll_and_jam.disable
- if (Attrs.UnrollAndJamEnable != LoopAttributes::Unspecified) {
- std::string Name;
- if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable)
- Name = "llvm.loop.unroll_and_jam.enable";
- else if (Attrs.UnrollAndJamEnable == LoopAttributes::Full)
- Name = "llvm.loop.unroll_and_jam.full";
- else
- Name = "llvm.loop.unroll_and_jam.disable";
- Metadata *Vals[] = {MDString::get(Ctx, Name)};
+ SmallVector<Metadata *, 4> FollowupLoopProperties;
+ FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
+ FollowupLoopProperties.push_back(
+ MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
+
+ bool FollowupHasTransforms = false;
+ MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
+ FollowupHasTransforms);
+
+ SmallVector<Metadata *, 4> Args;
+ TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
+ Args.push_back(TempNode.get());
+ Args.append(LoopProperties.begin(), LoopProperties.end());
+
+ // Setting unroll_and_jam.count
+ if (Attrs.UnrollAndJamCount > 0) {
+ Metadata *Vals[] = {
+ MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
+ ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
+ Attrs.UnrollAndJamCount))};
Args.push_back(MDNode::get(Ctx, Vals));
}
- if (Attrs.DistributeEnable != LoopAttributes::Unspecified) {
- Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
- ConstantAsMetadata::get(ConstantInt::get(
- Type::getInt1Ty(Ctx), (Attrs.DistributeEnable ==
- LoopAttributes::Enable)))};
+ if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
+ Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
Args.push_back(MDNode::get(Ctx, Vals));
}
- if (Attrs.IsParallel) {
- AccGroup = MDNode::getDistinct(Ctx, {});
+ if (FollowupHasTransforms)
Args.push_back(MDNode::get(
- Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
+ Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
+ Followup}));
+
+ if (UnrollAndJamInnerFollowup)
+ Args.push_back(MDNode::get(
+ Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
+ UnrollAndJamInnerFollowup}));
+
+ MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
+ LoopID->replaceOperandWith(0, LoopID);
+ HasUserTransforms = true;
+ return LoopID;
+}
+
+MDNode *
+LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
+ ArrayRef<Metadata *> LoopProperties,
+ bool &HasUserTransforms) {
+ LLVMContext &Ctx = Header->getContext();
+
+ Optional<bool> Enabled;
+ if (Attrs.VectorizeEnable == LoopAttributes::Disable)
+ Enabled = false;
+ else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
+ Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
+ Enabled = true;
+
+ if (Enabled != true) {
+ SmallVector<Metadata *, 4> NewLoopProperties;
+ if (Enabled == false) {
+ NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
+ NewLoopProperties.push_back(
+ MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
+ ConstantAsMetadata::get(ConstantInt::get(
+ llvm::Type::getInt1Ty(Ctx), 0))}));
+ LoopProperties = NewLoopProperties;
+ }
+ return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
+ }
+
+ SmallVector<Metadata *, 4> FollowupLoopProperties;
+ FollowupLoopProperties.append(
+ LoopProperties.begin(),
+ LoopProperties
+ .end()); // Apply all loop properties to the vectorized loop.
+ FollowupLoopProperties.push_back(MDNode::get(
+ Ctx, MDString::get(
+ Ctx, "llvm.loop.isvectorized"))); // Don't vectorize an already
+ // vectorized loop.
+
+ bool FollowupHasTransforms = false;
+ MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
+ FollowupHasTransforms);
+
+ SmallVector<Metadata *, 4> Args;
+ TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
+ Args.push_back(TempNode.get());
+ Args.append(LoopProperties.begin(), LoopProperties.end());
+
+ // Setting vectorize.width
+ if (Attrs.VectorizeWidth > 0) {
+ Metadata *Vals[] = {
+ MDString::get(Ctx, "llvm.loop.vectorize.width"),
+ ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
+ Attrs.VectorizeWidth))};
+ Args.push_back(MDNode::get(Ctx, Vals));
}
- if (Attrs.PipelineDisabled) {
+ // Setting interleave.count
+ if (Attrs.InterleaveCount > 0) {
Metadata *Vals[] = {
- MDString::get(Ctx, "llvm.loop.pipeline.disable"),
- ConstantAsMetadata::get(ConstantInt::get(
- Type::getInt1Ty(Ctx), (Attrs.PipelineDisabled == true)))};
+ MDString::get(Ctx, "llvm.loop.interleave.count"),
+ ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
+ Attrs.InterleaveCount))};
Args.push_back(MDNode::get(Ctx, Vals));
}
- if (Attrs.PipelineInitiationInterval > 0) {
+ // Setting vectorize.enable
+ if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
Metadata *Vals[] = {
- MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
+ MDString::get(Ctx, "llvm.loop.vectorize.enable"),
ConstantAsMetadata::get(ConstantInt::get(
- Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
+ llvm::Type::getInt1Ty(Ctx),
+ (Attrs.VectorizeEnable == LoopAttributes::Enable)))};
Args.push_back(MDNode::get(Ctx, Vals));
}
- // Set the first operand to itself.
+ if (FollowupHasTransforms)
+ Args.push_back(MDNode::get(
+ Ctx,
+ {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
+
+ MDNode *LoopID = MDNode::get(Ctx, Args);
+ LoopID->replaceOperandWith(0, LoopID);
+ HasUserTransforms = true;
+ return LoopID;
+}
+
+MDNode *
+LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
+ ArrayRef<Metadata *> LoopProperties,
+ bool &HasUserTransforms) {
+ LLVMContext &Ctx = Header->getContext();
+
+ Optional<bool> Enabled;
+ if (Attrs.DistributeEnable == LoopAttributes::Disable)
+ Enabled = false;
+ if (Attrs.DistributeEnable == LoopAttributes::Enable)
+ Enabled = true;
+
+ if (Enabled != true) {
+ SmallVector<Metadata *, 4> NewLoopProperties;
+ if (Enabled == false) {
+ NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
+ NewLoopProperties.push_back(
+ MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
+ ConstantAsMetadata::get(ConstantInt::get(
+ llvm::Type::getInt1Ty(Ctx), 0))}));
+ LoopProperties = NewLoopProperties;
+ }
+ return createLoopVectorizeMetadata(Attrs, LoopProperties,
+ HasUserTransforms);
+ }
+
+ bool FollowupHasTransforms = false;
+ MDNode *Followup =
+ createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
+
+ SmallVector<Metadata *, 4> Args;
+ TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
+ Args.push_back(TempNode.get());
+ Args.append(LoopProperties.begin(), LoopProperties.end());
+
+ Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
+ ConstantAsMetadata::get(ConstantInt::get(
+ llvm::Type::getInt1Ty(Ctx),
+ (Attrs.DistributeEnable == LoopAttributes::Enable)))};
+ Args.push_back(MDNode::get(Ctx, Vals));
+
+ if (FollowupHasTransforms)
+ Args.push_back(MDNode::get(
+ Ctx,
+ {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
+
MDNode *LoopID = MDNode::get(Ctx, Args);
LoopID->replaceOperandWith(0, LoopID);
+ HasUserTransforms = true;
+ return LoopID;
+}
+
+MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
+ ArrayRef<Metadata *> LoopProperties,
+ bool &HasUserTransforms) {
+ LLVMContext &Ctx = Header->getContext();
+
+ Optional<bool> Enabled;
+ if (Attrs.UnrollEnable == LoopAttributes::Disable)
+ Enabled = false;
+ else if (Attrs.UnrollEnable == LoopAttributes::Full)
+ Enabled = true;
+
+ if (Enabled != true) {
+ SmallVector<Metadata *, 4> NewLoopProperties;
+ if (Enabled == false) {
+ NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
+ NewLoopProperties.push_back(
+ MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
+ LoopProperties = NewLoopProperties;
+ }
+ return createLoopDistributeMetadata(Attrs, LoopProperties,
+ HasUserTransforms);
+ }
+
+ SmallVector<Metadata *, 4> Args;
+ TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
+ Args.push_back(TempNode.get());
+ Args.append(LoopProperties.begin(), LoopProperties.end());
+ Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
+
+ // No follow-up: there is no loop after full unrolling.
+ // TODO: Warn if there are transformations after full unrolling.
+
+ MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
+ LoopID->replaceOperandWith(0, LoopID);
+ HasUserTransforms = true;
return LoopID;
}
+MDNode *LoopInfo::createMetadata(const LoopAttributes &Attrs,
+ bool &HasUserTransforms) {
+ SmallVector<Metadata *, 3> LoopProperties;
+
+ // If we have a valid start debug location for the loop, add it.
+ if (StartLoc) {
+ LoopProperties.push_back(StartLoc.getAsMDNode());
+
+ // If we also have a valid end debug location for the loop, add it.
+ if (EndLoc)
+ LoopProperties.push_back(EndLoc.getAsMDNode());
+ }
+
+ assert(!!AccGroup == Attrs.IsParallel &&
+ "There must be an access group iff the loop is parallel");
+ if (Attrs.IsParallel) {
+ LLVMContext &Ctx = Header->getContext();
+ LoopProperties.push_back(MDNode::get(
+ Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
+ }
+
+ return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
+}
+
LoopAttributes::LoopAttributes(bool IsParallel)
: IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
UnrollEnable(LoopAttributes::Unspecified),
@@ -173,15 +429,104 @@
}
LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
- const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
- : LoopID(nullptr), Header(Header), Attrs(Attrs) {
- LoopID =
- createMetadata(Header->getContext(), Attrs, StartLoc, EndLoc, AccGroup);
+ const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
+ LoopInfo *Parent)
+ : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
+ Parent(Parent) {
+
+ if (Attrs.IsParallel) {
+ // Create an access group for this loop.
+ LLVMContext &Ctx = Header->getContext();
+ AccGroup = MDNode::getDistinct(Ctx, {});
+ }
+
+ if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
+ Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
+ Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
+ Attrs.PipelineInitiationInterval == 0 &&
+ Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
+ Attrs.UnrollEnable == LoopAttributes::Unspecified &&
+ Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
+ Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
+ !EndLoc)
+ return;
+
+ TempLoopID = MDNode::getTemporary(Header->getContext(), None);
+}
+
+void LoopInfo::finish() {
+ // We did not annote the loop body instructions because there are no
+ // attributes for this loop.
+ if (!TempLoopID)
+ return;
+
+ MDNode *LoopID;
+ LoopAttributes CurLoopAttr = Attrs;
+
+ if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
+ Parent->Attrs.UnrollAndJamCount != 0)) {
+ // Parent unroll-and-jams this loop.
+ // Split the transformations in those that happens before the unroll-and-jam
+ // and those after.
+
+ LoopAttributes BeforeJam, AfterJam;
+
+ BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
+
+ BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
+ BeforeJam.InterleaveCount = Attrs.InterleaveCount;
+ BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
+ BeforeJam.DistributeEnable = Attrs.DistributeEnable;
+
+ switch (Attrs.UnrollEnable) {
+ case LoopAttributes::Unspecified:
+ case LoopAttributes::Disable:
+ BeforeJam.UnrollEnable = Attrs.UnrollEnable;
+ AfterJam.UnrollEnable = Attrs.UnrollEnable;
+ break;
+ case LoopAttributes::Full:
+ BeforeJam.UnrollEnable = LoopAttributes::Full;
+ break;
+ case LoopAttributes::Enable:
+ AfterJam.UnrollEnable = LoopAttributes::Enable;
+ break;
+ }
+
+ AfterJam.UnrollCount = Attrs.UnrollCount;
+ AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
+ AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
+
+ // Unroll-and-jam of an inner loop and unroll-and-jam of the same loop as
+ // the outer loop does not make much sense, but we have to pick an order.
+ AfterJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
+ AfterJam.UnrollAndJamEnable = AfterJam.UnrollAndJamEnable;
+
+ // Set the inner followup metadata to process by the outer loop. Only
+ // consider the first inner loop.
+ if (!UnrollAndJamInnerFollowup) {
+ bool InnerFollowupHasTransform = false;
+ MDNode *InnerFollowup =
+ createMetadata(AfterJam, InnerFollowupHasTransform);
+ if (InnerFollowupHasTransform)
+ UnrollAndJamInnerFollowup = InnerFollowup;
+ }
+
+ CurLoopAttr = BeforeJam;
+ }
+
+ bool HasUserTransforms = false;
+ LoopID = createMetadata(CurLoopAttr, HasUserTransforms);
+
+ assert(LoopID->getNumOperands() >= 2 &&
+ "Should not have created an MDNode if there is no metadata to attach "
+ "to this loop");
+ TempLoopID->replaceAllUsesWith(LoopID);
}
void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
const llvm::DebugLoc &EndLoc) {
- Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc));
+ Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
+ Active.empty() ? nullptr : &Active.back()));
// Clear the attributes so nested loops do not inherit them.
StagedAttrs.clear();
}
@@ -364,6 +709,7 @@
void LoopInfoStack::pop() {
assert(!Active.empty() && "No active loops to pop");
+ Active.back().finish();
Active.pop_back();
}
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
