================
@@ -2030,32 +2031,68 @@ Value *llvm::addDiffRuntimeChecks(
// Map to keep track of created compares, The key is the pair of operands for
// the compare, to allow detecting and re-using redundant compares.
DenseMap<std::pair<Value *, Value *>, Value *> SeenCompares;
- for (const auto &[SrcStart, SinkStart, AccessSize, NeedsFreeze] : Checks) {
+ Value *AliasLaneMask = nullptr;
+ for (const auto &[SrcStart, SinkStart, AccessSize, NeedsFreeze,
+ WriteAfterRead] : Checks) {
Type *Ty = SinkStart->getType();
- // Compute VF * IC * AccessSize.
- auto *VFTimesICTimesSize =
- ChkBuilder.CreateMul(GetVF(ChkBuilder, Ty->getScalarSizeInBits()),
- ConstantInt::get(Ty, IC * AccessSize));
- Value *Diff =
- Expander.expandCodeFor(SE.getMinusSCEV(SinkStart, SrcStart), Ty, Loc);
-
- // Check if the same compare has already been created earlier. In that
case,
- // there is no need to check it again.
- Value *IsConflict = SeenCompares.lookup({Diff, VFTimesICTimesSize});
- if (IsConflict)
- continue;
+ if (!VF.isScalar() && UseSafeEltsMask) {
+ Value *Sink = Expander.expandCodeFor(SinkStart, Ty, Loc);
+ Value *Src = Expander.expandCodeFor(SrcStart, Ty, Loc);
+ unsigned IntOpc = WriteAfterRead ? Intrinsic::loop_dependence_war_mask
+ : Intrinsic::loop_dependence_raw_mask;
+ Value *SourceAsPtr = ChkBuilder.CreateCast(Instruction::IntToPtr, Src,
+ ChkBuilder.getPtrTy());
+ Value *SinkAsPtr = ChkBuilder.CreateCast(Instruction::IntToPtr, Sink,
+ ChkBuilder.getPtrTy());
+ Value *M = ChkBuilder.CreateIntrinsic(
+ IntOpc, {VectorType::get(ChkBuilder.getInt1Ty(), VF)},
+ {SourceAsPtr, SinkAsPtr, ChkBuilder.getInt64(AccessSize)}, nullptr,
+ "alias.lane.mask");
+ if (AliasLaneMask)
+ M = ChkBuilder.CreateAnd(AliasLaneMask, M);
+ else
+ AliasLaneMask = M;
+ } else {
+ // Compute VF * IC * AccessSize.
+ auto *VFTimesICTimesSize =
+ ChkBuilder.CreateMul(GetVF(ChkBuilder, Ty->getScalarSizeInBits()),
+ ConstantInt::get(Ty, IC * AccessSize));
+ Value *Diff =
+ Expander.expandCodeFor(SE.getMinusSCEV(SinkStart, SrcStart), Ty,
Loc);
+
+ // Check if the same compare has already been created earlier. In that
+ // case, there is no need to check it again.
+ Value *IsConflict = SeenCompares.lookup({Diff, VFTimesICTimesSize});
----------------
sdesmalen-arm wrote:
Do you need to use/check `SeenCompares` in the loop_dependence_war/raw_mask
path as well?
https://github.com/llvm/llvm-project/pull/100579
_______________________________________________
llvm-branch-commits mailing list
llvm-branch-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
