https://github.com/NewSigma updated https://github.com/llvm/llvm-project/pull/218309
>From ad992a806d847343379fa20fa7a0c16d691b9df3 Mon Sep 17 00:00:00 2001 From: NewSigma <[email protected]> Date: Fri, 21 Aug 2026 20:45:42 +0800 Subject: [PATCH 1/2] [Coroutines] Directly use live SSA values instead of reloading from frame --- .../coro-suspend-cleanups.cpp | 5 +- llvm/lib/Transforms/Coroutines/CoroFrame.cpp | 65 ++++++++++---- llvm/test/Transforms/Coroutines/coro-async.ll | 8 +- .../Coroutines/coro-catchswitch-cleanuppad.ll | 12 +-- .../Transforms/Coroutines/coro-catchswitch.ll | 3 +- llvm/test/Transforms/Coroutines/coro-frame.ll | 4 +- .../Coroutines/coro-retcon-once-value.ll | 8 +- .../Coroutines/coro-retcon-once-value2.ll | 8 +- .../Coroutines/coro-retcon-resume-values.ll | 10 +-- .../Coroutines/coro-retcon-resume-values2.ll | 10 +-- .../test/Transforms/Coroutines/coro-retcon.ll | 10 +-- .../Transforms/Coroutines/coro-spill-ramp.ll | 84 +++++++++++++++++++ 12 files changed, 172 insertions(+), 55 deletions(-) create mode 100644 llvm/test/Transforms/Coroutines/coro-spill-ramp.ll diff --git a/clang/test/CodeGenCoroutines/coro-suspend-cleanups.cpp b/clang/test/CodeGenCoroutines/coro-suspend-cleanups.cpp index 6c05d02d0642f..fa5d74e743b3f 100644 --- a/clang/test/CodeGenCoroutines/coro-suspend-cleanups.cpp +++ b/clang/test/CodeGenCoroutines/coro-suspend-cleanups.cpp @@ -59,8 +59,9 @@ coroutine ArrayInitCoro() { // CHECK: br label %cleanup{{.*}} // CHECK: await.ready: - // CHECK-NEXT: %arrayinit.element.reload.addr = getelementptr inbounds i8, ptr %0, i64 48 - // CHECK-NEXT: %arrayinit.element.reload = load ptr, ptr %arrayinit.element.reload.addr, align 8 + // CHECK-NEXT: br label %await.ready.after.spill + + // CHECK: await.ready.after.spill: // CHECK-NEXT: call void @_ZN7Awaiter12await_resumeEv // CHECK-NEXT: store i1 false, ptr %cleanup.isactive.reload.addr, align 1 // CHECK-NEXT: br label %cleanup{{.*}}.from.await.ready diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp index c59e65f270a23..2b5fec3ffaad3 100644 --- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp @@ -1086,20 +1086,48 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) { Builder.SetInsertPoint(coro::getSpillInsertionPt(Shape, Def, DT)); createStoreIntoFrame(Builder, Def, ByValTy, Shape, FrameData); - - BasicBlock *CurrentBlock = nullptr; + // Before insertSpills(): + // before.spill: + // ; use %def + // + // After insertSpills(): + // before.spill: + // (phis) + // %InRamp = call i1 @llvm.coro.is_in_ramp() + // br i1 %InRamp, label %after.spill, label %ssa.spill + // + // ssa.spill: + // ; gep and load from frame + // br label %after.spill + // + // after.spill: + // %MaybeReload = phi ptr [%def, %before.spill], [%reload, %ssa.spill] + // ; use %MaybeReload + BasicBlock *BeforeSpillBB = nullptr; + BasicBlock *SpillBB = nullptr; + BasicBlock *AfterSpillBB = nullptr; Value *CurrentReload = nullptr; for (auto *U : E.second) { // If we have not seen the use block, create a load instruction to reload // the spilled value from the coroutine frame. Populates the Value pointer // reference provided with the frame GEP. - if (CurrentBlock != U->getParent()) { - CurrentBlock = U->getParent(); - Builder.SetInsertPoint(CurrentBlock, - CurrentBlock->getFirstInsertionPt()); - - auto *GEP = createGEPToFramePointer(FrameData, Builder, Shape, E.first); - GEP->setName(E.first->getName() + Twine(".reload.addr")); + if (BeforeSpillBB != U->getParent()) { + BeforeSpillBB = U->getParent(); + AfterSpillBB = BeforeSpillBB->splitBasicBlock( + BeforeSpillBB->getFirstInsertionPt(), + BeforeSpillBB->getName() + Twine(".after.spill")); + SpillBB = BasicBlock::Create( + C, BeforeSpillBB->getName() + Twine(".spill"), F, AfterSpillBB); + + BeforeSpillBB->getTerminator()->eraseFromParent(); + Builder.SetInsertPoint(BeforeSpillBB); + auto *InRamp = Builder.CreateIntrinsic(Intrinsic::coro_is_in_ramp, {}); + Builder.CreateCondBr(InRamp, AfterSpillBB, SpillBB); + Shape.CoroIsInRampInsts.push_back(cast<CoroIsInRampInst>(InRamp)); + + Builder.SetInsertPoint(SpillBB); + auto *GEP = createGEPToFramePointer(FrameData, Builder, Shape, Def); + GEP->setName(Def->getName() + Twine(".reload.addr")); if (ByValTy) { CurrentReload = GEP; } else { @@ -1111,6 +1139,7 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) { LI->setMetadata(LLVMContext::MD_tbaa, TBAATag); CurrentReload = LI; } + Builder.CreateBr(AfterSpillBB); TinyPtrVector<DbgVariableRecord *> DVRs = findDbgRecordsThroughLoads< DbgVariableRecord::LocationType::Declare>(*F, Def); @@ -1123,8 +1152,8 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) { ValueAsMetadata::get(CurrentReload), DDI->getVariable(), DDI->getExpression(), DDI->getDebugLoc(), DbgVariableRecord::LocationType::Declare); - Builder.GetInsertPoint()->getParent()->insertDbgRecordBefore( - NewDVR, Builder.GetInsertPoint()); + BeforeSpillBB->insertDbgRecordBefore( + NewDVR, BeforeSpillBB->getFirstInsertionPt()); // This dbg.declare is for the main function entry point. It // will be deleted in all coro-split functions. coro::salvageDebugInfo(ArgToAllocaMap, *DDI, false /*UseEntryValue*/); @@ -1150,14 +1179,20 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) { DDI->getExpression(), DDI->getDebugLoc(), Ty->isPointerTy() ? DbgVariableRecord::LocationType::Declare : DbgVariableRecord::LocationType::Value); - Builder.GetInsertPoint()->getParent()->insertDbgRecordBefore( - NewDVR, Builder.GetInsertPoint()); + BeforeSpillBB->insertDbgRecordBefore( + NewDVR, BeforeSpillBB->getFirstInsertionPt()); // This dbg.declare_value is for the main function entry point. It // will be deleted in all coro-split functions. coro::salvageDebugInfo(ArgToAllocaMap, *DDI, false /*UseEntryValue*/); }; for_each(DVRDeclareValues, SalvageOneCoro); + Builder.SetInsertPoint(AfterSpillBB->getFirstInsertionPt()); + // No need to reload if the original SSA value is available + auto *MaybeReload = Builder.CreatePHI(Def->getType(), 2); + MaybeReload->addIncoming(CurrentReload, SpillBB); + MaybeReload->addIncoming(Def, BeforeSpillBB); + // If we have a single edge PHINode, remove it and replace it with a // reload from the coroutine frame. (We already took care of multi edge // PHINodes by normalizing them in the rewritePHIs function). @@ -1165,14 +1200,14 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) { assert(PN->getNumIncomingValues() == 1 && "unexpected number of incoming " "values in the PHINode"); - PN->replaceAllUsesWith(CurrentReload); + PN->replaceAllUsesWith(MaybeReload); PN->eraseFromParent(); continue; } // Replace all uses of CurrentValue in the current instruction with // reload. - U->replaceUsesOfWith(Def, CurrentReload); + U->replaceUsesOfWith(Def, MaybeReload); // Instructions are added to Def's user list if the attached // debug records use Def. Update those now. for (DbgVariableRecord &DVR : filterDbgVars(U->getDbgRecordRange())) diff --git a/llvm/test/Transforms/Coroutines/coro-async.ll b/llvm/test/Transforms/Coroutines/coro-async.ll index 3454737820b7a..fd433a225143f 100644 --- a/llvm/test/Transforms/Coroutines/coro-async.ll +++ b/llvm/test/Transforms/Coroutines/coro-async.ll @@ -151,12 +151,12 @@ define void @my_async_function_pa(ptr %ctxt, ptr %task, ptr %actor) { ; CHECK: [[FRAME_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[CALLER_CONTEXT]], i64 128 ; CHECK-O0: [[VECTOR_SPILL_ADDR:%.*]] = getelementptr inbounds i8, ptr {{.*}}, i64 32 ; CHECK-O0: load <4 x double>, ptr [[VECTOR_SPILL_ADDR]], align 16 -; CHECK: [[CALLEE_CTXT_SPILL_ADDR:%.*]] = getelementptr inbounds nuw i8, ptr [[CALLER_CONTEXT]], i64 160 -; CHECK: [[CALLEE_CTXT_RELOAD:%.*]] = load ptr, ptr [[CALLEE_CTXT_SPILL_ADDR]] -; CHECK: [[ACTOR_RELOAD_ADDR:%.*]] = getelementptr inbounds nuw i8, ptr [[CALLER_CONTEXT]], i64 152 -; CHECK: [[ACTOR_RELOAD:%.*]] = load ptr, ptr [[ACTOR_RELOAD_ADDR]] ; CHECK: [[ADDR1:%.*]] = getelementptr inbounds nuw i8, ptr [[CALLER_CONTEXT]], i64 144 ; CHECK: [[ASYNC_CTXT_RELOAD:%.*]] = load ptr, ptr [[ADDR1]] +; CHECK: [[ACTOR_RELOAD_ADDR:%.*]] = getelementptr inbounds nuw i8, ptr [[CALLER_CONTEXT]], i64 152 +; CHECK: [[ACTOR_RELOAD:%.*]] = load ptr, ptr [[ACTOR_RELOAD_ADDR]] +; CHECK: [[CALLEE_CTXT_SPILL_ADDR:%.*]] = getelementptr inbounds nuw i8, ptr [[CALLER_CONTEXT]], i64 160 +; CHECK: [[CALLEE_CTXT_RELOAD:%.*]] = load ptr, ptr [[CALLEE_CTXT_SPILL_ADDR]] ; CHECK: [[ALLOCA_PRJ2:%.*]] = getelementptr inbounds nuw i8, ptr [[CALLER_CONTEXT]], i64 136 ; CHECK: tail call void @llvm.coro.async.context.dealloc(ptr nonnull [[CALLEE_CTXT_RELOAD]]) ; CHECK: [[VAL1:%.*]] = load i64, ptr [[FRAME_PTR]] diff --git a/llvm/test/Transforms/Coroutines/coro-catchswitch-cleanuppad.ll b/llvm/test/Transforms/Coroutines/coro-catchswitch-cleanuppad.ll index 945b364cbad70..c7890620fced7 100644 --- a/llvm/test/Transforms/Coroutines/coro-catchswitch-cleanuppad.ll +++ b/llvm/test/Transforms/Coroutines/coro-catchswitch-cleanuppad.ll @@ -81,18 +81,18 @@ cleanup2: ; CHECK: %1 = phi i8 [ 0, %handler2 ], [ 1, %catch.dispatch.2 ] ; CHECK: %2 = cleanuppad within %h1 [] ; CHECK: %3 = icmp eq i8 %1, 0 -; CHECK: br i1 %3, label %cleanup2.from.handler2, label %cleanup2.from.catch.dispatch.2, !prof [[PROF1:![0-9]+]] +; CHECK: br i1 %3, label %[[FROM_HANDLER:.+]], label %[[FROM_DISPATCH:.+]], !prof [[PROF1:![0-9]+]] -; CHECK: cleanup2.from.handler2: -; CHECK: %valueB.reload = load i32, ptr %valueB.spill.addr, align 4 +; CHECK: [[FROM_HANDLER]]: +; CHECK: %valueB.reload = load i32, ptr %valueB.reload.addr, align 4 ; CHECK: br label %cleanup2 -; CHECK: cleanup2.from.catch.dispatch.2: -; CHECK: %valueA.reload = load i32, ptr %valueA.spill.addr, align 4 +; CHECK: [[FROM_DISPATCH]]: +; CHECK: %valueA.reload = load i32, ptr %valueA.reload.addr, align 4 ; CHECK: br label %cleanup2 ; CHECK: cleanup2: -; CHECK: %cleanupval2 = phi i32 [ %valueA.reload, %cleanup2.from.catch.dispatch.2 ], [ %valueB.reload, %cleanup2.from.handler2 ] +; CHECK: %cleanupval2 = phi i32 [ %valueA.reload, %[[FROM_DISPATCH]] ], [ %valueB.reload, %[[FROM_HANDLER]] ] ; CHECK: call void @print(i32 %cleanupval2) ; CHECK: br label %cleanup } diff --git a/llvm/test/Transforms/Coroutines/coro-catchswitch.ll b/llvm/test/Transforms/Coroutines/coro-catchswitch.ll index 776d2bfac2339..f09bb609e09da 100644 --- a/llvm/test/Transforms/Coroutines/coro-catchswitch.ll +++ b/llvm/test/Transforms/Coroutines/coro-catchswitch.ll @@ -38,8 +38,7 @@ define void @f(i1 %cond) presplitcoroutine personality i32 0 { ; CHECK-NEXT: store i1 false, ptr [[INDEX_ADDR3]], align 1 ; CHECK-NEXT: br i1 false, label %[[RESUME:.*]], label %[[AFTERCOROEND]] ; CHECK: [[RESUME]]: -; CHECK-NEXT: [[VAL_RELOAD:%.*]] = load i32, ptr [[VAL_SPILL_ADDR]], align 4 -; CHECK-NEXT: call void @print(i32 [[VAL_RELOAD]]) +; CHECK-NEXT: call void @print(i32 [[VAL]]) ; CHECK-NEXT: br label %[[AFTERCOROEND]] ; CHECK: [[AFTERCOROEND]]: ; CHECK-NEXT: ret void diff --git a/llvm/test/Transforms/Coroutines/coro-frame.ll b/llvm/test/Transforms/Coroutines/coro-frame.ll index 2af186e405d54..c7f6c4774de2e 100644 --- a/llvm/test/Transforms/Coroutines/coro-frame.ll +++ b/llvm/test/Transforms/Coroutines/coro-frame.ll @@ -75,10 +75,10 @@ declare void @free(ptr) ; CHECK-LABEL: define internal void @f.resume( ; CHECK-SAME: ptr noundef nonnull align 8 dereferenceable(40) [[HDL:%.*]]) personality i32 0 { ; CHECK-NEXT: [[ENTRY_RESUME:.*:]] -; CHECK-NEXT: [[R_RELOAD_ADDR:%.*]] = getelementptr inbounds i8, ptr [[HDL]], i64 16 -; CHECK-NEXT: [[R_RELOAD:%.*]] = load double, ptr [[R_RELOAD_ADDR]], align 8 ; CHECK-NEXT: [[THIS1_RELOAD_ADDR:%.*]] = getelementptr inbounds i8, ptr [[HDL]], i64 24 ; CHECK-NEXT: [[THIS1_RELOAD:%.*]] = load i64, ptr [[THIS1_RELOAD_ADDR]], align 4 +; CHECK-NEXT: [[R_RELOAD_ADDR:%.*]] = getelementptr inbounds i8, ptr [[HDL]], i64 16 +; CHECK-NEXT: [[R_RELOAD:%.*]] = load double, ptr [[R_RELOAD_ADDR]], align 8 ; CHECK-NEXT: [[TMP0:%.*]] = call double @print(double [[R_RELOAD]]) ; CHECK-NEXT: call void @print2(i64 [[THIS1_RELOAD]]) ; CHECK-NEXT: [[MEM:%.*]] = call ptr @llvm.coro.free(token poison, ptr [[HDL]]) diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-once-value.ll b/llvm/test/Transforms/Coroutines/coro-retcon-once-value.ll index 74a3f8d449d0c..e83d8f0e13690 100644 --- a/llvm/test/Transforms/Coroutines/coro-retcon-once-value.ll +++ b/llvm/test/Transforms/Coroutines/coro-retcon-once-value.ll @@ -123,7 +123,7 @@ declare void @print(i32) ; CHECK-LABEL: @f.resume.0( ; CHECK-NEXT: entryresume.0: ; CHECK-NEXT: br i1 [[TMP1:%.*]], label [[COROEND:%.*]], label [[CLEANUP_SINK_SPLIT:%.*]] -; CHECK: cleanup.sink.split: +; CHECK: cleanup.sink.split.after.spill: ; CHECK-NEXT: [[ARRAY_RELOAD:%.*]] = load ptr, ptr [[TMP0:%.*]], align 8 ; CHECK-NEXT: store i32 0, ptr [[ARRAY_RELOAD]], align 4 ; CHECK-NEXT: br label [[COROEND]] @@ -134,7 +134,7 @@ declare void @print(i32) ; CHECK-LABEL: @f.resume.1( ; CHECK-NEXT: entryresume.1: ; CHECK-NEXT: br i1 [[TMP1:%.*]], label [[COROEND:%.*]], label [[CLEANUP_SINK_SPLIT:%.*]] -; CHECK: cleanup.sink.split: +; CHECK: cleanup.sink.split.after.spill: ; CHECK-NEXT: [[ARRAY_RELOAD:%.*]] = load ptr, ptr [[TMP0:%.*]], align 8 ; CHECK-NEXT: store i32 10, ptr [[ARRAY_RELOAD]], align 4 ; CHECK-NEXT: br label [[COROEND]] @@ -175,7 +175,7 @@ declare void @print(i32) ; CHECK-NEXT: entryresume.0: ; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP0:%.*]], align 8 ; CHECK-NEXT: br i1 [[TMP1:%.*]], label [[COROEND:%.*]], label [[CLEANUP_SINK_SPLIT:%.*]] -; CHECK: cleanup.sink.split: +; CHECK: cleanup.sink.split.after.spill: ; CHECK-NEXT: [[ARRAY_RELOAD:%.*]] = load ptr, ptr [[TMP2]], align 8 ; CHECK-NEXT: store i32 0, ptr [[ARRAY_RELOAD]], align 4 ; CHECK-NEXT: br label [[COROEND]] @@ -193,7 +193,7 @@ declare void @print(i32) ; CHECK-NEXT: entryresume.1: ; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP0:%.*]], align 8 ; CHECK-NEXT: br i1 [[TMP1:%.*]], label [[COROEND:%.*]], label [[CLEANUP_SINK_SPLIT:%.*]] -; CHECK: cleanup.sink.split: +; CHECK: cleanup.sink.split.after.spill: ; CHECK-NEXT: [[ARRAY_RELOAD:%.*]] = load ptr, ptr [[TMP2]], align 8 ; CHECK-NEXT: store i32 10, ptr [[ARRAY_RELOAD]], align 4 ; CHECK-NEXT: br label [[COROEND]] diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-once-value2.ll b/llvm/test/Transforms/Coroutines/coro-retcon-once-value2.ll index bf95b2a74e6de..b9f271bbb5635 100644 --- a/llvm/test/Transforms/Coroutines/coro-retcon-once-value2.ll +++ b/llvm/test/Transforms/Coroutines/coro-retcon-once-value2.ll @@ -99,7 +99,7 @@ declare void @print(i32) ; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP0:%.*]], align 8 ; CHECK-NEXT: [[TEMP:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 8 ; CHECK-NEXT: br i1 [[TMP1:%.*]], label [[COROEND:%.*]], label [[CONT:%.*]] -; CHECK: cont: +; CHECK: cont.spill: ; CHECK-NEXT: [[PTR_RELOAD:%.*]] = load ptr, ptr [[TMP2]], align 8 ; CHECK-NEXT: [[NEWVALUE:%.*]] = load i32, ptr [[TEMP]], align 4 ; CHECK-NEXT: store i32 [[NEWVALUE]], ptr [[PTR_RELOAD]], align 4 @@ -128,12 +128,12 @@ declare void @print(i32) ; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP0:%.*]], align 8 ; CHECK-NEXT: [[TEMP:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 8 ; CHECK-NEXT: br i1 [[TMP1:%.*]], label [[CLEANUP:%.*]], label [[CONT:%.*]] -; CHECK: cont: +; CHECK: cont.spill: ; CHECK-NEXT: [[PTR_RELOAD:%.*]] = load ptr, ptr [[TMP2]], align 8 ; CHECK-NEXT: [[NEWVALUE:%.*]] = load i32, ptr [[TEMP]], align 4 ; CHECK-NEXT: store i32 [[NEWVALUE]], ptr [[PTR_RELOAD]], align 4 ; CHECK-NEXT: br label [[CLEANUP]] -; CHECK: cleanup: +; CHECK: cleanup.spill: ; CHECK-NEXT: [[VAL_RELOAD_ADDR:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 12 ; CHECK-NEXT: [[VAL_RELOAD:%.*]] = load i8, ptr [[VAL_RELOAD_ADDR]], align 1 ; CHECK-NEXT: call fastcc void @deallocate(ptr [[TMP2]]) @@ -157,7 +157,7 @@ declare void @print(i32) ; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP0:%.*]], align 8 ; CHECK-NEXT: [[TEMP:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 8 ; CHECK-NEXT: br i1 [[TMP1:%.*]], label [[COROEND:%.*]], label [[CONT:%.*]] -; CHECK: cont: +; CHECK: cont.spill: ; CHECK-NEXT: [[PTR_RELOAD:%.*]] = load ptr, ptr [[TMP2]], align 8 ; CHECK-NEXT: [[NEWVALUE:%.*]] = load i32, ptr [[TEMP]], align 4 ; CHECK-NEXT: store i32 [[NEWVALUE]], ptr [[PTR_RELOAD]], align 4 diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-resume-values.ll b/llvm/test/Transforms/Coroutines/coro-retcon-resume-values.ll index 2f04453d69c4b..489e9799a119f 100644 --- a/llvm/test/Transforms/Coroutines/coro-retcon-resume-values.ll +++ b/llvm/test/Transforms/Coroutines/coro-retcon-resume-values.ll @@ -40,14 +40,14 @@ define i32 @main() { ; CHECK-NEXT: store i32 1, ptr [[TMP0]], align 4 ; CHECK-NEXT: [[N_VAL3_SPILL_ADDR_I:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 4 ; CHECK-NEXT: store i32 1, ptr [[N_VAL3_SPILL_ADDR_I]], align 4, !noalias [[META0:![0-9]+]] -; CHECK-NEXT: [[INPUT_SPILL_ADDR_I:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8 -; CHECK-NEXT: store i32 2, ptr [[INPUT_SPILL_ADDR_I]], align 4, !noalias [[META0]] ; CHECK-NEXT: [[INPUT_RELOAD_ADDR13_I:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8 +; CHECK-NEXT: store i32 2, ptr [[INPUT_RELOAD_ADDR13_I]], align 4, !noalias [[META0]] ; CHECK-NEXT: [[N_VAL3_RELOAD_ADDR11_I:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 4 +; CHECK-NEXT: [[INPUT_RELOAD_ADDR14_I:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8 ; CHECK-NEXT: store i32 3, ptr [[N_VAL3_RELOAD_ADDR11_I]], align 4, !noalias [[META3:![0-9]+]] -; CHECK-NEXT: store i32 4, ptr [[INPUT_RELOAD_ADDR13_I]], align 4, !noalias [[META3]] -; CHECK-NEXT: tail call void @print(i32 7), !noalias [[META6:![0-9]+]] -; CHECK-NEXT: tail call void @deallocate(ptr nonnull [[TMP0]]), !noalias [[META6]] +; CHECK-NEXT: store i32 4, ptr [[INPUT_RELOAD_ADDR14_I]], align 4, !noalias [[META3]] +; CHECK-NEXT: tail call void @print(i32 7), !noalias [[META6:![0-9]+]], !inline_history [[META9:![0-9]+]] +; CHECK-NEXT: tail call void @deallocate(ptr nonnull [[TMP0]]), !noalias [[META6]], !inline_history [[META9]] ; CHECK-NEXT: ret i32 0 ; entry: diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-resume-values2.ll b/llvm/test/Transforms/Coroutines/coro-retcon-resume-values2.ll index b1dfbd1b6d4f6..6441456c5ac28 100644 --- a/llvm/test/Transforms/Coroutines/coro-retcon-resume-values2.ll +++ b/llvm/test/Transforms/Coroutines/coro-retcon-resume-values2.ll @@ -58,10 +58,10 @@ declare void @print(i32) ; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP0:%.*]], align 8 ; CHECK-NEXT: [[VALUE1_SPILL_ADDR:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 12 ; CHECK-NEXT: store i32 [[TMP1:%.*]], ptr [[VALUE1_SPILL_ADDR]], align 4 -; CHECK-NEXT: [[SUM0_RELOAD_ADDR:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 8 -; CHECK-NEXT: [[SUM0_RELOAD:%.*]] = load i32, ptr [[SUM0_RELOAD_ADDR]], align 4 ; CHECK-NEXT: [[VALUE0_RELOAD_ADDR5:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 4 ; CHECK-NEXT: [[VALUE0_RELOAD6:%.*]] = load i32, ptr [[VALUE0_RELOAD_ADDR5]], align 4 +; CHECK-NEXT: [[SUM0_RELOAD_ADDR:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 8 +; CHECK-NEXT: [[SUM0_RELOAD:%.*]] = load i32, ptr [[SUM0_RELOAD_ADDR]], align 4 ; CHECK-NEXT: [[SUM1:%.*]] = call i32 @add(i32 [[SUM0_RELOAD]], i32 [[VALUE0_RELOAD6]]) ; CHECK-NEXT: [[SUM2:%.*]] = call i32 @add(i32 [[SUM1]], i32 [[TMP1]]) ; CHECK-NEXT: [[SUM2_SPILL_ADDR:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 16 @@ -72,13 +72,13 @@ declare void @print(i32) ; CHECK-LABEL: @f.resume.2( ; CHECK-NEXT: entryresume.2: ; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP0:%.*]], align 8 -; CHECK-NEXT: [[SUM2_RELOAD_ADDR:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 16 +; CHECK-NEXT: [[SUM2_RELOAD_ADDR:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 4 ; CHECK-NEXT: [[SUM2_RELOAD:%.*]] = load i32, ptr [[SUM2_RELOAD_ADDR]], align 4 ; CHECK-NEXT: [[VALUE1_RELOAD_ADDR:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 12 ; CHECK-NEXT: [[VALUE1_RELOAD:%.*]] = load i32, ptr [[VALUE1_RELOAD_ADDR]], align 4 -; CHECK-NEXT: [[VALUE0_RELOAD_ADDR:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 4 +; CHECK-NEXT: [[VALUE0_RELOAD_ADDR:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 16 ; CHECK-NEXT: [[VALUE0_RELOAD:%.*]] = load i32, ptr [[VALUE0_RELOAD_ADDR]], align 4 -; CHECK-NEXT: [[SUM3:%.*]] = call i32 @add(i32 [[SUM2_RELOAD]], i32 [[VALUE0_RELOAD]]) +; CHECK-NEXT: [[SUM3:%.*]] = call i32 @add(i32 [[VALUE0_RELOAD]], i32 [[SUM2_RELOAD]]) ; CHECK-NEXT: [[SUM4:%.*]] = call i32 @add(i32 [[SUM3]], i32 [[VALUE1_RELOAD]]) ; CHECK-NEXT: [[SUM5:%.*]] = call i32 @add(i32 [[SUM4]], i32 [[TMP1:%.*]]) ; CHECK-NEXT: call void @print(i32 [[SUM5]]) diff --git a/llvm/test/Transforms/Coroutines/coro-retcon.ll b/llvm/test/Transforms/Coroutines/coro-retcon.ll index cd2488adc74be..99ec0d8986ed9 100644 --- a/llvm/test/Transforms/Coroutines/coro-retcon.ll +++ b/llvm/test/Transforms/Coroutines/coro-retcon.ll @@ -43,9 +43,9 @@ define i32 @main() { ; CHECK-NEXT: entry: ; CHECK-NEXT: tail call void @print(i32 4) ; CHECK-NEXT: tail call void @llvm.experimental.noalias.scope.decl(metadata [[META0:![0-9]+]]) -; CHECK-NEXT: tail call void @print(i32 5), !noalias [[META0]] -; CHECK-NEXT: tail call void @llvm.experimental.noalias.scope.decl(metadata [[META3:![0-9]+]]) -; CHECK-NEXT: tail call void @print(i32 6), !noalias [[META3]] +; CHECK-NEXT: tail call void @print(i32 5), !noalias [[META0]], !inline_history [[META3:![0-9]+]] +; CHECK-NEXT: tail call void @llvm.experimental.noalias.scope.decl(metadata [[META4:![0-9]+]]) +; CHECK-NEXT: tail call void @print(i32 6), !noalias [[META4]], !inline_history [[META3]] ; CHECK-NEXT: ret i32 0 ; ; CORO-LABEL: @main( @@ -84,10 +84,8 @@ define hidden { ptr, ptr } @g(ptr %buffer, ptr %ptr) { ; CORO-NEXT: [[TMP0:%.*]] = call ptr @allocate(i32 8) ; CORO-NEXT: store ptr [[TMP0]], ptr [[BUFFER:%.*]], align 8 ; CORO-NEXT: store ptr [[PTR:%.*]], ptr [[TMP0]], align 8 -; CORO-NEXT: [[PTR_RELOAD_ADDR:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 0 -; CORO-NEXT: [[PTR_RELOAD:%.*]] = load ptr, ptr [[PTR_RELOAD_ADDR]], align 8 ; CORO-NEXT: [[TMP1:%.*]] = insertvalue { ptr, ptr } poison, ptr @g.resume.0, 0 -; CORO-NEXT: [[TMP2:%.*]] = insertvalue { ptr, ptr } [[TMP1]], ptr [[PTR_RELOAD]], 1 +; CORO-NEXT: [[TMP2:%.*]] = insertvalue { ptr, ptr } [[TMP1]], ptr [[PTR]], 1 ; CORO-NEXT: ret { ptr, ptr } [[TMP2]] ; entry: diff --git a/llvm/test/Transforms/Coroutines/coro-spill-ramp.ll b/llvm/test/Transforms/Coroutines/coro-spill-ramp.ll new file mode 100644 index 0000000000000..087ed7568da22 --- /dev/null +++ b/llvm/test/Transforms/Coroutines/coro-spill-ramp.ll @@ -0,0 +1,84 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt < %s -passes='cgscc(instcombine,coro-split),simplifycfg,early-cse' -S | FileCheck %s + +; Regression test for GH192351 +; Other passes are free to sink instructions that do not access memory. +; Test that spills in the ramp function are not replaced with reloads from the coroutine frame, thereby preventing loads from a dead frame. +define void @f(i1 noundef zeroext %b) presplitcoroutine { +; CHECK-LABEL: define void @f( +; CHECK-SAME: i1 noundef zeroext [[B:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[TMP0:%.*]] = call token @llvm.coro.id(i32 16, ptr null, ptr nonnull @f, ptr nonnull @f.resumers) +; CHECK-NEXT: [[TMP1:%.*]] = call i1 @llvm.coro.alloc(token [[TMP0]]) +; CHECK-NEXT: br i1 [[TMP1]], label %[[CORO_ALLOC:.*]], label %[[CORO_INIT:.*]] +; CHECK: [[CORO_ALLOC]]: +; CHECK-NEXT: [[CALL:%.*]] = call ptr @malloc(i64 24) +; CHECK-NEXT: br label %[[CORO_INIT]] +; CHECK: [[CORO_INIT]]: +; CHECK-NEXT: [[TMP2:%.*]] = phi ptr [ @f.destroy, %[[CORO_ALLOC]] ], [ @f.cleanup, %[[ENTRY]] ] +; CHECK-NEXT: [[TMP3:%.*]] = phi ptr [ [[CALL]], %[[CORO_ALLOC]] ], [ null, %[[ENTRY]] ] +; CHECK-NEXT: [[TMP4:%.*]] = call noalias nonnull ptr @llvm.coro.begin(token [[TMP0]], ptr [[TMP3]]) +; CHECK-NEXT: store ptr @f.resume, ptr [[TMP4]], align 8 +; CHECK-NEXT: [[DESTROY_ADDR:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 8 +; CHECK-NEXT: store ptr [[TMP2]], ptr [[DESTROY_ADDR]], align 8 +; CHECK-NEXT: [[B_SPILL_ADDR:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 17 +; CHECK-NEXT: store i1 [[B]], ptr [[B_SPILL_ADDR]], align 1 +; CHECK-NEXT: br i1 [[B]], label %[[COROSAVE:.*]], label %[[CLEANUP17:.*]] +; CHECK: [[COROSAVE]]: +; CHECK-NEXT: [[INDEX_ADDR1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 16 +; CHECK-NEXT: store i1 false, ptr [[INDEX_ADDR1]], align 1 +; CHECK-NEXT: br label %[[AFTERCOROEND:.*]] +; CHECK: [[CLEANUP17]]: +; CHECK-NEXT: [[TMP5:%.*]] = call ptr @llvm.coro.free(token [[TMP0]], ptr nonnull [[TMP4]]) +; CHECK-NEXT: [[DOTNOT:%.*]] = icmp eq ptr [[TMP5]], null +; CHECK-NEXT: br i1 [[DOTNOT]], label %[[AFTERCOROEND]], label %[[CORO_FREE:.*]] +; CHECK: [[CORO_FREE]]: +; CHECK-NEXT: call void @free(ptr nonnull [[TMP5]], i64 24) +; CHECK-NEXT: br label %[[AFTERCOROEND]] +; CHECK: [[AFTERCOROEND]]: +; CHECK-NEXT: [[B8:%.*]] = zext i1 [[B]] to i8 +; CHECK-NEXT: notail call void (...) @llvm.fake.use(i8 [[B8]]) +; CHECK-NEXT: ret void +; +entry: + %b8 = zext i1 %b to i8 ; InstCombine sinks + %0 = call token @llvm.coro.id(i32 16, ptr null, ptr nonnull @f, ptr null) + %1 = call i1 @llvm.coro.alloc(token %0) + br i1 %1, label %coro.alloc, label %coro.init + +coro.alloc: + %2 = call i64 @llvm.coro.size.i64() + %call = call ptr @malloc(i64 %2) + br label %coro.init + +coro.init: + %3 = phi ptr [ null, %entry ], [ %call, %coro.alloc ] + %4 = call ptr @llvm.coro.begin(token %0, ptr %3) + br i1 %b, label %if.then, label %cleanup17 + +if.then: + %5 = call token @llvm.coro.save(ptr %4) + %6 = call i8 @llvm.coro.suspend(token %5, i1 false) + switch i8 %6, label %coro.ret [ + i8 0, label %cleanup17 + i8 1, label %cleanup17 + ] + +cleanup17: + %7 = call ptr @llvm.coro.free(token %0, ptr %4) + %.not = icmp eq ptr %7, null + br i1 %.not, label %coro.ret, label %coro.free + +coro.free: + %8 = call i64 @llvm.coro.size.i64() + call void @free(ptr %7, i64 %8) + br label %coro.ret + +coro.ret: + call void @llvm.coro.end(ptr null, i1 false, token none) + notail call void (...) @llvm.fake.use(i8 %b8) + ret void +} + +declare ptr @malloc(i64) +declare void @free(ptr, i64) >From 18878bc3f075bf96813be1a4f720a39423db15c8 Mon Sep 17 00:00:00 2001 From: NewSigma <[email protected]> Date: Mon, 24 Aug 2026 12:34:07 +0800 Subject: [PATCH 2/2] Update unittests --- llvm/unittests/Transforms/Coroutines/ExtraRematTest.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/unittests/Transforms/Coroutines/ExtraRematTest.cpp b/llvm/unittests/Transforms/Coroutines/ExtraRematTest.cpp index a34b630a34fb7..e2671d037d074 100644 --- a/llvm/unittests/Transforms/Coroutines/ExtraRematTest.cpp +++ b/llvm/unittests/Transforms/Coroutines/ExtraRematTest.cpp @@ -173,9 +173,9 @@ TEST_F(ExtraRematTest, TestCoroRematWithCallback) { Function *F = M->getFunction("f.resume"); ASSERT_TRUE(F) << "could not find split function f.resume"; - BasicBlock *Resume1 = getBasicBlockByName(F, "resume1"); + BasicBlock *Resume1 = getBasicBlockByName(F, "resume1.after.spill"); ASSERT_TRUE(Resume1) - << "could not find expected BB resume1 in split function"; + << "could not find expected BB in split function"; // With callback the extra rematerialization of the function should have // happened @@ -259,9 +259,9 @@ TEST_F(ExtraRematTest, TestCoroRematWithCustomABI) { Function *F = M->getFunction("f.resume"); ASSERT_TRUE(F) << "could not find split function f.resume"; - BasicBlock *Resume1 = getBasicBlockByName(F, "resume1"); + BasicBlock *Resume1 = getBasicBlockByName(F, "resume1.after.spill"); ASSERT_TRUE(Resume1) - << "could not find expected BB resume1 in split function"; + << "could not find expected BB in split function"; // With callback the extra rematerialization of the function should have // happened _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
