================ @@ -0,0 +1,371 @@ +//===----- RISCVLoadStoreOptimizer.cpp ------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Bundle loads and stores that operate on consecutive memory locations to take +// the advantage of hardware load/store bonding. +// +//===----------------------------------------------------------------------===// + +#include "RISCV.h" +#include "RISCVTargetMachine.h" +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/MC/TargetRegistry.h" +#include "llvm/Support/Debug.h" +#include "llvm/Target/TargetOptions.h" + +using namespace llvm; + +#define DEBUG_TYPE "riscv-load-store-opt" +#define RISCV_LOAD_STORE_OPT_NAME "RISCV Load / Store Optimizer" +namespace { + +struct RISCVLoadStoreOpt : public MachineFunctionPass { + static char ID; + bool runOnMachineFunction(MachineFunction &Fn) override; + + RISCVLoadStoreOpt() : MachineFunctionPass(ID) {} + + MachineFunctionProperties getRequiredProperties() const override { + return MachineFunctionProperties().set( + MachineFunctionProperties::Property::NoVRegs); + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired<AAResultsWrapperPass>(); + MachineFunctionPass::getAnalysisUsage(AU); + } + + StringRef getPassName() const override { return RISCV_LOAD_STORE_OPT_NAME; } + + // Find and pair load/store instructions. + bool tryToPairLdStInst(MachineBasicBlock::iterator &MBBI); + + // Convert load/store pairs to single instructions. + bool tryConvertToLdStPair(MachineBasicBlock::iterator First, + MachineBasicBlock::iterator Second); + + // Scan the instructions looking for a load/store that can be combined + // with the current instruction into a load/store pair. + // Return the matching instruction if one is found, else MBB->end(). + MachineBasicBlock::iterator findMatchingInsn(MachineBasicBlock::iterator I, + bool &MergeForward); + + MachineBasicBlock::iterator + mergePairedInsns(MachineBasicBlock::iterator I, + MachineBasicBlock::iterator Paired, bool MergeForward); + +private: + AliasAnalysis *AA; + MachineRegisterInfo *MRI; + const RISCVInstrInfo *TII; + const RISCVRegisterInfo *TRI; + LiveRegUnits ModifiedRegUnits, UsedRegUnits; + bool UseLoadStorePair = false; +}; +} // end anonymous namespace + +char RISCVLoadStoreOpt::ID = 0; +INITIALIZE_PASS(RISCVLoadStoreOpt, DEBUG_TYPE, RISCV_LOAD_STORE_OPT_NAME, false, + false) + +bool RISCVLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) { + if (skipFunction(Fn.getFunction())) + return false; + const RISCVSubtarget &Subtarget = Fn.getSubtarget<RISCVSubtarget>(); + + if (!Subtarget.useLoadStorePairs()) + return false; + + bool MadeChange = false; + TII = Subtarget.getInstrInfo(); + TRI = Subtarget.getRegisterInfo(); + MRI = &Fn.getRegInfo(); + AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); + ModifiedRegUnits.init(*TRI); + UsedRegUnits.init(*TRI); + UseLoadStorePair = Subtarget.useLoadStorePairs(); + + for (MachineBasicBlock &MBB : Fn) { + LLVM_DEBUG(dbgs() << "MBB: " << MBB.getName() << "\n"); + + for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); + MBBI != E;) { + if (TII->isPairableLdStInstOpc(MBBI->getOpcode()) && + tryToPairLdStInst(MBBI)) + MadeChange = true; + else + ++MBBI; + } + } + return MadeChange; +} + +// Find loads and stores that can be merged into a single load or store pair +// instruction. +bool RISCVLoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) { + MachineInstr &MI = *MBBI; + MachineBasicBlock::iterator E = MI.getParent()->end(); + + if (!TII->isLdStSafeToPair(MI, TRI)) + return false; + + // Look ahead for a pairable instruction. + bool MergeForward; + MachineBasicBlock::iterator Paired = findMatchingInsn(MBBI, MergeForward); + if (Paired != E) { + MBBI = mergePairedInsns(MBBI, Paired, MergeForward); + return true; + } + return false; +} + +bool RISCVLoadStoreOpt::tryConvertToLdStPair( + MachineBasicBlock::iterator First, MachineBasicBlock::iterator Second) { + if (!UseLoadStorePair) + return false; + + unsigned PairOpc; + switch (First->getOpcode()) { + default: + return false; + case RISCV::SW: + PairOpc = RISCV::SWP; + break; + case RISCV::LW: + PairOpc = RISCV::LWP; + break; + case RISCV::SD: + PairOpc = RISCV::SDP; + break; + case RISCV::LD: + PairOpc = RISCV::LDP; + break; + } + + MachineFunction *MF = First->getMF(); + const MachineMemOperand *MMO = *First->memoperands_begin(); + Align MMOAlign = MMO->getAlign(); + if (const PseudoSourceValue *Source = MMO->getPseudoValue()) + if (Source->kind() == PseudoSourceValue::FixedStack) + MMOAlign = MF->getSubtarget().getFrameLowering()->getStackAlign(); + + if (MMOAlign < Align(MMO->getSize().getValue() * 2)) + return false; + int64_t Offset = First->getOperand(2).getImm(); + if (!isUInt<7>(Offset) || + !isAligned(Align(MMO->getSize().getValue()), Offset)) + return false; + MachineInstrBuilder MIB = BuildMI( + *MF, + First->getDebugLoc().get() ? First->getDebugLoc() : Second->getDebugLoc(), + TII->get(PairOpc)); + MIB.add(First->getOperand(0)) + .add(Second->getOperand(0)) + .add(First->getOperand(1)) + .add(First->getOperand(2)) + .cloneMergedMemRefs({&*First, &*Second}); + + First->getParent()->insert(First, MIB); + + First->removeFromParent(); + Second->removeFromParent(); + + return true; +} + +/// TODO: Move to lambda +static bool mayAlias(MachineInstr &MIa, + SmallVectorImpl<MachineInstr *> &MemInsns, + AliasAnalysis *AA) { + for (MachineInstr *MIb : MemInsns) + if (MIa.mayAlias(AA, *MIb, /*UseTBAA*/ false)) + return true; + + return false; +} + +/// Scan the instructions looking for a load/store that can be combined with the +/// current instruction into a wider equivalent or a load/store pair. +MachineBasicBlock::iterator +RISCVLoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I, + bool &MergeForward) { + MachineBasicBlock::iterator E = I->getParent()->end(); + MachineBasicBlock::iterator MBBI = I; + MachineInstr &FirstMI = *I; + MBBI = next_nodbg(MBBI, E); + + bool MayLoad = FirstMI.mayLoad(); + Register Reg = FirstMI.getOperand(0).getReg(); + Register BaseReg = FirstMI.getOperand(1).getReg(); + int Offset = FirstMI.getOperand(2).getImm(); + int OffsetStride = (*FirstMI.memoperands_begin())->getSize().getValue(); + + LiveRegUnits UsedInBetween; + UsedInBetween.init(*TRI); + + MergeForward = false; + + // Track which register units have been modified and used between the first + // insn (inclusive) and the second insn. + ModifiedRegUnits.clear(); + UsedRegUnits.clear(); + + // Remember any instructions that read/write memory between FirstMI and MI. + SmallVector<MachineInstr *, 4> MemInsns; + + for (unsigned Count = 0; MBBI != E && Count < 128; + MBBI = next_nodbg(MBBI, E)) { + MachineInstr &MI = *MBBI; + + UsedInBetween.accumulate(MI); + + // Don't count transient instructions towards the search limit since there + // may be different numbers of them if e.g. debug information is present. + if (!MI.isTransient()) + ++Count; + + if (MI.getOpcode() == FirstMI.getOpcode() && + TII->isLdStSafeToPair(MI, TRI)) { + Register MIBaseReg = MI.getOperand(1).getReg(); + int MIOffset = MI.getOperand(2).getImm(); + + if (BaseReg == MIBaseReg) { + + if ((Offset != MIOffset + OffsetStride) && + (Offset + OffsetStride != MIOffset)) { + LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, + TRI); + MemInsns.push_back(&MI); + continue; + } + + // If the destination register of one load is the same register or a + // sub/super register of the other load, bail and keep looking. + if (MayLoad && + TRI->isSuperOrSubRegisterEq(Reg, MI.getOperand(0).getReg())) { + LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, + TRI); + MemInsns.push_back(&MI); + continue; + } + + // If the BaseReg has been modified, then we cannot do the optimization. + if (!ModifiedRegUnits.available(BaseReg)) + return E; + + // If the Rt of the second instruction was not modified or used between + // the two instructions and none of the instructions between the second + // and first alias with the second, we can combine the second into the + // first. + if (ModifiedRegUnits.available(MI.getOperand(0).getReg()) && + !(MI.mayLoad() && + !UsedRegUnits.available(MI.getOperand(0).getReg())) && + !mayAlias(MI, MemInsns, AA)) { + + MergeForward = false; + return MBBI; + } + + // Likewise, if the Rt of the first instruction is not modified or used + // between the two instructions and none of the instructions between the + // first and the second alias with the first, we can combine the first + // into the second. + if (!(MayLoad && + !UsedRegUnits.available(FirstMI.getOperand(0).getReg())) && + !mayAlias(FirstMI, MemInsns, AA)) { + + if (ModifiedRegUnits.available(FirstMI.getOperand(0).getReg())) { + MergeForward = true; + return MBBI; + } + } + // Unable to combine these instructions due to interference in between. + // Keep looking. + } + } + + // If the instruction wasn't a matching load or store. Stop searching if we + // encounter a call instruction that might modify memory. + if (MI.isCall()) + return E; + + // Update modified / uses register units. + LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI); + + // Otherwise, if the base register is modified, we have no match, so + // return early. + if (!ModifiedRegUnits.available(BaseReg)) + return E; + + // Update list of instructions that read/write memory. + if (MI.mayLoadOrStore()) + MemInsns.push_back(&MI); + } + return E; +} + +MachineBasicBlock::iterator __attribute__((noinline)) +RISCVLoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I, + MachineBasicBlock::iterator Paired, + bool MergeForward) { + MachineBasicBlock::iterator E = I->getParent()->end(); + MachineBasicBlock::iterator NextI = next_nodbg(I, E); + if (NextI == Paired) + NextI = next_nodbg(NextI, E); + + // Insert our new paired instruction after whichever of the paired + // instructions MergeForward indicates. + MachineBasicBlock::iterator InsertionPoint = MergeForward ? Paired : I; + MachineBasicBlock::iterator DeletionPoint = MergeForward ? I : Paired; + int Offset = I->getOperand(2).getImm(); + int PairedOffset = Paired->getOperand(2).getImm(); + bool InsertAfter = (Offset < PairedOffset) ^ MergeForward; + + if (!MergeForward) + Paired->getOperand(1).setIsKill(false); + + // Kill flags may become invalid when moving stores for pairing. + if (I->getOperand(0).isUse()) { + if (!MergeForward) { + // Clear kill flags on store if moving upwards. + I->getOperand(0).setIsKill(false); + Paired->getOperand(0).setIsKill(false); + } else { + // Clear kill flags of the first stores register. + Register Reg = I->getOperand(0).getReg(); + for (MachineInstr &MI : make_range(std::next(I), Paired)) ---------------- djtodoro wrote:
Hmm yes, nice catch, thanks, addressed in https://github.com/llvm/llvm-project/pull/121394 https://github.com/llvm/llvm-project/pull/117865 _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
