https://github.com/snehasish updated
https://github.com/llvm/llvm-project/pull/142811
>From 0ba44d66dce0260055d7dd576141e098f7e1c1d5 Mon Sep 17 00:00:00 2001
From: Snehasish Kumar <snehasi...@google.com>
Date: Tue, 3 Jun 2025 21:49:58 -0700
Subject: [PATCH] [MemProf] Split MemProfiler into Instrumentation and Use.
---
clang/lib/CodeGen/BackendUtil.cpp | 3 +-
.../Instrumentation/MemProfInstrumentation.h | 47 ++
.../{MemProfiler.h => MemProfUse.h} | 34 +-
llvm/lib/Passes/PassBuilder.cpp | 3 +-
llvm/lib/Passes/PassBuilderPipelines.cpp | 3 +-
.../Transforms/Instrumentation/CMakeLists.txt | 3 +-
.../MemProfInstrumentation.cpp | 658 ++++++++++++++++++
.../{MemProfiler.cpp => MemProfUse.cpp} | 634 +----------------
.../Instrumentation/MemProfUseTest.cpp | 2 +-
9 files changed, 724 insertions(+), 663 deletions(-)
create mode 100644
llvm/include/llvm/Transforms/Instrumentation/MemProfInstrumentation.h
rename llvm/include/llvm/Transforms/Instrumentation/{MemProfiler.h =>
MemProfUse.h} (61%)
create mode 100644
llvm/lib/Transforms/Instrumentation/MemProfInstrumentation.cpp
rename llvm/lib/Transforms/Instrumentation/{MemProfiler.cpp => MemProfUse.cpp}
(56%)
diff --git a/clang/lib/CodeGen/BackendUtil.cpp
b/clang/lib/CodeGen/BackendUtil.cpp
index cd5fc48c4a22b..9cecf3c0f586e 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -69,7 +69,8 @@
#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
#include "llvm/Transforms/Instrumentation/KCFI.h"
#include "llvm/Transforms/Instrumentation/LowerAllowCheckPass.h"
-#include "llvm/Transforms/Instrumentation/MemProfiler.h"
+#include "llvm/Transforms/Instrumentation/MemProfInstrumentation.h"
+#include "llvm/Transforms/Instrumentation/MemProfUse.h"
#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
#include "llvm/Transforms/Instrumentation/NumericalStabilitySanitizer.h"
#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
diff --git
a/llvm/include/llvm/Transforms/Instrumentation/MemProfInstrumentation.h
b/llvm/include/llvm/Transforms/Instrumentation/MemProfInstrumentation.h
new file mode 100644
index 0000000000000..ffc41993464fa
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Instrumentation/MemProfInstrumentation.h
@@ -0,0 +1,47 @@
+//===--- MemProfInstrumentation.h - Memory profiler instrumentation ----*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM
Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the MemProf instrumentation pass classes.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_MEMPROF_INSTRUMENTATION_H
+#define LLVM_TRANSFORMS_INSTRUMENTATION_MEMPROF_INSTRUMENTATION_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+class Function;
+class Module;
+
+/// Public interface to the memory profiler pass for instrumenting code to
+/// profile memory accesses.
+///
+/// The profiler itself is a function pass that works by inserting various
+/// calls to the MemProfiler runtime library functions. The runtime library
+/// essentially replaces malloc() and free() with custom implementations that
+/// record data about the allocations.
+class MemProfilerPass : public PassInfoMixin<MemProfilerPass> {
+public:
+ explicit MemProfilerPass();
+ PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+ static bool isRequired() { return true; }
+};
+
+/// Public interface to the memory profiler module pass for instrumenting code
+/// to profile memory allocations and accesses.
+class ModuleMemProfilerPass : public PassInfoMixin<ModuleMemProfilerPass> {
+public:
+ explicit ModuleMemProfilerPass();
+ PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+ static bool isRequired() { return true; }
+};
+
+} // namespace llvm
+
+#endif
\ No newline at end of file
diff --git a/llvm/include/llvm/Transforms/Instrumentation/MemProfiler.h
b/llvm/include/llvm/Transforms/Instrumentation/MemProfUse.h
similarity index 61%
rename from llvm/include/llvm/Transforms/Instrumentation/MemProfiler.h
rename to llvm/include/llvm/Transforms/Instrumentation/MemProfUse.h
index 169f757e580d2..91c6cdaecbce8 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/MemProfiler.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/MemProfUse.h
@@ -1,4 +1,4 @@
-//===--------- Definition of the MemProfiler class --------------*- C++
-*-===//
+//===--------- MemProfUse.h - Memory profiler use pass ----*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM
Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -6,11 +6,11 @@
//
//===----------------------------------------------------------------------===//
//
-// This file declares the MemProfiler class.
+// This file declares the MemProfUsePass class and related utilities.
//
//===----------------------------------------------------------------------===//
-#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_MEMPROFILER_H
-#define LLVM_TRANSFORMS_INSTRUMENTATION_MEMPROFILER_H
+#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_MEMPROFUSE_H
+#define LLVM_TRANSFORMS_INSTRUMENTATION_MEMPROFUSE_H
#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include "llvm/IR/PassManager.h"
@@ -19,7 +19,6 @@
#include <unordered_map>
namespace llvm {
-class Function;
class IndexedInstrProfReader;
class Module;
class TargetLibraryInfo;
@@ -28,29 +27,6 @@ namespace vfs {
class FileSystem;
} // namespace vfs
-/// Public interface to the memory profiler pass for instrumenting code to
-/// profile memory accesses.
-///
-/// The profiler itself is a function pass that works by inserting various
-/// calls to the MemProfiler runtime library functions. The runtime library
-/// essentially replaces malloc() and free() with custom implementations that
-/// record data about the allocations.
-class MemProfilerPass : public PassInfoMixin<MemProfilerPass> {
-public:
- explicit MemProfilerPass();
- PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
- static bool isRequired() { return true; }
-};
-
-/// Public interface to the memory profiler module pass for instrumenting code
-/// to profile memory allocations and accesses.
-class ModuleMemProfilerPass : public PassInfoMixin<ModuleMemProfilerPass> {
-public:
- explicit ModuleMemProfilerPass();
- PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
- static bool isRequired() { return true; }
-};
-
class MemProfUsePass : public PassInfoMixin<MemProfUsePass> {
public:
explicit MemProfUsePass(std::string MemoryProfileFile,
@@ -90,4 +66,4 @@ computeUndriftMap(Module &M, IndexedInstrProfReader
*MemProfReader,
} // namespace memprof
} // namespace llvm
-#endif
+#endif
\ No newline at end of file
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 1e4299a0b9803..a13519aae56a2 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -249,7 +249,8 @@
#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
#include "llvm/Transforms/Instrumentation/KCFI.h"
#include "llvm/Transforms/Instrumentation/LowerAllowCheckPass.h"
-#include "llvm/Transforms/Instrumentation/MemProfiler.h"
+#include "llvm/Transforms/Instrumentation/MemProfInstrumentation.h"
+#include "llvm/Transforms/Instrumentation/MemProfUse.h"
#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
#include "llvm/Transforms/Instrumentation/NumericalStabilitySanitizer.h"
#include "llvm/Transforms/Instrumentation/PGOCtxProfFlattening.h"
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp
b/llvm/lib/Passes/PassBuilderPipelines.cpp
index f3654600c5abb..683d5aca81b4b 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -77,7 +77,8 @@
#include "llvm/Transforms/Instrumentation/CGProfile.h"
#include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
-#include "llvm/Transforms/Instrumentation/MemProfiler.h"
+#include "llvm/Transforms/Instrumentation/MemProfInstrumentation.h"
+#include "llvm/Transforms/Instrumentation/MemProfUse.h"
#include "llvm/Transforms/Instrumentation/PGOCtxProfFlattening.h"
#include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h"
#include "llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h"
diff --git a/llvm/lib/Transforms/Instrumentation/CMakeLists.txt
b/llvm/lib/Transforms/Instrumentation/CMakeLists.txt
index 0b863f6fef460..15fd421a41b0f 100644
--- a/llvm/lib/Transforms/Instrumentation/CMakeLists.txt
+++ b/llvm/lib/Transforms/Instrumentation/CMakeLists.txt
@@ -6,7 +6,8 @@ add_llvm_component_library(LLVMInstrumentation
DataFlowSanitizer.cpp
GCOVProfiling.cpp
BlockCoverageInference.cpp
- MemProfiler.cpp
+ MemProfInstrumentation.cpp
+ MemProfUse.cpp
MemorySanitizer.cpp
NumericalStabilitySanitizer.cpp
IndirectCallPromotion.cpp
diff --git a/llvm/lib/Transforms/Instrumentation/MemProfInstrumentation.cpp
b/llvm/lib/Transforms/Instrumentation/MemProfInstrumentation.cpp
new file mode 100644
index 0000000000000..e462655c3bd94
--- /dev/null
+++ b/llvm/lib/Transforms/Instrumentation/MemProfInstrumentation.cpp
@@ -0,0 +1,658 @@
+//===- MemProfInstrumentation.cpp - memory allocation and access profiler
+//instrumentation ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM
Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProf. Memory accesses are instrumented
+// to increment the access count held in a shadow memory location, or
+// alternatively to call into the runtime. Memory intrinsic calls (memmove,
+// memcpy, memset) are changed to call the memory profiling runtime version
+// instead.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Instrumentation/MemProfInstrumentation.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Analysis/MemoryBuiltins.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/Constant.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
+#include "llvm/IR/Value.h"
+#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/ProfileData/MemProf.h"
+#include "llvm/ProfileData/MemProfCommon.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/TargetParser/Triple.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
+
+using namespace llvm;
+using namespace llvm::memprof;
+
+#define DEBUG_TYPE "memprof"
+
+constexpr int LLVM_MEM_PROFILER_VERSION = 1;
+
+// Size of memory mapped to a single shadow location.
+constexpr uint64_t DefaultMemGranularity = 64;
+
+// Size of memory mapped to a single histogram bucket.
+constexpr uint64_t HistogramGranularity = 8;
+
+// Scale from granularity down to shadow size.
+constexpr uint64_t DefaultShadowScale = 3;
+
+constexpr char MemProfModuleCtorName[] = "memprof.module_ctor";
+constexpr uint64_t MemProfCtorAndDtorPriority = 1;
+// On Emscripten, the system needs more than one priorities for constructors.
+constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = 50;
+constexpr char MemProfInitName[] = "__memprof_init";
+constexpr char MemProfVersionCheckNamePrefix[] =
+ "__memprof_version_mismatch_check_v";
+
+constexpr char MemProfShadowMemoryDynamicAddress[] =
+ "__memprof_shadow_memory_dynamic_address";
+
+constexpr char MemProfFilenameVar[] = "__memprof_profile_filename";
+
+constexpr char MemProfHistogramFlagVar[] = "__memprof_histogram";
+
+// Command-line flags.
+
+static cl::opt<bool> ClInsertVersionCheck(
+ "memprof-guard-against-version-mismatch",
+ cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden,
+ cl::init(true));
+
+// This flag may need to be replaced with -f[no-]memprof-reads.
+static cl::opt<bool> ClInstrumentReads("memprof-instrument-reads",
+ cl::desc("instrument read
instructions"),
+ cl::Hidden, cl::init(true));
+
+static cl::opt<bool>
+ ClInstrumentWrites("memprof-instrument-writes",
+ cl::desc("instrument write instructions"), cl::Hidden,
+ cl::init(true));
+
+static cl::opt<bool> ClInstrumentAtomics(
+ "memprof-instrument-atomics",
+ cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
+ cl::init(true));
+
+static cl::opt<bool> ClUseCalls(
+ "memprof-use-callbacks",
+ cl::desc("Use callbacks instead of inline instrumentation sequences."),
+ cl::Hidden, cl::init(false));
+
+static cl::opt<std::string>
+ ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix",
+ cl::desc("Prefix for memory access
callbacks"),
+ cl::Hidden, cl::init("__memprof_"));
+
+// These flags allow to change the shadow mapping.
+// The shadow mapping looks like
+// Shadow = ((Mem & mask) >> scale) + offset
+
+static cl::opt<int> ClMappingScale("memprof-mapping-scale",
+ cl::desc("scale of memprof shadow mapping"),
+ cl::Hidden, cl::init(DefaultShadowScale));
+
+static cl::opt<int>
+ ClMappingGranularity("memprof-mapping-granularity",
+ cl::desc("granularity of memprof shadow mapping"),
+ cl::Hidden, cl::init(DefaultMemGranularity));
+
+static cl::opt<bool> ClStack("memprof-instrument-stack",
+ cl::desc("Instrument scalar stack variables"),
+ cl::Hidden, cl::init(false));
+
+// Debug flags.
+
+static cl::opt<int> ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden,
+ cl::init(0));
+
+static cl::opt<std::string> ClDebugFunc("memprof-debug-func", cl::Hidden,
+ cl::desc("Debug func"));
+
+static cl::opt<int> ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"),
+ cl::Hidden, cl::init(-1));
+
+static cl::opt<int> ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"),
+ cl::Hidden, cl::init(-1));
+
+static cl::opt<bool> ClHistogram("memprof-histogram",
+ cl::desc("Collect access count histograms"),
+ cl::Hidden, cl::init(false));
+
+static cl::opt<std::string>
+ MemprofRuntimeDefaultOptions("memprof-runtime-default-options",
+ cl::desc("The default memprof options"),
+ cl::Hidden, cl::init(""));
+
+// Instrumentation statistics
+STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
+STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
+STATISTIC(NumSkippedStackReads, "Number of non-instrumented stack reads");
+STATISTIC(NumSkippedStackWrites, "Number of non-instrumented stack writes");
+
+namespace {
+
+/// This struct defines the shadow mapping using the rule:
+/// shadow = ((mem & mask) >> Scale) ADD DynamicShadowOffset.
+struct ShadowMapping {
+ ShadowMapping() {
+ Scale = ClMappingScale;
+ Granularity = ClHistogram ? HistogramGranularity : ClMappingGranularity;
+ Mask = ~(Granularity - 1);
+ }
+
+ int Scale;
+ int Granularity;
+ uint64_t Mask; // Computed as ~(Granularity-1)
+};
+
+static uint64_t getCtorAndDtorPriority(Triple &TargetTriple) {
+ return TargetTriple.isOSEmscripten() ? MemProfEmscriptenCtorAndDtorPriority
+ : MemProfCtorAndDtorPriority;
+}
+
+struct InterestingMemoryAccess {
+ Value *Addr = nullptr;
+ bool IsWrite;
+ Type *AccessTy;
+ Value *MaybeMask = nullptr;
+};
+
+/// Instrument the code in module to profile memory accesses.
+class MemProfiler {
+public:
+ MemProfiler(Module &M) {
+ C = &(M.getContext());
+ LongSize = M.getDataLayout().getPointerSizeInBits();
+ IntptrTy = Type::getIntNTy(*C, LongSize);
+ PtrTy = PointerType::getUnqual(*C);
+ }
+
+ /// If it is an interesting memory access, populate information
+ /// about the access and return a InterestingMemoryAccess struct.
+ /// Otherwise return std::nullopt.
+ std::optional<InterestingMemoryAccess>
+ isInterestingMemoryAccess(Instruction *I) const;
+
+ void instrumentMop(Instruction *I, const DataLayout &DL,
+ InterestingMemoryAccess &Access);
+ void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
+ Value *Addr, bool IsWrite);
+ void instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
+ Instruction *I, Value *Addr, Type *AccessTy,
+ bool IsWrite);
+ void instrumentMemIntrinsic(MemIntrinsic *MI);
+ Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
+ bool instrumentFunction(Function &F);
+ bool maybeInsertMemProfInitAtFunctionEntry(Function &F);
+ bool insertDynamicShadowAtFunctionEntry(Function &F);
+
+private:
+ void initializeCallbacks(Module &M);
+
+ LLVMContext *C;
+ int LongSize;
+ Type *IntptrTy;
+ PointerType *PtrTy;
+ ShadowMapping Mapping;
+
+ // These arrays is indexed by AccessIsWrite
+ FunctionCallee MemProfMemoryAccessCallback[2];
+
+ FunctionCallee MemProfMemmove, MemProfMemcpy, MemProfMemset;
+ Value *DynamicShadowOffset = nullptr;
+};
+
+class ModuleMemProfiler {
+public:
+ ModuleMemProfiler(Module &M) { TargetTriple = M.getTargetTriple(); }
+
+ bool instrumentModule(Module &);
+
+private:
+ Triple TargetTriple;
+ ShadowMapping Mapping;
+ Function *MemProfCtorFunction = nullptr;
+};
+
+} // end anonymous namespace
+
+MemProfilerPass::MemProfilerPass() = default;
+
+PreservedAnalyses MemProfilerPass::run(Function &F,
+ AnalysisManager<Function> &AM) {
+ assert((!ClHistogram || ClMappingGranularity == DefaultMemGranularity) &&
+ "Memprof with histogram only supports default mapping granularity");
+ Module &M = *F.getParent();
+ MemProfiler Profiler(M);
+ if (Profiler.instrumentFunction(F))
+ return PreservedAnalyses::none();
+ return PreservedAnalyses::all();
+}
+
+ModuleMemProfilerPass::ModuleMemProfilerPass() = default;
+
+PreservedAnalyses ModuleMemProfilerPass::run(Module &M,
+ AnalysisManager<Module> &AM) {
+
+ ModuleMemProfiler Profiler(M);
+ if (Profiler.instrumentModule(M))
+ return PreservedAnalyses::none();
+ return PreservedAnalyses::all();
+}
+
+Value *MemProfiler::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
+ // (Shadow & mask) >> scale
+ Shadow = IRB.CreateAnd(Shadow, Mapping.Mask);
+ Shadow = IRB.CreateLShr(Shadow, Mapping.Scale);
+ // (Shadow >> scale) | offset
+ assert(DynamicShadowOffset);
+ return IRB.CreateAdd(Shadow, DynamicShadowOffset);
+}
+
+// Instrument memset/memmove/memcpy
+void MemProfiler::instrumentMemIntrinsic(MemIntrinsic *MI) {
+ IRBuilder<> IRB(MI);
+ if (isa<MemTransferInst>(MI)) {
+ IRB.CreateCall(isa<MemMoveInst>(MI) ? MemProfMemmove : MemProfMemcpy,
+ {MI->getOperand(0), MI->getOperand(1),
+ IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
+ } else if (isa<MemSetInst>(MI)) {
+ IRB.CreateCall(
+ MemProfMemset,
+ {MI->getOperand(0),
+ IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
+ IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
+ }
+ MI->eraseFromParent();
+}
+
+std::optional<InterestingMemoryAccess>
+MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
+ // Do not instrument the load fetching the dynamic shadow address.
+ if (DynamicShadowOffset == I)
+ return std::nullopt;
+
+ InterestingMemoryAccess Access;
+
+ if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
+ if (!ClInstrumentReads)
+ return std::nullopt;
+ Access.IsWrite = false;
+ Access.AccessTy = LI->getType();
+ Access.Addr = LI->getPointerOperand();
+ } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
+ if (!ClInstrumentWrites)
+ return std::nullopt;
+ Access.IsWrite = true;
+ Access.AccessTy = SI->getValueOperand()->getType();
+ Access.Addr = SI->getPointerOperand();
+ } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
+ if (!ClInstrumentAtomics)
+ return std::nullopt;
+ Access.IsWrite = true;
+ Access.AccessTy = RMW->getValOperand()->getType();
+ Access.Addr = RMW->getPointerOperand();
+ } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
+ if (!ClInstrumentAtomics)
+ return std::nullopt;
+ Access.IsWrite = true;
+ Access.AccessTy = XCHG->getCompareOperand()->getType();
+ Access.Addr = XCHG->getPointerOperand();
+ } else if (auto *CI = dyn_cast<CallInst>(I)) {
+ auto *F = CI->getCalledFunction();
+ if (F && (F->getIntrinsicID() == Intrinsic::masked_load ||
+ F->getIntrinsicID() == Intrinsic::masked_store)) {
+ unsigned OpOffset = 0;
+ if (F->getIntrinsicID() == Intrinsic::masked_store) {
+ if (!ClInstrumentWrites)
+ return std::nullopt;
+ // Masked store has an initial operand for the value.
+ OpOffset = 1;
+ Access.AccessTy = CI->getArgOperand(0)->getType();
+ Access.IsWrite = true;
+ } else {
+ if (!ClInstrumentReads)
+ return std::nullopt;
+ Access.AccessTy = CI->getType();
+ Access.IsWrite = false;
+ }
+
+ auto *BasePtr = CI->getOperand(0 + OpOffset);
+ Access.MaybeMask = CI->getOperand(2 + OpOffset);
+ Access.Addr = BasePtr;
+ }
+ }
+
+ if (!Access.Addr)
+ return std::nullopt;
+
+ // Do not instrument accesses from different address spaces; we cannot deal
+ // with them.
+ Type *PtrTy = cast<PointerType>(Access.Addr->getType()->getScalarType());
+ if (PtrTy->getPointerAddressSpace() != 0)
+ return std::nullopt;
+
+ // Ignore swifterror addresses.
+ // swifterror memory addresses are mem2reg promoted by instruction
+ // selection. As such they cannot have regular uses like an instrumentation
+ // function and it makes no sense to track them as memory.
+ if (Access.Addr->isSwiftError())
+ return std::nullopt;
+
+ // Peel off GEPs and BitCasts.
+ auto *Addr = Access.Addr->stripInBoundsOffsets();
+
+ if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) {
+ // Do not instrument PGO counter updates.
+ if (GV->hasSection()) {
+ StringRef SectionName = GV->getSection();
+ // Check if the global is in the PGO counters section.
+ auto OF = I->getModule()->getTargetTriple().getObjectFormat();
+ if (SectionName.ends_with(
+ getInstrProfSectionName(IPSK_cnts, OF,
/*AddSegmentInfo=*/false)))
+ return std::nullopt;
+ }
+
+ // Do not instrument accesses to LLVM internal variables.
+ if (GV->getName().starts_with("__llvm"))
+ return std::nullopt;
+ }
+
+ return Access;
+}
+
+void MemProfiler::instrumentMaskedLoadOrStore(const DataLayout &DL, Value
*Mask,
+ Instruction *I, Value *Addr,
+ Type *AccessTy, bool IsWrite) {
+ auto *VTy = cast<FixedVectorType>(AccessTy);
+ unsigned Num = VTy->getNumElements();
+ auto *Zero = ConstantInt::get(IntptrTy, 0);
+ for (unsigned Idx = 0; Idx < Num; ++Idx) {
+ Value *InstrumentedAddress = nullptr;
+ Instruction *InsertBefore = I;
+ if (auto *Vector = dyn_cast<ConstantVector>(Mask)) {
+ // dyn_cast as we might get UndefValue
+ if (auto *Masked = dyn_cast<ConstantInt>(Vector->getOperand(Idx))) {
+ if (Masked->isZero())
+ // Mask is constant false, so no instrumentation needed.
+ continue;
+ // If we have a true or undef value, fall through to instrumentAddress.
+ // with InsertBefore == I
+ }
+ } else {
+ IRBuilder<> IRB(I);
+ Value *MaskElem = IRB.CreateExtractElement(Mask, Idx);
+ Instruction *ThenTerm = SplitBlockAndInsertIfThen(MaskElem, I, false);
+ InsertBefore = ThenTerm;
+ }
+
+ IRBuilder<> IRB(InsertBefore);
+ InstrumentedAddress =
+ IRB.CreateGEP(VTy, Addr, {Zero, ConstantInt::get(IntptrTy, Idx)});
+ instrumentAddress(I, InsertBefore, InstrumentedAddress, IsWrite);
+ }
+}
+
+void MemProfiler::instrumentMop(Instruction *I, const DataLayout &DL,
+ InterestingMemoryAccess &Access) {
+ // Skip instrumentation of stack accesses unless requested.
+ if (!ClStack && isa<AllocaInst>(getUnderlyingObject(Access.Addr))) {
+ if (Access.IsWrite)
+ ++NumSkippedStackWrites;
+ else
+ ++NumSkippedStackReads;
+ return;
+ }
+
+ if (Access.IsWrite)
+ NumInstrumentedWrites++;
+ else
+ NumInstrumentedReads++;
+
+ if (Access.MaybeMask) {
+ instrumentMaskedLoadOrStore(DL, Access.MaybeMask, I, Access.Addr,
+ Access.AccessTy, Access.IsWrite);
+ } else {
+ // Since the access counts will be accumulated across the entire
allocation,
+ // we only update the shadow access count for the first location and thus
+ // don't need to worry about alignment and type size.
+ instrumentAddress(I, I, Access.Addr, Access.IsWrite);
+ }
+}
+
+void MemProfiler::instrumentAddress(Instruction *OrigIns,
+ Instruction *InsertBefore, Value *Addr,
+ bool IsWrite) {
+ IRBuilder<> IRB(InsertBefore);
+ Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
+
+ if (ClUseCalls) {
+ IRB.CreateCall(MemProfMemoryAccessCallback[IsWrite], AddrLong);
+ return;
+ }
+
+ Type *ShadowTy = ClHistogram ? Type::getInt8Ty(*C) : Type::getInt64Ty(*C);
+ Type *ShadowPtrTy = PointerType::get(*C, 0);
+
+ Value *ShadowPtr = memToShadow(AddrLong, IRB);
+ Value *ShadowAddr = IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy);
+ Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);
+ // If we are profiling with histograms, add overflow protection at 255.
+ if (ClHistogram) {
+ Value *MaxCount = ConstantInt::get(Type::getInt8Ty(*C), 255);
+ Value *Cmp = IRB.CreateICmpULT(ShadowValue, MaxCount);
+ Instruction *IncBlock =
+ SplitBlockAndInsertIfThen(Cmp, InsertBefore, /*Unreachable=*/false);
+ IRB.SetInsertPoint(IncBlock);
+ }
+ Value *Inc = ConstantInt::get(ShadowTy, 1);
+ ShadowValue = IRB.CreateAdd(ShadowValue, Inc);
+ IRB.CreateStore(ShadowValue, ShadowAddr);
+}
+
+// Create the variable for the profile file name.
+void createProfileFileNameVar(Module &M) {
+ const MDString *MemProfFilename =
+ dyn_cast_or_null<MDString>(M.getModuleFlag("MemProfProfileFilename"));
+ if (!MemProfFilename)
+ return;
+ assert(!MemProfFilename->getString().empty() &&
+ "Unexpected MemProfProfileFilename metadata with empty string");
+ Constant *ProfileNameConst = ConstantDataArray::getString(
+ M.getContext(), MemProfFilename->getString(), true);
+ GlobalVariable *ProfileNameVar = new GlobalVariable(
+ M, ProfileNameConst->getType(), /*isConstant=*/true,
+ GlobalValue::WeakAnyLinkage, ProfileNameConst, MemProfFilenameVar);
+ const Triple &TT = M.getTargetTriple();
+ if (TT.supportsCOMDAT()) {
+ ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
+ ProfileNameVar->setComdat(M.getOrInsertComdat(MemProfFilenameVar));
+ }
+}
+
+// Set MemprofHistogramFlag as a Global veriable in IR. This makes it
accessible
+// to the runtime, changing shadow count behavior.
+void createMemprofHistogramFlagVar(Module &M) {
+ const StringRef VarName(MemProfHistogramFlagVar);
+ Type *IntTy1 = Type::getInt1Ty(M.getContext());
+ auto MemprofHistogramFlag = new GlobalVariable(
+ M, IntTy1, true, GlobalValue::WeakAnyLinkage,
+ Constant::getIntegerValue(IntTy1, APInt(1, ClHistogram)), VarName);
+ const Triple &TT = M.getTargetTriple();
+ if (TT.supportsCOMDAT()) {
+ MemprofHistogramFlag->setLinkage(GlobalValue::ExternalLinkage);
+ MemprofHistogramFlag->setComdat(M.getOrInsertComdat(VarName));
+ }
+ appendToCompilerUsed(M, MemprofHistogramFlag);
+}
+
+void createMemprofDefaultOptionsVar(Module &M) {
+ Constant *OptionsConst = ConstantDataArray::getString(
+ M.getContext(), MemprofRuntimeDefaultOptions, /*AddNull=*/true);
+ GlobalVariable *OptionsVar =
+ new GlobalVariable(M, OptionsConst->getType(), /*isConstant=*/true,
+ GlobalValue::WeakAnyLinkage, OptionsConst,
+ memprof::getMemprofOptionsSymbolName());
+ const Triple &TT = M.getTargetTriple();
+ if (TT.supportsCOMDAT()) {
+ OptionsVar->setLinkage(GlobalValue::ExternalLinkage);
+ OptionsVar->setComdat(M.getOrInsertComdat(OptionsVar->getName()));
+ }
+}
+
+bool ModuleMemProfiler::instrumentModule(Module &M) {
+
+ // Create a module constructor.
+ std::string MemProfVersion = std::to_string(LLVM_MEM_PROFILER_VERSION);
+ std::string VersionCheckName =
+ ClInsertVersionCheck ? (MemProfVersionCheckNamePrefix + MemProfVersion)
+ : "";
+ std::tie(MemProfCtorFunction, std::ignore) =
+ createSanitizerCtorAndInitFunctions(M, MemProfModuleCtorName,
+ MemProfInitName, /*InitArgTypes=*/{},
+ /*InitArgs=*/{}, VersionCheckName);
+
+ const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);
+ appendToGlobalCtors(M, MemProfCtorFunction, Priority);
+
+ createProfileFileNameVar(M);
+
+ createMemprofHistogramFlagVar(M);
+
+ createMemprofDefaultOptionsVar(M);
+
+ return true;
+}
+
+void MemProfiler::initializeCallbacks(Module &M) {
+ IRBuilder<> IRB(*C);
+
+ for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
+ const std::string TypeStr = AccessIsWrite ? "store" : "load";
+ const std::string HistPrefix = ClHistogram ? "hist_" : "";
+
+ SmallVector<Type *, 2> Args1{1, IntptrTy};
+ MemProfMemoryAccessCallback[AccessIsWrite] = M.getOrInsertFunction(
+ ClMemoryAccessCallbackPrefix + HistPrefix + TypeStr,
+ FunctionType::get(IRB.getVoidTy(), Args1, false));
+ }
+ MemProfMemmove = M.getOrInsertFunction(
+ ClMemoryAccessCallbackPrefix + "memmove", PtrTy, PtrTy, PtrTy, IntptrTy);
+ MemProfMemcpy = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix +
"memcpy",
+ PtrTy, PtrTy, PtrTy, IntptrTy);
+ MemProfMemset =
+ M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memset", PtrTy,
+ PtrTy, IRB.getInt32Ty(), IntptrTy);
+}
+
+bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) {
+ // For each NSObject descendant having a +load method, this method is invoked
+ // by the ObjC runtime before any of the static constructors is called.
+ // Therefore we need to instrument such methods with a call to __memprof_init
+ // at the beginning in order to initialize our runtime before any access to
+ // the shadow memory.
+ // We cannot just ignore these methods, because they may call other
+ // instrumented functions.
+ if (F.getName().contains(" load]")) {
+ FunctionCallee MemProfInitFunction =
+ declareSanitizerInitFunction(*F.getParent(), MemProfInitName, {});
+ IRBuilder<> IRB(&F.front(), F.front().begin());
+ IRB.CreateCall(MemProfInitFunction, {});
+ return true;
+ }
+ return false;
+}
+
+bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) {
+ IRBuilder<> IRB(&F.front().front());
+ Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
+ MemProfShadowMemoryDynamicAddress, IntptrTy);
+ if (F.getParent()->getPICLevel() == PICLevel::NotPIC)
+ cast<GlobalVariable>(GlobalDynamicAddress)->setDSOLocal(true);
+ DynamicShadowOffset = IRB.CreateLoad(IntptrTy, GlobalDynamicAddress);
+ return true;
+}
+
+bool MemProfiler::instrumentFunction(Function &F) {
+ if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
+ return false;
+ if (ClDebugFunc == F.getName())
+ return false;
+ if (F.getName().starts_with("__memprof_"))
+ return false;
+
+ bool FunctionModified = false;
+
+ // If needed, insert __memprof_init.
+ // This function needs to be called even if the function body is not
+ // instrumented.
+ if (maybeInsertMemProfInitAtFunctionEntry(F))
+ FunctionModified = true;
+
+ LLVM_DEBUG(dbgs() << "MEMPROF instrumenting:\n" << F << "\n");
+
+ initializeCallbacks(*F.getParent());
+
+ SmallVector<Instruction *, 16> ToInstrument;
+
+ // Fill the set of memory operations to instrument.
+ for (auto &BB : F) {
+ for (auto &Inst : BB) {
+ if (isInterestingMemoryAccess(&Inst) || isa<MemIntrinsic>(Inst))
+ ToInstrument.push_back(&Inst);
+ }
+ }
+
+ if (ToInstrument.empty()) {
+ LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified
+ << " " << F << "\n");
+
+ return FunctionModified;
+ }
+
+ FunctionModified |= insertDynamicShadowAtFunctionEntry(F);
+
+ int NumInstrumented = 0;
+ for (auto *Inst : ToInstrument) {
+ if (ClDebugMin < 0 || ClDebugMax < 0 ||
+ (NumInstrumented >= ClDebugMin && NumInstrumented <= ClDebugMax)) {
+ std::optional<InterestingMemoryAccess> Access =
+ isInterestingMemoryAccess(Inst);
+ if (Access)
+ instrumentMop(Inst, F.getDataLayout(), *Access);
+ else
+ instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
+ }
+ NumInstrumented++;
+ }
+
+ if (NumInstrumented > 0)
+ FunctionModified = true;
+
+ LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified << "
"
+ << F << "\n");
+
+ return FunctionModified;
+}
\ No newline at end of file
diff --git a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
b/llvm/lib/Transforms/Instrumentation/MemProfUse.cpp
similarity index 56%
rename from llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
rename to llvm/lib/Transforms/Instrumentation/MemProfUse.cpp
index e06add7cf313b..17a745a4b3163 100644
--- a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemProfUse.cpp
@@ -1,4 +1,4 @@
-//===- MemProfiler.cpp - memory allocation and access profiler
------------===//
+//===- MemProfUse.cpp - memory allocation profile use pass --*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM
Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -6,34 +6,22 @@
//
//===----------------------------------------------------------------------===//
//
-// This file is a part of MemProfiler. Memory accesses are instrumented
-// to increment the access count held in a shadow memory location, or
-// alternatively to call into the runtime. Memory intrinsic calls (memmove,
-// memcpy, memset) are changed to call the memory profiling runtime version
-// instead.
+// This file implements the MemProfUsePass which reads memory profiling data
+// and uses it to add metadata to instructions to guide optimization.
//
//===----------------------------------------------------------------------===//
-#include "llvm/Transforms/Instrumentation/MemProfiler.h"
+#include "llvm/Transforms/Instrumentation/MemProfUse.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
-#include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/MemoryProfileInfo.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
-#include "llvm/Analysis/ValueTracking.h"
-#include "llvm/IR/Constant.h"
-#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Function.h"
-#include "llvm/IR/GlobalValue.h"
-#include "llvm/IR/IRBuilder.h"
-#include "llvm/IR/Instruction.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Module.h"
-#include "llvm/IR/Type.h"
-#include "llvm/IR/Value.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/InstrProfReader.h"
#include "llvm/ProfileData/MemProfCommon.h"
@@ -42,10 +30,7 @@
#include "llvm/Support/Debug.h"
#include "llvm/Support/HashBuilder.h"
#include "llvm/Support/VirtualFileSystem.h"
-#include "llvm/TargetParser/Triple.h"
-#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/LongestCommonSequence.h"
-#include "llvm/Transforms/Utils/ModuleUtils.h"
#include <map>
#include <set>
@@ -60,95 +45,6 @@ extern cl::opt<bool> NoPGOWarnMismatch;
extern cl::opt<bool> NoPGOWarnMismatchComdatWeak;
} // namespace llvm
-constexpr int LLVM_MEM_PROFILER_VERSION = 1;
-
-// Size of memory mapped to a single shadow location.
-constexpr uint64_t DefaultMemGranularity = 64;
-
-// Size of memory mapped to a single histogram bucket.
-constexpr uint64_t HistogramGranularity = 8;
-
-// Scale from granularity down to shadow size.
-constexpr uint64_t DefaultShadowScale = 3;
-
-constexpr char MemProfModuleCtorName[] = "memprof.module_ctor";
-constexpr uint64_t MemProfCtorAndDtorPriority = 1;
-// On Emscripten, the system needs more than one priorities for constructors.
-constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = 50;
-constexpr char MemProfInitName[] = "__memprof_init";
-constexpr char MemProfVersionCheckNamePrefix[] =
- "__memprof_version_mismatch_check_v";
-
-constexpr char MemProfShadowMemoryDynamicAddress[] =
- "__memprof_shadow_memory_dynamic_address";
-
-constexpr char MemProfFilenameVar[] = "__memprof_profile_filename";
-
-constexpr char MemProfHistogramFlagVar[] = "__memprof_histogram";
-
-// Command-line flags.
-
-static cl::opt<bool> ClInsertVersionCheck(
- "memprof-guard-against-version-mismatch",
- cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden,
- cl::init(true));
-
-// This flag may need to be replaced with -f[no-]memprof-reads.
-static cl::opt<bool> ClInstrumentReads("memprof-instrument-reads",
- cl::desc("instrument read
instructions"),
- cl::Hidden, cl::init(true));
-
-static cl::opt<bool>
- ClInstrumentWrites("memprof-instrument-writes",
- cl::desc("instrument write instructions"), cl::Hidden,
- cl::init(true));
-
-static cl::opt<bool> ClInstrumentAtomics(
- "memprof-instrument-atomics",
- cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
- cl::init(true));
-
-static cl::opt<bool> ClUseCalls(
- "memprof-use-callbacks",
- cl::desc("Use callbacks instead of inline instrumentation sequences."),
- cl::Hidden, cl::init(false));
-
-static cl::opt<std::string>
- ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix",
- cl::desc("Prefix for memory access
callbacks"),
- cl::Hidden, cl::init("__memprof_"));
-
-// These flags allow to change the shadow mapping.
-// The shadow mapping looks like
-// Shadow = ((Mem & mask) >> scale) + offset
-
-static cl::opt<int> ClMappingScale("memprof-mapping-scale",
- cl::desc("scale of memprof shadow mapping"),
- cl::Hidden, cl::init(DefaultShadowScale));
-
-static cl::opt<int>
- ClMappingGranularity("memprof-mapping-granularity",
- cl::desc("granularity of memprof shadow mapping"),
- cl::Hidden, cl::init(DefaultMemGranularity));
-
-static cl::opt<bool> ClStack("memprof-instrument-stack",
- cl::desc("Instrument scalar stack variables"),
- cl::Hidden, cl::init(false));
-
-// Debug flags.
-
-static cl::opt<int> ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden,
- cl::init(0));
-
-static cl::opt<std::string> ClDebugFunc("memprof-debug-func", cl::Hidden,
- cl::desc("Debug func"));
-
-static cl::opt<int> ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"),
- cl::Hidden, cl::init(-1));
-
-static cl::opt<int> ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"),
- cl::Hidden, cl::init(-1));
-
// By default disable matching of allocation profiles onto operator new that
// already explicitly pass a hot/cold hint, since we don't currently
// override these hints anyway.
@@ -158,21 +54,12 @@ static cl::opt<bool> ClMemProfMatchHotColdNew(
"Match allocation profiles onto existing hot/cold operator new calls"),
cl::Hidden, cl::init(false));
-static cl::opt<bool> ClHistogram("memprof-histogram",
- cl::desc("Collect access count histograms"),
- cl::Hidden, cl::init(false));
-
static cl::opt<bool>
ClPrintMemProfMatchInfo("memprof-print-match-info",
cl::desc("Print matching stats for each allocation
"
"context in this module's profiles"),
cl::Hidden, cl::init(false));
-static cl::opt<std::string>
- MemprofRuntimeDefaultOptions("memprof-runtime-default-options",
- cl::desc("The default memprof options"),
- cl::Hidden, cl::init(""));
-
static cl::opt<bool>
SalvageStaleProfile("memprof-salvage-stale-profile",
cl::desc("Salvage stale MemProf profile"),
@@ -192,12 +79,6 @@ static cl::opt<unsigned> MinMatchedColdBytePercent(
"memprof-matching-cold-threshold", cl::init(100), cl::Hidden,
cl::desc("Min percent of cold bytes matched to hint allocation cold"));
-// Instrumentation statistics
-STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
-STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
-STATISTIC(NumSkippedStackReads, "Number of non-instrumented stack reads");
-STATISTIC(NumSkippedStackWrites, "Number of non-instrumented stack writes");
-
// Matching statistics
STATISTIC(NumOfMemProfMissing, "Number of functions without memory profile.");
STATISTIC(NumOfMemProfMismatch,
@@ -214,91 +95,6 @@ STATISTIC(NumOfMemProfMatchedAllocs,
STATISTIC(NumOfMemProfMatchedCallSites,
"Number of matched memory profile callsites.");
-namespace {
-
-/// This struct defines the shadow mapping using the rule:
-/// shadow = ((mem & mask) >> Scale) ADD DynamicShadowOffset.
-struct ShadowMapping {
- ShadowMapping() {
- Scale = ClMappingScale;
- Granularity = ClHistogram ? HistogramGranularity : ClMappingGranularity;
- Mask = ~(Granularity - 1);
- }
-
- int Scale;
- int Granularity;
- uint64_t Mask; // Computed as ~(Granularity-1)
-};
-
-static uint64_t getCtorAndDtorPriority(Triple &TargetTriple) {
- return TargetTriple.isOSEmscripten() ? MemProfEmscriptenCtorAndDtorPriority
- : MemProfCtorAndDtorPriority;
-}
-
-struct InterestingMemoryAccess {
- Value *Addr = nullptr;
- bool IsWrite;
- Type *AccessTy;
- Value *MaybeMask = nullptr;
-};
-
-/// Instrument the code in module to profile memory accesses.
-class MemProfiler {
-public:
- MemProfiler(Module &M) {
- C = &(M.getContext());
- LongSize = M.getDataLayout().getPointerSizeInBits();
- IntptrTy = Type::getIntNTy(*C, LongSize);
- PtrTy = PointerType::getUnqual(*C);
- }
-
- /// If it is an interesting memory access, populate information
- /// about the access and return a InterestingMemoryAccess struct.
- /// Otherwise return std::nullopt.
- std::optional<InterestingMemoryAccess>
- isInterestingMemoryAccess(Instruction *I) const;
-
- void instrumentMop(Instruction *I, const DataLayout &DL,
- InterestingMemoryAccess &Access);
- void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
- Value *Addr, bool IsWrite);
- void instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
- Instruction *I, Value *Addr, Type *AccessTy,
- bool IsWrite);
- void instrumentMemIntrinsic(MemIntrinsic *MI);
- Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
- bool instrumentFunction(Function &F);
- bool maybeInsertMemProfInitAtFunctionEntry(Function &F);
- bool insertDynamicShadowAtFunctionEntry(Function &F);
-
-private:
- void initializeCallbacks(Module &M);
-
- LLVMContext *C;
- int LongSize;
- Type *IntptrTy;
- PointerType *PtrTy;
- ShadowMapping Mapping;
-
- // These arrays is indexed by AccessIsWrite
- FunctionCallee MemProfMemoryAccessCallback[2];
-
- FunctionCallee MemProfMemmove, MemProfMemcpy, MemProfMemset;
- Value *DynamicShadowOffset = nullptr;
-};
-
-class ModuleMemProfiler {
-public:
- ModuleMemProfiler(Module &M) { TargetTriple = M.getTargetTriple(); }
-
- bool instrumentModule(Module &);
-
-private:
- Triple TargetTriple;
- ShadowMapping Mapping;
- Function *MemProfCtorFunction = nullptr;
-};
-
// Options under which we need to record the context size info in the alloc
trie
// used to build metadata.
bool recordContextSizeInfo() {
@@ -306,426 +102,6 @@ bool recordContextSizeInfo() {
MinCallsiteColdBytePercent < 100;
}
-} // end anonymous namespace
-
-MemProfilerPass::MemProfilerPass() = default;
-
-PreservedAnalyses MemProfilerPass::run(Function &F,
- AnalysisManager<Function> &AM) {
- assert((!ClHistogram || ClMappingGranularity == DefaultMemGranularity) &&
- "Memprof with histogram only supports default mapping granularity");
- Module &M = *F.getParent();
- MemProfiler Profiler(M);
- if (Profiler.instrumentFunction(F))
- return PreservedAnalyses::none();
- return PreservedAnalyses::all();
-}
-
-ModuleMemProfilerPass::ModuleMemProfilerPass() = default;
-
-PreservedAnalyses ModuleMemProfilerPass::run(Module &M,
- AnalysisManager<Module> &AM) {
-
- ModuleMemProfiler Profiler(M);
- if (Profiler.instrumentModule(M))
- return PreservedAnalyses::none();
- return PreservedAnalyses::all();
-}
-
-Value *MemProfiler::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
- // (Shadow & mask) >> scale
- Shadow = IRB.CreateAnd(Shadow, Mapping.Mask);
- Shadow = IRB.CreateLShr(Shadow, Mapping.Scale);
- // (Shadow >> scale) | offset
- assert(DynamicShadowOffset);
- return IRB.CreateAdd(Shadow, DynamicShadowOffset);
-}
-
-// Instrument memset/memmove/memcpy
-void MemProfiler::instrumentMemIntrinsic(MemIntrinsic *MI) {
- IRBuilder<> IRB(MI);
- if (isa<MemTransferInst>(MI)) {
- IRB.CreateCall(isa<MemMoveInst>(MI) ? MemProfMemmove : MemProfMemcpy,
- {MI->getOperand(0), MI->getOperand(1),
- IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
- } else if (isa<MemSetInst>(MI)) {
- IRB.CreateCall(
- MemProfMemset,
- {MI->getOperand(0),
- IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
- IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
- }
- MI->eraseFromParent();
-}
-
-std::optional<InterestingMemoryAccess>
-MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
- // Do not instrument the load fetching the dynamic shadow address.
- if (DynamicShadowOffset == I)
- return std::nullopt;
-
- InterestingMemoryAccess Access;
-
- if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
- if (!ClInstrumentReads)
- return std::nullopt;
- Access.IsWrite = false;
- Access.AccessTy = LI->getType();
- Access.Addr = LI->getPointerOperand();
- } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
- if (!ClInstrumentWrites)
- return std::nullopt;
- Access.IsWrite = true;
- Access.AccessTy = SI->getValueOperand()->getType();
- Access.Addr = SI->getPointerOperand();
- } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
- if (!ClInstrumentAtomics)
- return std::nullopt;
- Access.IsWrite = true;
- Access.AccessTy = RMW->getValOperand()->getType();
- Access.Addr = RMW->getPointerOperand();
- } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
- if (!ClInstrumentAtomics)
- return std::nullopt;
- Access.IsWrite = true;
- Access.AccessTy = XCHG->getCompareOperand()->getType();
- Access.Addr = XCHG->getPointerOperand();
- } else if (auto *CI = dyn_cast<CallInst>(I)) {
- auto *F = CI->getCalledFunction();
- if (F && (F->getIntrinsicID() == Intrinsic::masked_load ||
- F->getIntrinsicID() == Intrinsic::masked_store)) {
- unsigned OpOffset = 0;
- if (F->getIntrinsicID() == Intrinsic::masked_store) {
- if (!ClInstrumentWrites)
- return std::nullopt;
- // Masked store has an initial operand for the value.
- OpOffset = 1;
- Access.AccessTy = CI->getArgOperand(0)->getType();
- Access.IsWrite = true;
- } else {
- if (!ClInstrumentReads)
- return std::nullopt;
- Access.AccessTy = CI->getType();
- Access.IsWrite = false;
- }
-
- auto *BasePtr = CI->getOperand(0 + OpOffset);
- Access.MaybeMask = CI->getOperand(2 + OpOffset);
- Access.Addr = BasePtr;
- }
- }
-
- if (!Access.Addr)
- return std::nullopt;
-
- // Do not instrument accesses from different address spaces; we cannot deal
- // with them.
- Type *PtrTy = cast<PointerType>(Access.Addr->getType()->getScalarType());
- if (PtrTy->getPointerAddressSpace() != 0)
- return std::nullopt;
-
- // Ignore swifterror addresses.
- // swifterror memory addresses are mem2reg promoted by instruction
- // selection. As such they cannot have regular uses like an instrumentation
- // function and it makes no sense to track them as memory.
- if (Access.Addr->isSwiftError())
- return std::nullopt;
-
- // Peel off GEPs and BitCasts.
- auto *Addr = Access.Addr->stripInBoundsOffsets();
-
- if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) {
- // Do not instrument PGO counter updates.
- if (GV->hasSection()) {
- StringRef SectionName = GV->getSection();
- // Check if the global is in the PGO counters section.
- auto OF = I->getModule()->getTargetTriple().getObjectFormat();
- if (SectionName.ends_with(
- getInstrProfSectionName(IPSK_cnts, OF,
/*AddSegmentInfo=*/false)))
- return std::nullopt;
- }
-
- // Do not instrument accesses to LLVM internal variables.
- if (GV->getName().starts_with("__llvm"))
- return std::nullopt;
- }
-
- return Access;
-}
-
-void MemProfiler::instrumentMaskedLoadOrStore(const DataLayout &DL, Value
*Mask,
- Instruction *I, Value *Addr,
- Type *AccessTy, bool IsWrite) {
- auto *VTy = cast<FixedVectorType>(AccessTy);
- unsigned Num = VTy->getNumElements();
- auto *Zero = ConstantInt::get(IntptrTy, 0);
- for (unsigned Idx = 0; Idx < Num; ++Idx) {
- Value *InstrumentedAddress = nullptr;
- Instruction *InsertBefore = I;
- if (auto *Vector = dyn_cast<ConstantVector>(Mask)) {
- // dyn_cast as we might get UndefValue
- if (auto *Masked = dyn_cast<ConstantInt>(Vector->getOperand(Idx))) {
- if (Masked->isZero())
- // Mask is constant false, so no instrumentation needed.
- continue;
- // If we have a true or undef value, fall through to instrumentAddress.
- // with InsertBefore == I
- }
- } else {
- IRBuilder<> IRB(I);
- Value *MaskElem = IRB.CreateExtractElement(Mask, Idx);
- Instruction *ThenTerm = SplitBlockAndInsertIfThen(MaskElem, I, false);
- InsertBefore = ThenTerm;
- }
-
- IRBuilder<> IRB(InsertBefore);
- InstrumentedAddress =
- IRB.CreateGEP(VTy, Addr, {Zero, ConstantInt::get(IntptrTy, Idx)});
- instrumentAddress(I, InsertBefore, InstrumentedAddress, IsWrite);
- }
-}
-
-void MemProfiler::instrumentMop(Instruction *I, const DataLayout &DL,
- InterestingMemoryAccess &Access) {
- // Skip instrumentation of stack accesses unless requested.
- if (!ClStack && isa<AllocaInst>(getUnderlyingObject(Access.Addr))) {
- if (Access.IsWrite)
- ++NumSkippedStackWrites;
- else
- ++NumSkippedStackReads;
- return;
- }
-
- if (Access.IsWrite)
- NumInstrumentedWrites++;
- else
- NumInstrumentedReads++;
-
- if (Access.MaybeMask) {
- instrumentMaskedLoadOrStore(DL, Access.MaybeMask, I, Access.Addr,
- Access.AccessTy, Access.IsWrite);
- } else {
- // Since the access counts will be accumulated across the entire
allocation,
- // we only update the shadow access count for the first location and thus
- // don't need to worry about alignment and type size.
- instrumentAddress(I, I, Access.Addr, Access.IsWrite);
- }
-}
-
-void MemProfiler::instrumentAddress(Instruction *OrigIns,
- Instruction *InsertBefore, Value *Addr,
- bool IsWrite) {
- IRBuilder<> IRB(InsertBefore);
- Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
-
- if (ClUseCalls) {
- IRB.CreateCall(MemProfMemoryAccessCallback[IsWrite], AddrLong);
- return;
- }
-
- Type *ShadowTy = ClHistogram ? Type::getInt8Ty(*C) : Type::getInt64Ty(*C);
- Type *ShadowPtrTy = PointerType::get(*C, 0);
-
- Value *ShadowPtr = memToShadow(AddrLong, IRB);
- Value *ShadowAddr = IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy);
- Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);
- // If we are profiling with histograms, add overflow protection at 255.
- if (ClHistogram) {
- Value *MaxCount = ConstantInt::get(Type::getInt8Ty(*C), 255);
- Value *Cmp = IRB.CreateICmpULT(ShadowValue, MaxCount);
- Instruction *IncBlock =
- SplitBlockAndInsertIfThen(Cmp, InsertBefore, /*Unreachable=*/false);
- IRB.SetInsertPoint(IncBlock);
- }
- Value *Inc = ConstantInt::get(ShadowTy, 1);
- ShadowValue = IRB.CreateAdd(ShadowValue, Inc);
- IRB.CreateStore(ShadowValue, ShadowAddr);
-}
-
-// Create the variable for the profile file name.
-void createProfileFileNameVar(Module &M) {
- const MDString *MemProfFilename =
- dyn_cast_or_null<MDString>(M.getModuleFlag("MemProfProfileFilename"));
- if (!MemProfFilename)
- return;
- assert(!MemProfFilename->getString().empty() &&
- "Unexpected MemProfProfileFilename metadata with empty string");
- Constant *ProfileNameConst = ConstantDataArray::getString(
- M.getContext(), MemProfFilename->getString(), true);
- GlobalVariable *ProfileNameVar = new GlobalVariable(
- M, ProfileNameConst->getType(), /*isConstant=*/true,
- GlobalValue::WeakAnyLinkage, ProfileNameConst, MemProfFilenameVar);
- const Triple &TT = M.getTargetTriple();
- if (TT.supportsCOMDAT()) {
- ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
- ProfileNameVar->setComdat(M.getOrInsertComdat(MemProfFilenameVar));
- }
-}
-
-// Set MemprofHistogramFlag as a Global veriable in IR. This makes it
accessible
-// to the runtime, changing shadow count behavior.
-void createMemprofHistogramFlagVar(Module &M) {
- const StringRef VarName(MemProfHistogramFlagVar);
- Type *IntTy1 = Type::getInt1Ty(M.getContext());
- auto MemprofHistogramFlag = new GlobalVariable(
- M, IntTy1, true, GlobalValue::WeakAnyLinkage,
- Constant::getIntegerValue(IntTy1, APInt(1, ClHistogram)), VarName);
- const Triple &TT = M.getTargetTriple();
- if (TT.supportsCOMDAT()) {
- MemprofHistogramFlag->setLinkage(GlobalValue::ExternalLinkage);
- MemprofHistogramFlag->setComdat(M.getOrInsertComdat(VarName));
- }
- appendToCompilerUsed(M, MemprofHistogramFlag);
-}
-
-void createMemprofDefaultOptionsVar(Module &M) {
- Constant *OptionsConst = ConstantDataArray::getString(
- M.getContext(), MemprofRuntimeDefaultOptions, /*AddNull=*/true);
- GlobalVariable *OptionsVar = new GlobalVariable(
- M, OptionsConst->getType(), /*isConstant=*/true,
- GlobalValue::WeakAnyLinkage, OptionsConst,
getMemprofOptionsSymbolName());
- const Triple &TT = M.getTargetTriple();
- if (TT.supportsCOMDAT()) {
- OptionsVar->setLinkage(GlobalValue::ExternalLinkage);
- OptionsVar->setComdat(M.getOrInsertComdat(OptionsVar->getName()));
- }
-}
-
-bool ModuleMemProfiler::instrumentModule(Module &M) {
-
- // Create a module constructor.
- std::string MemProfVersion = std::to_string(LLVM_MEM_PROFILER_VERSION);
- std::string VersionCheckName =
- ClInsertVersionCheck ? (MemProfVersionCheckNamePrefix + MemProfVersion)
- : "";
- std::tie(MemProfCtorFunction, std::ignore) =
- createSanitizerCtorAndInitFunctions(M, MemProfModuleCtorName,
- MemProfInitName, /*InitArgTypes=*/{},
- /*InitArgs=*/{}, VersionCheckName);
-
- const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);
- appendToGlobalCtors(M, MemProfCtorFunction, Priority);
-
- createProfileFileNameVar(M);
-
- createMemprofHistogramFlagVar(M);
-
- createMemprofDefaultOptionsVar(M);
-
- return true;
-}
-
-void MemProfiler::initializeCallbacks(Module &M) {
- IRBuilder<> IRB(*C);
-
- for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
- const std::string TypeStr = AccessIsWrite ? "store" : "load";
- const std::string HistPrefix = ClHistogram ? "hist_" : "";
-
- SmallVector<Type *, 2> Args1{1, IntptrTy};
- MemProfMemoryAccessCallback[AccessIsWrite] = M.getOrInsertFunction(
- ClMemoryAccessCallbackPrefix + HistPrefix + TypeStr,
- FunctionType::get(IRB.getVoidTy(), Args1, false));
- }
- MemProfMemmove = M.getOrInsertFunction(
- ClMemoryAccessCallbackPrefix + "memmove", PtrTy, PtrTy, PtrTy, IntptrTy);
- MemProfMemcpy = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix +
"memcpy",
- PtrTy, PtrTy, PtrTy, IntptrTy);
- MemProfMemset =
- M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memset", PtrTy,
- PtrTy, IRB.getInt32Ty(), IntptrTy);
-}
-
-bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) {
- // For each NSObject descendant having a +load method, this method is invoked
- // by the ObjC runtime before any of the static constructors is called.
- // Therefore we need to instrument such methods with a call to __memprof_init
- // at the beginning in order to initialize our runtime before any access to
- // the shadow memory.
- // We cannot just ignore these methods, because they may call other
- // instrumented functions.
- if (F.getName().contains(" load]")) {
- FunctionCallee MemProfInitFunction =
- declareSanitizerInitFunction(*F.getParent(), MemProfInitName, {});
- IRBuilder<> IRB(&F.front(), F.front().begin());
- IRB.CreateCall(MemProfInitFunction, {});
- return true;
- }
- return false;
-}
-
-bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) {
- IRBuilder<> IRB(&F.front().front());
- Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
- MemProfShadowMemoryDynamicAddress, IntptrTy);
- if (F.getParent()->getPICLevel() == PICLevel::NotPIC)
- cast<GlobalVariable>(GlobalDynamicAddress)->setDSOLocal(true);
- DynamicShadowOffset = IRB.CreateLoad(IntptrTy, GlobalDynamicAddress);
- return true;
-}
-
-bool MemProfiler::instrumentFunction(Function &F) {
- if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
- return false;
- if (ClDebugFunc == F.getName())
- return false;
- if (F.getName().starts_with("__memprof_"))
- return false;
-
- bool FunctionModified = false;
-
- // If needed, insert __memprof_init.
- // This function needs to be called even if the function body is not
- // instrumented.
- if (maybeInsertMemProfInitAtFunctionEntry(F))
- FunctionModified = true;
-
- LLVM_DEBUG(dbgs() << "MEMPROF instrumenting:\n" << F << "\n");
-
- initializeCallbacks(*F.getParent());
-
- SmallVector<Instruction *, 16> ToInstrument;
-
- // Fill the set of memory operations to instrument.
- for (auto &BB : F) {
- for (auto &Inst : BB) {
- if (isInterestingMemoryAccess(&Inst) || isa<MemIntrinsic>(Inst))
- ToInstrument.push_back(&Inst);
- }
- }
-
- if (ToInstrument.empty()) {
- LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified
- << " " << F << "\n");
-
- return FunctionModified;
- }
-
- FunctionModified |= insertDynamicShadowAtFunctionEntry(F);
-
- int NumInstrumented = 0;
- for (auto *Inst : ToInstrument) {
- if (ClDebugMin < 0 || ClDebugMax < 0 ||
- (NumInstrumented >= ClDebugMin && NumInstrumented <= ClDebugMax)) {
- std::optional<InterestingMemoryAccess> Access =
- isInterestingMemoryAccess(Inst);
- if (Access)
- instrumentMop(Inst, F.getDataLayout(), *Access);
- else
- instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
- }
- NumInstrumented++;
- }
-
- if (NumInstrumented > 0)
- FunctionModified = true;
-
- LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified << "
"
- << F << "\n");
-
- return FunctionModified;
-}
-
static void addCallsiteMetadata(Instruction &I,
ArrayRef<uint64_t> InlinedCallStack,
LLVMContext &Ctx) {
@@ -1367,4 +743,4 @@ PreservedAnalyses MemProfUsePass::run(Module &M,
ModuleAnalysisManager &AM) {
}
return PreservedAnalyses::none();
-}
+}
\ No newline at end of file
diff --git a/llvm/unittests/Transforms/Instrumentation/MemProfUseTest.cpp
b/llvm/unittests/Transforms/Instrumentation/MemProfUseTest.cpp
index c4f38863e6f0f..0dd819a10e0b1 100644
--- a/llvm/unittests/Transforms/Instrumentation/MemProfUseTest.cpp
+++ b/llvm/unittests/Transforms/Instrumentation/MemProfUseTest.cpp
@@ -6,6 +6,7 @@
//
//===----------------------------------------------------------------------===//
+#include "llvm/Transforms/Instrumentation/MemProfUse.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/AsmParser/Parser.h"
#include "llvm/IR/LLVMContext.h"
@@ -17,7 +18,6 @@
#include "llvm/ProfileData/MemProf.h"
#include "llvm/Support/SourceMgr.h"
#include "llvm/Testing/Support/Error.h"
-#include "llvm/Transforms/Instrumentation/MemProfiler.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
