================
@@ -1513,6 +1522,171 @@ class InProcessThinBackend : public ThinBackendProc {
return Error::success();
}
};
+
+/// This backend is utilized in the first round of a two-codegen round process.
+/// It first saves optimized bitcode files to disk before the codegen process
+/// begins. After codegen, it stores the resulting object files in a scratch
+/// buffer. Note the codegen data stored in the scratch buffer will be
extracted
+/// and merged in the subsequent step.
+class FirstRoundThinBackend : public InProcessThinBackend {
+ AddStreamFn IRAddStream;
+ FileCache IRCache;
+
+public:
+ FirstRoundThinBackend(
+ const Config &Conf, ModuleSummaryIndex &CombinedIndex,
+ ThreadPoolStrategy ThinLTOParallelism,
+ const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
+ AddStreamFn CGAddStream, FileCache CGCache, AddStreamFn IRAddStream,
+ FileCache IRCache)
+ : InProcessThinBackend(Conf, CombinedIndex, ThinLTOParallelism,
+ ModuleToDefinedGVSummaries,
std::move(CGAddStream),
+ std::move(CGCache), /*OnWrite=*/nullptr,
+ /*ShouldEmitIndexFiles=*/false,
+ /*ShouldEmitImportsFiles=*/false),
+ IRAddStream(std::move(IRAddStream)), IRCache(std::move(IRCache)) {}
+
+ Error runThinLTOBackendThread(
+ AddStreamFn CGAddStream, FileCache CGCache, unsigned Task,
+ BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,
+ const FunctionImporter::ImportMapTy &ImportList,
+ const FunctionImporter::ExportSetTy &ExportList,
+ const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>
&ResolvedODR,
+ const GVSummaryMapTy &DefinedGlobals,
+ MapVector<StringRef, BitcodeModule> &ModuleMap) override {
+ auto RunThinBackend = [&](AddStreamFn CGAddStream,
+ AddStreamFn IRAddStream) {
+ LTOLLVMContext BackendContext(Conf);
+ Expected<std::unique_ptr<Module>> MOrErr =
BM.parseModule(BackendContext);
+ if (!MOrErr)
+ return MOrErr.takeError();
+
+ return thinBackend(Conf, Task, CGAddStream, **MOrErr, CombinedIndex,
+ ImportList, DefinedGlobals, &ModuleMap,
+ Conf.CodeGenOnly, IRAddStream);
+ };
+
+ auto ModuleID = BM.getModuleIdentifier();
+ // Like InProcessThinBackend, we produce index files as needed for
+ // FirstRoundThinBackend. However, these files are not generated for
+ // SecondRoundThinBackend.
+ if (ShouldEmitIndexFiles) {
+ if (auto E = emitFiles(ImportList, ModuleID, ModuleID.str()))
+ return E;
+ }
+
+ assert((CGCache.isValid() == IRCache.isValid()) &&
+ "Both caches for CG and IR should have matching availability");
+ if (!CGCache.isValid() || !CombinedIndex.modulePaths().count(ModuleID) ||
+ all_of(CombinedIndex.getModuleHash(ModuleID),
+ [](uint32_t V) { return V == 0; }))
+ // Cache disabled or no entry for this module in the combined index or
+ // no module hash.
+ return RunThinBackend(CGAddStream, IRAddStream);
+
+ // Get CGKey for caching object in CGCache.
+ std::string CGKey = computeLTOCacheKey(
+ Conf, CombinedIndex, ModuleID, ImportList, ExportList, ResolvedODR,
+ DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls);
+ Expected<AddStreamFn> CacheCGAddStreamOrErr =
+ CGCache(Task, CGKey, ModuleID);
+ if (Error Err = CacheCGAddStreamOrErr.takeError())
+ return Err;
+ AddStreamFn &CacheCGAddStream = *CacheCGAddStreamOrErr;
+
+ // Get IRKey for caching (optimized) IR in IRCache with an extra ID.
+ std::string IRKey = computeLTOCacheKey(
+ Conf, CombinedIndex, ModuleID, ImportList, ExportList, ResolvedODR,
+ DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls, /*ExtraID=*/"IR");
+ Expected<AddStreamFn> CacheIRAddStreamOrErr =
+ IRCache(Task, IRKey, ModuleID);
+ if (Error Err = CacheIRAddStreamOrErr.takeError())
+ return Err;
+ AddStreamFn &CacheIRAddStream = *CacheIRAddStreamOrErr;
+
+ assert((CacheCGAddStream == nullptr) == (CacheIRAddStream == nullptr) &&
+ "Both CG and IR caching should be matched");
+ if (CacheIRAddStream) {
+ LLVM_DEBUG(dbgs() << "[FirstRound] Cache Miss for "
+ << BM.getModuleIdentifier() << "\n");
+ return RunThinBackend(CacheCGAddStream, CacheIRAddStream);
+ }
+
+ return Error::success();
+ }
+};
+
+/// This backend operates in the second round of a two-codegen round process.
+/// It starts by reading the optimized bitcode files that were saved during the
+/// first round. The backend then executes the codegen only to further optimize
+/// the code, utilizing the codegen data merged from the first round. Finally,
+/// it writes the resulting object files as usual.
+class SecondRoundThinBackend : public InProcessThinBackend {
+ std::unique_ptr<SmallVector<StringRef>> IRFiles;
+ stable_hash CombinedCGDataHash;
+
+public:
+ SecondRoundThinBackend(
+ const Config &Conf, ModuleSummaryIndex &CombinedIndex,
+ ThreadPoolStrategy ThinLTOParallelism,
+ const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
+ AddStreamFn AddStream, FileCache Cache,
+ std::unique_ptr<SmallVector<StringRef>> IRFiles,
+ stable_hash CombinedCGDataHash)
+ : InProcessThinBackend(Conf, CombinedIndex, ThinLTOParallelism,
+ ModuleToDefinedGVSummaries, std::move(AddStream),
+ std::move(Cache),
+ /*OnWrite=*/nullptr,
+ /*ShouldEmitIndexFiles=*/false,
+ /*ShouldEmitImportsFiles=*/false),
+ IRFiles(std::move(IRFiles)), CombinedCGDataHash(CombinedCGDataHash) {}
+
+ virtual Error runThinLTOBackendThread(
+ AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM,
+ ModuleSummaryIndex &CombinedIndex,
+ const FunctionImporter::ImportMapTy &ImportList,
+ const FunctionImporter::ExportSetTy &ExportList,
+ const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>
&ResolvedODR,
+ const GVSummaryMapTy &DefinedGlobals,
+ MapVector<StringRef, BitcodeModule> &ModuleMap) override {
+ auto RunThinBackend = [&](AddStreamFn AddStream) {
+ LTOLLVMContext BackendContext(Conf);
+ std::unique_ptr<Module> LoadedModule =
+ cgdata::loadModuleForTwoRounds(BM, Task, BackendContext, *IRFiles);
+
+ return thinBackend(Conf, Task, AddStream, *LoadedModule, CombinedIndex,
+ ImportList, DefinedGlobals, &ModuleMap,
+ /*CodeGenOnly=*/true);
+ };
+
+ auto ModuleID = BM.getModuleIdentifier();
+ if (!Cache.isValid() || !CombinedIndex.modulePaths().count(ModuleID) ||
+ all_of(CombinedIndex.getModuleHash(ModuleID),
+ [](uint32_t V) { return V == 0; }))
+ // Cache disabled or no entry for this module in the combined index or
+ // no module hash.
+ return RunThinBackend(AddStream);
+
+ // Get Key for caching the final object file in Cache with the combined
+ // CGData hash.
+ std::string Key = computeLTOCacheKey(
+ Conf, CombinedIndex, ModuleID, ImportList, ExportList, ResolvedODR,
+ DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls,
+ /*ExtraID=*/std::to_string(CombinedCGDataHash));
----------------
kyulee-com wrote:
I think most developer builds benefit from using previous codegen summaries in
incremental builds, which assists with scaling. For final builds or getting the
optimal binary, this two-round support is quite useful as it runs well on a
single machine without requiring changes to the build system. I plan to use the
two-round support, particularly in scenarios without dthin-lto for now, but I'm
also open to explore this case for the dthin-lto scenario.
https://github.com/llvm/llvm-project/pull/90933
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
