gamesh411 created this revision.
Herald added subscribers: cfe-commits, jdoerfert, dkrupp, donat.nagy,
Szelethus, a.sidorin, baloghadamsoftware, whisperity.
Herald added a reviewer: martong.
Herald added a project: clang.
During CTU analysis of complex projects, the loaded AST-contents of
imported TUs can grow bigger than available system memory. This option
introduces a threshold on the number of TUs to be imported for a single
TU in order to prevent such cases.
Repository:
rC Clang
https://reviews.llvm.org/D59798
Files:
include/clang/CrossTU/CrossTranslationUnit.h
include/clang/Driver/CC1Options.td
include/clang/StaticAnalyzer/Core/AnalyzerOptions.h
lib/CrossTU/CrossTranslationUnit.cpp
lib/Frontend/CompilerInvocation.cpp
lib/StaticAnalyzer/Core/CallEvent.cpp
test/Analysis/ctu-import-threshold.c
unittests/CrossTU/CrossTranslationUnitTest.cpp
Index: unittests/CrossTU/CrossTranslationUnitTest.cpp
===================================================================
--- unittests/CrossTU/CrossTranslationUnitTest.cpp
+++ unittests/CrossTU/CrossTranslationUnitTest.cpp
@@ -23,8 +23,9 @@
class CTUASTConsumer : public clang::ASTConsumer {
public:
- explicit CTUASTConsumer(clang::CompilerInstance &CI, bool *Success)
- : CTU(CI), Success(Success) {}
+ explicit CTUASTConsumer(clang::CompilerInstance &CI, bool *Success,
+ unsigned ImportLimit)
+ : CTU(CI), Success(Success), ImportLimit(ImportLimit) {}
void HandleTranslationUnit(ASTContext &Ctx) {
const TranslationUnitDecl *TU = Ctx.getTranslationUnitDecl();
@@ -70,41 +71,54 @@
EXPECT_TRUE(llvm::sys::fs::exists(ASTFileName));
// Load the definition from the AST file.
- llvm::Expected<const FunctionDecl *> NewFDorError =
- CTU.getCrossTUDefinition(FD, "", IndexFileName);
- EXPECT_TRUE((bool)NewFDorError);
- const FunctionDecl *NewFD = *NewFDorError;
+ llvm::Expected<const FunctionDecl *> NewFDorError = handleExpected(
+ CTU.getCrossTUDefinition(FD, "", IndexFileName, false, ImportLimit),
+ []() { return nullptr; }, [](IndexError &) {});
- *Success = NewFD && NewFD->hasBody() && !OrigFDHasBody;
+ if (NewFDorError) {
+ const FunctionDecl *NewFD = *NewFDorError;
+ *Success = NewFD && NewFD->hasBody() && !OrigFDHasBody;
+ }
}
private:
CrossTranslationUnitContext CTU;
bool *Success;
+ unsigned ImportLimit;
};
class CTUAction : public clang::ASTFrontendAction {
public:
- CTUAction(bool *Success) : Success(Success) {}
+ CTUAction(bool *Success, unsigned ImportLimit)
+ : Success(Success), ImportLimit(ImportLimit) {}
protected:
std::unique_ptr<clang::ASTConsumer>
CreateASTConsumer(clang::CompilerInstance &CI, StringRef) override {
- return llvm::make_unique<CTUASTConsumer>(CI, Success);
+ return llvm::make_unique<CTUASTConsumer>(CI, Success, ImportLimit);
}
private:
bool *Success;
+ unsigned ImportLimit;
};
} // end namespace
TEST(CrossTranslationUnit, CanLoadFunctionDefinition) {
bool Success = false;
- EXPECT_TRUE(tooling::runToolOnCode(new CTUAction(&Success), "int f(int);"));
+ EXPECT_TRUE(
+ tooling::runToolOnCode(new CTUAction(&Success, 1u), "int f(int);"));
EXPECT_TRUE(Success);
}
+TEST(CrossTranslationUnit, RespectsLoadThreshold) {
+ bool Success = false;
+ EXPECT_TRUE(
+ tooling::runToolOnCode(new CTUAction(&Success, 0u), "int f(int);"));
+ EXPECT_FALSE(Success);
+}
+
TEST(CrossTranslationUnit, IndexFormatCanBeParsed) {
llvm::StringMap<std::string> Index;
Index["a"] = "/b/f1";
Index: test/Analysis/ctu-import-threshold.c
===================================================================
--- /dev/null
+++ test/Analysis/ctu-import-threshold.c
@@ -0,0 +1,6 @@
+// Ensure analyzer-ctu-import-threshold option is a recognized option.
+//
+// RUN: %clang_cc1 -analyze -analyzer-ctu-import-threshold 500 -verify %s
+// RUN: %clang_cc1 -analyze -analyzer-ctu-import-threshold=500 -verify %s
+//
+// expected-no-diagnostics
Index: lib/StaticAnalyzer/Core/CallEvent.cpp
===================================================================
--- lib/StaticAnalyzer/Core/CallEvent.cpp
+++ lib/StaticAnalyzer/Core/CallEvent.cpp
@@ -571,8 +571,9 @@
cross_tu::CrossTranslationUnitContext &CTUCtx =
*Engine.getCrossTranslationUnitContext();
llvm::Expected<const FunctionDecl *> CTUDeclOrError =
- CTUCtx.getCrossTUDefinition(FD, Opts.CTUDir, Opts.CTUIndexName,
- Opts.DisplayCTUProgress);
+ CTUCtx.getCrossTUDefinition(FD, Opts.getCTUDir(), Opts.getCTUIndexName(),
+ Opts.AnalyzerDisplayCtuProgress,
+ Opts.CTUImportThreshold);
if (!CTUDeclOrError) {
handleAllErrors(CTUDeclOrError.takeError(),
Index: lib/Frontend/CompilerInvocation.cpp
===================================================================
--- lib/Frontend/CompilerInvocation.cpp
+++ lib/Frontend/CompilerInvocation.cpp
@@ -301,6 +301,10 @@
Opts.NoRetryExhausted = Args.hasArg(OPT_analyzer_disable_retry_exhausted);
Opts.AnalyzeAll = Args.hasArg(OPT_analyzer_opt_analyze_headers);
Opts.AnalyzerDisplayProgress = Args.hasArg(OPT_analyzer_display_progress);
+ Opts.AnalyzerDisplayCtuProgress =
+ Args.hasArg(OPT_analyzer_display_ctu_progress);
+ Opts.CTUImportThreshold = getLastArgIntValue(
+ Args, OPT_analyzer_ctu_import_threshold, Opts.CTUImportThreshold, Diags);
Opts.AnalyzeNestedBlocks =
Args.hasArg(OPT_analyzer_opt_analyze_nested_blocks);
Opts.AnalyzeSpecificFunction = Args.getLastArgValue(OPT_analyze_function);
Index: lib/CrossTU/CrossTranslationUnit.cpp
===================================================================
--- lib/CrossTU/CrossTranslationUnit.cpp
+++ lib/CrossTU/CrossTranslationUnit.cpp
@@ -43,6 +43,8 @@
STATISTIC(NumTripleMismatch, "The # of triple mismatches");
STATISTIC(NumLangMismatch, "The # of language mismatches");
STATISTIC(NumLangDialectMismatch, "The # of language dialect mismatches");
+STATISTIC(NumASTLoadThresholdReached,
+ "The # of ASTs not loaded because of threshold");
// Same as Triple's equality operator, but we check a field only if that is
// known in both instances.
@@ -102,6 +104,8 @@
return "Language mismatch";
case index_error_code::lang_dialect_mismatch:
return "Language dialect mismatch";
+ case index_error_code::load_threshold_reached:
+ return "Load threshold reached";
}
llvm_unreachable("Unrecognized index_error_code.");
}
@@ -197,7 +201,8 @@
CrossTranslationUnitContext::getCrossTUDefinition(const FunctionDecl *FD,
StringRef CrossTUDir,
StringRef IndexName,
- bool DisplayCTUProgress) {
+ bool DisplayCTUProgress,
+ unsigned CTULoadThreshold) {
assert(FD && "FD is missing, bad call to this function!");
assert(!FD->hasBody() && "FD has a definition in current translation unit!");
++NumGetCTUCalled;
@@ -206,7 +211,8 @@
return llvm::make_error<IndexError>(
index_error_code::failed_to_generate_usr);
llvm::Expected<ASTUnit *> ASTUnitOrError =
- loadExternalAST(LookupFnName, CrossTUDir, IndexName, DisplayCTUProgress);
+ loadExternalAST(LookupFnName, CrossTUDir, IndexName, DisplayCTUProgress,
+ CTULoadThreshold);
if (!ASTUnitOrError)
return ASTUnitOrError.takeError();
ASTUnit *Unit = *ASTUnitOrError;
@@ -293,11 +299,18 @@
llvm::Expected<ASTUnit *> CrossTranslationUnitContext::loadExternalAST(
StringRef LookupName, StringRef CrossTUDir, StringRef IndexName,
- bool DisplayCTUProgress) {
+ bool DisplayCTUProgress, unsigned CTULoadThreshold) {
// FIXME: The current implementation only supports loading functions with
// a lookup name from a single translation unit. If multiple
// translation units contains functions with the same lookup name an
// error will be returned.
+
+ if (NumASTLoaded >= CTULoadThreshold) {
+ ++NumASTLoadThresholdReached;
+ return llvm::make_error<IndexError>(
+ index_error_code::load_threshold_reached);
+ }
+
ASTUnit *Unit = nullptr;
auto FnUnitCacheEntry = FunctionASTUnitMap.find(LookupName);
if (FnUnitCacheEntry == FunctionASTUnitMap.end()) {
@@ -335,6 +348,7 @@
ASTUnit::LoadEverything, Diags, CI.getFileSystemOpts()));
Unit = LoadedUnit.get();
FileASTUnitMap[ASTFileName] = std::move(LoadedUnit);
+ ++NumASTLoaded;
if (DisplayCTUProgress) {
llvm::errs() << "CTU loaded AST file: "
<< ASTFileName << "\n";
Index: include/clang/StaticAnalyzer/Core/AnalyzerOptions.h
===================================================================
--- include/clang/StaticAnalyzer/Core/AnalyzerOptions.h
+++ include/clang/StaticAnalyzer/Core/AnalyzerOptions.h
@@ -219,6 +219,10 @@
// Cap the stack depth at 4 calls (5 stack frames, base + 4 calls).
unsigned InlineMaxStackDepth = 5;
+ /// The maximal amount of translation units that is considered for import
+ /// when inlining functions during CTU analysis.
+ unsigned CTUImportThreshold{100u};
+
/// The mode of function selection used during inlining.
AnalysisInliningMode InliningMode = NoRedundancy;
Index: include/clang/Driver/CC1Options.td
===================================================================
--- include/clang/Driver/CC1Options.td
+++ include/clang/Driver/CC1Options.td
@@ -85,6 +85,13 @@
HelpText<"Dump exploded graph to the specified file">;
def analyzer_dump_egraph_EQ : Joined<["-"], "analyzer-dump-egraph=">, Alias<analyzer_dump_egraph>;
+def analyzer_display_ctu_progress : Flag<["-"], "analyzer-display-ctu-progress">,
+ HelpText<"Emit verbose output about the analyzer's progress related to ctu">;
+def analyzer_ctu_import_threshold : Separate<["-"], "analyzer-ctu-import-threshold">,
+ HelpText<"Limit the number of TUs imported during the CTU analysis of a single TU">;
+def analyzer_ctu_import_threshold_EQ : Joined<["-"], "analyzer-ctu-import-threshold=">,
+ Alias<analyzer_ctu_import_threshold>;
+
def analyzer_inline_max_stack_depth : Separate<["-"], "analyzer-inline-max-stack-depth">,
HelpText<"Bound on stack depth while inlining (4 by default)">;
def analyzer_inline_max_stack_depth_EQ : Joined<["-"], "analyzer-inline-max-stack-depth=">,
Index: include/clang/CrossTU/CrossTranslationUnit.h
===================================================================
--- include/clang/CrossTU/CrossTranslationUnit.h
+++ include/clang/CrossTU/CrossTranslationUnit.h
@@ -44,7 +44,8 @@
failed_to_generate_usr,
triple_mismatch,
lang_mismatch,
- lang_dialect_mismatch
+ lang_dialect_mismatch,
+ load_threshold_reached
};
class IndexError : public llvm::ErrorInfo<IndexError> {
@@ -111,7 +112,9 @@
/// \p CrossTUDir directory, called \p IndexName. In case the declaration is
/// found in the index the corresponding AST file will be loaded and the
/// definition of the function will be merged into the original AST using
- /// the AST Importer.
+ /// the AST Importer. \p CTULoadTreshold should serve as an upper limit to the
+ /// number of TUs imported in order to reduce the memory footprint of CTU
+ /// analysis.
///
/// \return The declaration with the definition will be returned.
/// If no suitable definition is found in the index file or multiple
@@ -120,7 +123,8 @@
/// Note that the AST files should also be in the \p CrossTUDir.
llvm::Expected<const FunctionDecl *>
getCrossTUDefinition(const FunctionDecl *FD, StringRef CrossTUDir,
- StringRef IndexName, bool DisplayCTUProgress = false);
+ StringRef IndexName, bool DisplayCTUProgress,
+ unsigned CTULoadThreshold);
/// This function loads a function definition from an external AST
/// file.
@@ -128,7 +132,8 @@
/// A function definition with the same declaration will be looked up in the
/// index file which should be in the \p CrossTUDir directory, called
/// \p IndexName. In case the declaration is found in the index the
- /// corresponding AST file will be loaded.
+ /// corresponding AST file will be loaded. If the number of TUs imported
+ /// reaches \p CTULoadTreshold, no loading is performed.
///
/// \return Returns a pointer to the ASTUnit that contains the definition of
/// the looked up function or an Error.
@@ -138,7 +143,8 @@
llvm::Expected<ASTUnit *> loadExternalAST(StringRef LookupName,
StringRef CrossTUDir,
StringRef IndexName,
- bool DisplayCTUProgress = false);
+ bool DisplayCTUProgress,
+ unsigned CTULoadThreshold);
/// This function merges a definition from a separate AST Unit into
/// the current one which was created by the compiler instance that
@@ -167,6 +173,7 @@
CompilerInstance &CI;
ASTContext &Context;
std::unique_ptr<ASTImporterLookupTable> LookupTable;
+ unsigned NumASTLoaded{0u};
};
} // namespace cross_tu
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
