https://github.com/Bigcheese created
https://github.com/llvm/llvm-project/pull/144000
While the source code isn't supposed to change during a build, in some
environments it does. This adds an option that disables caching of stat
failures, meaning that source files can be added to the build during scanning.
This adds a `-no-cache-negative-stats` option to clang-scan-deps to enable this
behavior. There are no tests for clang-scan-deps as there's no reliable way to
do so from it. A unit test has been added that modifies the filesystem between
scans to test it.
>From 4eef87a1fc7cd4144c9be591c1c56ed5f105e7df Mon Sep 17 00:00:00 2001
From: Michael Spencer <bigchees...@gmail.com>
Date: Thu, 12 Jun 2025 17:13:52 -0700
Subject: [PATCH] [clang][scan-deps] Add option to disable caching stat
failures
While the source code isn't supposed to change during a build, in some
environments it does. This adds an option that disables caching of
stat failures, meaning that source files can be added to the build
during scanning.
This adds a `-no-cache-negative-stats` option to clang-scan-deps to
enable this behavior. There are no tests for clang-scan-deps as
there's no reliable way to do so from it. A unit test has been added
that modifies the filesystem between scans to test it.
---
.../DependencyScanningFilesystem.h | 5 +-
.../DependencyScanningService.h | 6 ++-
.../DependencyScanningFilesystem.cpp | 7 ++-
.../DependencyScanningService.cpp | 3 +-
.../DependencyScanningWorker.cpp | 4 +-
clang/tools/clang-scan-deps/ClangScanDeps.cpp | 6 ++-
clang/tools/clang-scan-deps/Opts.td | 1 +
.../DependencyScannerTest.cpp | 50 +++++++++++++++++++
8 files changed, 74 insertions(+), 8 deletions(-)
diff --git
a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
index a20a89a4c2b76..7a688082a55e1 100644
---
a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++
b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -350,7 +350,8 @@ class DependencyScanningWorkerFilesystem
DependencyScanningWorkerFilesystem(
DependencyScanningFilesystemSharedCache &SharedCache,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS);
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
+ bool CacheNegativeStats = true);
llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override;
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
@@ -491,6 +492,8 @@ class DependencyScanningWorkerFilesystem
/// using them for cache lookups.
llvm::ErrorOr<std::string> WorkingDirForCacheLookup;
+ bool CacheNegativeStats;
+
void updateWorkingDirForCacheLookup();
llvm::ErrorOr<StringRef>
diff --git
a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
index 4e97c7bc9f36e..ceaf3c2279e7f 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
@@ -87,7 +87,8 @@ class DependencyScanningService {
ScanningOptimizations OptimizeArgs = ScanningOptimizations::Default,
bool EagerLoadModules = false, bool TraceVFS = false,
std::time_t BuildSessionTimestamp =
- llvm::sys::toTimeT(std::chrono::system_clock::now()));
+ llvm::sys::toTimeT(std::chrono::system_clock::now()),
+ bool CacheNegativeStats = true);
ScanningMode getMode() const { return Mode; }
@@ -99,6 +100,8 @@ class DependencyScanningService {
bool shouldTraceVFS() const { return TraceVFS; }
+ bool shouldCacheNegativeStats() const { return CacheNegativeStats; }
+
DependencyScanningFilesystemSharedCache &getSharedCache() {
return SharedCache;
}
@@ -116,6 +119,7 @@ class DependencyScanningService {
const bool EagerLoadModules;
/// Whether to trace VFS accesses.
const bool TraceVFS;
+ const bool CacheNegativeStats;
/// The global file system cache.
DependencyScanningFilesystemSharedCache SharedCache;
/// The global module cache entries.
diff --git
a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index 140833050f4e9..e28c4362a0f2f 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -233,11 +233,12 @@ bool
DependencyScanningWorkerFilesystem::shouldBypass(StringRef Path) const {
DependencyScanningWorkerFilesystem::DependencyScanningWorkerFilesystem(
DependencyScanningFilesystemSharedCache &SharedCache,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, bool CacheNegativeStats)
: llvm::RTTIExtends<DependencyScanningWorkerFilesystem,
llvm::vfs::ProxyFileSystem>(std::move(FS)),
SharedCache(SharedCache),
- WorkingDirForCacheLookup(llvm::errc::invalid_argument) {
+ WorkingDirForCacheLookup(llvm::errc::invalid_argument),
+ CacheNegativeStats(CacheNegativeStats) {
updateWorkingDirForCacheLookup();
}
@@ -267,6 +268,8 @@ DependencyScanningWorkerFilesystem::computeAndStoreResult(
llvm::ErrorOr<llvm::vfs::Status> Stat =
getUnderlyingFS().status(OriginalFilename);
if (!Stat) {
+ if (!CacheNegativeStats)
+ return Stat.getError();
const auto &Entry =
getOrEmplaceSharedEntryForFilename(FilenameForLookup, Stat.getError());
return insertLocalEntryForFilename(FilenameForLookup, Entry);
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
index 7f40c99f07287..c2f3cdbb02e37 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
@@ -15,7 +15,8 @@ using namespace dependencies;
DependencyScanningService::DependencyScanningService(
ScanningMode Mode, ScanningOutputFormat Format,
ScanningOptimizations OptimizeArgs, bool EagerLoadModules, bool TraceVFS,
- std::time_t BuildSessionTimestamp)
+ std::time_t BuildSessionTimestamp, bool CacheNegativeStats)
: Mode(Mode), Format(Format), OptimizeArgs(OptimizeArgs),
EagerLoadModules(EagerLoadModules), TraceVFS(TraceVFS),
+ CacheNegativeStats(CacheNegativeStats),
BuildSessionTimestamp(BuildSessionTimestamp) {}
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
index 9bd85479d9810..a54bcfdb509a0 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -611,8 +611,8 @@ DependencyScanningWorker::DependencyScanningWorker(
switch (Service.getMode()) {
case ScanningMode::DependencyDirectivesScan:
- DepFS =
- new DependencyScanningWorkerFilesystem(Service.getSharedCache(), FS);
+ DepFS = new DependencyScanningWorkerFilesystem(
+ Service.getSharedCache(), FS, Service.shouldCacheNegativeStats());
BaseFS = DepFS;
break;
case ScanningMode::CanonicalPreprocessing:
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index 921ba7aadd67d..bb42f2f43aee6 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -85,6 +85,7 @@ static ScanningOutputFormat Format =
ScanningOutputFormat::Make;
static ScanningOptimizations OptimizeArgs;
static std::string ModuleFilesDir;
static bool EagerLoadModules;
+static bool CacheNegativeStats = true;
static unsigned NumThreads = 0;
static std::string CompilationDB;
static std::optional<std::string> ModuleName;
@@ -191,6 +192,8 @@ static void ParseArgs(int argc, char **argv) {
EagerLoadModules = Args.hasArg(OPT_eager_load_pcm);
+ CacheNegativeStats = !Args.hasArg(OPT_no_cache_negative_stats);
+
if (const llvm::opt::Arg *A = Args.getLastArg(OPT_j)) {
StringRef S{A->getValue()};
if (!llvm::to_integer(S, NumThreads, 0)) {
@@ -1081,7 +1084,8 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
};
DependencyScanningService Service(ScanMode, Format, OptimizeArgs,
- EagerLoadModules, /*TraceVFS=*/Verbose);
+ EagerLoadModules, /*TraceVFS=*/Verbose,
+ CacheNegativeStats);
llvm::Timer T;
T.startTimer();
diff --git a/clang/tools/clang-scan-deps/Opts.td
b/clang/tools/clang-scan-deps/Opts.td
index 9cccbb3aaf0c8..582ae60851e1e 100644
--- a/clang/tools/clang-scan-deps/Opts.td
+++ b/clang/tools/clang-scan-deps/Opts.td
@@ -22,6 +22,7 @@ defm module_files_dir : Eq<"module-files-dir",
def optimize_args_EQ : CommaJoined<["-", "--"], "optimize-args=">,
HelpText<"Which command-line arguments of modules to optimize">;
def eager_load_pcm : F<"eager-load-pcm", "Load PCM files eagerly (instead of
lazily on import)">;
+def no_cache_negative_stats : F<"no-cache-negative-stats", "Don't cache stat
failures">;
def j : Arg<"j", "Number of worker threads to use (default: use all concurrent
threads)">;
diff --git
a/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp
b/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp
index 683d9070b1dcf..3535f1a04f2be 100644
--- a/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp
+++ b/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp
@@ -384,3 +384,53 @@ TEST(DependencyScanner, ScanDepsWithDiagConsumer) {
EXPECT_TRUE(DiagConsumer.Finished);
}
}
+
+TEST(DependencyScanner, NoNegativeCache) {
+ StringRef CWD = "/root";
+
+ auto VFS = new llvm::vfs::InMemoryFileSystem();
+ VFS->setCurrentWorkingDirectory(CWD);
+ auto Sept = llvm::sys::path::get_separator();
+ std::string HeaderPath =
+ std::string(llvm::formatv("{0}root{0}header.h", Sept));
+ std::string Test0Path = std::string(llvm::formatv("{0}root{0}test0.cpp",
Sept));
+ std::string Test1Path = std::string(llvm::formatv("{0}root{0}test1.cpp",
Sept));
+
+ VFS->addFile(Test0Path, 0,
+ llvm::MemoryBuffer::getMemBuffer(
+ "#if __has_include(\"header.h\")\n#endif"));
+ VFS->addFile(Test1Path, 0,
+ llvm::MemoryBuffer::getMemBuffer("#include \"header.h\""));
+
+ DependencyScanningService Service(
+ ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make,
+ ScanningOptimizations::All, false, false,
+ llvm::sys::toTimeT(std::chrono::system_clock::now()), false);
+ DependencyScanningTool ScanTool(Service, VFS);
+
+ std::vector<std::string> CommandLine0 = {"clang",
+ "-target",
+ "x86_64-apple-macosx10.7",
+ "-c",
+ "test0.cpp",
+ "-o"
+ "test0.cpp.o"};
+ std::vector<std::string> CommandLine1 = {"clang",
+ "-target",
+ "x86_64-apple-macosx10.7",
+ "-c",
+ "test1.cpp",
+ "-o"
+ "test1.cpp.o"};
+
+ std::string Result;
+ ASSERT_THAT_ERROR(
+ ScanTool.getDependencyFile(CommandLine0, CWD).moveInto(Result),
+ llvm::Succeeded());
+
+ VFS->addFile(HeaderPath, 0, llvm::MemoryBuffer::getMemBuffer(""));
+
+ ASSERT_THAT_ERROR(
+ ScanTool.getDependencyFile(CommandLine1, CWD).moveInto(Result),
+ llvm::Succeeded());
+}
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
