This revision was landed with ongoing or failed builds.
This revision was automatically updated to reflect the committed changes.
Closed by commit rGbc1a2979fc70: [clang][deps] Separate filesystem caches for
minimized and original files (authored by jansvoboda11).
Changed prior to commit:
https://reviews.llvm.org/D106146?vs=359294&id=360065#toc
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D106146/new/
https://reviews.llvm.org/D106146
Files:
clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
clang/unittests/Tooling/CMakeLists.txt
clang/unittests/Tooling/DependencyScannerTest.cpp
Index: clang/unittests/Tooling/DependencyScannerTest.cpp
===================================================================
--- clang/unittests/Tooling/DependencyScannerTest.cpp
+++ clang/unittests/Tooling/DependencyScannerTest.cpp
@@ -15,6 +15,7 @@
#include "clang/Frontend/FrontendActions.h"
#include "clang/Tooling/CompilationDatabase.h"
#include "clang/Tooling/Tooling.h"
+#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/Path.h"
@@ -203,5 +204,35 @@
EXPECT_EQ(convert_to_slash(Deps[5]), "/root/symlink.h");
}
+namespace dependencies {
+TEST(DependencyScanningFilesystem, IgnoredFilesHaveSeparateCache) {
+ auto VFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+ VFS->addFile("/mod.h", 0, llvm::MemoryBuffer::getMemBuffer("// hi there!\n"));
+
+ DependencyScanningFilesystemSharedCache SharedCache;
+ auto Mappings = std::make_unique<ExcludedPreprocessorDirectiveSkipMapping>();
+ DependencyScanningWorkerFilesystem DepFS(SharedCache, VFS, Mappings.get());
+
+ auto StatusMinimized0 = DepFS.status("/mod.h");
+ DepFS.ignoreFile("/mod.h");
+ auto StatusFull1 = DepFS.status("/mod.h");
+ DepFS.clearIgnoredFiles();
+
+ auto StatusMinimized2 = DepFS.status("/mod.h");
+ DepFS.ignoreFile("/mod.h");
+ auto StatusFull3 = DepFS.status("/mod.h");
+
+ EXPECT_TRUE(StatusMinimized0);
+ EXPECT_EQ(StatusMinimized0->getSize(), 0u);
+ EXPECT_TRUE(StatusFull1);
+ EXPECT_EQ(StatusFull1->getSize(), 13u);
+
+ EXPECT_TRUE(StatusMinimized2);
+ EXPECT_EQ(StatusMinimized2->getSize(), 0u);
+ EXPECT_TRUE(StatusFull3);
+ EXPECT_EQ(StatusFull3->getSize(), 13u);
+}
+
+} // end namespace dependencies
} // end namespace tooling
} // end namespace clang
Index: clang/unittests/Tooling/CMakeLists.txt
===================================================================
--- clang/unittests/Tooling/CMakeLists.txt
+++ clang/unittests/Tooling/CMakeLists.txt
@@ -68,6 +68,7 @@
clangAST
clangASTMatchers
clangBasic
+ clangDependencyScanning
clangFormat
clangFrontend
clangLex
Index: clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
===================================================================
--- clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -99,8 +99,7 @@
return Result;
}
-DependencyScanningFilesystemSharedCache::
- DependencyScanningFilesystemSharedCache() {
+DependencyScanningFilesystemSharedCache::SingleCache::SingleCache() {
// This heuristic was chosen using a empirical testing on a
// reasonably high core machine (iMacPro 18 cores / 36 threads). The cache
// sharding gives a performance edge by reducing the lock contention.
@@ -111,18 +110,20 @@
CacheShards = std::make_unique<CacheShard[]>(NumShards);
}
-/// Returns a cache entry for the corresponding key.
-///
-/// A new cache entry is created if the key is not in the cache. This is a
-/// thread safe call.
DependencyScanningFilesystemSharedCache::SharedFileSystemEntry &
-DependencyScanningFilesystemSharedCache::get(StringRef Key) {
+DependencyScanningFilesystemSharedCache::SingleCache::get(StringRef Key) {
CacheShard &Shard = CacheShards[llvm::hash_value(Key) % NumShards];
std::unique_lock<std::mutex> LockGuard(Shard.CacheLock);
auto It = Shard.Cache.try_emplace(Key);
return It.first->getValue();
}
+DependencyScanningFilesystemSharedCache::SharedFileSystemEntry &
+DependencyScanningFilesystemSharedCache::get(StringRef Key, bool Minimized) {
+ SingleCache &Cache = Minimized ? CacheMinimized : CacheOriginal;
+ return Cache.get(Key);
+}
+
/// Whitelist file extensions that should be minimized, treating no extension as
/// a source file that should be minimized.
///
@@ -165,17 +166,17 @@
llvm::ErrorOr<const CachedFileSystemEntry *>
DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry(
const StringRef Filename) {
- if (const CachedFileSystemEntry *Entry = getCachedEntry(Filename)) {
+ bool ShouldMinimize =
+ !IgnoredFiles.count(Filename) && shouldMinimize(Filename);
+
+ if (const auto *Entry = Cache.getCachedEntry(Filename, ShouldMinimize))
return Entry;
- }
// FIXME: Handle PCM/PCH files.
// FIXME: Handle module map files.
- bool KeepOriginalSource = shouldIgnoreFile(Filename) ||
- !shouldMinimize(Filename);
DependencyScanningFilesystemSharedCache::SharedFileSystemEntry
- &SharedCacheEntry = SharedCache.get(Filename);
+ &SharedCacheEntry = SharedCache.get(Filename, ShouldMinimize);
const CachedFileSystemEntry *Result;
{
std::unique_lock<std::mutex> LockGuard(SharedCacheEntry.ValueLock);
@@ -197,15 +198,15 @@
CacheEntry = CachedFileSystemEntry::createDirectoryEntry(
std::move(*MaybeStatus));
else
- CacheEntry = CachedFileSystemEntry::createFileEntry(
- Filename, FS, !KeepOriginalSource);
+ CacheEntry = CachedFileSystemEntry::createFileEntry(Filename, FS,
+ ShouldMinimize);
}
Result = &CacheEntry;
}
// Store the result in the local cache.
- setCachedEntry(Filename, Result);
+ Cache.setCachedEntry(Filename, ShouldMinimize, Result);
return Result;
}
Index: clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
===================================================================
--- clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++ clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -103,7 +103,8 @@
};
/// This class is a shared cache, that caches the 'stat' and 'open' calls to the
-/// underlying real file system.
+/// underlying real file system. It distinguishes between minimized and original
+/// files.
///
/// It is sharded based on the hash of the key to reduce the lock contention for
/// the worker threads.
@@ -114,21 +115,62 @@
CachedFileSystemEntry Value;
};
- DependencyScanningFilesystemSharedCache();
-
/// Returns a cache entry for the corresponding key.
///
/// A new cache entry is created if the key is not in the cache. This is a
/// thread safe call.
- SharedFileSystemEntry &get(StringRef Key);
+ SharedFileSystemEntry &get(StringRef Key, bool Minimized);
private:
- struct CacheShard {
- std::mutex CacheLock;
- llvm::StringMap<SharedFileSystemEntry, llvm::BumpPtrAllocator> Cache;
+ class SingleCache {
+ public:
+ SingleCache();
+
+ SharedFileSystemEntry &get(StringRef Key);
+
+ private:
+ struct CacheShard {
+ std::mutex CacheLock;
+ llvm::StringMap<SharedFileSystemEntry, llvm::BumpPtrAllocator> Cache;
+ };
+ std::unique_ptr<CacheShard[]> CacheShards;
+ unsigned NumShards;
};
- std::unique_ptr<CacheShard[]> CacheShards;
- unsigned NumShards;
+
+ SingleCache CacheMinimized;
+ SingleCache CacheOriginal;
+};
+
+/// This class is a local cache, that caches the 'stat' and 'open' calls to the
+/// underlying real file system. It distinguishes between minimized and original
+/// files.
+class DependencyScanningFilesystemLocalCache {
+private:
+ using SingleCache =
+ llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator>;
+
+ SingleCache CacheMinimized;
+ SingleCache CacheOriginal;
+
+ SingleCache &selectCache(bool Minimized) {
+ return Minimized ? CacheMinimized : CacheOriginal;
+ }
+
+public:
+ void setCachedEntry(StringRef Filename, bool Minimized,
+ const CachedFileSystemEntry *Entry) {
+ SingleCache &Cache = selectCache(Minimized);
+ bool IsInserted = Cache.try_emplace(Filename, Entry).second;
+ (void)IsInserted;
+ assert(IsInserted && "local cache is updated more than once");
+ }
+
+ const CachedFileSystemEntry *getCachedEntry(StringRef Filename,
+ bool Minimized) {
+ SingleCache &Cache = selectCache(Minimized);
+ auto It = Cache.find(Filename);
+ return It == Cache.end() ? nullptr : It->getValue();
+ }
};
/// A virtual file system optimized for the dependency discovery.
@@ -159,24 +201,14 @@
private:
bool shouldIgnoreFile(StringRef Filename);
- void setCachedEntry(StringRef Filename, const CachedFileSystemEntry *Entry) {
- bool IsInserted = Cache.try_emplace(Filename, Entry).second;
- (void)IsInserted;
- assert(IsInserted && "local cache is updated more than once");
- }
-
- const CachedFileSystemEntry *getCachedEntry(StringRef Filename) {
- auto It = Cache.find(Filename);
- return It == Cache.end() ? nullptr : It->getValue();
- }
-
llvm::ErrorOr<const CachedFileSystemEntry *>
getOrCreateFileSystemEntry(const StringRef Filename);
+ /// The global cache shared between worker threads.
DependencyScanningFilesystemSharedCache &SharedCache;
/// The local cache is used by the worker thread to cache file system queries
/// locally instead of querying the global cache every time.
- llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator> Cache;
+ DependencyScanningFilesystemLocalCache Cache;
/// The optional mapping structure which records information about the
/// excluded conditional directive skip mappings that are used by the
/// currently active preprocessor.
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
