kadircet updated this revision to Diff 174052.
kadircet added a comment.
Herald added a subscriber: mgorny.
- Address comments.
- Move storage related things to new files.
Repository:
rCTE Clang Tools Extra
https://reviews.llvm.org/D54269
Files:
clangd/CMakeLists.txt
clangd/index/Background.cpp
clangd/index/Background.h
clangd/index/BackgroundIndexStorage.cpp
clangd/index/BackgroundIndexStorage.h
unittests/clangd/BackgroundIndexTests.cpp
Index: unittests/clangd/BackgroundIndexTests.cpp
===================================================================
--- unittests/clangd/BackgroundIndexTests.cpp
+++ unittests/clangd/BackgroundIndexTests.cpp
@@ -43,7 +43,9 @@
void f_b() {
(void)common;
})cpp";
- BackgroundIndex Idx(Context::empty(), "", FS, /*URISchemes=*/{"unittest"});
+ BackgroundIndex Idx(
+ Context::empty(), "", FS, /*URISchemes=*/{"unittest"},
+ llvm::make_unique<BackgroundIndexStorageFactory>(nullptr));
tooling::CompileCommand Cmd;
Cmd.Filename = testPath("root/A.cc");
@@ -76,5 +78,75 @@
FileURI("unittest:///root/B.cc")}));
}
+TEST(BackgroundIndexTest, ShardStorageWriteTest) {
+ class MemoryShardStorage : public BackgroundIndexStorage {
+ mutable std::mutex StorageMu;
+ llvm::StringMap<std::string> &Storage;
+ size_t &CacheHits;
+
+ public:
+ MemoryShardStorage(llvm::StringMap<std::string> &Storage, size_t &CacheHits)
+ : Storage(Storage), CacheHits(CacheHits) {}
+ llvm::Error storeShard(llvm::StringRef ShardIdentifier,
+ IndexFileOut Shard) const override {
+ std::lock_guard<std::mutex> Lock(StorageMu);
+ std::string &str = Storage[ShardIdentifier];
+ llvm::raw_string_ostream OS(str);
+ OS << Shard;
+ OS.flush();
+ return llvm::Error::success();
+ }
+ std::unique_ptr<IndexFileIn>
+ loadShard(llvm::StringRef ShardIdentifier) const override {
+ std::lock_guard<std::mutex> Lock(StorageMu);
+ if (Storage.find(ShardIdentifier) == Storage.end()) {
+ ADD_FAILURE() << "Shard " << ShardIdentifier << ": not found.";
+ return nullptr;
+ }
+ auto IndexFile = readIndexFile(Storage[ShardIdentifier]);
+ if (!IndexFile) {
+ ADD_FAILURE() << "Error while reading " << ShardIdentifier << ':'
+ << IndexFile.takeError();
+ return nullptr;
+ }
+ CacheHits++;
+ return llvm::make_unique<IndexFileIn>(std::move(*IndexFile));
+ }
+ };
+ MockFSProvider FS;
+ FS.Files[testPath("root/A.h")] = R"cpp(
+ void common();
+ void f_b();
+ class A_CC {};
+ )cpp";
+ FS.Files[testPath("root/A.cc")] =
+ "#include \"A.h\"\nvoid g() { (void)common; }";
+
+ llvm::StringMap<std::string> Storage;
+ size_t CacheHits = 0;
+ BackgroundIndexStorageFactory::IndexStorageCreator MemoryStorageCreator =
+ [&](llvm::StringRef) {
+ return llvm::make_unique<MemoryShardStorage>(Storage, CacheHits);
+ };
+
+ tooling::CompileCommand Cmd;
+ Cmd.Filename = testPath("root/A.cc");
+ Cmd.Directory = testPath("root");
+ Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
+ std::unique_ptr<BackgroundIndexStorageFactory> BISF =
+ llvm::make_unique<BackgroundIndexStorageFactory>(
+ std::move(MemoryStorageCreator));
+ // Check nothing is loaded from Storage, but A.cc and A.h has been stored.
+ {
+ BackgroundIndex Idx(Context::empty(), "", FS, /*URISchemes=*/{"unittest"},
+ std::move(BISF));
+ Idx.enqueue(testPath("root"), Cmd);
+ Idx.blockUntilIdleForTest();
+ }
+ EXPECT_EQ(Storage.size(), 2U);
+ EXPECT_NE(Storage.find(testPath("root/A.h")), Storage.end());
+ EXPECT_NE(Storage.find(testPath("root/A.cc")), Storage.end());
+}
+
} // namespace clangd
} // namespace clang
Index: clangd/index/BackgroundIndexStorage.h
===================================================================
--- /dev/null
+++ clangd/index/BackgroundIndexStorage.h
@@ -0,0 +1,63 @@
+//===--- Background.h - Build an index in a background thread ----*- C++-*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_BACKGROUND_INDEX_STORAGE_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_BACKGROUND_INDEX_STORAGE_H
+
+#include "index/Serialization.h"
+#include "llvm/ADT/FunctionExtras.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/Support/Error.h"
+
+namespace clang {
+namespace clangd {
+
+// Handles storage and retrieval of index shards. Both store and load
+// operations can be called from multiple-threads concurrently.
+class BackgroundIndexStorage {
+public:
+ // Shards of the index are stored and retrieved independently, keyed by shard
+ // identifier - in practice this is a source file name
+ virtual llvm::Error storeShard(llvm::StringRef ShardIdentifier,
+ IndexFileOut Shard) const = 0;
+
+ // Tries to load shard with given identifier, returns nullptr if shard
+ // couldn't be loaded.
+ virtual std::unique_ptr<IndexFileIn>
+ loadShard(llvm::StringRef ShardIdentifier) const = 0;
+
+ // Creates an Index Storage that saves shards into disk. Index storage uses
+ // CDBDirectory + ".clangd-index/" as the folder to save shards.
+ static std::unique_ptr<BackgroundIndexStorage>
+ createDiskStorage(llvm::StringRef CDBDirectory);
+};
+
+// Creates and owns IndexStorages for multiple CDBs.
+class BackgroundIndexStorageFactory {
+public:
+ // The function to be used to create IndexStorage units for each compilation
+ // database. Those databases are identified by the directory they are found
+ // in. The function must be thread-safe.
+ using IndexStorageCreator =
+ llvm::unique_function<std::unique_ptr<BackgroundIndexStorage>(
+ llvm::StringRef)>;
+
+ explicit BackgroundIndexStorageFactory(IndexStorageCreator Creator);
+
+ // Creates or fetches to storage from cache for the specified CDB.
+ BackgroundIndexStorage *createStorage(llvm::StringRef CDBDirectory);
+
+private:
+ llvm::StringMap<std::unique_ptr<BackgroundIndexStorage>> IndexStorageMap;
+ std::mutex IndexStorageMapMu;
+ IndexStorageCreator Creator;
+};
+} // namespace clangd
+} // namespace clang
+#endif
Index: clangd/index/BackgroundIndexStorage.cpp
===================================================================
--- /dev/null
+++ clangd/index/BackgroundIndexStorage.cpp
@@ -0,0 +1,123 @@
+//== BackgroundIndexStorage.cpp - Provide caching support to BackgroundIndex ==/
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "index/BackgroundIndexStorage.h"
+#include "Logger.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/SHA1.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+using FileDigest = decltype(llvm::SHA1::hash({}));
+
+static FileDigest digest(StringRef Content) {
+ return llvm::SHA1::hash({(const uint8_t *)Content.data(), Content.size()});
+}
+
+std::string getShardPathFromFilePath(llvm::StringRef ShardRoot,
+ llvm::StringRef FilePath) {
+ llvm::SmallString<128> ShardRootSS(ShardRoot);
+ llvm::sys::path::append(ShardRootSS, llvm::sys::path::filename(FilePath) +
+ llvm::toHex(digest(FilePath)) +
+ ".idx");
+ return ShardRoot.str();
+}
+
+// Uses disk as a storage for index shards. Creates a directory called
+// ".clangd-index/" under the path provided during construction.
+class DiskBackedIndexStorage : public BackgroundIndexStorage {
+ std::string DiskShardRoot;
+
+public:
+ // Sets DiskShardRoot to (Directory + ".clangd-index/") which is the base
+ // directory for all shard files.
+ DiskBackedIndexStorage(llvm::StringRef Directory) {
+ llvm::SmallString<128> CDBDirectory(Directory);
+ llvm::sys::path::append(CDBDirectory, ".clangd-index/");
+ DiskShardRoot = CDBDirectory.str();
+ std::error_code OK;
+ std::error_code EC = llvm::sys::fs::create_directory(DiskShardRoot);
+ if (EC != OK) {
+ elog("Failed to create directory {0} for disk backed index storage: {1}",
+ DiskShardRoot, EC.message());
+ }
+ }
+
+ std::unique_ptr<IndexFileIn>
+ loadShard(llvm::StringRef ShardIdentifier) const override {
+ const std::string ShardPath =
+ getShardPathFromFilePath(DiskShardRoot, ShardIdentifier);
+ auto Buffer = llvm::MemoryBuffer::getFile(ShardPath);
+ if (!Buffer)
+ return nullptr;
+ if (auto I = readIndexFile(Buffer->get()->getBuffer()))
+ return llvm::make_unique<IndexFileIn>(std::move(*I));
+ else
+ elog("Error while reading shard {0}: {1}", ShardIdentifier,
+ I.takeError());
+ return nullptr;
+ }
+
+ llvm::Error storeShard(llvm::StringRef ShardIdentifier,
+ IndexFileOut Shard) const override {
+ auto ShardPath = getShardPathFromFilePath(DiskShardRoot, ShardIdentifier);
+ std::error_code EC;
+ llvm::raw_fd_ostream OS(ShardPath, EC);
+ if (EC)
+ return llvm::errorCodeToError(EC);
+ OS << Shard;
+ return llvm::Error::success();
+ }
+};
+class DummyIndexStorage : public BackgroundIndexStorage {
+public:
+ std::unique_ptr<IndexFileIn>
+ loadShard(llvm::StringRef ShardIdentifier) const override {
+ vlog("Dropping loadShard({0})", ShardIdentifier);
+ return llvm::make_unique<IndexFileIn>();
+ }
+ llvm::Error storeShard(llvm::StringRef ShardIdentifier,
+ IndexFileOut Shard) const override {
+ vlog("Dropping storeShard({0}, ...)", ShardIdentifier);
+ return llvm::Error::success();
+ }
+ static std::unique_ptr<BackgroundIndexStorage>
+ createDummyStorage(llvm::StringRef CDBDirectory) {
+ vlog("Creating dummy storage for: {0}", CDBDirectory);
+ return llvm::make_unique<DummyIndexStorage>();
+ }
+};
+
+} // namespace
+
+std::unique_ptr<BackgroundIndexStorage>
+BackgroundIndexStorage::createDiskStorage(llvm::StringRef CDBDirectory) {
+ return llvm::make_unique<DiskBackedIndexStorage>(CDBDirectory);
+}
+
+BackgroundIndexStorageFactory::BackgroundIndexStorageFactory(
+ IndexStorageCreator Creator)
+ : Creator(Creator ? std::move(Creator)
+ : DummyIndexStorage::createDummyStorage) {}
+
+BackgroundIndexStorage *
+BackgroundIndexStorageFactory::createStorage(llvm::StringRef CDBDirectory) {
+ std::lock_guard<std::mutex> Lock(IndexStorageMapMu);
+ auto &IndexStorage = IndexStorageMap[CDBDirectory];
+ if (!IndexStorage)
+ IndexStorage = Creator(CDBDirectory);
+ return IndexStorage.get();
+}
+
+} // namespace clangd
+} // namespace clang
Index: clangd/index/Background.h
===================================================================
--- clangd/index/Background.h
+++ clangd/index/Background.h
@@ -12,8 +12,10 @@
#include "Context.h"
#include "FSProvider.h"
+#include "index/BackgroundIndexStorage.h"
#include "index/FileIndex.h"
#include "index/Index.h"
+#include "index/Serialization.h"
#include "clang/Tooling/CompilationDatabase.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/Support/SHA1.h"
@@ -34,9 +36,11 @@
class BackgroundIndex : public SwapIndex {
public:
// FIXME: resource-dir injection should be hoisted somewhere common.
- BackgroundIndex(Context BackgroundContext, StringRef ResourceDir,
- const FileSystemProvider &, ArrayRef<std::string> URISchemes,
- size_t ThreadPoolSize = llvm::hardware_concurrency());
+ BackgroundIndex(
+ Context BackgroundContext, llvm::StringRef ResourceDir,
+ const FileSystemProvider &, ArrayRef<std::string> URISchemes,
+ std::unique_ptr<BackgroundIndexStorageFactory> IndexStorageFactory,
+ size_t ThreadPoolSize = llvm::hardware_concurrency());
~BackgroundIndex(); // Blocks while the current task finishes.
// Enqueue a translation unit for indexing.
@@ -59,25 +63,31 @@
private:
/// Given index results from a TU, only update files in \p FilesToUpdate.
void update(llvm::StringRef MainFile, SymbolSlab Symbols, RefSlab Refs,
- const llvm::StringMap<FileDigest> &FilesToUpdate);
+ const llvm::StringMap<FileDigest> &FilesToUpdate,
+ BackgroundIndexStorage *IndexStorage);
// configuration
std::string ResourceDir;
const FileSystemProvider &FSProvider;
Context BackgroundContext;
std::vector<std::string> URISchemes;
// index state
- llvm::Error index(tooling::CompileCommand);
+ llvm::Error index(tooling::CompileCommand,
+ BackgroundIndexStorage *IndexStorage);
FileSymbols IndexedSymbols;
llvm::StringMap<FileDigest> IndexedFileDigests; // Key is absolute file path.
std::mutex DigestsMu;
+ // index storage
+ std::unique_ptr<BackgroundIndexStorageFactory> IndexStorageFactory;
+
// queue management
using Task = std::function<void()>;
void run(); // Main loop executed by Thread. Runs tasks from Queue.
- void enqueueLocked(tooling::CompileCommand Cmd);
+ void enqueueLocked(tooling::CompileCommand Cmd,
+ BackgroundIndexStorage *IndexStorage);
std::mutex QueueMu;
unsigned NumActiveTasks = 0; // Only idle when queue is empty *and* no tasks.
std::condition_variable QueueCV;
Index: clangd/index/Background.cpp
===================================================================
--- clangd/index/Background.cpp
+++ clangd/index/Background.cpp
@@ -24,28 +24,61 @@
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/SHA1.h"
+
+#include <memory>
+#include <queue>
#include <random>
#include <string>
using namespace llvm;
namespace clang {
namespace clangd {
-BackgroundIndex::BackgroundIndex(Context BackgroundContext,
- StringRef ResourceDir,
- const FileSystemProvider &FSProvider,
- ArrayRef<std::string> URISchemes,
- size_t ThreadPoolSize)
+namespace {
+
+static BackgroundIndex::FileDigest digest(StringRef Content) {
+ return SHA1::hash({(const uint8_t *)Content.data(), Content.size()});
+}
+
+static Optional<BackgroundIndex::FileDigest> digestFile(const SourceManager &SM,
+ FileID FID) {
+ bool Invalid = false;
+ StringRef Content = SM.getBufferData(FID, &Invalid);
+ if (Invalid)
+ return None;
+ return digest(Content);
+}
+
+std::string getAbsoluteFilePath(const tooling::CompileCommand &Cmd) {
+ SmallString<128> AbsolutePath;
+ if (sys::path::is_absolute(Cmd.Filename)) {
+ AbsolutePath = Cmd.Filename;
+ } else {
+ AbsolutePath = Cmd.Directory;
+ sys::path::append(AbsolutePath, Cmd.Filename);
+ }
+ return AbsolutePath.str();
+}
+
+} // namespace
+
+BackgroundIndex::BackgroundIndex(
+ Context BackgroundContext, StringRef ResourceDir,
+ const FileSystemProvider &FSProvider, ArrayRef<std::string> URISchemes,
+ std::unique_ptr<BackgroundIndexStorageFactory> IndexStorageFactory,
+ size_t ThreadPoolSize)
: SwapIndex(make_unique<MemIndex>()), ResourceDir(ResourceDir),
FSProvider(FSProvider), BackgroundContext(std::move(BackgroundContext)),
- URISchemes(URISchemes) {
+ URISchemes(URISchemes),
+ IndexStorageFactory(std::move(IndexStorageFactory)) {
assert(ThreadPoolSize > 0 && "Thread pool size can't be zero.");
+ assert(IndexStorageFactory && "Storage factory can not be null!");
while (ThreadPoolSize--) {
ThreadPool.emplace_back([this] { run(); });
// Set priority to low, since background indexing is a long running task we
// do not want to eat up cpu when there are any other high priority threads.
// FIXME: In the future we might want a more general way of handling this to
- // support a tasks with various priorities.
+ // support tasks with various priorities.
setThreadPriority(ThreadPool.back(), ThreadPriority::Low);
}
}
@@ -97,9 +130,11 @@
void BackgroundIndex::enqueue(StringRef Directory,
tooling::CompileCommand Cmd) {
+ BackgroundIndexStorage *IndexStorage =
+ IndexStorageFactory->createStorage(Directory);
{
std::lock_guard<std::mutex> Lock(QueueMu);
- enqueueLocked(std::move(Cmd));
+ enqueueLocked(std::move(Cmd), IndexStorage);
}
QueueCV.notify_all();
}
@@ -110,42 +145,32 @@
// FIXME: this function may be slow. Perhaps enqueue a task to re-read the CDB
// from disk and enqueue the commands asynchronously?
auto Cmds = CDB.getAllCompileCommands();
+ BackgroundIndexStorage *IndexStorage =
+ IndexStorageFactory->createStorage(Directory);
SPAN_ATTACH(Tracer, "commands", int64_t(Cmds.size()));
std::mt19937 Generator(std::random_device{}());
std::shuffle(Cmds.begin(), Cmds.end(), Generator);
log("Enqueueing {0} commands for indexing from {1}", Cmds.size(), Directory);
{
std::lock_guard<std::mutex> Lock(QueueMu);
for (auto &Cmd : Cmds)
- enqueueLocked(std::move(Cmd));
+ enqueueLocked(std::move(Cmd), IndexStorage);
}
QueueCV.notify_all();
}
-void BackgroundIndex::enqueueLocked(tooling::CompileCommand Cmd) {
+void BackgroundIndex::enqueueLocked(tooling::CompileCommand Cmd,
+ BackgroundIndexStorage *IndexStorage) {
Queue.push_back(Bind(
- [this](tooling::CompileCommand Cmd) {
+ [this, IndexStorage](tooling::CompileCommand Cmd) {
std::string Filename = Cmd.Filename;
Cmd.CommandLine.push_back("-resource-dir=" + ResourceDir);
- if (auto Error = index(std::move(Cmd)))
+ if (auto Error = index(std::move(Cmd), IndexStorage))
log("Indexing {0} failed: {1}", Filename, std::move(Error));
},
std::move(Cmd)));
}
-static BackgroundIndex::FileDigest digest(StringRef Content) {
- return SHA1::hash({(const uint8_t *)Content.data(), Content.size()});
-}
-
-static Optional<BackgroundIndex::FileDigest> digestFile(const SourceManager &SM,
- FileID FID) {
- bool Invalid = false;
- StringRef Content = SM.getBufferData(FID, &Invalid);
- if (Invalid)
- return None;
- return digest(Content);
-}
-
// Resolves URI to file paths with cache.
class URIToFileCache {
public:
@@ -179,7 +204,8 @@
/// Given index results from a TU, only update files in \p FilesToUpdate.
void BackgroundIndex::update(StringRef MainFile, SymbolSlab Symbols,
RefSlab Refs,
- const StringMap<FileDigest> &FilesToUpdate) {
+ const StringMap<FileDigest> &FilesToUpdate,
+ BackgroundIndexStorage *IndexStorage) {
// Partition symbols/references into files.
struct File {
DenseSet<const Symbol *> Symbols;
@@ -227,20 +253,35 @@
for (const auto *R : F.second.Refs)
Refs.insert(RefToIDs[R], *R);
+ auto SS = llvm::make_unique<SymbolSlab>(std::move(Syms).build());
+ auto RS = llvm::make_unique<RefSlab>(std::move(Refs).build());
+
+ auto Hash = FilesToUpdate.lookup(Path);
+ // We need to store shards before updating the index, since the latter
+ // consumes slabs.
+ // FIXME: Store Hash in the Shard.
+ if (IndexStorage) {
+ IndexFileOut Shard;
+ Shard.Symbols = SS.get();
+ Shard.Refs = RS.get();
+ if (auto Error = IndexStorage->storeShard(Path, Shard))
+ elog("Failed to write background-index shard for file {0}: {1}", Path,
+ std::move(Error));
+ }
+
std::lock_guard<std::mutex> Lock(DigestsMu);
// This can override a newer version that is added in another thread,
// if this thread sees the older version but finishes later. This should be
// rare in practice.
- IndexedFileDigests[Path] = FilesToUpdate.lookup(Path);
- IndexedSymbols.update(Path,
- make_unique<SymbolSlab>(std::move(Syms).build()),
- make_unique<RefSlab>(std::move(Refs).build()));
+ IndexedFileDigests[Path] = Hash;
+ IndexedSymbols.update(Path, std::move(SS), std::move(RS));
}
}
// Creates a filter to not collect index results from files with unchanged
// digests.
-// \p FileDigests contains file digests for the current indexed files, and all changed files will be added to \p FilesToUpdate.
+// \p FileDigests contains file digests for the current indexed files, and all
+// changed files will be added to \p FilesToUpdate.
decltype(SymbolCollector::Options::FileFilter) createFileFilter(
const llvm::StringMap<BackgroundIndex::FileDigest> &FileDigests,
llvm::StringMap<BackgroundIndex::FileDigest> &FilesToUpdate) {
@@ -269,16 +310,11 @@
};
}
-Error BackgroundIndex::index(tooling::CompileCommand Cmd) {
+Error BackgroundIndex::index(tooling::CompileCommand Cmd,
+ BackgroundIndexStorage *IndexStorage) {
trace::Span Tracer("BackgroundIndex");
SPAN_ATTACH(Tracer, "file", Cmd.Filename);
- SmallString<128> AbsolutePath;
- if (sys::path::is_absolute(Cmd.Filename)) {
- AbsolutePath = Cmd.Filename;
- } else {
- AbsolutePath = Cmd.Directory;
- sys::path::append(AbsolutePath, Cmd.Filename);
- }
+ const std::string AbsolutePath = getAbsoluteFilePath(Cmd);
auto FS = FSProvider.getFileSystem();
auto Buf = FS->getBufferForFile(AbsolutePath);
@@ -342,7 +378,8 @@
Symbols.size(), Refs.numRefs());
SPAN_ATTACH(Tracer, "symbols", int(Symbols.size()));
SPAN_ATTACH(Tracer, "refs", int(Refs.numRefs()));
- update(AbsolutePath, std::move(Symbols), std::move(Refs), FilesToUpdate);
+ update(AbsolutePath, std::move(Symbols), std::move(Refs), FilesToUpdate,
+ IndexStorage);
{
// Make sure hash for the main file is always updated even if there is no
// index data in it.
Index: clangd/CMakeLists.txt
===================================================================
--- clangd/CMakeLists.txt
+++ clangd/CMakeLists.txt
@@ -38,6 +38,7 @@
XRefs.cpp
index/Background.cpp
+ index/BackgroundIndexStorage.cpp
index/CanonicalIncludes.cpp
index/FileIndex.cpp
index/Index.cpp
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
