Author: kadircet
Date: Thu Nov 15 02:31:23 2018
New Revision: 346941
URL: http://llvm.org/viewvc/llvm-project?rev=346941&view=rev
Log:
Address comments.
Modified:
clang-tools-extra/trunk/clangd/index/Background.cpp
clang-tools-extra/trunk/clangd/index/Background.h
clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp
Modified: clang-tools-extra/trunk/clangd/index/Background.cpp
URL:
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Background.cpp?rev=346941&r1=346940&r2=346941&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/Background.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/Background.cpp Thu Nov 15 02:31:23 2018
@@ -48,54 +48,49 @@ static Optional<BackgroundIndex::FileDig
return digest(Content);
}
-llvm::SmallString<128>
-getShardPathFromFilePath(llvm::SmallString<128> ShardRoot,
- llvm::StringRef FilePath) {
- sys::path::append(ShardRoot, sys::path::filename(FilePath) +
- toHex(digest(FilePath)) + ".idx");
- return ShardRoot;
+std::string getShardPathFromFilePath(llvm::StringRef ShardRoot,
+ llvm::StringRef FilePath) {
+ llvm::SmallString<128> ShardRootSS(ShardRoot);
+ sys::path::append(ShardRootSS, sys::path::filename(FilePath) +
+ toHex(digest(FilePath)) + ".idx");
+ return ShardRoot.str();
}
// Uses disk as a storage for index shards. Creates a directory called
-// ".clangd-index/" under the path provided during initialize.
-// Note: Requires initialize to be called before storing or retrieval.
+// ".clangd-index/" under the path provided during construction.
// This class is thread-safe.
class DiskBackedIndexStorage : public BackgroundIndexStorage {
- llvm::SmallString<128> DiskShardRoot;
+ std::string DiskShardRoot;
public:
- llvm::Expected<IndexFileIn>
- retrieveShard(llvm::StringRef ShardIdentifier) const override {
- auto ShardPath = getShardPathFromFilePath(DiskShardRoot, ShardIdentifier);
+ llvm::Expected<IndexFileIn> loadShard(llvm::StringRef ShardIdentifier) const
{
+ const std::string ShardPath =
+ getShardPathFromFilePath(DiskShardRoot, ShardIdentifier);
auto Buffer = MemoryBuffer::getFile(ShardPath);
if (!Buffer) {
- elog("Couldn't retrieve {0}: {1}", ShardPath,
- Buffer.getError().message());
+ elog("Couldn't load {0}: {1}", ShardPath, Buffer.getError().message());
return llvm::make_error<llvm::StringError>(Buffer.getError());
}
- // FIXME: Change readIndexFile to also look at Hash of the source that
- // generated index and skip if there is a mismatch.
return readIndexFile(Buffer->get()->getBuffer());
}
- bool storeShard(llvm::StringRef ShardIdentifier,
- IndexFileOut Shard) const override {
+ llvm::Error storeShard(llvm::StringRef ShardIdentifier,
+ IndexFileOut Shard) const override {
auto ShardPath = getShardPathFromFilePath(DiskShardRoot, ShardIdentifier);
std::error_code EC;
llvm::raw_fd_ostream OS(ShardPath, EC);
- if (EC) {
- elog("Failed to open {0} for writing: {1}", ShardPath, EC.message());
- return false;
- }
+ if (EC)
+ return errorCodeToError(EC);
OS << Shard;
- return true;
+ return llvm::Error::success();
}
- // Initializes DiskShardRoot to (Directory + ".clangd-index/") which is the
- // base directory for all shard files. After the initialization succeeds all
- // subsequent calls are no-op.
- DiskBackedIndexStorage(llvm::StringRef Directory) : DiskShardRoot(Directory)
{
- sys::path::append(DiskShardRoot, ".clangd-index/");
+ // Sets DiskShardRoot to (Directory + ".clangd-index/") which is the base
+ // directory for all shard files.
+ DiskBackedIndexStorage(llvm::StringRef Directory) {
+ llvm::SmallString<128> CDBDirectory(Directory);
+ sys::path::append(CDBDirectory, ".clangd-index/");
+ DiskShardRoot = CDBDirectory.str();
if (!llvm::sys::fs::exists(DiskShardRoot)) {
std::error_code OK;
std::error_code EC = llvm::sys::fs::create_directory(DiskShardRoot);
@@ -106,7 +101,7 @@ public:
}
};
-SmallString<128> getAbsolutePath(const tooling::CompileCommand &Cmd) {
+std::string getAbsoluteFilePath(const tooling::CompileCommand &Cmd) {
SmallString<128> AbsolutePath;
if (sys::path::is_absolute(Cmd.Filename)) {
AbsolutePath = Cmd.Filename;
@@ -114,7 +109,7 @@ SmallString<128> getAbsolutePath(const t
AbsolutePath = Cmd.Directory;
sys::path::append(AbsolutePath, Cmd.Filename);
}
- return AbsolutePath;
+ return AbsolutePath.str();
}
} // namespace
@@ -123,10 +118,12 @@ BackgroundIndex::BackgroundIndex(Context
StringRef ResourceDir,
const FileSystemProvider &FSProvider,
ArrayRef<std::string> URISchemes,
+ IndexStorageFactory IndexStorageCreator,
size_t ThreadPoolSize)
: SwapIndex(make_unique<MemIndex>()), ResourceDir(ResourceDir),
FSProvider(FSProvider), BackgroundContext(std::move(BackgroundContext)),
- URISchemes(URISchemes) {
+ URISchemes(URISchemes),
+ IndexStorageCreator(std::move(IndexStorageCreator)) {
assert(ThreadPoolSize > 0 && "Thread pool size can't be zero.");
while (ThreadPoolSize--) {
ThreadPool.emplace_back([this] { run(); });
@@ -185,57 +182,24 @@ void BackgroundIndex::blockUntilIdleForT
void BackgroundIndex::enqueue(StringRef Directory,
tooling::CompileCommand Cmd) {
- auto IndexStorage = BackgroundIndexStorage::getForDirectory(Directory);
- if (IndexStorage)
- loadShard(IndexStorage.get(), Cmd);
- else
+ BackgroundIndexStorage *IndexStorage = getIndexStorage(Directory);
+ if (!IndexStorage)
elog("No index storage for: {0}", Directory);
{
std::lock_guard<std::mutex> Lock(QueueMu);
- enqueueLocked(std::move(Cmd), std::move(IndexStorage));
+ enqueueLocked(std::move(Cmd), IndexStorage);
}
QueueCV.notify_all();
}
-void BackgroundIndex::loadShard(BackgroundIndexStorage *IndexStorage,
- const tooling::CompileCommand &Cmd) {
- assert(IndexStorage && "No index storage to load from?");
- auto AbsolutePath = getAbsolutePath(Cmd);
- auto Shard = IndexStorage->retrieveShard(AbsolutePath);
- if (Shard) {
- // FIXME: Updated hashes once we have them in serialized format.
- // IndexedFileDigests[AbsolutePath] = Hash;
- IndexedSymbols.update(AbsolutePath,
- make_unique<SymbolSlab>(std::move(*Shard->Symbols)),
- make_unique<RefSlab>(std::move(*Shard->Refs)));
-
- vlog("Loaded {0} from storage", AbsolutePath);
- }
-}
-
-void BackgroundIndex::loadShards(
- BackgroundIndexStorage *IndexStorage,
- const std::vector<tooling::CompileCommand> &Cmds) {
- assert(IndexStorage && "No index storage to load from?");
- for (const auto &Cmd : Cmds)
- loadShard(IndexStorage, Cmd);
- // FIXME: Maybe we should get rid of this one once we start building index
- // periodically? Especially if we also offload this task onto the queue.
- vlog("Rebuilding automatic index");
- reset(IndexedSymbols.buildIndex(IndexType::Light, DuplicateHandling::Merge,
- URISchemes));
-}
-
void BackgroundIndex::enqueueAll(StringRef Directory,
const tooling::CompilationDatabase &CDB) {
trace::Span Tracer("BackgroundIndexEnqueueCDB");
// FIXME: this function may be slow. Perhaps enqueue a task to re-read the
CDB
// from disk and enqueue the commands asynchronously?
auto Cmds = CDB.getAllCompileCommands();
- auto IndexStorage = BackgroundIndexStorage::getForDirectory(Directory);
- if (IndexStorage)
- loadShards(IndexStorage.get(), Cmds);
- else
+ BackgroundIndexStorage *IndexStorage = getIndexStorage(Directory);
+ if (!IndexStorage)
elog("No index storage for: {0}", Directory);
SPAN_ATTACH(Tracer, "commands", int64_t(Cmds.size()));
std::mt19937 Generator(std::random_device{}());
@@ -249,18 +213,16 @@ void BackgroundIndex::enqueueAll(StringR
QueueCV.notify_all();
}
-void BackgroundIndex::enqueueLocked(
- tooling::CompileCommand Cmd,
- std::shared_ptr<BackgroundIndexStorage> IndexStorage) {
+void BackgroundIndex::enqueueLocked(tooling::CompileCommand Cmd,
+ BackgroundIndexStorage *IndexStorage) {
Queue.push_back(Bind(
- [this](tooling::CompileCommand Cmd,
- std::shared_ptr<BackgroundIndexStorage> IndexStorage) {
+ [this, IndexStorage](tooling::CompileCommand Cmd) {
std::string Filename = Cmd.Filename;
Cmd.CommandLine.push_back("-resource-dir=" + ResourceDir);
- if (auto Error = index(std::move(Cmd), IndexStorage.get()))
+ if (auto Error = index(std::move(Cmd), IndexStorage))
log("Indexing {0} failed: {1}", Filename, std::move(Error));
},
- std::move(Cmd), std::move(IndexStorage)));
+ std::move(Cmd)));
}
// Resolves URI to file paths with cache.
@@ -356,7 +318,8 @@ void BackgroundIndex::update(StringRef M
IndexFileOut Shard;
Shard.Symbols = SS.get();
Shard.Refs = RS.get();
- IndexStorage->storeShard(Path, Shard);
+ if (auto Error = IndexStorage->storeShard(Path, Shard))
+ elog("Failed to store shard for {0}: {1}", Path, std::move(Error));
}
std::lock_guard<std::mutex> Lock(DigestsMu);
@@ -404,7 +367,7 @@ Error BackgroundIndex::index(tooling::Co
BackgroundIndexStorage *IndexStorage) {
trace::Span Tracer("BackgroundIndex");
SPAN_ATTACH(Tracer, "file", Cmd.Filename);
- SmallString<128> AbsolutePath = getAbsolutePath(Cmd);
+ const std::string AbsolutePath = getAbsoluteFilePath(Cmd);
auto FS = FSProvider.getFileSystem();
auto Buf = FS->getBufferForFile(AbsolutePath);
@@ -486,8 +449,18 @@ Error BackgroundIndex::index(tooling::Co
return Error::success();
}
-std::function<std::shared_ptr<BackgroundIndexStorage>(llvm::StringRef)>
- BackgroundIndexStorage::Factory = nullptr;
+BackgroundIndexStorage *
+BackgroundIndex::getIndexStorage(llvm::StringRef CDBDirectory) {
+ if (!IndexStorageCreator)
+ return nullptr;
+ auto IndexStorageIt = IndexStorageMap.find(CDBDirectory);
+ if (IndexStorageIt == IndexStorageMap.end())
+ IndexStorageIt =
+ IndexStorageMap
+ .insert({CDBDirectory, IndexStorageCreator(CDBDirectory)})
+ .first;
+ return IndexStorageIt->second.get();
+}
} // namespace clangd
} // namespace clang
Modified: clang-tools-extra/trunk/clangd/index/Background.h
URL:
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Background.h?rev=346941&r1=346940&r2=346941&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/Background.h (original)
+++ clang-tools-extra/trunk/clangd/index/Background.h Thu Nov 15 02:31:23 2018
@@ -30,44 +30,30 @@ namespace clangd {
// Handles storage and retrieval of index shards.
class BackgroundIndexStorage {
-private:
- static
std::function<std::shared_ptr<BackgroundIndexStorage>(llvm::StringRef)>
- Factory;
-
public:
- using FileDigest = decltype(llvm::SHA1::hash({}));
-
// Stores given shard associationg with ShardIdentifier, which can be
// retrieved later on with the same identifier.
- virtual bool storeShard(llvm::StringRef ShardIdentifier,
- IndexFileOut Shard) const = 0;
+ virtual llvm::Error storeShard(llvm::StringRef ShardIdentifier,
+ IndexFileOut Shard) const = 0;
- // Retrieves the shard if found, also returns hash for the source file that
- // generated this shard.
- virtual llvm::Expected<IndexFileIn>
- retrieveShard(llvm::StringRef ShardIdentifier) const = 0;
-
- template <typename T> static void setStorageFactory(T Factory) {
- BackgroundIndexStorage::Factory = Factory;
- }
-
- static std::shared_ptr<BackgroundIndexStorage>
- getForDirectory(llvm::StringRef Directory) {
- if (!Factory)
- return nullptr;
- return Factory(Directory);
- }
+ static std::unique_ptr<BackgroundIndexStorage>
+ createDiskStorage(llvm::StringRef CDBDirectory);
};
// Builds an in-memory index by by running the static indexer action over
// all commands in a compilation database. Indexing happens in the background.
+// Takes a factory function to create IndexStorage units for each compilation
+// database. Those databases are identified by directory they are found.
// FIXME: it should also persist its state on disk for fast start.
// FIXME: it should watch for changes to files on disk.
class BackgroundIndex : public SwapIndex {
public:
+ using IndexStorageFactory =
+ std::function<std::unique_ptr<BackgroundIndexStorage>(llvm::StringRef)>;
// FIXME: resource-dir injection should be hoisted somewhere common.
BackgroundIndex(Context BackgroundContext, llvm::StringRef ResourceDir,
const FileSystemProvider &, ArrayRef<std::string> URISchemes,
+ IndexStorageFactory IndexStorageCreator = nullptr,
size_t ThreadPoolSize = llvm::hardware_concurrency());
~BackgroundIndex(); // Blocks while the current task finishes.
@@ -93,10 +79,6 @@ private:
void update(llvm::StringRef MainFile, SymbolSlab Symbols, RefSlab Refs,
const llvm::StringMap<FileDigest> &FilesToUpdate,
BackgroundIndexStorage *IndexStorage);
- void loadShard(BackgroundIndexStorage *IndexStorage,
- const tooling::CompileCommand &Cmd);
- void loadShards(BackgroundIndexStorage *IndexStorage,
- const std::vector<tooling::CompileCommand> &Cmds);
// configuration
std::string ResourceDir;
@@ -112,11 +94,17 @@ private:
llvm::StringMap<FileDigest> IndexedFileDigests; // Key is absolute file path.
std::mutex DigestsMu;
+ // index storage
+ BackgroundIndexStorage *getIndexStorage(llvm::StringRef CDBDirectory);
+ // Maps CDB Directory to index storage.
+ llvm::StringMap<std::unique_ptr<BackgroundIndexStorage>> IndexStorageMap;
+ IndexStorageFactory IndexStorageCreator;
+
// queue management
using Task = std::function<void()>;
void run(); // Main loop executed by Thread. Runs tasks from Queue.
void enqueueLocked(tooling::CompileCommand Cmd,
- std::shared_ptr<BackgroundIndexStorage> IndexStorage);
+ BackgroundIndexStorage *IndexStorage);
std::mutex QueueMu;
unsigned NumActiveTasks = 0; // Only idle when queue is empty *and* no tasks.
std::condition_variable QueueCV;
Modified: clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp
URL:
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp?rev=346941&r1=346940&r2=346941&view=diff
==============================================================================
--- clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp (original)
+++ clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp Thu Nov
15 02:31:23 2018
@@ -78,38 +78,23 @@ TEST(BackgroundIndexTest, IndexTwoFiles)
FileURI("unittest:///root/B.cc")}));
}
-TEST(BackgroundIndexTest, ShardStorageTest) {
+TEST(BackgroundIndexTest, ShardStorageWriteTest) {
class MemoryShardStorage : public BackgroundIndexStorage {
mutable std::mutex StorageMu;
llvm::StringMap<std::string> &Storage;
- size_t &CacheHits;
public:
- MemoryShardStorage(llvm::StringMap<std::string> &Storage, size_t
&CacheHits)
- : Storage(Storage), CacheHits(CacheHits) {}
-
- bool storeShard(llvm::StringRef ShardIdentifier,
- IndexFileOut Shard) const override {
+ MemoryShardStorage(llvm::StringMap<std::string> &Storage)
+ : Storage(Storage) {}
+ llvm::Error storeShard(llvm::StringRef ShardIdentifier,
+ IndexFileOut Shard) const override {
std::lock_guard<std::mutex> Lock(StorageMu);
std::string &str = Storage[ShardIdentifier];
llvm::raw_string_ostream OS(str);
OS << Shard;
OS.flush();
- return true;
- }
- llvm::Expected<IndexFileIn>
- retrieveShard(llvm::StringRef ShardIdentifier) const override {
- std::lock_guard<std::mutex> Lock(StorageMu);
- if (Storage.find(ShardIdentifier) == Storage.end())
- return llvm::make_error<llvm::StringError>(
- "Shard not found.", llvm::inconvertibleErrorCode());
- auto IndexFile = readIndexFile(Storage[ShardIdentifier]);
- if (!IndexFile)
- return IndexFile;
- CacheHits++;
- return IndexFile;
+ return llvm::Error::success();
}
- bool initialize(llvm::StringRef Directory) { return true; }
};
MockFSProvider FS;
FS.Files[testPath("root/A.h")] = R"cpp(
@@ -121,11 +106,10 @@ TEST(BackgroundIndexTest, ShardStorageTe
"#include \"A.h\"\nvoid g() { (void)common; }";
llvm::StringMap<std::string> Storage;
- size_t CacheHits = 0;
- BackgroundIndexStorage::setStorageFactory(
- [&Storage, &CacheHits](llvm::StringRef) {
- return std::make_shared<MemoryShardStorage>(Storage, CacheHits);
- });
+ BackgroundIndex::IndexStorageFactory MemoryStorageFactory =
+ [&Storage](llvm::StringRef) {
+ return llvm::make_unique<MemoryShardStorage>(Storage);
+ };
tooling::CompileCommand Cmd;
Cmd.Filename = testPath("root/A.cc");
@@ -133,22 +117,11 @@ TEST(BackgroundIndexTest, ShardStorageTe
Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
// Check nothing is loaded from Storage, but A.cc and A.h has been stored.
{
- BackgroundIndex Idx(Context::empty(), "", FS, /*URISchemes=*/{"unittest"});
- Idx.enqueue(testPath("root"), Cmd);
- Idx.blockUntilIdleForTest();
- }
- EXPECT_EQ(CacheHits, 0U);
- EXPECT_EQ(Storage.size(), 2U);
- EXPECT_NE(Storage.find(testPath("root/A.h")), Storage.end());
- EXPECT_NE(Storage.find(testPath("root/A.cc")), Storage.end());
-
- // Check A.cc has been loaded from cache.
- {
- BackgroundIndex Idx(Context::empty(), "", FS, /*URISchemes=*/{"unittest"});
+ BackgroundIndex Idx(Context::empty(), "", FS, /*URISchemes=*/{"unittest"},
+ MemoryStorageFactory);
Idx.enqueue(testPath("root"), Cmd);
Idx.blockUntilIdleForTest();
}
- EXPECT_EQ(CacheHits, 1U);
EXPECT_EQ(Storage.size(), 2U);
EXPECT_NE(Storage.find(testPath("root/A.h")), Storage.end());
EXPECT_NE(Storage.find(testPath("root/A.cc")), Storage.end());
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
