kadircet created this revision.
kadircet added a reviewer: ilya-biryukov.
Herald added subscribers: cfe-commits, jdoerfert, erik.pilkington, arphaman,
jkorous, MaskRay, ioeric, mgorny.
Herald added a project: clang.
Sample output:
I[18:37:50.083] BackgroundIndex: build symbol index periodically every 86400000
ms.
I[18:37:50.115] Enqueueing 3742 commands for indexing
I[18:37:50.280] [0/3742] Loaded shards from storage
I[18:37:51.026] [1/3742] Indexed
/usr/local/google/home/kadircet/repos/llvm/llvm/lib/Demangle/MicrosoftDemangleNodes.cpp
(6422 symbols, 18236 refs, 161 files)
I[18:37:51.581] [2/3742] Indexed
/usr/local/google/home/kadircet/repos/llvm/llvm/lib/Support/MD5.cpp (10204
symbols, 28897 refs, 232 files)
I[18:37:51.607] [3/3742] Indexed
/usr/local/google/home/kadircet/repos/llvm/llvm/lib/MC/MCSchedule.cpp (11461
symbols, 32420 refs, 247 files)
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D59605
Files:
clang-tools-extra/clangd/CMakeLists.txt
clang-tools-extra/clangd/background-indexer/BackgroundIndexer.cpp
clang-tools-extra/clangd/background-indexer/CMakeLists.txt
clang-tools-extra/clangd/index/Background.cpp
clang-tools-extra/clangd/index/Background.h
Index: clang-tools-extra/clangd/index/Background.h
===================================================================
--- clang-tools-extra/clangd/index/Background.h
+++ clang-tools-extra/clangd/index/Background.h
@@ -144,6 +144,11 @@
std::deque<std::pair<Task, ThreadPriority>> Queue;
std::vector<std::thread> ThreadPool; // FIXME: Abstract this away.
GlobalCompilationDatabase::CommandChanged::Subscription CommandsChanged;
+
+ // For logging
+ size_t EnqueuedTUs = 0;
+ size_t IndexedTUs = 0;
+ std::mutex TUCounterMutex;
};
} // namespace clangd
Index: clang-tools-extra/clangd/index/Background.cpp
===================================================================
--- clang-tools-extra/clangd/index/Background.cpp
+++ clang-tools-extra/clangd/index/Background.cpp
@@ -29,6 +29,7 @@
#include <chrono>
#include <memory>
+#include <mutex>
#include <numeric>
#include <queue>
#include <random>
@@ -215,6 +216,10 @@
// We're doing this asynchronously, because we'll read shards here too.
log("Enqueueing {0} commands for indexing", ChangedFiles.size());
SPAN_ATTACH(Tracer, "files", int64_t(ChangedFiles.size()));
+ {
+ std::lock_guard<std::mutex> Lock(TUCounterMutex);
+ EnqueuedTUs += ChangedFiles.size();
+ }
auto NeedsReIndexing = loadShards(std::move(ChangedFiles));
// Run indexing for files that need to be updated.
@@ -447,9 +452,13 @@
assert(Index.Symbols && Index.Refs && Index.Sources &&
"Symbols, Refs and Sources must be set.");
- log("Indexed {0} ({1} symbols, {2} refs, {3} files)",
- Inputs.CompileCommand.Filename, Index.Symbols->size(),
- Index.Refs->numRefs(), Index.Sources->size());
+ {
+ std::lock_guard<std::mutex> Lock(TUCounterMutex);
+ ++IndexedTUs;
+ log("[{0}/{1}] Indexed {2} ({3} symbols, {4} refs, {5} files)", IndexedTUs,
+ EnqueuedTUs, Inputs.CompileCommand.Filename, Index.Symbols->size(),
+ Index.Refs->numRefs(), Index.Sources->size());
+ }
SPAN_ATTACH(Tracer, "symbols", int(Index.Symbols->size()));
SPAN_ATTACH(Tracer, "refs", int(Index.Refs->numRefs()));
SPAN_ATTACH(Tracer, "sources", int(Index.Sources->size()));
@@ -581,20 +590,23 @@
// Keeps track of the loaded shards to make sure we don't perform redundant
// disk IO. Keys are absolute paths.
llvm::StringSet<> LoadedShards;
+ size_t UpToDateTUs = 0;
for (const auto &File : ChangedFiles) {
ProjectInfo PI;
auto Cmd = CDB.getCompileCommand(File, &PI);
if (!Cmd)
continue;
BackgroundIndexStorage *IndexStorage = IndexStorageFactory(PI.SourceRoot);
+ ++UpToDateTUs;
auto Dependencies = loadShard(*Cmd, IndexStorage, LoadedShards);
for (const auto &Dependency : Dependencies) {
if (!Dependency.NeedsReIndexing || FilesToIndex.count(Dependency.Path))
continue;
+ --UpToDateTUs;
// FIXME: Currently, we simply schedule indexing on a TU whenever any of
// its dependencies needs re-indexing. We might do it smarter by figuring
// out a minimal set of TUs that will cover all the stale dependencies.
- vlog("Enqueueing TU {0} because its dependency {1} needs re-indexing.",
+ dlog("Enqueueing TU {0} because its dependency {1} needs re-indexing.",
Cmd->Filename, Dependency.Path);
NeedsReIndexing.push_back({std::move(*Cmd), IndexStorage});
// Mark all of this TU's dependencies as to-be-indexed so that we won't
@@ -604,7 +616,11 @@
break;
}
}
- vlog("Loaded all shards");
+ {
+ std::lock_guard<std::mutex> Lock(TUCounterMutex);
+ IndexedTUs += UpToDateTUs;
+ log("[{0}/{1}] Loaded shards from storage", IndexedTUs, EnqueuedTUs);
+ }
reset(IndexedSymbols.buildIndex(IndexType::Heavy, DuplicateHandling::Merge));
vlog("BackgroundIndex: built symbol index with estimated memory {0} "
"bytes.",
Index: clang-tools-extra/clangd/background-indexer/CMakeLists.txt
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/background-indexer/CMakeLists.txt
@@ -0,0 +1,20 @@
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../)
+
+set(LLVM_LINK_COMPONENTS
+ Support
+ )
+
+add_clang_executable(clangd-background-indexer
+ BackgroundIndexer.cpp
+)
+
+target_link_libraries(clangd-background-indexer
+ PRIVATE
+ clangAST
+ clangIndex
+ clangDaemon
+ clangBasic
+ clangFrontend
+ clangLex
+ clangTooling
+)
Index: clang-tools-extra/clangd/background-indexer/BackgroundIndexer.cpp
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/background-indexer/BackgroundIndexer.cpp
@@ -0,0 +1,64 @@
+//===--- BackgroundIndexer.cpp -----------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// A standalone binary to run clangd's background indexer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "/usr/local/google/home/kadircet/repos/llvm/clang-tools-extra/clangd/Context.h"
+#include "/usr/local/google/home/kadircet/repos/llvm/clang-tools-extra/clangd/Logger.h"
+#include "FSProvider.h"
+#include "GlobalCompilationDatabase.h"
+#include "index/Background.h"
+#include "clang/Tooling/CompilationDatabase.h"
+#include "llvm/ADT/None.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Signals.h"
+#include "llvm/Support/raw_ostream.h"
+#include <memory>
+
+static llvm::cl::opt<std::string> CompileCommandsDir(
+ llvm::cl::Positional,
+ llvm::cl::desc("Directory containing compilation database"),
+ llvm::cl::init("."));
+
+int main(int Argc, const char **Argv) {
+ llvm::sys::PrintStackTraceOnErrorSignal(Argv[0]);
+ llvm::cl::ParseCommandLineOptions(
+ Argc, Argv, "A standalone binary to run clangd's background indexer.");
+ clang::clangd::RealFileSystemProvider FSProvider;
+
+ llvm::errs().SetBuffered();
+ clang::clangd::StreamLogger Logger(llvm::errs(), clang::clangd::Logger::Info);
+ clang::clangd::LoggingSession LoggingSession(Logger);
+
+ clang::clangd::DirectoryBasedGlobalCompilationDatabase DirectoryBasedCDB(
+ /*CompileCommandsDir=*/llvm::None);
+ clang::clangd::OverlayCDB CDB(&DirectoryBasedCDB);
+ // FIXME: Maybe add a way to not lower thread priority?
+ clang::clangd::BackgroundIndex BackgroundIdx(
+ clang::clangd::Context::empty(), FSProvider, CDB,
+ clang::clangd::BackgroundIndexStorage::createDiskBackedStorageFactory(),
+ // We don't want to build dex/mem index until every file has been
+ // indexed/loaded.
+ // FIXME: Change BackgroundIndex to only built after all files have been
+ // read if build period is zero, which should only be used by
+ // non-interactive tools like this one.
+ 24 * 60 * 60 * 1000);
+ llvm::SmallString<128> DummyFile(CompileCommandsDir);
+ llvm::sys::fs::make_absolute(DummyFile);
+ llvm::sys::path::remove_dots(DummyFile, true);
+ llvm::sys::path::append(DummyFile, "dummy.cpp");
+ CDB.getCompileCommand(DummyFile);
+ BackgroundIdx.blockUntilIdleForTest(llvm::None);
+ return 0;
+}
Index: clang-tools-extra/clangd/CMakeLists.txt
===================================================================
--- clang-tools-extra/clangd/CMakeLists.txt
+++ clang-tools-extra/clangd/CMakeLists.txt
@@ -121,6 +121,7 @@
endif()
add_subdirectory(tool)
add_subdirectory(indexer)
+add_subdirectory(background-indexer)
add_subdirectory(index/dex/dexp)
if (LLVM_INCLUDE_BENCHMARKS)
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
