ilya-biryukov updated this revision to Diff 122845.
ilya-biryukov added a comment.
Made in-memory preambles optional (on-disk by default).
https://reviews.llvm.org/D39843
Files:
clangd/ClangdLSPServer.cpp
clangd/ClangdLSPServer.h
clangd/ClangdServer.cpp
clangd/ClangdServer.h
clangd/ClangdUnit.cpp
clangd/ClangdUnit.h
clangd/ClangdUnitStore.cpp
clangd/ClangdUnitStore.h
clangd/tool/ClangdMain.cpp
test/clangd/completion.test
test/clangd/diagnostics-preamble.test
unittests/clangd/ClangdTests.cpp
Index: unittests/clangd/ClangdTests.cpp
===================================================================
--- unittests/clangd/ClangdTests.cpp
+++ unittests/clangd/ClangdTests.cpp
@@ -332,6 +332,7 @@
ErrorCheckingDiagConsumer DiagConsumer;
MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true);
ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(),
+ /*StorePreamblesInMemory=*/true,
clangd::CodeCompleteOptions(),
EmptyLogger::getInstance());
for (const auto &FileWithContents : ExtraFiles)
@@ -396,6 +397,7 @@
ErrorCheckingDiagConsumer DiagConsumer;
MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true);
ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(),
+ /*StorePreamblesInMemory=*/true,
clangd::CodeCompleteOptions(),
EmptyLogger::getInstance());
@@ -442,6 +444,7 @@
MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true);
ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(),
+ /*StorePreamblesInMemory=*/true,
clangd::CodeCompleteOptions(),
EmptyLogger::getInstance());
@@ -490,7 +493,9 @@
MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true);
// Run ClangdServer synchronously.
ClangdServer Server(CDB, DiagConsumer, FS,
- /*AsyncThreadsCount=*/0, clangd::CodeCompleteOptions(),
+ /*AsyncThreadsCount=*/0,
+ /*StorePreamblesInMemory=*/true,
+ clangd::CodeCompleteOptions(),
EmptyLogger::getInstance());
auto FooCpp = getVirtualTestFilePath("foo.cpp");
@@ -524,7 +529,9 @@
"-stdlib=libstdc++"});
// Run ClangdServer synchronously.
ClangdServer Server(CDB, DiagConsumer, FS,
- /*AsyncThreadsCount=*/0, clangd::CodeCompleteOptions(),
+ /*AsyncThreadsCount=*/0,
+ /*StorePreamblesInMemory=*/true,
+ clangd::CodeCompleteOptions(),
EmptyLogger::getInstance());
// Just a random gcc version string
@@ -573,7 +580,9 @@
ErrorCheckingDiagConsumer DiagConsumer;
MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true);
ClangdServer Server(CDB, DiagConsumer, FS,
- /*AsyncThreadsCount=*/0, clangd::CodeCompleteOptions(),
+ /*AsyncThreadsCount=*/0,
+ /*StorePreamblesInMemory=*/true,
+ clangd::CodeCompleteOptions(),
EmptyLogger::getInstance());
// No need to sync reparses, because reparses are performed on the calling
// thread to true.
@@ -642,6 +651,7 @@
MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true);
ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(),
+ /*StorePreamblesInMemory=*/true,
clangd::CodeCompleteOptions(),
EmptyLogger::getInstance());
@@ -759,7 +769,8 @@
auto TestWithOpts = [&](clangd::CodeCompleteOptions Opts) {
ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(),
- Opts, EmptyLogger::getInstance());
+ /*StorePreamblesInMemory=*/true, Opts,
+ EmptyLogger::getInstance());
// No need to sync reparses here as there are no asserts on diagnostics (or
// other async operations).
Server.addDocument(FooCpp, GlobalCompletion.Text);
@@ -952,6 +963,7 @@
{
MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true);
ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(),
+ /*StorePreamblesInMemory=*/true,
clangd::CodeCompleteOptions(),
EmptyLogger::getInstance());
@@ -1112,6 +1124,7 @@
MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true);
ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(),
+ /*StorePreamblesInMemory=*/true,
clangd::CodeCompleteOptions(),
EmptyLogger::getInstance());
@@ -1238,7 +1251,8 @@
std::move(StartSecondReparsePromise));
MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true);
- ClangdServer Server(CDB, DiagConsumer, FS, 4, clangd::CodeCompleteOptions(),
+ ClangdServer Server(CDB, DiagConsumer, FS, 4, /*StorePreamblesInMemory=*/true,
+ clangd::CodeCompleteOptions(),
EmptyLogger::getInstance());
Server.addDocument(FooCpp, SourceContentsWithErrors);
StartSecondReparse.wait();
Index: test/clangd/diagnostics-preamble.test
===================================================================
--- test/clangd/diagnostics-preamble.test
+++ test/clangd/diagnostics-preamble.test
@@ -1,4 +1,5 @@
# RUN: clangd -pretty -run-synchronously < %s | FileCheck -strict-whitespace %s
+# RUN: clangd -pretty -run-synchronously -in-memory-pchs < %s | FileCheck -strict-whitespace %s
# It is absolutely vital that this file has CRLF line endings.
#
Content-Length: 125
Index: test/clangd/completion.test
===================================================================
--- test/clangd/completion.test
+++ test/clangd/completion.test
@@ -1,4 +1,5 @@
# RUN: clangd -pretty -run-synchronously < %s | FileCheck -strict-whitespace %s
+# RUN: clangd -pretty -run-synchronously -in-memory-pchs < %s | FileCheck -strict-whitespace %s
# It is absolutely vital that this file has CRLF line endings.
#
Content-Length: 125
Index: clangd/tool/ClangdMain.cpp
===================================================================
--- clangd/tool/ClangdMain.cpp
+++ clangd/tool/ClangdMain.cpp
@@ -45,6 +45,13 @@
PrettyPrint("pretty", llvm::cl::desc("Pretty-print JSON output"),
llvm::cl::init(false));
+static llvm::cl::opt<bool> StorePreamblesInMemory(
+ "in-memory-pchs",
+ llvm::cl::desc("Store PCHs in memory instead of disk. Enabling this option "
+ "increases memory consumption considerably, but also "
+ "increases performance"),
+ llvm::cl::init(false));
+
static llvm::cl::opt<bool> RunSynchronously(
"run-synchronously",
llvm::cl::desc("Parse on main thread. If set, -j is ignored"),
@@ -135,8 +142,9 @@
llvm::sys::ChangeStdinToBinary();
// Initialize and run ClangdLSPServer.
- ClangdLSPServer LSPServer(Out, WorkerThreadsCount, EnableSnippets,
- ResourceDirRef, CompileCommandsDirPath);
+ ClangdLSPServer LSPServer(Out, WorkerThreadsCount, StorePreamblesInMemory,
+ EnableSnippets, ResourceDirRef,
+ CompileCommandsDirPath);
constexpr int NoShutdownRequestErrorCode = 1;
llvm::set_thread_name("clangd.main");
return LSPServer.run(std::cin) ? 0 : NoShutdownRequestErrorCode;
Index: clangd/ClangdUnitStore.h
===================================================================
--- clangd/ClangdUnitStore.h
+++ clangd/ClangdUnitStore.h
@@ -25,17 +25,20 @@
/// Thread-safe mapping from FileNames to CppFile.
class CppFileCollection {
public:
- std::shared_ptr<CppFile> getOrCreateFile(
- PathRef File, PathRef ResourceDir, GlobalCompilationDatabase &CDB,
- std::shared_ptr<PCHContainerOperations> PCHs, clangd::Logger &Logger) {
+ std::shared_ptr<CppFile>
+ getOrCreateFile(PathRef File, PathRef ResourceDir,
+ GlobalCompilationDatabase &CDB, bool StorePreamblesInMemory,
+ std::shared_ptr<PCHContainerOperations> PCHs,
+ clangd::Logger &Logger) {
std::lock_guard<std::mutex> Lock(Mutex);
auto It = OpenedFiles.find(File);
if (It == OpenedFiles.end()) {
auto Command = getCompileCommand(CDB, File, ResourceDir);
It = OpenedFiles
.try_emplace(File, CppFile::Create(File, std::move(Command),
+ StorePreamblesInMemory,
std::move(PCHs), Logger))
.first;
}
@@ -58,7 +61,8 @@
/// will be returned in RecreateResult.RemovedFile.
RecreateResult recreateFileIfCompileCommandChanged(
PathRef File, PathRef ResourceDir, GlobalCompilationDatabase &CDB,
- std::shared_ptr<PCHContainerOperations> PCHs, clangd::Logger &Logger);
+ bool StorePreamblesInMemory, std::shared_ptr<PCHContainerOperations> PCHs,
+ clangd::Logger &Logger);
std::shared_ptr<CppFile> getFile(PathRef File) {
std::lock_guard<std::mutex> Lock(Mutex);
Index: clangd/ClangdUnitStore.cpp
===================================================================
--- clangd/ClangdUnitStore.cpp
+++ clangd/ClangdUnitStore.cpp
@@ -29,7 +29,8 @@
CppFileCollection::RecreateResult
CppFileCollection::recreateFileIfCompileCommandChanged(
PathRef File, PathRef ResourceDir, GlobalCompilationDatabase &CDB,
- std::shared_ptr<PCHContainerOperations> PCHs, clangd::Logger &Logger) {
+ bool StorePreamblesInMemory, std::shared_ptr<PCHContainerOperations> PCHs,
+ clangd::Logger &Logger) {
auto NewCommand = getCompileCommand(CDB, File, ResourceDir);
std::lock_guard<std::mutex> Lock(Mutex);
@@ -40,13 +41,15 @@
if (It == OpenedFiles.end()) {
It = OpenedFiles
.try_emplace(File, CppFile::Create(File, std::move(NewCommand),
+ StorePreamblesInMemory,
std::move(PCHs), Logger))
.first;
} else if (!compileCommandsAreEqual(It->second->getCompileCommand(),
NewCommand)) {
Result.RemovedFile = std::move(It->second);
It->second =
- CppFile::Create(File, std::move(NewCommand), std::move(PCHs), Logger);
+ CppFile::Create(File, std::move(NewCommand), StorePreamblesInMemory,
+ std::move(PCHs), Logger);
}
Result.FileInCollection = It->second;
return Result;
Index: clangd/ClangdUnit.h
===================================================================
--- clangd/ClangdUnit.h
+++ clangd/ClangdUnit.h
@@ -143,10 +143,12 @@
// deferRebuild will hold references to it.
static std::shared_ptr<CppFile>
Create(PathRef FileName, tooling::CompileCommand Command,
+ bool StorePreamblesInMemory,
std::shared_ptr<PCHContainerOperations> PCHs, clangd::Logger &Logger);
private:
CppFile(PathRef FileName, tooling::CompileCommand Command,
+ bool StorePreamblesInMemory,
std::shared_ptr<PCHContainerOperations> PCHs, clangd::Logger &Logger);
public:
@@ -222,6 +224,7 @@
Path FileName;
tooling::CompileCommand Command;
+ bool StorePreamblesInMemory;
/// Mutex protects all fields, declared below it, FileName and Command are not
/// mutated.
Index: clangd/ClangdUnit.cpp
===================================================================
--- clangd/ClangdUnit.cpp
+++ clangd/ClangdUnit.cpp
@@ -235,7 +235,7 @@
// NOTE: we use Buffer.get() when adding remapped files, so we have to make
// sure it will be released if no error is emitted.
if (Preamble) {
- Preamble->AddImplicitPreamble(*CI, Buffer.get());
+ Preamble->AddImplicitPreamble(*CI, /*ref*/ VFS, Buffer.get());
} else {
CI->getPreprocessorOpts().addRemappedFile(
CI->getFrontendOpts().Inputs[0].getFile(), Buffer.get());
@@ -1110,16 +1110,20 @@
std::shared_ptr<CppFile>
CppFile::Create(PathRef FileName, tooling::CompileCommand Command,
+ bool StorePreamblesInMemory,
std::shared_ptr<PCHContainerOperations> PCHs,
clangd::Logger &Logger) {
- return std::shared_ptr<CppFile>(
- new CppFile(FileName, std::move(Command), std::move(PCHs), Logger));
+ return std::shared_ptr<CppFile>(new CppFile(FileName, std::move(Command),
+ StorePreamblesInMemory,
+ std::move(PCHs), Logger));
}
CppFile::CppFile(PathRef FileName, tooling::CompileCommand Command,
+ bool StorePreamblesInMemory,
std::shared_ptr<PCHContainerOperations> PCHs,
clangd::Logger &Logger)
- : FileName(FileName), Command(std::move(Command)), RebuildCounter(0),
+ : FileName(FileName), Command(std::move(Command)),
+ StorePreamblesInMemory(StorePreamblesInMemory), RebuildCounter(0),
RebuildInProgress(false), PCHs(std::move(PCHs)), Logger(Logger) {
std::lock_guard<std::mutex> Lock(Mutex);
@@ -1263,7 +1267,7 @@
CppFilePreambleCallbacks SerializedDeclsCollector;
auto BuiltPreamble = PrecompiledPreamble::Build(
*CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine, VFS, PCHs,
- SerializedDeclsCollector);
+ /*StoreInMemory=*/true, SerializedDeclsCollector);
if (BuiltPreamble) {
return std::make_shared<PreambleData>(
Index: clangd/ClangdServer.h
===================================================================
--- clangd/ClangdServer.h
+++ clangd/ClangdServer.h
@@ -205,10 +205,14 @@
/// worker thread. Therefore, instances of \p DiagConsumer must properly
/// synchronize access to shared state.
///
+ /// \p StorePreamblesInMemory defines whether the Preambles generated by
+ /// clangd are stored in-memory or on disk.
+ ///
/// Various messages are logged using \p Logger.
ClangdServer(GlobalCompilationDatabase &CDB,
DiagnosticsConsumer &DiagConsumer,
FileSystemProvider &FSProvider, unsigned AsyncThreadsCount,
+ bool StorePreamblesInMemory,
clangd::CodeCompleteOptions CodeCompleteOpts,
clangd::Logger &Logger,
llvm::Optional<StringRef> ResourceDir = llvm::None);
@@ -326,6 +330,7 @@
// If set, this represents the workspace path.
llvm::Optional<std::string> RootPath;
std::shared_ptr<PCHContainerOperations> PCHs;
+ bool StorePreamblesInMemory;
clangd::CodeCompleteOptions CodeCompleteOpts;
/// Used to serialize diagnostic callbacks.
/// FIXME(ibiryukov): get rid of an extra map and put all version counters
Index: clangd/ClangdServer.cpp
===================================================================
--- clangd/ClangdServer.cpp
+++ clangd/ClangdServer.cpp
@@ -169,17 +169,16 @@
Worker.join();
}
-ClangdServer::ClangdServer(GlobalCompilationDatabase &CDB,
- DiagnosticsConsumer &DiagConsumer,
- FileSystemProvider &FSProvider,
- unsigned AsyncThreadsCount,
- clangd::CodeCompleteOptions CodeCompleteOpts,
- clangd::Logger &Logger,
- llvm::Optional<StringRef> ResourceDir)
+ClangdServer::ClangdServer(
+ GlobalCompilationDatabase &CDB, DiagnosticsConsumer &DiagConsumer,
+ FileSystemProvider &FSProvider, unsigned AsyncThreadsCount,
+ bool StorePreamblesInMemory, clangd::CodeCompleteOptions CodeCompleteOpts,
+ clangd::Logger &Logger, llvm::Optional<StringRef> ResourceDir)
: Logger(Logger), CDB(CDB), DiagConsumer(DiagConsumer),
FSProvider(FSProvider),
ResourceDir(ResourceDir ? ResourceDir->str() : getStandardResourceDir()),
PCHs(std::make_shared<PCHContainerOperations>()),
+ StorePreamblesInMemory(StorePreamblesInMemory),
CodeCompleteOpts(CodeCompleteOpts), WorkScheduler(AsyncThreadsCount) {}
void ClangdServer::setRootPath(PathRef RootPath) {
@@ -193,8 +192,8 @@
DocVersion Version = DraftMgr.updateDraft(File, Contents);
auto TaggedFS = FSProvider.getTaggedFileSystem(File);
- std::shared_ptr<CppFile> Resources =
- Units.getOrCreateFile(File, ResourceDir, CDB, PCHs, Logger);
+ std::shared_ptr<CppFile> Resources = Units.getOrCreateFile(
+ File, ResourceDir, CDB, StorePreamblesInMemory, PCHs, Logger);
return scheduleReparseAndDiags(File, VersionedDraft{Version, Contents.str()},
std::move(Resources), std::move(TaggedFS));
}
@@ -211,8 +210,8 @@
"forceReparse() was called for non-added document");
auto TaggedFS = FSProvider.getTaggedFileSystem(File);
- auto Recreated = Units.recreateFileIfCompileCommandChanged(File, ResourceDir,
- CDB, PCHs, Logger);
+ auto Recreated = Units.recreateFileIfCompileCommandChanged(
+ File, ResourceDir, CDB, StorePreamblesInMemory, PCHs, Logger);
// Note that std::future from this cleanup action is ignored.
scheduleCancelRebuild(std::move(Recreated.RemovedFile));
Index: clangd/ClangdLSPServer.h
===================================================================
--- clangd/ClangdLSPServer.h
+++ clangd/ClangdLSPServer.h
@@ -31,7 +31,7 @@
/// loaded only from \p CompileCommandsDir. Otherwise, clangd will look
/// for compile_commands.json in all parent directories of each file.
ClangdLSPServer(JSONOutput &Out, unsigned AsyncThreadsCount,
- bool SnippetCompletions,
+ bool StorePreamblesInMemory, bool SnippetCompletions,
llvm::Optional<StringRef> ResourceDir,
llvm::Optional<Path> CompileCommandsDir);
Index: clangd/ClangdLSPServer.cpp
===================================================================
--- clangd/ClangdLSPServer.cpp
+++ clangd/ClangdLSPServer.cpp
@@ -236,11 +236,13 @@
}
ClangdLSPServer::ClangdLSPServer(JSONOutput &Out, unsigned AsyncThreadsCount,
+ bool StorePreamblesInMemory,
bool SnippetCompletions,
llvm::Optional<StringRef> ResourceDir,
llvm::Optional<Path> CompileCommandsDir)
: Out(Out), CDB(/*Logger=*/Out, std::move(CompileCommandsDir)),
Server(CDB, /*DiagConsumer=*/*this, FSProvider, AsyncThreadsCount,
+ StorePreamblesInMemory,
clangd::CodeCompleteOptions(
/*EnableSnippetsAndCodePatterns=*/SnippetCompletions),
/*Logger=*/Out, ResourceDir) {}
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
