dexonsmith created this revision.
dexonsmith added reviewers: arphaman, JDevlieghere, jansvoboda11, akyrtzi.
Herald added a subscriber: ributzka.
dexonsmith requested review of this revision.
Re-implement the `RemappedFiles` parameter to various `ASTUnit` entry points by
storing buffers in an `InMemoryFileSystem`. This parameter is commonly used by
indexing tools to pass in source files that haven't been saved to disk (yet).
This is the same strategy `ClangTool` switched to in 2015.
The previous implementation added them to
`PreprocessorOptions::RemappedFileBuffers`, which eventually triggers calls to
`FileManager::getVirtualFile` and `SourceManager::overrideFileContents`. That's
one of the blockers for sinking the content cache from `SourceManager` down to
`FileManager`. The new strategy is also a bit cleaner (the old one predates
`llvm::vfs::FileSystem`).
Follow-ups will handle the other two uses of `RemappedFileBuffers` in
`ASTUnit`. One is for the preamble's serialized AST. The other is for
overriding the main file buffer, which is a bit tricky since we need the
`ASTWriter` to save the preamble-only version.
WIP: I'm not totally happy with this patch yet, but tests are finally passing
so I'm posting this for early review. In the meantime I'll continue peeling off
what I can for prep patches (or there may be changes I can just delete). Among
other things, I want to add a test (in a prep patch) for the handling of a file
showing up twice in the same `RemappedFiles` vector, since it was pretty tough
to discover I needed special logic for that.
https://reviews.llvm.org/D91300
Files:
clang/include/clang/Frontend/ASTUnit.h
clang/include/clang/Frontend/PrecompiledPreamble.h
clang/lib/Frontend/ASTUnit.cpp
clang/lib/Frontend/PrecompiledPreamble.cpp
clang/tools/libclang/CIndex.cpp
clang/tools/libclang/CIndexCodeCompletion.cpp
clang/unittests/Frontend/PCHPreambleTest.cpp
Index: clang/unittests/Frontend/PCHPreambleTest.cpp
===================================================================
--- clang/unittests/Frontend/PCHPreambleTest.cpp
+++ clang/unittests/Frontend/PCHPreambleTest.cpp
@@ -85,11 +85,6 @@
CI->getTargetOpts().Triple = "i386-unknown-linux-gnu";
- CI->getPreprocessorOpts().RemappedFileBuffers = GetRemappedFiles();
-
- PreprocessorOptions &PPOpts = CI->getPreprocessorOpts();
- PPOpts.RemappedFilesKeepOriginalName = true;
-
IntrusiveRefCntPtr<DiagnosticsEngine>
Diags(CompilerInstance::createDiagnostics(new DiagnosticOptions, new DiagnosticConsumer));
@@ -97,7 +92,8 @@
std::unique_ptr<ASTUnit> AST = ASTUnit::LoadFromCompilerInvocation(
CI, PCHContainerOpts, Diags, FileMgr,
- /*PrecompilePreambleAfterNParses=*/1);
+ /*PrecompilePreambleAfterNParses=*/1, /*UserFilesAreVolatile=*/false,
+ GetRemappedFiles());
return AST;
}
@@ -111,9 +107,8 @@
}
private:
- std::vector<std::pair<std::string, llvm::MemoryBuffer *>>
- GetRemappedFiles() const {
- std::vector<std::pair<std::string, llvm::MemoryBuffer *>> Remapped;
+ std::vector<ASTUnit::RemappedFile> GetRemappedFiles() const {
+ std::vector<ASTUnit::RemappedFile> Remapped;
for (const auto &RemappedFile : RemappedFiles) {
std::unique_ptr<MemoryBuffer> buf = MemoryBuffer::getMemBufferCopy(
RemappedFile.second, RemappedFile.first());
Index: clang/tools/libclang/CIndexCodeCompletion.cpp
===================================================================
--- clang/tools/libclang/CIndexCodeCompletion.cpp
+++ clang/tools/libclang/CIndexCodeCompletion.cpp
@@ -247,7 +247,7 @@
/// AllocatedCXCodeCompleteResults outlives the CXTranslationUnit, so we can
/// not rely on the StringPool in the TU.
struct AllocatedCXCodeCompleteResults : public CXCodeCompleteResults {
- AllocatedCXCodeCompleteResults(IntrusiveRefCntPtr<FileManager> FileMgr);
+ AllocatedCXCodeCompleteResults();
~AllocatedCXCodeCompleteResults();
/// Diagnostics produced while performing code completion.
@@ -354,8 +354,7 @@
/// Used for debugging purposes only.
static std::atomic<unsigned> CodeCompletionResultObjects;
-AllocatedCXCodeCompleteResults::AllocatedCXCodeCompleteResults(
- IntrusiveRefCntPtr<FileManager> FileMgr)
+AllocatedCXCodeCompleteResults::AllocatedCXCodeCompleteResults()
: CXCodeCompleteResults(), DiagOpts(new DiagnosticOptions),
Diag(new DiagnosticsEngine(
IntrusiveRefCntPtr<DiagnosticIDs>(new DiagnosticIDs), &*DiagOpts)),
@@ -717,21 +716,19 @@
ASTUnit::ConcurrencyCheck Check(*AST);
// Perform the remapping of source files.
- SmallVector<ASTUnit::RemappedFile, 4> RemappedFiles;
+ std::vector<ASTUnit::RemappedFile> RemappedFiles;
- for (auto &UF : unsaved_files) {
- std::unique_ptr<llvm::MemoryBuffer> MB =
- llvm::MemoryBuffer::getMemBufferCopy(getContents(UF), UF.Filename);
- RemappedFiles.push_back(std::make_pair(UF.Filename, MB.release()));
- }
+ for (auto &UF : unsaved_files)
+ RemappedFiles.push_back(std::make_pair(
+ UF.Filename,
+ llvm::MemoryBuffer::getMemBufferCopy(getContents(UF), UF.Filename)));
if (EnableLogging) {
// FIXME: Add logging.
}
// Parse the resulting source file to find code-completion results.
- AllocatedCXCodeCompleteResults *Results = new AllocatedCXCodeCompleteResults(
- &AST->getFileManager());
+ AllocatedCXCodeCompleteResults *Results = new AllocatedCXCodeCompleteResults;
Results->Results = nullptr;
Results->NumResults = 0;
@@ -754,8 +751,8 @@
*CXXIdx, LibclangInvocationReporter::OperationKind::CompletionOperation,
TU->ParsingOptions, CArgs, CompletionInvocation, unsaved_files);
AST->CodeComplete(
- complete_filename, complete_line, complete_column, RemappedFiles,
- (options & CXCodeComplete_IncludeMacros),
+ complete_filename, complete_line, complete_column,
+ std::move(RemappedFiles), (options & CXCodeComplete_IncludeMacros),
(options & CXCodeComplete_IncludeCodePatterns), IncludeBriefComments,
Capture, CXXIdx->getPCHContainerOperations(), *Results->Diag,
Results->LangOpts, Results->Diagnostics, Results->TemporaryBuffers);
Index: clang/tools/libclang/CIndex.cpp
===================================================================
--- clang/tools/libclang/CIndex.cpp
+++ clang/tools/libclang/CIndex.cpp
@@ -3564,11 +3564,10 @@
llvm::CrashRecoveryContextCleanupRegistrar<std::vector<ASTUnit::RemappedFile>>
RemappedCleanup(RemappedFiles.get());
- for (auto &UF : unsaved_files) {
- std::unique_ptr<llvm::MemoryBuffer> MB =
- llvm::MemoryBuffer::getMemBufferCopy(getContents(UF), UF.Filename);
- RemappedFiles->push_back(std::make_pair(UF.Filename, MB.release()));
- }
+ for (auto &UF : unsaved_files)
+ RemappedFiles->push_back(std::make_pair(
+ UF.Filename,
+ llvm::MemoryBuffer::getMemBufferCopy(getContents(UF), UF.Filename)));
std::unique_ptr<std::vector<const char *>> Args(
new std::vector<const char *>());
@@ -3629,8 +3628,9 @@
Args->data(), Args->data() + Args->size(),
CXXIdx->getPCHContainerOperations(), Diags,
CXXIdx->getClangResourcesPath(), CXXIdx->getOnlyLocalDecls(),
- CaptureDiagnostics, *RemappedFiles.get(), PrecompilePreambleAfterNParses,
- TUKind, CacheCodeCompletionResults, IncludeBriefCommentsInCodeCompletion,
+ CaptureDiagnostics, std::move(*RemappedFiles),
+ PrecompilePreambleAfterNParses, TUKind, CacheCodeCompletionResults,
+ IncludeBriefCommentsInCodeCompletion,
/*AllowPCHWithCompilerErrors=*/true, SkipFunctionBodies, SingleFileParse,
/*UserFilesAreVolatile=*/true, ForSerialization, RetainExcludedCB,
CXXIdx->getPCHContainerOperations()->getRawReader().getFormat(),
@@ -4214,14 +4214,13 @@
llvm::CrashRecoveryContextCleanupRegistrar<std::vector<ASTUnit::RemappedFile>>
RemappedCleanup(RemappedFiles.get());
- for (auto &UF : unsaved_files) {
- std::unique_ptr<llvm::MemoryBuffer> MB =
- llvm::MemoryBuffer::getMemBufferCopy(getContents(UF), UF.Filename);
- RemappedFiles->push_back(std::make_pair(UF.Filename, MB.release()));
- }
+ for (auto &UF : unsaved_files)
+ RemappedFiles->push_back(std::make_pair(
+ UF.Filename,
+ llvm::MemoryBuffer::getMemBufferCopy(getContents(UF), UF.Filename)));
if (!CXXUnit->Reparse(CXXIdx->getPCHContainerOperations(),
- *RemappedFiles.get()))
+ std::move(*RemappedFiles)))
return CXError_Success;
if (isASTReadError(CXXUnit))
return CXError_ASTReadError;
Index: clang/lib/Frontend/PrecompiledPreamble.cpp
===================================================================
--- clang/lib/Frontend/PrecompiledPreamble.cpp
+++ clang/lib/Frontend/PrecompiledPreamble.cpp
@@ -388,6 +388,7 @@
Diagnostics.Reset();
ProcessWarningOptions(Diagnostics, Clang->getDiagnosticOpts());
+ // FIXME: should this be deleted? Can we trust that the caller has done this?
VFS =
createVFSFromCompilerInvocation(Clang->getInvocation(), Diagnostics, VFS);
@@ -404,6 +405,9 @@
Clang->getLangOpts().CompilingPCH = true;
// Remap the main source file to the preamble buffer.
+ //
+ // Note: the buffer needs to be overridden in the SourceManager so that the
+ // preamble is serialized the AST file.
StringRef MainFilePath = FrontendOpts.Inputs[0].getFile();
auto PreambleInputBuffer = llvm::MemoryBuffer::getMemBufferCopy(
MainFileBuffer->getBuffer().slice(0, Bounds.Size), MainFilePath);
@@ -500,11 +504,11 @@
llvm_unreachable("Unhandled storage kind");
}
-bool PrecompiledPreamble::CanReuse(const CompilerInvocation &Invocation,
- const llvm::MemoryBufferRef &MainFileBuffer,
- PreambleBounds Bounds,
- llvm::vfs::FileSystem &VFS) const {
-
+bool PrecompiledPreamble::CanReuse(
+ const CompilerInvocation &Invocation,
+ const llvm::MemoryBufferRef &MainFileBuffer, PreambleBounds Bounds,
+ llvm::vfs::FileSystem &VFS,
+ llvm::vfs::FileSystem *OverriddenFilesFS) const {
assert(
Bounds.Size <= MainFileBuffer.getBufferSize() &&
"Buffer is too large. Bounds were calculated from a different buffer?");
@@ -564,6 +568,15 @@
// Check whether anything has changed.
for (const auto &F : FilesInPreamble) {
+ if (OverriddenFilesFS) {
+ std::unique_ptr<llvm::MemoryBuffer> Buffer;
+ if (moveOnNoError(OverriddenFilesFS->getBufferForFile(F.first()),
+ Buffer)) {
+ if (PreambleFileHash::createForMemoryBuffer(*Buffer) != F.second)
+ return false;
+ continue;
+ }
+ }
auto OverridenFileBuffer = OverridenFileBuffers.find(F.first());
if (OverridenFileBuffer != OverridenFileBuffers.end()) {
// The file's buffer was remapped and the file was not found in VFS.
@@ -804,7 +817,7 @@
auto &PreprocessorOpts = CI.getPreprocessorOpts();
- // Remap main file to point to MainFileBuffer.
+ // Remap main file to point to MainFileBuffer (instead of the preamble).
auto MainFilePath = CI.getFrontendOpts().Inputs[0].getFile();
PreprocessorOpts.addRemappedFile(MainFilePath, MainFileBuffer);
Index: clang/lib/Frontend/ASTUnit.cpp
===================================================================
--- clang/lib/Frontend/ASTUnit.cpp
+++ clang/lib/Frontend/ASTUnit.cpp
@@ -775,9 +775,8 @@
AST->OnlyLocalDecls = OnlyLocalDecls;
AST->CaptureDiagnostics = CaptureDiagnostics;
AST->Diagnostics = Diags;
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
- llvm::vfs::getRealFileSystem();
- AST->FileMgr = new FileManager(FileSystemOpts, VFS);
+ AST->initializeBaseFS(llvm::vfs::getRealFileSystem());
+ AST->FileMgr = new FileManager(FileSystemOpts, AST->FS);
AST->UserFilesAreVolatile = UserFilesAreVolatile;
AST->SourceMgr = new SourceManager(AST->getDiagnostics(),
AST->getFileManager(),
@@ -1094,21 +1093,21 @@
/// \returns True if a failure occurred that causes the ASTUnit not to
/// contain any translation-unit information, false otherwise.
bool ASTUnit::Parse(std::shared_ptr<PCHContainerOperations> PCHContainerOps,
- std::unique_ptr<llvm::MemoryBuffer> OverrideMainBuffer,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
+ std::unique_ptr<llvm::MemoryBuffer> OverrideMainBuffer) {
if (!Invocation)
return true;
- if (VFS && FileMgr)
- assert(VFS == &FileMgr->getVirtualFileSystem() &&
- "VFS passed to Parse and VFS in FileMgr are different");
+ assert((!FileMgr || &FileMgr->getVirtualFileSystem() == FS) &&
+ "ASTUnit has a FileManager without initializing FS");
auto CCInvocation = std::make_shared<CompilerInvocation>(*Invocation);
if (OverrideMainBuffer) {
assert(Preamble &&
"No preamble was built, but OverrideMainBuffer is not null");
- Preamble->AddImplicitPreamble(*CCInvocation, VFS, OverrideMainBuffer.get());
+ Preamble->AddImplicitPreamble(*CCInvocation, FS, OverrideMainBuffer.get());
// VFS may have changed...
+ if (FileMgr && &FileMgr->getVirtualFileSystem() != FS)
+ FileMgr->setVirtualFileSystem(FS);
}
// Create the compiler instance to use for building the AST.
@@ -1131,10 +1130,10 @@
// Ensure that Clang has a FileManager with the right VFS, which may have
// changed above in AddImplicitPreamble. If VFS is nullptr, rely on
// createFileManager to create one.
- if (VFS && FileMgr && &FileMgr->getVirtualFileSystem() == VFS)
+ if (FileMgr && &FileMgr->getVirtualFileSystem() == FS)
Clang->setFileManager(&*FileMgr);
else
- FileMgr = Clang->createFileManager(std::move(VFS));
+ FileMgr = Clang->createFileManager(FS);
// Recover resources if we crash before exiting this method.
llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance>
@@ -1299,13 +1298,12 @@
std::unique_ptr<llvm::MemoryBuffer>
ASTUnit::getMainBufferWithPrecompiledPreamble(
std::shared_ptr<PCHContainerOperations> PCHContainerOps,
- CompilerInvocation &PreambleInvocationIn,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS, bool AllowRebuild,
+ CompilerInvocation &PreambleInvocationIn, bool AllowRebuild,
unsigned MaxLines) {
auto MainFilePath =
PreambleInvocationIn.getFrontendOpts().Inputs[0].getFile();
std::unique_ptr<llvm::MemoryBuffer> MainFileBuffer =
- getBufferForFileHandlingRemapping(PreambleInvocationIn, VFS.get(),
+ getBufferForFileHandlingRemapping(PreambleInvocationIn, FS.get(),
MainFilePath, UserFilesAreVolatile);
if (!MainFileBuffer)
return nullptr;
@@ -1317,7 +1315,7 @@
if (Preamble) {
if (Preamble->CanReuse(PreambleInvocationIn, *MainFileBuffer, Bounds,
- *VFS)) {
+ *FS, &*RemappedFilesFS)) {
// Okay! We can re-use the precompiled preamble.
// Set the state of the diagnostic object to mimic its state
@@ -1373,8 +1371,8 @@
PreambleInvocationIn.getFrontendOpts().SkipFunctionBodies = true;
llvm::ErrorOr<PrecompiledPreamble> NewPreamble = PrecompiledPreamble::Build(
- PreambleInvocationIn, MainFileBuffer.get(), Bounds, *Diagnostics, VFS,
- PCHContainerOps, /*StoreInMemory=*/false, Callbacks);
+ PreambleInvocationIn, MainFileBuffer.get(), Bounds, *Diagnostics,
+ FS.get(), PCHContainerOps, /*StoreInMemory=*/false, Callbacks);
PreambleInvocationIn.getFrontendOpts().SkipFunctionBodies =
PreviousSkipFunctionBodies;
@@ -1492,12 +1490,11 @@
bool UserFilesAreVolatile) {
std::unique_ptr<ASTUnit> AST(new ASTUnit(false));
ConfigureDiags(Diags, *AST, CaptureDiagnostics);
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
- createVFSFromCompilerInvocation(*CI, *Diags);
AST->Diagnostics = Diags;
AST->FileSystemOpts = CI->getFileSystemOpts();
AST->Invocation = std::move(CI);
- AST->FileMgr = new FileManager(AST->FileSystemOpts, VFS);
+ AST->computeAndInitializeFS(*AST->Invocation, *Diags);
+ AST->FileMgr = new FileManager(AST->FileSystemOpts, AST->FS);
AST->UserFilesAreVolatile = UserFilesAreVolatile;
AST->SourceMgr = new SourceManager(AST->getDiagnostics(), *AST->FileMgr,
UserFilesAreVolatile);
@@ -1653,13 +1650,10 @@
bool ASTUnit::LoadFromCompilerInvocation(
std::shared_ptr<PCHContainerOperations> PCHContainerOps,
- unsigned PrecompilePreambleAfterNParses,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
+ unsigned PrecompilePreambleAfterNParses) {
if (!Invocation)
return true;
- assert(VFS && "VFS is null");
-
// We'll manage file buffers ourselves.
Invocation->getPreprocessorOpts().RetainRemappedFileBuffers = true;
Invocation->getFrontendOpts().DisableFree = false;
@@ -1670,7 +1664,7 @@
if (PrecompilePreambleAfterNParses > 0) {
PreambleRebuildCountdown = PrecompilePreambleAfterNParses;
OverrideMainBuffer =
- getMainBufferWithPrecompiledPreamble(PCHContainerOps, *Invocation, VFS);
+ getMainBufferWithPrecompiledPreamble(PCHContainerOps, *Invocation);
getDiagnostics().Reset();
ProcessWarningOptions(getDiagnostics(), Invocation->getDiagnosticOpts());
}
@@ -1682,15 +1676,15 @@
llvm::CrashRecoveryContextCleanupRegistrar<llvm::MemoryBuffer>
MemBufferCleanup(OverrideMainBuffer.get());
- return Parse(std::move(PCHContainerOps), std::move(OverrideMainBuffer), VFS);
+ return Parse(std::move(PCHContainerOps), std::move(OverrideMainBuffer));
}
std::unique_ptr<ASTUnit> ASTUnit::LoadFromCompilerInvocation(
std::shared_ptr<CompilerInvocation> CI,
std::shared_ptr<PCHContainerOperations> PCHContainerOps,
IntrusiveRefCntPtr<DiagnosticsEngine> Diags, FileManager *FileMgr,
- unsigned PrecompilePreambleAfterNParses,
- bool UserFilesAreVolatile) {
+ unsigned PrecompilePreambleAfterNParses, bool UserFilesAreVolatile,
+ Optional<std::vector<RemappedFile>> RemappedFiles) {
// Create the AST unit.
std::unique_ptr<ASTUnit> AST(new ASTUnit(false));
ConfigureDiags(Diags, *AST, CaptureDiagsKind::None);
@@ -1705,6 +1699,12 @@
AST->FileMgr = FileMgr;
AST->UserFilesAreVolatile = UserFilesAreVolatile;
+ // If we have a file manager, this will update its VFS.
+ AST->computeAndInitializeFS(*AST->Invocation, *Diags);
+
+ // Override any files that need remapping.
+ AST->remapFiles(std::move(RemappedFiles));
+
// Recover resources if we crash before exiting this method.
llvm::CrashRecoveryContextCleanupRegistrar<ASTUnit>
ASTUnitCleanup(AST.get());
@@ -1713,8 +1713,7 @@
DiagCleanup(Diags.get());
if (AST->LoadFromCompilerInvocation(std::move(PCHContainerOps),
- PrecompilePreambleAfterNParses,
- &AST->FileMgr->getVirtualFileSystem()))
+ PrecompilePreambleAfterNParses))
return nullptr;
return AST;
}
@@ -1724,7 +1723,7 @@
std::shared_ptr<PCHContainerOperations> PCHContainerOps,
IntrusiveRefCntPtr<DiagnosticsEngine> Diags, StringRef ResourceFilesPath,
bool OnlyLocalDecls, CaptureDiagsKind CaptureDiagnostics,
- ArrayRef<RemappedFile> RemappedFiles,
+ Optional<std::vector<RemappedFile>> RemappedFiles,
unsigned PrecompilePreambleAfterNParses, TranslationUnitKind TUKind,
bool CacheCodeCompletionResults, bool IncludeBriefCommentsInCodeCompletion,
bool AllowPCHWithCompilerErrors, SkipFunctionBodiesScope SkipFunctionBodies,
@@ -1747,11 +1746,6 @@
return nullptr;
}
- // Override any files that need remapping
- for (const auto &RemappedFile : RemappedFiles) {
- CI->getPreprocessorOpts().addRemappedFile(RemappedFile.first,
- RemappedFile.second);
- }
PreprocessorOptions &PPOpts = CI->getPreprocessorOpts();
PPOpts.RemappedFilesKeepOriginalName = true;
PPOpts.AllowPCHWithCompilerErrors = AllowPCHWithCompilerErrors;
@@ -1771,15 +1765,17 @@
// Create the AST unit.
std::unique_ptr<ASTUnit> AST;
AST.reset(new ASTUnit(false));
+
+ // Override any files that need remapping
+ AST->initializeBaseFS(createVFSFromCompilerInvocation(*CI, *Diags));
+ AST->remapFiles(std::move(RemappedFiles));
+
AST->NumStoredDiagnosticsFromDriver = StoredDiagnostics.size();
AST->StoredDiagnostics.swap(StoredDiagnostics);
ConfigureDiags(Diags, *AST, CaptureDiagnostics);
AST->Diagnostics = Diags;
AST->FileSystemOpts = CI->getFileSystemOpts();
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
- llvm::vfs::getRealFileSystem();
- VFS = createVFSFromCompilerInvocation(*CI, *Diags, VFS);
- AST->FileMgr = new FileManager(AST->FileSystemOpts, VFS);
+ AST->FileMgr = new FileManager(AST->FileSystemOpts, AST->FS);
AST->ModuleCache = new InMemoryModuleCache;
AST->OnlyLocalDecls = OnlyLocalDecls;
AST->CaptureDiagnostics = CaptureDiagnostics;
@@ -1801,8 +1797,7 @@
ASTUnitCleanup(AST.get());
if (AST->LoadFromCompilerInvocation(std::move(PCHContainerOps),
- PrecompilePreambleAfterNParses,
- VFS)) {
+ PrecompilePreambleAfterNParses)) {
// Some error occurred, if caller wants to examine diagnostics, pass it the
// ASTUnit.
if (ErrAST) {
@@ -1816,38 +1811,33 @@
}
bool ASTUnit::Reparse(std::shared_ptr<PCHContainerOperations> PCHContainerOps,
- ArrayRef<RemappedFile> RemappedFiles,
+ Optional<std::vector<RemappedFile>> RemappedFiles,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
if (!Invocation)
return true;
- if (!VFS) {
- assert(FileMgr && "FileMgr is null on Reparse call");
- VFS = &FileMgr->getVirtualFileSystem();
- }
-
clearFileLevelDecls();
SimpleTimer ParsingTimer(WantTiming);
ParsingTimer.setOutput("Reparsing " + getMainFileName());
// Remap files.
+ reinitializeBaseFS(std::move(VFS));
+ remapFiles(std::move(RemappedFiles));
+
+ // Clear out buffers in the Invocation.
PreprocessorOptions &PPOpts = Invocation->getPreprocessorOpts();
for (const auto &RB : PPOpts.RemappedFileBuffers)
delete RB.second;
Invocation->getPreprocessorOpts().clearRemappedFiles();
- for (const auto &RemappedFile : RemappedFiles) {
- Invocation->getPreprocessorOpts().addRemappedFile(RemappedFile.first,
- RemappedFile.second);
- }
// If we have a preamble file lying around, or if we might try to
// build a precompiled preamble, do so now.
std::unique_ptr<llvm::MemoryBuffer> OverrideMainBuffer;
if (Preamble || PreambleRebuildCountdown > 0)
OverrideMainBuffer =
- getMainBufferWithPrecompiledPreamble(PCHContainerOps, *Invocation, VFS);
+ getMainBufferWithPrecompiledPreamble(PCHContainerOps, *Invocation);
// Clear out the diagnostics state.
FileMgr.reset();
@@ -1858,7 +1848,7 @@
// Parse the sources
bool Result =
- Parse(std::move(PCHContainerOps), std::move(OverrideMainBuffer), VFS);
+ Parse(std::move(PCHContainerOps), std::move(OverrideMainBuffer));
// If we're caching global code-completion results, and the top-level
// declarations have changed, clear out the code-completion cache.
@@ -2128,7 +2118,7 @@
void ASTUnit::CodeComplete(
StringRef File, unsigned Line, unsigned Column,
- ArrayRef<RemappedFile> RemappedFiles, bool IncludeMacros,
+ Optional<std::vector<RemappedFile>> RemappedFiles, bool IncludeMacros,
bool IncludeCodePatterns, bool IncludeBriefComments,
CodeCompleteConsumer &Consumer,
std::shared_ptr<PCHContainerOperations> PCHContainerOps,
@@ -2211,20 +2201,21 @@
Language::LLVM_IR &&
"IR inputs not support here!");
+ // Remap files.
+ reinitializeBaseFS();
+ remapFiles(std::move(RemappedFiles));
+
// Initialize file and source managers up front so that the caller can access
// them after.
Clang->createFileManager(FS);
+ Clang->createSourceManager(Clang->getFileManager());
FileMgr = &Clang->getFileManager();
- Clang->createSourceManager(*FileMgr);
SourceMgr = &Clang->getSourceManager();
- // Remap files.
- PreprocessorOpts.clearRemappedFiles();
+ // Clear out buffers in the invocation and prepare for any overridden buffers
+ // there by requesting they be retained.
PreprocessorOpts.RetainRemappedFileBuffers = true;
- for (const auto &RemappedFile : RemappedFiles) {
- PreprocessorOpts.addRemappedFile(RemappedFile.first, RemappedFile.second);
- OwnedBuffers.push_back(RemappedFile.second);
- }
+ PreprocessorOpts.clearRemappedFiles();
// Use the code completion consumer we were given, but adding any cached
// code-completion results.
@@ -2233,8 +2224,8 @@
Clang->setCodeCompletionConsumer(AugmentedConsumer);
auto getUniqueID =
- [&FileMgr](StringRef Filename) -> Optional<llvm::sys::fs::UniqueID> {
- if (auto Status = FileMgr.getVirtualFileSystem().status(Filename))
+ [this](StringRef Filename) -> Optional<llvm::sys::fs::UniqueID> {
+ if (auto Status = BaseFS->status(Filename))
return Status->getUniqueID();
return None;
};
@@ -2255,7 +2246,7 @@
std::unique_ptr<llvm::MemoryBuffer> OverrideMainBuffer;
if (Preamble && Line > 1 && hasSameUniqueID(File, OriginalSourceFile)) {
OverrideMainBuffer = getMainBufferWithPrecompiledPreamble(
- PCHContainerOps, Inv, &FileMgr.getVirtualFileSystem(), false, Line - 1);
+ PCHContainerOps, Inv, false, Line - 1);
}
// If the main file has been overridden due to the use of a preamble,
@@ -2264,9 +2255,7 @@
assert(Preamble &&
"No preamble was built, but OverrideMainBuffer is not null");
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
- &FileMgr.getVirtualFileSystem();
- Preamble->AddImplicitPreamble(Clang->getInvocation(), VFS,
+ Preamble->AddImplicitPreamble(Clang->getInvocation(), FS,
OverrideMainBuffer.get());
// FIXME: there is no way to update VFS if it was changed by
// AddImplicitPreamble as FileMgr is accepted as a parameter by this method.
@@ -2288,6 +2277,9 @@
if (llvm::Error Err = Act->Execute()) {
consumeError(std::move(Err)); // FIXME this drops errors on the floor.
}
+
+ transferASTDataFromCompilerInstance(*Clang);
+
Act->EndSourceFile();
}
}
@@ -2711,3 +2703,64 @@
void ASTUnit::ConcurrencyState::finish() {}
#endif // NDEBUG
+
+void ASTUnit::initializeBaseFS(IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS) {
+ assert(BaseFS && "Given empty base filesystem?");
+ if (this->BaseFS == BaseFS || this->FS == BaseFS)
+ return;
+
+ llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS(
+ new llvm::vfs::OverlayFileSystem(BaseFS));
+ RemappedFilesFS = new llvm::vfs::InMemoryFileSystem;
+ OverlayFS->pushOverlay(RemappedFilesFS);
+ FS = std::move(OverlayFS);
+ this->BaseFS = std::move(BaseFS);
+ if (FileMgr)
+ FileMgr->setVirtualFileSystem(FS);
+}
+
+void ASTUnit::reinitializeBaseFS(
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> MaybeFS) {
+ assert(FS && "Resetting without initializing?");
+ assert(BaseFS && "Resetting without initializing?");
+ assert(RemappedFilesFS && "Resetting without initializing?");
+ if (!MaybeFS)
+ MaybeFS = std::move(BaseFS);
+ else
+ BaseFS.reset();
+
+ RemappedFilesFS.reset();
+ FS.reset();
+
+ // Reinitialize.
+ initializeBaseFS(std::move(MaybeFS));
+}
+
+IntrusiveRefCntPtr<llvm::vfs::FileSystem> ASTUnit::computeBaseFS(
+ const CompilerInvocation &CI, DiagnosticsEngine &Diags,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> MaybeFS) const {
+ if (MaybeFS)
+ return FS == MaybeFS ? BaseFS : MaybeFS;
+ if (!FileMgr)
+ return FS ? BaseFS : createVFSFromCompilerInvocation(CI, Diags);
+
+ llvm::vfs::FileSystem &FMFS = FileMgr->getVirtualFileSystem();
+ return FS == &FMFS ? BaseFS : &FMFS;
+}
+
+void ASTUnit::computeAndInitializeFS(
+ const CompilerInvocation &CI, DiagnosticsEngine &Diags,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> MaybeFS) {
+ initializeBaseFS(computeBaseFS(CI, Diags, std::move(MaybeFS)));
+}
+
+void ASTUnit::remapFiles(Optional<std::vector<RemappedFile>> RemappedFiles) {
+ if (!RemappedFiles || RemappedFiles->empty())
+ return;
+
+ // Add files to the in-memory filesystem in reverse order to ensure the last
+ // mapping "wins". This works because InMemoryFileSystem::addFile ignores
+ // subsequent calls.
+ for (auto &File : llvm::reverse(*RemappedFiles))
+ RemappedFilesFS->addFile(File.first, 0, std::move(File.second));
+}
Index: clang/include/clang/Frontend/PrecompiledPreamble.h
===================================================================
--- clang/include/clang/Frontend/PrecompiledPreamble.h
+++ clang/include/clang/Frontend/PrecompiledPreamble.h
@@ -106,7 +106,8 @@
/// MainFileBuffer) of the main file.
bool CanReuse(const CompilerInvocation &Invocation,
const llvm::MemoryBufferRef &MainFileBuffer,
- PreambleBounds Bounds, llvm::vfs::FileSystem &VFS) const;
+ PreambleBounds Bounds, llvm::vfs::FileSystem &VFS,
+ llvm::vfs::FileSystem *OverriddenFilesFS = nullptr) const;
/// Changes options inside \p CI to use PCH from this preamble. Also remaps
/// main file to \p MainFileBuffer and updates \p VFS to ensure the preamble
Index: clang/include/clang/Frontend/ASTUnit.h
===================================================================
--- clang/include/clang/Frontend/ASTUnit.h
+++ clang/include/clang/Frontend/ASTUnit.h
@@ -53,6 +53,7 @@
namespace vfs {
class FileSystem;
+class InMemoryFileSystem;
} // namespace vfs
} // namespace llvm
@@ -108,6 +109,16 @@
private:
std::shared_ptr<LangOptions> LangOpts;
IntrusiveRefCntPtr<DiagnosticsEngine> Diagnostics;
+
+ /// The top-level filesystem.
+ llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS;
+
+ /// Filesystem for remapped file buffers.
+ llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> RemappedFilesFS;
+
+ /// The base filesystem from the CompilerInvocation.
+ llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS;
+
IntrusiveRefCntPtr<FileManager> FileMgr;
IntrusiveRefCntPtr<SourceManager> SourceMgr;
IntrusiveRefCntPtr<InMemoryModuleCache> ModuleCache;
@@ -368,13 +379,11 @@
explicit ASTUnit(bool MainFileIsAST);
bool Parse(std::shared_ptr<PCHContainerOperations> PCHContainerOps,
- std::unique_ptr<llvm::MemoryBuffer> OverrideMainBuffer,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS);
+ std::unique_ptr<llvm::MemoryBuffer> OverrideMainBuffer);
std::unique_ptr<llvm::MemoryBuffer> getMainBufferWithPrecompiledPreamble(
std::shared_ptr<PCHContainerOperations> PCHContainerOps,
- CompilerInvocation &PreambleInvocationIn,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS, bool AllowRebuild = true,
+ CompilerInvocation &PreambleInvocationIn, bool AllowRebuild = true,
unsigned MaxLines = 0);
void RealizeTopLevelDeclsFromPreamble();
@@ -659,7 +668,8 @@
/// A mapping from a file name to the memory buffer that stores the
/// remapped contents of that file.
- using RemappedFile = std::pair<std::string, llvm::MemoryBuffer *>;
+ using RemappedFile =
+ std::pair<std::string, std::unique_ptr<llvm::MemoryBuffer>>;
/// Create a ASTUnit. Gets ownership of the passed CompilerInvocation.
static std::unique_ptr<ASTUnit>
@@ -706,17 +716,11 @@
/// of this translation unit should be precompiled, to improve the performance
/// of reparsing. Set to zero to disable preambles.
///
- /// \param VFS - A llvm::vfs::FileSystem to be used for all file accesses.
- /// Note that preamble is saved to a temporary directory on a RealFileSystem,
- /// so in order for it to be loaded correctly, VFS should have access to
- /// it(i.e., be an overlay over RealFileSystem).
- ///
/// \returns \c true if a catastrophic failure occurred (which means that the
/// \c ASTUnit itself is invalid), or \c false otherwise.
bool LoadFromCompilerInvocation(
std::shared_ptr<PCHContainerOperations> PCHContainerOps,
- unsigned PrecompilePreambleAfterNParses,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS);
+ unsigned PrecompilePreambleAfterNParses);
public:
/// Create an ASTUnit from a source file, via a CompilerInvocation
@@ -779,7 +783,8 @@
std::shared_ptr<PCHContainerOperations> PCHContainerOps,
IntrusiveRefCntPtr<DiagnosticsEngine> Diags, FileManager *FileMgr,
unsigned PrecompilePreambleAfterNParses = 0,
- bool UserFilesAreVolatile = false);
+ bool UserFilesAreVolatile = false,
+ Optional<std::vector<RemappedFile>> RemappedFiles = None);
/// LoadFromCommandLine - Create an ASTUnit from a vector of command line
/// arguments, which must specify exactly one source file.
@@ -802,12 +807,6 @@
/// (e.g. because the PCH could not be loaded), this accepts the ASTUnit
/// mainly to allow the caller to see the diagnostics.
///
- /// \param VFS - A llvm::vfs::FileSystem to be used for all file accesses.
- /// Note that preamble is saved to a temporary directory on a RealFileSystem,
- /// so in order for it to be loaded correctly, VFS should have access to
- /// it(i.e., be an overlay over RealFileSystem). RealFileSystem will be used
- /// if \p VFS is nullptr.
- ///
// FIXME: Move OnlyLocalDecls, UseBumpAllocator to setters on the ASTUnit, we
// shouldn't need to specify them at construction time.
static ASTUnit *LoadFromCommandLine(
@@ -816,7 +815,7 @@
IntrusiveRefCntPtr<DiagnosticsEngine> Diags, StringRef ResourceFilesPath,
bool OnlyLocalDecls = false,
CaptureDiagsKind CaptureDiagnostics = CaptureDiagsKind::None,
- ArrayRef<RemappedFile> RemappedFiles = None,
+ Optional<std::vector<RemappedFile>> RemappedFiles = None,
unsigned PrecompilePreambleAfterNParses = 0,
TranslationUnitKind TUKind = TU_Complete,
bool CacheCodeCompletionResults = false,
@@ -842,7 +841,7 @@
/// \returns True if a failure occurred that causes the ASTUnit not to
/// contain any translation-unit information, false otherwise.
bool Reparse(std::shared_ptr<PCHContainerOperations> PCHContainerOps,
- ArrayRef<RemappedFile> RemappedFiles = None,
+ Optional<std::vector<RemappedFile>> RemappedFiles = None,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS = nullptr);
/// Free data that will be re-generated on the next parse.
@@ -871,9 +870,9 @@
/// FIXME: The Diag, LangOpts, SourceMgr, FileMgr, StoredDiagnostics, and
/// OwnedBuffers parameters are all disgusting hacks. They will go away.
void CodeComplete(StringRef File, unsigned Line, unsigned Column,
- ArrayRef<RemappedFile> RemappedFiles, bool IncludeMacros,
- bool IncludeCodePatterns, bool IncludeBriefComments,
- CodeCompleteConsumer &Consumer,
+ Optional<std::vector<RemappedFile>> RemappedFiles,
+ bool IncludeMacros, bool IncludeCodePatterns,
+ bool IncludeBriefComments, CodeCompleteConsumer &Consumer,
std::shared_ptr<PCHContainerOperations> PCHContainerOps,
DiagnosticsEngine &Diag, LangOptions &LangOpts,
SmallVectorImpl<StoredDiagnostic> &StoredDiagnostics,
@@ -889,6 +888,18 @@
///
/// \returns True if an error occurred, false otherwise.
bool serialize(raw_ostream &OS);
+
+private:
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> computeBaseFS(
+ const CompilerInvocation &CI, DiagnosticsEngine &Diags,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> MaybeFS = nullptr) const;
+ void initializeBaseFS(IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS);
+ void computeAndInitializeFS(
+ const CompilerInvocation &CI, DiagnosticsEngine &Diags,
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> MaybeFS = nullptr);
+ void reinitializeBaseFS(
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> MaybeFS = nullptr);
+ void remapFiles(Optional<std::vector<RemappedFile>> RemappedFiles);
};
} // namespace clang
_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
