arphaman updated this revision to Diff 218778.
arphaman marked 2 inline comments as done.
arphaman added a comment.
Add support for chained PP callbacks in setAdditionalPPCallbacks.
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D67127/new/
https://reviews.llvm.org/D67127
Files:
clang/include/clang/Frontend/CompilerInstance.h
clang/include/clang/Lex/DependencyDirectivesSourceMinimizer.h
clang/include/clang/Lex/Lexer.h
clang/include/clang/Lex/PPCallbacks.h
clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
clang/include/clang/Tooling/DependencyScanning/PPRangeSkipping.h
clang/lib/Frontend/CompilerInstance.cpp
clang/lib/Lex/DependencyDirectivesSourceMinimizer.cpp
clang/lib/Lex/Lexer.cpp
clang/lib/Lex/PPDirectives.cpp
clang/lib/Tooling/DependencyScanning/CMakeLists.txt
clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
clang/lib/Tooling/DependencyScanning/PPRangeSkipping.cpp
clang/test/ClangScanDeps/regular_cdb.cpp
clang/tools/clang-scan-deps/ClangScanDeps.cpp
clang/unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp
Index: clang/unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp
===================================================================
--- clang/unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp
+++ clang/unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp
@@ -617,4 +617,46 @@
minimize_source_to_dependency_directives::cxx_module_decl);
}
+TEST(MinimizeSourceToDependencyDirectivesTest, SkippedPPRangesBasic) {
+ SmallString<128> Out;
+ SmallVector<Token, 32> Toks;
+ StringRef Source = "#ifndef GUARD\n"
+ "#define GUARD\n"
+ "void foo();\n"
+ "#endif\n";
+ ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Toks));
+ SmallVector<SkippedRange, 4> Ranges;
+ ASSERT_FALSE(computeSkippedRanges(Toks, Ranges));
+ EXPECT_EQ(Ranges.size(), 1u);
+ EXPECT_EQ(Ranges[0].Offset, 0);
+ EXPECT_EQ(Ranges[0].Length, (int)Out.find("#endif"));
+}
+
+TEST(MinimizeSourceToDependencyDirectivesTest, SkippedPPRangesNested) {
+ SmallString<128> Out;
+ SmallVector<Token, 32> Toks;
+ StringRef Source = "#ifndef GUARD\n"
+ "#define GUARD\n"
+ "#if FOO\n"
+ "#include hello\n"
+ "#elif BAR\n"
+ "#include bye\n"
+ "#endif\n"
+ "#else\n"
+ "#include nothing\n"
+ "#endif\n";
+ ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Toks));
+ SmallVector<SkippedRange, 4> Ranges;
+ ASSERT_FALSE(computeSkippedRanges(Toks, Ranges));
+ EXPECT_EQ(Ranges.size(), 4u);
+ EXPECT_EQ(Ranges[0].Offset, (int)Out.find("#if FOO"));
+ EXPECT_EQ(Ranges[0].Offset + Ranges[0].Length, (int)Out.find("#elif"));
+ EXPECT_EQ(Ranges[1].Offset, (int)Out.find("#elif BAR"));
+ EXPECT_EQ(Ranges[1].Offset + Ranges[1].Length, (int)Out.find("#endif"));
+ EXPECT_EQ(Ranges[2].Offset, 0);
+ EXPECT_EQ(Ranges[2].Length, (int)Out.find("#else"));
+ EXPECT_EQ(Ranges[3].Offset, (int)Out.find("#else"));
+ EXPECT_EQ(Ranges[3].Offset + Ranges[3].Length, (int)Out.rfind("#endif"));
+}
+
} // end anonymous namespace
Index: clang/tools/clang-scan-deps/ClangScanDeps.cpp
===================================================================
--- clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -168,6 +168,14 @@
llvm::cl::desc("Reuse the file manager and its cache between invocations."),
llvm::cl::init(true), llvm::cl::cat(DependencyScannerCategory));
+llvm::cl::opt<bool> SkipExcludedPPRanges(
+ "skip-excluded-pp-ranges",
+ llvm::cl::desc(
+ "Use the preprocessor optimization that skips excluded conditionals by "
+ "bumping the buffer pointer in the lexer instead of lexing the tokens "
+ "until reaching the end directive."),
+ llvm::cl::init(true), llvm::cl::cat(DependencyScannerCategory));
+
} // end anonymous namespace
int main(int argc, const char **argv) {
@@ -214,7 +222,8 @@
// Print out the dependency results to STDOUT by default.
SharedStream DependencyOS(llvm::outs());
- DependencyScanningService Service(ScanMode, ReuseFileManager);
+ DependencyScanningService Service(ScanMode, ReuseFileManager,
+ SkipExcludedPPRanges);
#if LLVM_ENABLE_THREADS
unsigned NumWorkers =
NumThreads == 0 ? llvm::hardware_concurrency() : NumThreads;
Index: clang/test/ClangScanDeps/regular_cdb.cpp
===================================================================
--- clang/test/ClangScanDeps/regular_cdb.cpp
+++ clang/test/ClangScanDeps/regular_cdb.cpp
@@ -12,6 +12,8 @@
// RUN: FileCheck --check-prefixes=CHECK1,CHECK2,CHECK2NO %s
// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess | \
// RUN: FileCheck --check-prefixes=CHECK1,CHECK2,CHECK2NO %s
+// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess-minimized-sources \
+// RUN: -skip-excluded-pp-ranges=0 | FileCheck --check-prefixes=CHECK1,CHECK2,CHECK2NO %s
//
// Make sure we didn't produce any dependency files!
// RUN: not cat %t.dir/regular_cdb.d
Index: clang/lib/Tooling/DependencyScanning/PPRangeSkipping.cpp
===================================================================
--- /dev/null
+++ clang/lib/Tooling/DependencyScanning/PPRangeSkipping.cpp
@@ -0,0 +1,71 @@
+//===- PPRangeSkipping.cpp - PP Callbacks to skip inactive range ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/DependencyScanning/PPRangeSkipping.h"
+#include "clang/Lex/Preprocessor.h"
+
+using namespace clang;
+using namespace tooling;
+using namespace dependencies;
+
+Optional<unsigned>
+PPRangeSkippingCallbacks::getSkippedRangeForExcludedConditionalBlock(
+ Preprocessor &PP, SourceLocation HashLoc, unsigned CurLexerBufferOffset) {
+ if (FileStack.empty())
+ return None;
+ const SourceManager &SM = PP.getSourceManager();
+
+ // Retrieve the information about the file we're currently lexing.
+ auto &Info = FileStack.back();
+ if (!Info.isValid()) {
+ std::pair<FileID, unsigned> F = SM.getDecomposedLoc(Info.Loc);
+ Info.File = F.first;
+ const auto *Buf = SM.getBuffer(F.first);
+ if (const auto *PPSkippedRanges = BufferMappings.getSkippedRanges(Buf)) {
+ Info.PPSkippedRanges = PPSkippedRanges;
+ }
+ }
+
+ if (!Info.PPSkippedRanges)
+ return None;
+
+ auto HashFileOffset = SM.getDecomposedLoc(HashLoc);
+ if (HashFileOffset.first != Info.File)
+ return None;
+ // Check if the offset of '#' is mapped in the skipped ranges.
+ auto It = Info.PPSkippedRanges->find(HashFileOffset.second);
+ if (It == Info.PPSkippedRanges->end()) {
+ return None;
+ }
+ unsigned BytesToSkip = It->getSecond();
+ assert(CurLexerBufferOffset >= HashFileOffset.second &&
+ "lexer is before the hash?");
+ // Take into account the fact that the lexer has already advanced, so the
+ // number of bytes to skip must be adjusted.
+ unsigned LengthDiff = CurLexerBufferOffset - HashFileOffset.second;
+ assert(BytesToSkip >= LengthDiff && "lexer is after the skipped range?");
+ return BytesToSkip - LengthDiff;
+}
+
+void PPRangeSkippingCallbacks::FileChanged(SourceLocation Loc,
+ FileChangeReason Reason,
+ SrcMgr::CharacteristicKind FileType,
+ FileID PrevFID) {
+ switch (Reason) {
+ case EnterFile:
+ if (Loc.isValid())
+ FileStack.push_back(Loc);
+ break;
+ case ExitFile:
+ if (!FileStack.empty())
+ FileStack.pop_back();
+ break;
+ default:
+ break;
+ }
+}
Index: clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
===================================================================
--- clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -66,9 +66,10 @@
public:
DependencyScanningAction(
StringRef WorkingDirectory, DependencyConsumer &Consumer,
- llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS)
+ llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
+ PreprocessorSkippedMappings *PPSkippedRanges)
: WorkingDirectory(WorkingDirectory), Consumer(Consumer),
- DepFS(std::move(DepFS)) {}
+ DepFS(std::move(DepFS)), PPSkippedRanges(PPSkippedRanges) {}
bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
FileManager *FileMgr,
@@ -107,6 +108,14 @@
Compiler.setFileManager(FileMgr);
Compiler.createSourceManager(*FileMgr);
+ // Set the PP Callbacks which should speed up excluded conditional block
+ // skipping in the preprocessor.
+ if (PPSkippedRanges) {
+ PPSkippedRanges->reset();
+ Compiler.setAdditionalPPCallbacks(
+ std::make_unique<PPRangeSkippingCallbacks>(*PPSkippedRanges));
+ }
+
// Create the dependency collector that will collect the produced
// dependencies.
//
@@ -134,18 +143,20 @@
StringRef WorkingDirectory;
DependencyConsumer &Consumer;
llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
+ PreprocessorSkippedMappings *PPSkippedRanges;
};
} // end anonymous namespace
DependencyScanningWorker::DependencyScanningWorker(
- DependencyScanningService &Service) {
+ DependencyScanningService &Service)
+ : SkipExcludedPPRanges(Service.canSkipExcludedPPRanges()) {
DiagOpts = new DiagnosticOptions();
PCHContainerOps = std::make_shared<PCHContainerOperations>();
RealFS = new ProxyFileSystemWithoutChdir(llvm::vfs::getRealFileSystem());
if (Service.getMode() == ScanningMode::MinimizedSourcePreprocessing)
DepFS = new DependencyScanningWorkerFilesystem(Service.getSharedCache(),
- RealFS);
+ RealFS, PPSkippedRanges);
if (Service.canReuseFileManager())
Files = new FileManager(FileSystemOptions(), RealFS);
}
@@ -178,7 +189,9 @@
Tool.setRestoreWorkingDir(false);
Tool.setPrintErrorMessage(false);
Tool.setDiagnosticConsumer(&DC);
- DependencyScanningAction Action(WorkingDirectory, Consumer, DepFS);
+ DependencyScanningAction Action(WorkingDirectory, Consumer, DepFS,
+ SkipExcludedPPRanges ? &PPSkippedRanges
+ : nullptr);
return !Tool.run(&Action);
});
}
Index: clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
===================================================================
--- clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
+++ clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
@@ -13,5 +13,7 @@
using namespace dependencies;
DependencyScanningService::DependencyScanningService(ScanningMode Mode,
- bool ReuseFileManager)
- : Mode(Mode), ReuseFileManager(ReuseFileManager) {}
+ bool ReuseFileManager,
+ bool SkipExcludedPPRanges)
+ : Mode(Mode), ReuseFileManager(ReuseFileManager),
+ SkipExcludedPPRanges(SkipExcludedPPRanges) {}
Index: clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
===================================================================
--- clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -69,6 +69,25 @@
// Now make the null terminator implicit again, so that Clang's lexer can find
// it right where the buffer ends.
Result.Contents.pop_back();
+
+ // Compute the skipped PP ranges that speedup skipping over inactive
+ // preprocessor blocks.
+ llvm::SmallVector<minimize_source_to_dependency_directives::SkippedRange, 32>
+ SkippedRanges;
+ minimize_source_to_dependency_directives::computeSkippedRanges(Tokens,
+ SkippedRanges);
+ SkippedRangeMapping Mapping;
+ for (const auto &Range : SkippedRanges) {
+ if (Range.Length < 16) {
+ // Ignore small ranges as non-profitable.
+ // FIXME: This is a heuristic, its worth investigating the tradeoffs
+ // when it should be applied.
+ continue;
+ }
+ Mapping[Range.Offset] = Range.Length;
+ }
+ Result.PPSkippedRangeMapping = std::move(Mapping);
+
return Result;
}
@@ -172,14 +191,19 @@
};
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
-createFile(const CachedFileSystemEntry *Entry) {
+createFile(const CachedFileSystemEntry *Entry,
+ PreprocessorSkippedMappings &PPSkippedRanges) {
llvm::ErrorOr<StringRef> Contents = Entry->getContents();
if (!Contents)
return Contents.getError();
- return std::make_unique<MinimizedVFSFile>(
+ auto Result = std::make_unique<MinimizedVFSFile>(
llvm::MemoryBuffer::getMemBuffer(*Contents, Entry->getName(),
/*RequiresNullTerminator=*/false),
*Entry->getStatus());
+ if (!Entry->getPPSkippedRangeMapping().empty())
+ PPSkippedRanges.setSkippedRanges(Result->getBufferPtr(),
+ &Entry->getPPSkippedRangeMapping());
+ return Result;
}
} // end anonymous namespace
@@ -191,7 +215,7 @@
// Check the local cache first.
if (const CachedFileSystemEntry *Entry = getCachedEntry(Filename))
- return createFile(Entry);
+ return createFile(Entry, PPSkippedRanges);
// FIXME: Handle PCM/PCH files.
// FIXME: Handle module map files.
@@ -214,5 +238,5 @@
// Store the result in the local cache.
setCachedEntry(Filename, Result);
- return createFile(Result);
+ return createFile(Result, PPSkippedRanges);
}
Index: clang/lib/Tooling/DependencyScanning/CMakeLists.txt
===================================================================
--- clang/lib/Tooling/DependencyScanning/CMakeLists.txt
+++ clang/lib/Tooling/DependencyScanning/CMakeLists.txt
@@ -7,6 +7,7 @@
DependencyScanningFilesystem.cpp
DependencyScanningService.cpp
DependencyScanningWorker.cpp
+ PPRangeSkipping.cpp
DEPENDS
ClangDriverOptions
Index: clang/lib/Lex/PPDirectives.cpp
===================================================================
--- clang/lib/Lex/PPDirectives.cpp
+++ clang/lib/Lex/PPDirectives.cpp
@@ -396,6 +396,13 @@
// disabling warnings, etc.
CurPPLexer->LexingRawMode = true;
Token Tok;
+ if (Callbacks && HashTokenLoc.isFileID()) {
+ if (auto SkipLength = Callbacks->getSkippedRangeForExcludedConditionalBlock(
+ *this, HashTokenLoc, CurLexer->getCurrentBufferOffset())) {
+ // Skip to the next '#endif' / '#else' / '#elif'.
+ CurLexer->skipOver(*SkipLength);
+ }
+ }
while (true) {
CurLexer->Lex(Tok);
Index: clang/lib/Lex/Lexer.cpp
===================================================================
--- clang/lib/Lex/Lexer.cpp
+++ clang/lib/Lex/Lexer.cpp
@@ -218,6 +218,15 @@
return L;
}
+bool Lexer::skipOver(unsigned NumBytes) {
+ IsAtPhysicalStartOfLine = true;
+ IsAtStartOfLine = true;
+ if ((BufferPtr + NumBytes) > BufferEnd)
+ return true;
+ BufferPtr += NumBytes;
+ return false;
+}
+
template <typename T> static void StringifyImpl(T &Str, char Quote) {
typename T::size_type i = 0, e = Str.size();
while (i < e) {
Index: clang/lib/Lex/DependencyDirectivesSourceMinimizer.cpp
===================================================================
--- clang/lib/Lex/DependencyDirectivesSourceMinimizer.cpp
+++ clang/lib/Lex/DependencyDirectivesSourceMinimizer.cpp
@@ -865,6 +865,54 @@
return Error;
}
+bool clang::minimize_source_to_dependency_directives::computeSkippedRanges(
+ ArrayRef<Token> Input, llvm::SmallVectorImpl<SkippedRange> &Range) {
+ struct Directive {
+ enum DirectiveKind {
+ If, // if/ifdef/ifndef
+ Else // elif,else
+ };
+ int Offset;
+ DirectiveKind Kind;
+ };
+ llvm::SmallVector<Directive, 32> Offsets;
+ for (const Token &T : Input) {
+ switch (T.K) {
+ case pp_if:
+ case pp_ifdef:
+ case pp_ifndef:
+ Offsets.push_back({T.Offset, Directive::If});
+ break;
+
+ case pp_elif:
+ case pp_else: {
+ if (Offsets.empty())
+ return true;
+ int PreviousOffset = Offsets.back().Offset;
+ Range.push_back({PreviousOffset, T.Offset - PreviousOffset});
+ Offsets.push_back({T.Offset, Directive::Else});
+ break;
+ }
+
+ case pp_endif: {
+ if (Offsets.empty())
+ return true;
+ int PreviousOffset = Offsets.back().Offset;
+ Range.push_back({PreviousOffset, T.Offset - PreviousOffset});
+ do {
+ Directive::DirectiveKind Kind = Offsets.pop_back_val().Kind;
+ if (Kind == Directive::If)
+ break;
+ } while (!Offsets.empty());
+ break;
+ }
+ default:
+ break;
+ }
+ }
+ return false;
+}
+
bool clang::minimizeSourceToDependencyDirectives(
StringRef Input, SmallVectorImpl<char> &Output,
SmallVectorImpl<Token> &Tokens, DiagnosticsEngine *Diags,
Index: clang/lib/Frontend/CompilerInstance.cpp
===================================================================
--- clang/lib/Frontend/CompilerInstance.cpp
+++ clang/lib/Frontend/CompilerInstance.cpp
@@ -75,6 +75,15 @@
Invocation = std::move(Value);
}
+void CompilerInstance::setAdditionalPPCallbacks(
+ std::unique_ptr<PPCallbacks> PPC) {
+ if (AdditionalPPCallbacks)
+ AdditionalPPCallbacks = std::make_unique<PPChainedCallbacks>(
+ std::move(AdditionalPPCallbacks), std::move(PPC));
+ else
+ AdditionalPPCallbacks = std::move(PPC);
+}
+
bool CompilerInstance::shouldBuildGlobalModuleIndex() const {
return (BuildGlobalModuleIndex ||
(ModuleManager && ModuleManager->isGlobalIndexUnavailable() &&
@@ -457,6 +466,9 @@
/*ShowAllHeaders=*/true, /*OutputPath=*/"",
/*ShowDepth=*/true, /*MSStyle=*/true);
}
+
+ if (AdditionalPPCallbacks)
+ PP->addPPCallbacks(std::move(AdditionalPPCallbacks));
}
std::string CompilerInstance::getSpecificModuleCachePath() {
Index: clang/include/clang/Tooling/DependencyScanning/PPRangeSkipping.h
===================================================================
--- /dev/null
+++ clang/include/clang/Tooling/DependencyScanning/PPRangeSkipping.h
@@ -0,0 +1,87 @@
+//===- PPRangeSkipping.h - PP Callbacks to skip inactive range --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_PP_RANGE_SKIPPING_H
+#define LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_PP_RANGE_SKIPPING_H
+
+#include "clang/Basic/LLVM.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Lex/PPCallbacks.h"
+#include "llvm/ADT/DenseMap.h"
+#include <string>
+
+namespace clang {
+namespace tooling {
+namespace dependencies {
+
+/// A mapping from an offset into a buffer to the number of bytes that can be
+/// skipped by the preprocessor when skipping over inactive preprocessor ranges.
+using SkippedRangeMapping = llvm::DenseMap<unsigned, unsigned>;
+
+/// Contains the currently active skipped range mappings for a particular
+/// preprocessor instance.
+class PreprocessorSkippedMappings {
+public:
+ /// Clears the buffer -> skipped range mappings.
+ void reset() { MappingForBuffer.clear(); }
+
+ /// Inserts a new mapping that maps from the specified memory buffer to the
+ /// specified skipped range mappings.
+ void setSkippedRanges(const llvm::MemoryBuffer *Buf,
+ const SkippedRangeMapping *Mapping) {
+ MappingForBuffer[Buf] = Mapping;
+ }
+
+ /// Returns the skipped range mappings for the given buffer or null if none
+ /// exist.
+ const SkippedRangeMapping *
+ getSkippedRanges(const llvm::MemoryBuffer *Buf) const {
+ auto It = MappingForBuffer.find(Buf);
+ if (It != MappingForBuffer.end())
+ return It->getSecond();
+ return nullptr;
+ }
+
+private:
+ llvm::DenseMap<const llvm::MemoryBuffer *, const SkippedRangeMapping *>
+ MappingForBuffer;
+};
+
+/// A PP callbacks instance that implements fast PP skipping.
+class PPRangeSkippingCallbacks final : public PPCallbacks {
+public:
+ PPRangeSkippingCallbacks(PreprocessorSkippedMappings &BufferMappings)
+ : BufferMappings(BufferMappings) {}
+
+ Optional<unsigned> getSkippedRangeForExcludedConditionalBlock(
+ Preprocessor &PP, SourceLocation HashLoc,
+ unsigned CurLexerBufferOffset) override;
+
+ void FileChanged(SourceLocation Loc, FileChangeReason Reason,
+ SrcMgr::CharacteristicKind FileType,
+ FileID PrevFID) override;
+
+private:
+ PreprocessorSkippedMappings &BufferMappings;
+ struct FileInfo {
+ SourceLocation Loc;
+ FileID File;
+ const SkippedRangeMapping *PPSkippedRanges = nullptr;
+
+ FileInfo(SourceLocation Loc) : Loc(Loc) {}
+
+ bool isValid() const { return File.isValid(); }
+ };
+ llvm::SmallVector<FileInfo, 8> FileStack;
+};
+
+} // end namespace dependencies
+} // end namespace tooling
+} // end namespace clang
+
+#endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_PP_RANGE_SKIPPING_H
Index: clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
===================================================================
--- clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
+++ clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
@@ -14,6 +14,7 @@
#include "clang/Basic/LLVM.h"
#include "clang/Frontend/PCHContainerOperations.h"
#include "clang/Tooling/CompilationDatabase.h"
+#include "clang/Tooling/DependencyScanning/PPRangeSkipping.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
#include <string>
@@ -62,6 +63,10 @@
private:
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts;
std::shared_ptr<PCHContainerOperations> PCHContainerOps;
+ /// The mapping that holds the active 'skip' mappings that are used by a
+ /// particular preprocessor.
+ PreprocessorSkippedMappings PPSkippedRanges;
+ const bool SkipExcludedPPRanges;
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> RealFS;
/// The file system that is used by each worker when scanning for
Index: clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
===================================================================
--- clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
+++ clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
@@ -34,12 +34,15 @@
/// the invidual dependency scanning workers.
class DependencyScanningService {
public:
- DependencyScanningService(ScanningMode Mode, bool ReuseFileManager = true);
+ DependencyScanningService(ScanningMode Mode, bool ReuseFileManager = true,
+ bool SkipExcludedPPRanges = true);
ScanningMode getMode() const { return Mode; }
bool canReuseFileManager() const { return ReuseFileManager; }
+ bool canSkipExcludedPPRanges() const { return SkipExcludedPPRanges; }
+
DependencyScanningFilesystemSharedCache &getSharedCache() {
return SharedCache;
}
@@ -47,6 +50,10 @@
private:
const ScanningMode Mode;
const bool ReuseFileManager;
+ /// Set to true to use the preprocessor optimization that skips excluded PP
+ /// ranges by bumping the buffer pointer in the lexer instead of lexing the
+ /// tokens in the range until reaching the corresponding directive.
+ const bool SkipExcludedPPRanges;
/// The global file system cache.
DependencyScanningFilesystemSharedCache SharedCache;
};
Index: clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
===================================================================
--- clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++ clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -10,6 +10,7 @@
#define LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H
#include "clang/Basic/LLVM.h"
+#include "clang/Tooling/DependencyScanning/PPRangeSkipping.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringSet.h"
#include "llvm/Support/Allocator.h"
@@ -76,6 +77,12 @@
return MaybeStat->getName();
}
+ /// Return the mapping between location -> distance that is used to speed up
+ /// the block skipping in the preprocessor.
+ const SkippedRangeMapping &getPPSkippedRangeMapping() const {
+ return PPSkippedRangeMapping;
+ }
+
CachedFileSystemEntry(CachedFileSystemEntry &&) = default;
CachedFileSystemEntry &operator=(CachedFileSystemEntry &&) = default;
@@ -89,6 +96,7 @@
// Note: small size of 1 allows us to store an empty string with an implicit
// null terminator without any allocations.
llvm::SmallString<1> Contents;
+ SkippedRangeMapping PPSkippedRangeMapping;
};
/// This class is a shared cache, that caches the 'stat' and 'open' calls to the
@@ -133,8 +141,10 @@
public:
DependencyScanningWorkerFilesystem(
DependencyScanningFilesystemSharedCache &SharedCache,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
- : ProxyFileSystem(std::move(FS)), SharedCache(SharedCache) {}
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
+ PreprocessorSkippedMappings &PPSkippedRanges)
+ : ProxyFileSystem(std::move(FS)), SharedCache(SharedCache),
+ PPSkippedRanges(PPSkippedRanges) {}
llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override;
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
@@ -159,6 +169,7 @@
/// The local cache is used by the worker thread to cache file system queries
/// locally instead of querying the global cache every time.
llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator> Cache;
+ PreprocessorSkippedMappings &PPSkippedRanges;
};
} // end namespace dependencies
Index: clang/include/clang/Lex/PPCallbacks.h
===================================================================
--- clang/include/clang/Lex/PPCallbacks.h
+++ clang/include/clang/Lex/PPCallbacks.h
@@ -370,6 +370,12 @@
/// \param IfLoc the source location of the \#if/\#ifdef/\#ifndef directive.
virtual void Endif(SourceLocation Loc, SourceLocation IfLoc) {
}
+
+ /// This skipped range class returns an optional thing.
+ virtual Optional<unsigned> getSkippedRangeForExcludedConditionalBlock(
+ Preprocessor &PP, SourceLocation HashLoc, unsigned CurLexerBufferOffset) {
+ return None;
+ }
};
/// Simple wrapper class for chaining callbacks.
@@ -606,6 +612,16 @@
First->Endif(Loc, IfLoc);
Second->Endif(Loc, IfLoc);
}
+
+ Optional<unsigned> getSkippedRangeForExcludedConditionalBlock(
+ Preprocessor &PP, SourceLocation HashLoc,
+ unsigned CurLexerBufferOffset) override {
+ if (auto I = First->getSkippedRangeForExcludedConditionalBlock(
+ PP, HashLoc, CurLexerBufferOffset))
+ return I;
+ return Second->getSkippedRangeForExcludedConditionalBlock(
+ PP, HashLoc, CurLexerBufferOffset);
+ }
};
} // end namespace clang
Index: clang/include/clang/Lex/Lexer.h
===================================================================
--- clang/include/clang/Lex/Lexer.h
+++ clang/include/clang/Lex/Lexer.h
@@ -265,6 +265,21 @@
/// Return the current location in the buffer.
const char *getBufferLocation() const { return BufferPtr; }
+ /// Returns the current lexing offset.
+ unsigned getCurrentBufferOffset() {
+ assert(BufferPtr >= BufferStart && "Invalid buffer state");
+ return BufferPtr - BufferStart;
+ }
+
+ /// Skip over \p NumBytes bytes.
+ ///
+ /// If the skip is successful, the next token will be lexed from the new
+ /// offset. The lexer also assumes that we skipped to the start of the line.
+ ///
+ /// \returns true if the skip failed (new offset would have been past the
+ /// end of the buffer), false otherwise.
+ bool skipOver(unsigned NumBytes);
+
/// Stringify - Convert the specified string into a C string by i) escaping
/// '\\' and " characters and ii) replacing newline character(s) with "\\n".
/// If Charify is true, this escapes the ' character instead of ".
Index: clang/include/clang/Lex/DependencyDirectivesSourceMinimizer.h
===================================================================
--- clang/include/clang/Lex/DependencyDirectivesSourceMinimizer.h
+++ clang/include/clang/Lex/DependencyDirectivesSourceMinimizer.h
@@ -66,6 +66,24 @@
Token(TokenKind K, int Offset) : K(K), Offset(Offset) {}
};
+/// Simplified token range to track the range of a potentially skippable PP
+/// directive.
+struct SkippedRange {
+ /// Offset into the output byte stream of where the skipped directive begins.
+ int Offset;
+
+ /// The number of bytes that can be skipped before the preprocessing must
+ /// resume.
+ int Length;
+};
+
+/// Computes the potential source ranges that can be skipped by the preprocessor
+/// when skipping a directive like #if, #ifdef or #elsif.
+///
+/// \returns false on success, true on error.
+bool computeSkippedRanges(ArrayRef<Token> Input,
+ llvm::SmallVectorImpl<SkippedRange> &Range);
+
} // end namespace minimize_source_to_dependency_directives
/// Minimize the input down to the preprocessor directives that might have
Index: clang/include/clang/Frontend/CompilerInstance.h
===================================================================
--- clang/include/clang/Frontend/CompilerInstance.h
+++ clang/include/clang/Frontend/CompilerInstance.h
@@ -46,6 +46,7 @@
class FrontendAction;
class InMemoryModuleCache;
class Module;
+class PPCallbacks;
class Preprocessor;
class Sema;
class SourceManager;
@@ -180,6 +181,9 @@
/// Force an output buffer.
std::unique_ptr<llvm::raw_pwrite_stream> OutputStream;
+ /// Additional PPCallbacks to add to the created preprocessor.
+ std::unique_ptr<PPCallbacks> AdditionalPPCallbacks;
+
CompilerInstance(const CompilerInstance &) = delete;
void operator=(const CompilerInstance &) = delete;
public:
@@ -247,6 +251,10 @@
BuildGlobalModuleIndex = Build;
}
+ /// Set the additional PP callbacks that will be added to the created
+ /// preprocessor.
+ void setAdditionalPPCallbacks(std::unique_ptr<PPCallbacks> PPC);
+
/// }
/// @name Forwarding Methods
/// {
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
