kadircet created this revision.
kadircet added a reviewer: sammccall.
Herald added subscribers: cfe-commits, arphaman, jkorous, MaskRay,
ilya-biryukov.
Herald added a project: clang.
Some custom toolchains come with their own header files and compiler
drivers. Those compiler drivers implicitly know about include search path for
those headers. This patch aims to extract that information from drivers and add
it to the command line when invoking clang frontend.
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D62804
Files:
clang-tools-extra/clangd/ClangdLSPServer.cpp
clang-tools-extra/clangd/ClangdServer.h
clang-tools-extra/clangd/GlobalCompilationDatabase.cpp
clang-tools-extra/clangd/GlobalCompilationDatabase.h
clang-tools-extra/clangd/test/system-include-extractor.test
clang-tools-extra/clangd/tool/ClangdMain.cpp
Index: clang-tools-extra/clangd/tool/ClangdMain.cpp
===================================================================
--- clang-tools-extra/clangd/tool/ClangdMain.cpp
+++ clang-tools-extra/clangd/tool/ClangdMain.cpp
@@ -268,6 +268,13 @@
"Always used text-based completion")),
llvm::cl::init(CodeCompleteOptions().RunParser), llvm::cl::Hidden);
+static llvm::cl::opt<std::string> TrustedCompilerDriverGlob(
+ "trusted-compiler-driver-glob",
+ llvm::cl::desc("Tells clangd to extract system includes from drivers "
+ "maching the glob. Only accepts ** for sequence match and * "
+ "for non-sequence match."),
+ llvm::cl::init(""));
+
namespace {
/// \brief Supports a test URI scheme with relaxed constraints for lit tests.
@@ -521,6 +528,7 @@
return ClangTidyOptProvider->getOptions(File);
};
Opts.SuggestMissingIncludes = SuggestMissingIncludes;
+ Opts.TrustedCompilerDriverGlob = TrustedCompilerDriverGlob;
llvm::Optional<OffsetEncoding> OffsetEncodingFromFlag;
if (ForceOffsetEncoding != OffsetEncoding::UnsupportedEncoding)
OffsetEncodingFromFlag = ForceOffsetEncoding;
Index: clang-tools-extra/clangd/test/system-include-extractor.test
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/test/system-include-extractor.test
@@ -0,0 +1,41 @@
+# RUN: rm -rf %t.dir && mkdir -p %t.dir
+
+# RUN: echo '#!/bin/bash' >> %t.dir/my_driver.sh
+# RUN: echo 'echo line to ignore' >> %t.dir/my_driver.sh
+# RUN: echo 'echo \#include \<...\> search starts here:' >> %t.dir/my_driver.sh
+# RUN: echo 'echo %t.dir/my/dir/' >> %t.dir/my_driver.sh
+# RUN: echo 'echo End of search list.' >> %t.dir/my_driver.sh
+# RUN: chmod +x %t.dir/my_driver.sh
+
+# RUN: mkdir -p %t.dir/my/dir
+# RUN: touch %t.dir/my/dir/a.h
+
+# RUN: echo '[{"directory": "%/t.dir", "command": "%/t.dir/my_driver.sh the-file.cpp", "file": "the-file.cpp"}]' > %t.dir/compile_commands.json
+
+# RUN: sed -e "s|INPUT_DIR|%/t.dir|g" %s > %t.test.1
+# On Windows, we need the URI in didOpen to look like "uri":"file:///C:/..."
+# (with the extra slash in the front), so we add it here.
+# RUN: sed -e "s|file://\([A-Z]\):/|file:///\1:/|g" %t.test.1 > %t.test
+
+# RUN: clangd -lit-test -trusted-compiler-driver-glob="**/*.sh" < %t.test | FileCheck -strict-whitespace %t.test
+{"jsonrpc":"2.0","id":0,"method":"initialize","params":{}}
+---
+{
+ "jsonrpc":"2.0",
+ "method":"textDocument/didOpen",
+ "params": {
+ "textDocument": {
+ "uri": "file://INPUT_DIR/the-file.cpp",
+ "languageId":"cpp",
+ "version":1,
+ "text":"#include <a.h>"
+ }
+ }
+}
+# CHECK: "method": "textDocument/publishDiagnostics",
+# CHECK-NEXT: "params": {
+# CHECK-NEXT: "diagnostics": [],
+---
+{"jsonrpc":"2.0","id":10000,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}
Index: clang-tools-extra/clangd/GlobalCompilationDatabase.h
===================================================================
--- clang-tools-extra/clangd/GlobalCompilationDatabase.h
+++ clang-tools-extra/clangd/GlobalCompilationDatabase.h
@@ -91,6 +91,14 @@
llvm::Optional<Path> CompileCommandsDir;
};
+/// Extracts system include search path from trusted drivers and adds them to
+/// the compile flags.
+/// Returns nullptr when \p TrustedCompilerDriverGlob is empty or \p Base is
+/// nullptr.
+std::unique_ptr<GlobalCompilationDatabase> getSystemIncludeExtractorDatabase(
+ llvm::StringRef TrustedCompilerDriverGlob,
+ std::unique_ptr<GlobalCompilationDatabase> Base);
+
/// Wraps another compilation database, and supports overriding the commands
/// using an in-memory mapping.
class OverlayCDB : public GlobalCompilationDatabase {
Index: clang-tools-extra/clangd/GlobalCompilationDatabase.cpp
===================================================================
--- clang-tools-extra/clangd/GlobalCompilationDatabase.cpp
+++ clang-tools-extra/clangd/GlobalCompilationDatabase.cpp
@@ -8,12 +8,31 @@
#include "GlobalCompilationDatabase.h"
#include "Logger.h"
+#include "Path.h"
+#include "Trace.h"
+#include "clang/Driver/Types.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Tooling/ArgumentsAdjusters.h"
#include "clang/Tooling/CompilationDatabase.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/Regex.h"
+#include "llvm/Support/ScopedPrinter.h"
+#include "llvm/Support/raw_ostream.h"
+#include <memory>
+#include <mutex>
+#include <string>
+#include <vector>
namespace clang {
namespace clangd {
@@ -43,6 +62,157 @@
return CompilerInvocation::GetResourcesPath("clangd", (void *)&Dummy);
}
+std::vector<std::string> parseDriverOutput(llvm::StringRef Output) {
+ std::vector<std::string> SystemIncludes;
+ constexpr char const *SIS = "#include <...> search starts here:";
+ constexpr char const *SIE = "End of search list.";
+ llvm::SmallVector<llvm::StringRef, 8> Lines;
+ Output.split(Lines, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
+ bool FoundStart = false;
+ for (llvm::StringRef Line : Lines) {
+ if (Line == SIS) {
+ FoundStart = true;
+ continue;
+ }
+ if (Line == SIE)
+ break;
+ if (!FoundStart)
+ continue;
+
+ if (!llvm::sys::fs::exists(Line))
+ elog("Parsed non-existing system include: {0}", Line);
+ SystemIncludes.push_back(Line.str());
+ vlog("Added: {0}", Line);
+ }
+ return SystemIncludes;
+}
+
+std::vector<std::string> extractSystemIncludes(PathRef Driver, PathRef File) {
+ trace::Span Tracer("Extract system includes");
+ SPAN_ATTACH(Tracer, "driver", Driver);
+ SPAN_ATTACH(Tracer, "file", File);
+
+ llvm::SmallString<128> OutputPath;
+ const auto EC = llvm::sys::fs::createTemporaryFile("system-includes",
+ "clangd", OutputPath);
+ if (EC) {
+ elog("Couldn't create temporary file: {0}", EC.message());
+ return {};
+ }
+ const llvm::Optional<llvm::StringRef> Redirects[] = {
+ {""}, llvm::StringRef(OutputPath), {""}};
+
+ const llvm::StringRef Ext = llvm::sys::path::extension(File).trim('.');
+ const auto Type = driver::types::lookupTypeForExtension(Ext);
+ // Should we also preserve flags like "-sysroot", "-nostdinc" ?
+ const llvm::StringRef Args[] = {"-E", "-x", driver::types::getTypeName(Type),
+ "-", "-v"};
+
+ const int RC =
+ llvm::sys::ExecuteAndWait(Driver, Args, /*Env=*/llvm::None, Redirects);
+ if (RC) {
+ elog("Execution failed with return code: {0}", llvm::to_string(RC));
+ return {};
+ }
+
+ auto BufOrError = llvm::MemoryBuffer::getFile(OutputPath);
+ if (!BufOrError) {
+ elog("Failed to read {0}: {1}", OutputPath,
+ BufOrError.getError().message());
+ return {};
+ }
+
+ return parseDriverOutput(BufOrError->get()->getBuffer());
+}
+
+tooling::CompileCommand
+addSystemIncludes(tooling::CompileCommand Cmd,
+ llvm::ArrayRef<std::string> SystemIncludes) {
+ for (llvm::StringRef Include : SystemIncludes) {
+ Cmd.CommandLine.push_back("-isystem");
+ Cmd.CommandLine.push_back(Include.str());
+ }
+ return Cmd;
+}
+
+/// Converts a glob containing only ** or * into a regex.
+llvm::Regex convertGlobToRegex(llvm::StringRef Glob) {
+ const llvm::StringRef MetaChars("()^$|*+?.[]\\{}");
+ std::string RegText;
+ llvm::raw_string_ostream RegStream(RegText);
+ RegStream << '^';
+ for (size_t I = 0, E = Glob.size(); I < E; ++I) {
+ if (Glob[I] == '*') {
+ if (I + 1 < E && Glob[I + 1] == '*') {
+ // Double star, accept any sequence.
+ RegStream << ".*";
+ // Also skip the second star.
+ ++I;
+ } else {
+ // Single star, accept any sequence without a slash.
+ RegStream << "[^/]*";
+ }
+ } else if (MetaChars.find(Glob[I]) != MetaChars.npos)
+ RegStream << '\\' << Glob[I];
+ else
+ RegStream << Glob[I];
+ }
+ RegStream << '$';
+ RegStream.flush();
+
+ llvm::Regex Result(RegText);
+ assert(Result.isValid(RegText) && "Glob to regex conversion failed");
+ return Result;
+}
+
+/// Extracts system includes from a trusted driver by parsing the output of
+/// include search path and appends them to the commands coming from underlying
+/// compilation database.
+class SystemIncludeExtractorDatabase : public GlobalCompilationDatabase {
+public:
+ SystemIncludeExtractorDatabase(
+ llvm::StringRef TrustedCompilerDriverGlob,
+ std::unique_ptr<GlobalCompilationDatabase> Base)
+ : TrustedCompilerDriverRegex(
+ convertGlobToRegex(TrustedCompilerDriverGlob)),
+ Base(std::move(Base)) {
+ assert(this->Base && "Base cannot be null!");
+ assert(!TrustedCompilerDriverGlob.empty() &&
+ "TrustedCompilerDriverGlob cannot be empty!");
+ }
+
+ llvm::Optional<tooling::CompileCommand>
+ getCompileCommand(PathRef File, ProjectInfo *PI = nullptr) const override {
+ auto Cmd = Base->getCompileCommand(File, PI);
+ if (!Cmd)
+ return llvm::None;
+
+ llvm::StringRef Driver = Cmd->CommandLine.front();
+
+ llvm::ArrayRef<std::string> SystemIncludes;
+ {
+ std::lock_guard<std::mutex> Lock(Mu);
+ if (!TrustedCompilerDriverRegex.match(Driver))
+ return Cmd;
+
+ auto It = DriverToIncludes.try_emplace(Driver);
+ if (It.second)
+ It.first->second = extractSystemIncludes(Driver, File);
+ SystemIncludes = It.first->second;
+ }
+
+ return addSystemIncludes(*Cmd, SystemIncludes);
+ }
+
+private:
+ mutable std::mutex Mu;
+ // Caches includes extracted from a driver.
+ mutable llvm::StringMap<std::vector<std::string>> DriverToIncludes;
+ mutable llvm::Regex TrustedCompilerDriverRegex;
+
+ std::unique_ptr<GlobalCompilationDatabase> Base;
+};
+
} // namespace
static std::string getFallbackClangPath() {
@@ -188,5 +358,12 @@
OnCommandChanged.broadcast({File});
}
+std::unique_ptr<GlobalCompilationDatabase> getSystemIncludeExtractorDatabase(
+ llvm::StringRef TrustedCompilerDriverGlob,
+ std::unique_ptr<GlobalCompilationDatabase> Base) {
+ return llvm::make_unique<SystemIncludeExtractorDatabase>(
+ TrustedCompilerDriverGlob, std::move(Base));
+}
+
} // namespace clangd
} // namespace clang
Index: clang-tools-extra/clangd/ClangdServer.h
===================================================================
--- clang-tools-extra/clangd/ClangdServer.h
+++ clang-tools-extra/clangd/ClangdServer.h
@@ -123,6 +123,10 @@
std::chrono::milliseconds(500);
bool SuggestMissingIncludes = false;
+
+ /// If set clangd will execute compiler drivers matching the following regex
+ /// to fetch system include path.
+ std::string TrustedCompilerDriverGlob;
};
// Sensible default options for use in tests.
// Features like indexing must be enabled if desired.
Index: clang-tools-extra/clangd/ClangdLSPServer.cpp
===================================================================
--- clang-tools-extra/clangd/ClangdLSPServer.cpp
+++ clang-tools-extra/clangd/ClangdLSPServer.cpp
@@ -8,6 +8,7 @@
#include "ClangdLSPServer.h"
#include "Diagnostics.h"
+#include "GlobalCompilationDatabase.h"
#include "Protocol.h"
#include "SourceCode.h"
#include "Trace.h"
@@ -336,9 +337,13 @@
ErrorCode::InvalidRequest));
if (const auto &Dir = Params.initializationOptions.compilationDatabasePath)
CompileCommandsDir = Dir;
- if (UseDirBasedCDB)
+ if (UseDirBasedCDB) {
BaseCDB = llvm::make_unique<DirectoryBasedGlobalCompilationDatabase>(
CompileCommandsDir);
+ if (!ClangdServerOpts.TrustedCompilerDriverGlob.empty())
+ BaseCDB = getSystemIncludeExtractorDatabase(
+ ClangdServerOpts.TrustedCompilerDriverGlob, std::move(BaseCDB));
+ }
CDB.emplace(BaseCDB.get(), Params.initializationOptions.fallbackFlags,
ClangdServerOpts.ResourceDir);
Server.emplace(*CDB, FSProvider, static_cast<DiagnosticsConsumer &>(*this),
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
