kadircet created this revision.
kadircet added a reviewer: sammccall.
Herald added subscribers: cfe-commits, usaxena95, jfb, arphaman, jkorous,
MaskRay, javed.absar, ilya-biryukov.
Herald added a project: clang.
First step to enable deferred preamble builds. Not intending to land it
alone, will have follow-ups that will implement full deferred build
functionality and will land after all of them are ready.
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D75995
Files:
clang-tools-extra/clangd/Compiler.cpp
clang-tools-extra/clangd/Preamble.cpp
clang-tools-extra/clangd/Preamble.h
clang-tools-extra/clangd/TUScheduler.cpp
clang-tools-extra/clangd/Threading.h
Index: clang-tools-extra/clangd/Threading.h
===================================================================
--- clang-tools-extra/clangd/Threading.h
+++ clang-tools-extra/clangd/Threading.h
@@ -107,7 +107,7 @@
/// Destructor waits for all pending tasks to finish.
~AsyncTaskRunner();
- void wait() const { (void)wait(Deadline::infinity()); }
+ void wait() const { (void)wait(timeoutSeconds(20)); }
LLVM_NODISCARD bool wait(Deadline D) const;
// The name is used for tracing and debugging (e.g. to name a spawned thread).
void runAsync(const llvm::Twine &Name, llvm::unique_function<void()> Action);
Index: clang-tools-extra/clangd/TUScheduler.cpp
===================================================================
--- clang-tools-extra/clangd/TUScheduler.cpp
+++ clang-tools-extra/clangd/TUScheduler.cpp
@@ -49,28 +49,41 @@
#include "GlobalCompilationDatabase.h"
#include "Logger.h"
#include "ParsedAST.h"
+#include "Path.h"
#include "Preamble.h"
+#include "Threading.h"
#include "Trace.h"
#include "index/CanonicalIncludes.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Tooling/CompilationDatabase.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/FunctionExtras.h"
+#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Errc.h"
+#include "llvm/Support/Error.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Threading.h"
#include <algorithm>
+#include <condition_variable>
+#include <deque>
#include <memory>
#include <mutex>
#include <queue>
+#include <string>
#include <thread>
+#include <utility>
+#include <vector>
namespace clang {
namespace clangd {
using std::chrono::steady_clock;
namespace {
-class ASTWorker;
+class MainFileWorker;
} // namespace
static clang::clangd::Key<std::string> kFileBeingProcessed;
@@ -87,7 +100,7 @@
/// Workers borrow ASTs when active, and return them when done.
class TUScheduler::ASTCache {
public:
- using Key = const ASTWorker *;
+ using Key = const MainFileWorker *;
ASTCache(unsigned MaxRetainedASTs) : MaxRetainedASTs(MaxRetainedASTs) {}
@@ -149,363 +162,389 @@
};
namespace {
-class ASTWorkerHandle;
-
-/// Owns one instance of the AST, schedules updates and reads of it.
-/// Also responsible for building and providing access to the preamble.
-/// Each ASTWorker processes the async requests sent to it on a separate
-/// dedicated thread.
-/// The ASTWorker that manages the AST is shared by both the processing thread
-/// and the TUScheduler. The TUScheduler should discard an ASTWorker when
-/// remove() is called, but its thread may be busy and we don't want to block.
-/// So the workers are accessed via an ASTWorkerHandle. Destroying the handle
-/// signals the worker to exit its run loop and gives up shared ownership of the
-/// worker.
-class ASTWorker {
- friend class ASTWorkerHandle;
- ASTWorker(PathRef FileName, const GlobalCompilationDatabase &CDB,
- TUScheduler::ASTCache &LRUCache, Semaphore &Barrier, bool RunSync,
- DebouncePolicy UpdateDebounce, bool StorePreamblesInMemory,
- ParsingCallbacks &Callbacks);
+/// Common base class for MainFile and Preamble workers. Holds information
+/// needed for both and supports triggering a stop.
+class StoppableTUWorker {
public:
- /// Create a new ASTWorker and return a handle to it.
- /// The processing thread is spawned using \p Tasks. However, when \p Tasks
- /// is null, all requests will be processed on the calling thread
- /// synchronously instead. \p Barrier is acquired when processing each
- /// request, it is used to limit the number of actively running threads.
- static ASTWorkerHandle
- create(PathRef FileName, const GlobalCompilationDatabase &CDB,
- TUScheduler::ASTCache &IdleASTs, AsyncTaskRunner *Tasks,
- Semaphore &Barrier, DebouncePolicy UpdateDebounce,
- bool StorePreamblesInMemory, ParsingCallbacks &Callbacks);
- ~ASTWorker();
-
- void update(ParseInputs Inputs, WantDiagnostics);
- void
- runWithAST(llvm::StringRef Name,
- llvm::unique_function<void(llvm::Expected<InputsAndAST>)> Action,
- TUScheduler::ASTActionInvalidation);
- bool blockUntilIdle(Deadline Timeout) const;
-
- std::shared_ptr<const PreambleData> getPossiblyStalePreamble() const;
-
- /// Obtain a preamble reflecting all updates so far. Threadsafe.
- /// It may be delivered immediately, or later on the worker thread.
- void getCurrentPreamble(
- llvm::unique_function<void(std::shared_ptr<const PreambleData>)>);
- /// Returns compile command from the current file inputs.
- tooling::CompileCommand getCurrentCompileCommand() const;
+ StoppableTUWorker(ParsingCallbacks &Callbacks, PathRef FileName, bool RunSync)
+ : Callbacks(Callbacks), FileName(FileName), RunSync(RunSync),
+ Done(false) {}
+ bool isDone() { return Done.load(); }
+ /// Must be called exactly once on processing thread. Should return in a
+ /// bounded time after stop() is called on a separate thread.
+ virtual void run() = 0;
+ virtual void stop() {
+ assert(!Done.load() && "Already stopped");
+ Done.store(true);
+ }
+ virtual ~StoppableTUWorker() {}
- /// Wait for the first build of preamble to finish. Preamble itself can be
- /// accessed via getPossiblyStalePreamble(). Note that this function will
- /// return after an unsuccessful build of the preamble too, i.e. result of
- /// getPossiblyStalePreamble() can be null even after this function returns.
- void waitForFirstPreamble() const;
+ virtual size_t getUsedBytes() const = 0;
- std::size_t getUsedBytes() const;
- bool isASTCached() const;
+ /// Should only be used by tests. Blocks until deadline is reached or worker
+ /// went idle. Returns true for the latter.
+ virtual bool blockUntilIdle(Deadline Timeout) const = 0;
-private:
- // Must be called exactly once on processing thread. Will return after
- // stop() is called on a separate thread and all pending requests are
- // processed.
- void run();
- /// Signal that run() should finish processing pending requests and exit.
- void stop();
- /// Adds a new task to the end of the request queue.
- void startTask(llvm::StringRef Name, llvm::unique_function<void()> Task,
- llvm::Optional<WantDiagnostics> UpdateType,
- TUScheduler::ASTActionInvalidation);
+protected:
/// Updates the TUStatus and emits it. Only called in the worker thread.
- void emitTUStatus(TUAction FAction,
- const TUStatus::BuildDetails *Detail = nullptr);
-
- /// Determines the next action to perform.
- /// All actions that should never run are discarded.
- /// Returns a deadline for the next action. If it's expired, run now.
- /// scheduleLocked() is called again at the deadline, or if requests arrive.
- Deadline scheduleLocked();
- /// Should the first task in the queue be skipped instead of run?
- bool shouldSkipHeadLocked() const;
- /// This is private because `FileInputs.FS` is not thread-safe and thus not
- /// safe to share. Callers should make sure not to expose `FS` via a public
- /// interface.
- std::shared_ptr<const ParseInputs> getCurrentFileInputs() const;
-
- struct Request {
- llvm::unique_function<void()> Action;
- std::string Name;
- steady_clock::time_point AddTime;
- Context Ctx;
- llvm::Optional<WantDiagnostics> UpdateType;
- TUScheduler::ASTActionInvalidation InvalidationPolicy;
- Canceler Invalidate;
- };
+ void emitTUStatus(TUAction Action,
+ const TUStatus::BuildDetails *Details = nullptr) {
+ TUStatus Status({std::move(Action), {}});
+ if (Details)
+ Status.Details = *Details;
+ // Do not emit TU statuses when the ASTWorker is shutting down.
+ if (!isDone())
+ Callbacks.onFileUpdated(FileName, Status);
+ }
- /// Handles retention of ASTs.
- TUScheduler::ASTCache &IdleASTs;
- const bool RunSync;
- /// Time to wait after an update to see whether another update obsoletes it.
- const DebouncePolicy UpdateDebounce;
- /// File that ASTWorker is responsible for.
- const Path FileName;
- const GlobalCompilationDatabase &CDB;
- /// Whether to keep the built preambles in memory or on disk.
- const bool StorePreambleInMemory;
- /// Callback invoked when preamble or main file AST is built.
+ /// Callbacks invoked to report state of the worker.
ParsingCallbacks &Callbacks;
- /// Only accessed by the worker thread.
- TUStatus Status;
+ /// Root of the translation unit.
+ llvm::StringRef FileName;
+ const bool RunSync;
- Semaphore &Barrier;
- /// Whether the 'onMainAST' callback ran for the current FileInputs.
- bool RanASTCallback = false;
- /// Guards members used by both TUScheduler and the worker thread.
- mutable std::mutex Mutex;
- /// File inputs, currently being used by the worker.
- /// Inputs are written and read by the worker thread, compile command can also
- /// be consumed by clients of ASTWorker.
- std::shared_ptr<const ParseInputs> FileInputs; /* GUARDED_BY(Mutex) */
- std::shared_ptr<const PreambleData> LastBuiltPreamble; /* GUARDED_BY(Mutex) */
- /// Times of recent AST rebuilds, used for UpdateDebounce computation.
- llvm::SmallVector<DebouncePolicy::clock::duration, 8>
- RebuildTimes; /* GUARDED_BY(Mutex) */
- /// Becomes ready when the first preamble build finishes.
- Notification PreambleWasBuilt;
- /// Set to true to signal run() to finish processing.
- bool Done; /* GUARDED_BY(Mutex) */
- std::deque<Request> Requests; /* GUARDED_BY(Mutex) */
- llvm::Optional<Request> CurrentRequest; /* GUARDED_BY(Mutex) */
- mutable std::condition_variable RequestsCV;
- /// Guards the callback that publishes results of AST-related computations
- /// (diagnostics, highlightings) and file statuses.
- std::mutex PublishMu;
- // Used to prevent remove document + add document races that lead to
- // out-of-order callbacks for publishing results of onMainAST callback.
- //
- // The lifetime of the old/new ASTWorkers will overlap, but their handles
- // don't. When the old handle is destroyed, the old worker will stop reporting
- // any results to the user.
- bool CanPublishResults = true; /* GUARDED_BY(PublishMu) */
+private:
+ std::atomic<bool> Done;
};
-/// A smart-pointer-like class that points to an active ASTWorker.
-/// In destructor, signals to the underlying ASTWorker that no new requests will
-/// be sent and the processing loop may exit (after running all pending
-/// requests).
-class ASTWorkerHandle {
- friend class ASTWorker;
- ASTWorkerHandle(std::shared_ptr<ASTWorker> Worker)
- : Worker(std::move(Worker)) {
- assert(this->Worker);
+/// Responsible for building and providing access to the preamble of a TU. This
+/// worker doesn't guarantee that each preamble request will be built, in case
+/// of multiple preamble requests, it will only build the latest one. Once stop
+/// is called it will build any remaining request and will exit the run loop.
+class PreambleWorker : public StoppableTUWorker {
+public:
+ PreambleWorker(ParsingCallbacks &Callbacks, PathRef FileName,
+ bool StorePreambleInMemory, bool RunSync)
+ : StoppableTUWorker(Callbacks, FileName, RunSync),
+ StorePreambleInMemory(StorePreambleInMemory) {}
+
+ size_t getUsedBytes() const override {
+ auto Preamble = getLatestBuiltPreamble();
+ return Preamble ? Preamble->Preamble.getSize() : 0;
}
-public:
- ASTWorkerHandle(const ASTWorkerHandle &) = delete;
- ASTWorkerHandle &operator=(const ASTWorkerHandle &) = delete;
- ASTWorkerHandle(ASTWorkerHandle &&) = default;
- ASTWorkerHandle &operator=(ASTWorkerHandle &&) = default;
+ void requestBuild(CompilerInvocation *CI, ParseInputs PI) {
+ assert(!isDone() && "Build request to PreambleWorker after stop");
+ // If compiler invocation was broken, just fail out early.
+ if (!CI) {
+ TUStatus::BuildDetails Details;
+ Details.BuildFailed = true;
+ std::string TaskName = llvm::formatv("Update ({0})", PI.Version);
+ emitTUStatus({TUAction::BuildingPreamble, std::move(TaskName)}, &Details);
+ // Make sure anyone waiting for the preamble gets notified it could not be
+ // built.
+ BuiltFirstPreamble.notify();
+ return;
+ }
+ // Make possibly expensive copy while not holding the lock.
+ Request Req = {std::make_unique<CompilerInvocation>(*CI), std::move(PI)};
+ if (RunSync) {
+ updateLatestBuiltPreamble(buildPreamble(std::move(Req)));
+ return;
+ }
+ {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ PreambleReq = std::move(Req);
+ }
+ // Let the worker thread know there's a request, notify_one is safe as there
+ // should be a single worker thread waiting on it.
+ PreambleRequested.notify_one();
+ }
- ~ASTWorkerHandle() {
- if (Worker)
- Worker->stop();
+ /// Blocks until at least a single request has been processed. Note that it
+ /// will unblock even after an unsuccessful build.
+ void waitForFirstPreamble() const { BuiltFirstPreamble.wait(); }
+
+ /// Returns the latest built preamble, might be null if no preamble has been
+ /// built or latest attempt resulted in a failure.
+ std::shared_ptr<const PreambleData> getLatestBuiltPreamble() const {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ return LastBuiltPreamble;
}
- ASTWorker &operator*() {
- assert(Worker && "Handle was moved from");
- return *Worker;
+ void run() override {
+ dlog("Starting preamble worker for {0}", FileName);
+ while (true) {
+ {
+ std::unique_lock<std::mutex> Lock(Mutex);
+ assert(!CurrentReq && "Already processing a request?");
+ // Wait until stop is called or there is a request.
+ PreambleRequested.wait(Lock,
+ [this] { return PreambleReq || isDone(); });
+ // No request means we are done.
+ if (!PreambleReq)
+ break;
+ CurrentReq = std::move(*PreambleReq);
+ // Reset it here, before we build the request to not overwrite any new
+ // request that might arrive while we are still building this one.
+ PreambleReq.reset();
+ }
+ // Build the preamble and let the waiters know about it.
+ updateLatestBuiltPreamble(buildPreamble(std::move(*CurrentReq)));
+ }
+ // We are no longer going to build any preambles, let the waiters know that.
+ BuiltFirstPreamble.notify();
+ dlog("Preamble worker for {0} finished", FileName);
}
- ASTWorker *operator->() {
- assert(Worker && "Handle was moved from");
- return Worker.get();
+ virtual void stop() override {
+ dlog("Stopping preamble worker for {0}", FileName);
+ StoppableTUWorker::stop();
+ // Let the worker thread know we've been stopped. notify_one is safe as
+ // there should be only a single worker.
+ PreambleRequested.notify_one();
}
- /// Returns an owning reference to the underlying ASTWorker that can outlive
- /// the ASTWorkerHandle. However, no new requests to an active ASTWorker can
- /// be schedule via the returned reference, i.e. only reads of the preamble
- /// are possible.
- std::shared_ptr<const ASTWorker> lock() { return Worker; }
+ bool blockUntilIdle(Deadline Timeout) const override {
+ std::unique_lock<std::mutex> Lock(Mutex);
+ return wait(Lock, PreambleBuilt, Timeout,
+ [&] { return !PreambleReq && !CurrentReq; });
+ }
private:
- std::shared_ptr<ASTWorker> Worker;
+ /// Holds inputs required for building a preamble, CI is guaranteed to be
+ /// non-null.
+ struct Request {
+ std::unique_ptr<CompilerInvocation> CI;
+ ParseInputs Inputs;
+ };
+
+ /// Caches the latest built preamble and singals waiters about the built.
+ /// FIXME: We shouldn't cache failed preambles, if we've got a successful
+ /// build before.
+ void updateLatestBuiltPreamble(std::shared_ptr<const PreambleData> Preamble) {
+ {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ LastBuiltPreamble = std::move(Preamble);
+ CurrentReq.reset();
+ }
+ BuiltFirstPreamble.notify();
+ PreambleBuilt.notify_all();
+ }
+
+ /// Builds a preamble for Req, might re-use the latest built preamble if it is
+ /// valid for Req.
+ std::shared_ptr<const PreambleData> buildPreamble(Request Req) {
+ assert(Req.CI && "Got preamble request with null compiler invocation");
+ const ParseInputs &Inputs = Req.Inputs;
+ std::shared_ptr<const PreambleData> OldPreamble =
+ Inputs.ForceRebuild ? std::shared_ptr<const PreambleData>()
+ : getLatestBuiltPreamble();
+ // FIXME: We should rather use the compile command in the OldPreamble, but
+ // currently it is not populated.
+ const tooling::CompileCommand OldCmd = LastCmd;
+ LastCmd = Inputs.CompileCommand;
+
+ std::string TaskName = llvm::formatv("Update ({0})", Inputs.Version);
+ emitTUStatus({TUAction::BuildingPreamble, std::move(TaskName)});
+
+ return clang::clangd::buildPreamble(
+ FileName, std::move(*Req.CI), OldPreamble, OldCmd, Inputs,
+ StorePreambleInMemory,
+ [this, Version(Inputs.Version)](
+ ASTContext &Ctx, std::shared_ptr<clang::Preprocessor> PP,
+ const CanonicalIncludes &CanonIncludes) {
+ Callbacks.onPreambleAST(FileName, Version, Ctx, std::move(PP),
+ CanonIncludes);
+ });
+ }
+
+ mutable std::mutex Mutex;
+ llvm::Optional<Request> PreambleReq;
+ // Written by only worker thread, might be read by multiple threads through
+ // BlockUntilIdle.
+ llvm::Optional<Request> CurrentReq;
+ // Signaled whenever a thread populates PreambleReq.
+ mutable std::condition_variable PreambleRequested;
+ // Signaled whenever a LastBuiltPreamble is replaced.
+ mutable std::condition_variable PreambleBuilt;
+ std::shared_ptr<const PreambleData> LastBuiltPreamble;
+
+ Notification BuiltFirstPreamble;
+
+ // Only used by current worker thread, so no data races.
+ tooling::CompileCommand LastCmd;
+ const bool StorePreambleInMemory;
};
-ASTWorkerHandle
-ASTWorker::create(PathRef FileName, const GlobalCompilationDatabase &CDB,
- TUScheduler::ASTCache &IdleASTs, AsyncTaskRunner *Tasks,
- Semaphore &Barrier, DebouncePolicy UpdateDebounce,
- bool StorePreamblesInMemory, ParsingCallbacks &Callbacks) {
- std::shared_ptr<ASTWorker> Worker(
- new ASTWorker(FileName, CDB, IdleASTs, Barrier, /*RunSync=*/!Tasks,
- UpdateDebounce, StorePreamblesInMemory, Callbacks));
- if (Tasks)
- Tasks->runAsync("worker:" + llvm::sys::path::filename(FileName),
- [Worker]() { Worker->run(); });
-
- return ASTWorkerHandle(std::move(Worker));
-}
+/// Responsible for building and providing access to the main file AST of a TU.
+/// It serializes build requests and makes sure any access will happen after the
+/// preceding build succeeds. It might discard build requests that doesn't have
+/// any depending reads, or read requests that should be invalidated after a
+/// write. Once stop is called it will serve any remaining requests and will
+/// exit the run loop.
+class MainFileWorker : public StoppableTUWorker {
+public:
+ MainFileWorker(PathRef FileName, TUScheduler::ASTCache &LRUCache,
+ Semaphore &Barrier, bool RunSync,
+ DebouncePolicy UpdateDebounce, ParsingCallbacks &Callbacks)
+ : StoppableTUWorker(Callbacks, FileName, RunSync), Barrier(Barrier),
+ UpdateDebounce(UpdateDebounce), IdleASTs(LRUCache) {}
+
+ size_t getUsedBytes() const override { return IdleASTs.getUsedBytes(this); }
+ bool isASTCached() const { return IdleASTs.getUsedBytes(this) != 0; }
+
+ // Must be called exactly once on processing thread. Will return after stop()
+ // is called on a separate thread and all pending requests are processed.
+ void run() override {
+ dlog("Starting mainfile worker for {0}", FileName);
+ while (true) {
+ {
+ std::unique_lock<std::mutex> Lock(Mutex);
+ assert(!CurrentRequest && "A task is running, multiple workers?");
+ for (auto Wait = scheduleLocked(); !Wait.expired();
+ Wait = scheduleLocked()) {
+ if (isDone()) {
+ if (Requests.empty())
+ return;
+ // Even though Done is set, finish pending requests. However, skip
+ // delays to shutdown fast.
+ break;
+ }
-ASTWorker::ASTWorker(PathRef FileName, const GlobalCompilationDatabase &CDB,
- TUScheduler::ASTCache &LRUCache, Semaphore &Barrier,
- bool RunSync, DebouncePolicy UpdateDebounce,
- bool StorePreamblesInMemory, ParsingCallbacks &Callbacks)
- : IdleASTs(LRUCache), RunSync(RunSync), UpdateDebounce(UpdateDebounce),
- FileName(FileName), CDB(CDB),
- StorePreambleInMemory(StorePreamblesInMemory),
- Callbacks(Callbacks), Status{TUAction(TUAction::Idle, ""),
- TUStatus::BuildDetails()},
- Barrier(Barrier), Done(false) {
- auto Inputs = std::make_shared<ParseInputs>();
- // Set a fallback command because compile command can be accessed before
- // `Inputs` is initialized. Other fields are only used after initialization
- // from client inputs.
- Inputs->CompileCommand = CDB.getFallbackCommand(FileName);
- FileInputs = std::move(Inputs);
-}
+ // Tracing: we have a next request, attribute this sleep to it.
+ llvm::Optional<WithContext> Ctx;
+ llvm::Optional<trace::Span> Tracer;
+ if (!Requests.empty()) {
+ Ctx.emplace(Requests.front().Ctx.clone());
+ Tracer.emplace("Debounce");
+ SPAN_ATTACH(*Tracer, "next_request", Requests.front().Name);
+ if (!(Wait == Deadline::infinity())) {
+ emitTUStatus({TUAction::Queued, Requests.front().Name});
+ SPAN_ATTACH(*Tracer, "sleep_ms",
+ std::chrono::duration_cast<std::chrono::milliseconds>(
+ Wait.time() - steady_clock::now())
+ .count());
+ }
+ }
-ASTWorker::~ASTWorker() {
- // Make sure we remove the cached AST, if any.
- IdleASTs.take(this);
-#ifndef NDEBUG
- std::lock_guard<std::mutex> Lock(Mutex);
- assert(Done && "handle was not destroyed");
- assert(Requests.empty() && !CurrentRequest &&
- "unprocessed requests when destroying ASTWorker");
-#endif
-}
+ wait(Lock, RequestsCV, Wait);
+ }
+ CurrentRequest = std::move(Requests.front());
+ Requests.pop_front();
+ } // unlock Mutex
-void ASTWorker::update(ParseInputs Inputs, WantDiagnostics WantDiags) {
- std::string TaskName = llvm::formatv("Update ({0})", Inputs.Version);
- auto Task = [=]() mutable {
- auto RunPublish = [&](llvm::function_ref<void()> Publish) {
+ // It is safe to perform reads to CurrentRequest without holding the lock
+ // as only writer is also this thread.
+ {
+ std::unique_lock<Semaphore> Lock(Barrier, std::try_to_lock);
+ if (!Lock.owns_lock()) {
+ emitTUStatus({TUAction::Queued, CurrentRequest->Name});
+ Lock.lock();
+ }
+ WithContext Guard(std::move(CurrentRequest->Ctx));
+ trace::Span Tracer(CurrentRequest->Name);
+ emitTUStatus({TUAction::RunningAction, CurrentRequest->Name});
+ CurrentRequest->Action();
+ }
+
+ bool IsEmpty = false;
+ {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ CurrentRequest.reset();
+ IsEmpty = Requests.empty();
+ }
+ if (IsEmpty)
+ emitTUStatus({TUAction::Idle, /*Name=*/""});
+ RequestsCV.notify_all();
+ }
+ }
+
+ /// Adds a request to the queue, which will be executed after any requests
+ /// issued so far.
+ void dispatchUpdate(llvm::unique_function<void()> Update,
+ WantDiagnostics WantDiags, llvm::StringRef Version) {
+ std::string TaskName = llvm::formatv("Update ({0})", Version);
+ addToQueue(TaskName, std::move(Update), std::move(WantDiags),
+ TUScheduler::NoInvalidation);
+ }
+
+ /// Builds an AST for provided inputs and invokes the relevant callbacks.
+ /// Might re-use a cached AST, and won't invoke callbacks if they've already
+ /// been invoked while building the cached AST. Should only be invoked by
+ /// worker thread.
+ void buildAST(std::unique_ptr<CompilerInvocation> Invocation,
+ std::vector<Diag> CompilerInvocationDiags, ParseInputs Inputs,
+ WantDiagnostics WantDiags,
+ std::shared_ptr<const PreambleData> Preamble) {
+ std::string TaskName = llvm::formatv("Update ({0})", Inputs.Version);
+
+ auto RunPublish = [this](llvm::function_ref<void()> Publish) {
// Ensure we only publish results from the worker if the file was not
// removed, making sure there are not race conditions.
- std::lock_guard<std::mutex> Lock(PublishMu);
- if (CanPublishResults)
+ if (!isDone())
Publish();
};
-
- // Get the actual command as `Inputs` does not have a command.
- // FIXME: some build systems like Bazel will take time to preparing
- // environment to build the file, it would be nice if we could emit a
- // "PreparingBuild" status to inform users, it is non-trivial given the
- // current implementation.
- if (auto Cmd = CDB.getCompileCommand(FileName))
- Inputs.CompileCommand = *Cmd;
- else
- // FIXME: consider using old command if it's not a fallback one.
- Inputs.CompileCommand = CDB.getFallbackCommand(FileName);
- auto PrevInputs = getCurrentFileInputs();
+ auto PrevInputs = getCurrentInputs();
// Will be used to check if we can avoid rebuilding the AST.
- bool InputsAreTheSame =
- std::tie(PrevInputs->CompileCommand, PrevInputs->Contents) ==
+ const bool InputsAreTheSame =
+ std::tie(PrevInputs.CompileCommand, PrevInputs.Contents) ==
std::tie(Inputs.CompileCommand, Inputs.Contents);
- tooling::CompileCommand OldCommand = PrevInputs->CompileCommand;
bool RanCallbackForPrevInputs = RanASTCallback;
- {
- std::lock_guard<std::mutex> Lock(Mutex);
- FileInputs = std::make_shared<ParseInputs>(Inputs);
- }
+ setCurrentInputs(Inputs);
RanASTCallback = false;
- emitTUStatus({TUAction::BuildingPreamble, TaskName});
- log("ASTWorker building file {0} version {1} with command {2}\n[{3}]\n{4}",
+
+ log("ASTWorker building file {0} version {1} with command "
+ "{2}\n[{3}]\n{4}",
FileName, Inputs.Version, Inputs.CompileCommand.Heuristic,
Inputs.CompileCommand.Directory,
llvm::join(Inputs.CompileCommand.CommandLine, " "));
- // Rebuild the preamble and the AST.
- StoreDiags CompilerInvocationDiagConsumer;
- std::vector<std::string> CC1Args;
- std::unique_ptr<CompilerInvocation> Invocation = buildCompilerInvocation(
- Inputs, CompilerInvocationDiagConsumer, &CC1Args);
- // Log cc1 args even (especially!) if creating invocation failed.
- if (!CC1Args.empty())
- vlog("Driver produced command: cc1 {0}", llvm::join(CC1Args, " "));
- std::vector<Diag> CompilerInvocationDiags =
- CompilerInvocationDiagConsumer.take();
+
if (!Invocation) {
- elog("Could not build CompilerInvocation for file {0}", FileName);
- // Remove the old AST if it's still in cache.
IdleASTs.take(this);
- TUStatus::BuildDetails Details;
- Details.BuildFailed = true;
- emitTUStatus({TUAction::BuildingPreamble, TaskName}, &Details);
// Report the diagnostics we collected when parsing the command line.
Callbacks.onFailedAST(FileName, Inputs.Version,
std::move(CompilerInvocationDiags), RunPublish);
- // Make sure anyone waiting for the preamble gets notified it could not
- // be built.
- PreambleWasBuilt.notify();
return;
}
- std::shared_ptr<const PreambleData> OldPreamble =
- Inputs.ForceRebuild ? std::shared_ptr<const PreambleData>()
- : getPossiblyStalePreamble();
- std::shared_ptr<const PreambleData> NewPreamble = buildPreamble(
- FileName, *Invocation, OldPreamble, OldCommand, Inputs,
- StorePreambleInMemory,
- [this, Version(Inputs.Version)](
- ASTContext &Ctx, std::shared_ptr<clang::Preprocessor> PP,
- const CanonicalIncludes &CanonIncludes) {
- Callbacks.onPreambleAST(FileName, Version, Ctx, std::move(PP),
- CanonIncludes);
- });
-
- bool CanReuseAST = InputsAreTheSame && (OldPreamble == NewPreamble);
- {
- std::lock_guard<std::mutex> Lock(Mutex);
- LastBuiltPreamble = NewPreamble;
- }
- // Before doing the expensive AST reparse, we want to release our reference
- // to the old preamble, so it can be freed if there are no other references
- // to it.
- OldPreamble.reset();
- PreambleWasBuilt.notify();
+ bool CanReuseAST = InputsAreTheSame && (CurrentPreamble == Preamble);
+ // Before doing the expensive AST reparse, we want to release our
+ // reference to the old preamble, so it can be freed if there are no other
+ // references to it.
+ CurrentPreamble = std::move(Preamble);
emitTUStatus({TUAction::BuildingFile, TaskName});
- if (!CanReuseAST) {
- IdleASTs.take(this); // Remove the old AST if it's still in cache.
- } else {
- // We don't need to rebuild the AST, check if we need to run the callback.
- if (RanCallbackForPrevInputs) {
- RanASTCallback = true;
- // Take a shortcut and don't report the diagnostics, since they should
- // not changed. All the clients should handle the lack of OnUpdated()
- // call anyway to handle empty result from buildAST.
- // FIXME(ibiryukov): the AST could actually change if non-preamble
- // includes changed, but we choose to ignore it.
- // FIXME(ibiryukov): should we refresh the cache in IdleASTs for the
- // current file at this point?
- log("Skipping rebuild of the AST for {0}, inputs are the same.",
- FileName);
- TUStatus::BuildDetails Details;
- Details.ReuseAST = true;
- emitTUStatus({TUAction::BuildingFile, TaskName}, &Details);
- return;
- }
+ // We don't need to rebuild the AST, check if we need to run the
+ // callback.
+ if (CanReuseAST && RanCallbackForPrevInputs) {
+ RanASTCallback = true;
+ // Take a shortcut and don't report the diagnostics, since they should
+ // not changed. All the clients should handle the lack of OnUpdated()
+ // call anyway to handle empty result from buildAST.
+ // FIXME(ibiryukov): the AST could actually change if non-preamble
+ // includes changed, but we choose to ignore it.
+ // FIXME(ibiryukov): should we refresh the cache in IdleASTs for the
+ // current file at this point?
+ log("Skipping rebuild of the AST for {0}, inputs are the same.",
+ FileName);
+ TUStatus::BuildDetails Details;
+ Details.ReuseAST = true;
+ emitTUStatus({TUAction::BuildingFile, TaskName}, &Details);
+ return;
}
+ // We either need a new AST or haven't published diags yet.
+ // Remove the old AST if we need to build a new one.
+ if (!CanReuseAST)
+ IdleASTs.take(this);
// We only need to build the AST if diagnostics were requested.
if (WantDiags == WantDiagnostics::No)
return;
- {
- std::lock_guard<std::mutex> Lock(PublishMu);
- // No need to rebuild the AST if we won't send the diagnostics. However,
- // note that we don't prevent preamble rebuilds.
- if (!CanPublishResults)
- return;
- }
+ // Don't send the diagnostics if worker has stopped, as there might be a new
+ // worker and we shouldn't race with it.
+ if (isDone())
+ return;
// Get the AST for diagnostics.
llvm::Optional<std::unique_ptr<ParsedAST>> AST = IdleASTs.take(this);
auto RebuildStartTime = DebouncePolicy::clock::now();
if (!AST) {
- llvm::Optional<ParsedAST> NewAST =
- buildAST(FileName, std::move(Invocation), CompilerInvocationDiags,
- Inputs, NewPreamble);
+ llvm::Optional<ParsedAST> NewAST = clang::clangd::buildAST(
+ FileName, std::move(Invocation), CompilerInvocationDiags, Inputs,
+ CurrentPreamble);
AST = NewAST ? std::make_unique<ParsedAST>(std::move(*NewAST)) : nullptr;
if (!(*AST)) { // buildAST fails.
TUStatus::BuildDetails Details;
@@ -525,12 +564,14 @@
// Note *AST can still be null if buildAST fails.
if (*AST) {
{
- // Try to record the AST-build time, to inform future update debouncing.
- // This is best-effort only: if the lock is held, don't bother.
+ // Try to record the AST-build time, to inform future update
+ // debouncing. This is best-effort only: if the lock is held, don't
+ // bother.
auto RebuildDuration = DebouncePolicy::clock::now() - RebuildStartTime;
std::unique_lock<std::mutex> Lock(Mutex, std::try_to_lock);
if (Lock.owns_lock()) {
- // Do not let RebuildTimes grow beyond its small-size (i.e. capacity).
+ // Do not let RebuildTimes grow beyond its small-size (i.e.
+ // capacity).
if (RebuildTimes.size() == RebuildTimes.capacity())
RebuildTimes.erase(RebuildTimes.begin());
RebuildTimes.push_back(RebuildDuration);
@@ -550,307 +591,474 @@
}
// Stash the AST in the cache for further use.
IdleASTs.put(this, std::move(*AST));
- };
- startTask(TaskName, std::move(Task), WantDiags, TUScheduler::NoInvalidation);
-}
+ }
-void ASTWorker::runWithAST(
- llvm::StringRef Name,
- llvm::unique_function<void(llvm::Expected<InputsAndAST>)> Action,
- TUScheduler::ASTActionInvalidation Invalidation) {
- auto Task = [=, Action = std::move(Action)]() mutable {
- if (isCancelled())
- return Action(llvm::make_error<CancelledError>());
- llvm::Optional<std::unique_ptr<ParsedAST>> AST = IdleASTs.take(this);
- auto CurrentInputs = getCurrentFileInputs();
- if (!AST) {
- StoreDiags CompilerInvocationDiagConsumer;
- std::unique_ptr<CompilerInvocation> Invocation = buildCompilerInvocation(
- *CurrentInputs, CompilerInvocationDiagConsumer);
- // Try rebuilding the AST.
- vlog("ASTWorker rebuilding evicted AST to run {0}: {1} version {2}", Name,
- FileName, CurrentInputs->Version);
- llvm::Optional<ParsedAST> NewAST =
- Invocation
- ? buildAST(FileName,
- std::make_unique<CompilerInvocation>(*Invocation),
- CompilerInvocationDiagConsumer.take(), *CurrentInputs,
- getPossiblyStalePreamble())
- : None;
- AST = NewAST ? std::make_unique<ParsedAST>(std::move(*NewAST)) : nullptr;
+ void stop() override {
+ dlog("Stopping mainfile worker for {0}", FileName);
+ StoppableTUWorker::stop();
+ // Let the worker thread know we've stopped. We can't use notify_one, as
+ // there might be other threads calling blockUntilIdle.
+ RequestsCV.notify_all();
+ }
+
+ bool blockUntilIdle(Deadline Timeout) const override {
+ std::unique_lock<std::mutex> Lock(Mutex);
+ return wait(Lock, RequestsCV, Timeout,
+ [&] { return Requests.empty() && !CurrentRequest; });
+ }
+
+ ~MainFileWorker() {
+ // Make sure we remove the cached AST, if any.
+ IdleASTs.take(this);
+#ifndef NDEBUG
+ std::lock_guard<std::mutex> Lock(Mutex);
+ assert(isDone() && "handle was not destroyed");
+ assert(Requests.empty() && !CurrentRequest &&
+ "unprocessed requests when destroying ASTWorker");
+#endif
+ }
+
+ /// Adds a read request to the queue. Might end up building the AST if it is
+ /// evicted from the cache.
+ /// GetPreamble is used to fetch the latest build preamble while executing
+ /// this action, rather than using the latest build preamble while issuing the
+ /// request.
+ void
+ runAction(llvm::StringRef Name,
+ llvm::unique_function<void(llvm::Expected<InputsAndAST>)> Action,
+ TUScheduler::ASTActionInvalidation Invalidation,
+ llvm::unique_function<std::shared_ptr<const PreambleData>()>
+ GetPreamble) {
+ auto Task = [Action = std::move(Action),
+ GetPreamble = std::move(GetPreamble), Name, this]() mutable {
+ if (isCancelled())
+ return Action(llvm::make_error<CancelledError>());
+ auto CurrentInputs = getCurrentInputs();
+ auto AST = IdleASTs.take(this);
+ if (!AST) {
+ StoreDiags CompilerInvocationDiagConsumer;
+ std::unique_ptr<CompilerInvocation> Invocation =
+ buildCompilerInvocation(CurrentInputs,
+ CompilerInvocationDiagConsumer);
+ // Try rebuilding the AST.
+ vlog("ASTWorker rebuilding evicted AST to run {0}: {1} version {2}",
+ Name, FileName, CurrentInputs.Version);
+ /// FIXME: We should be updating CurrentInputs and Preamble here.
+ llvm::Optional<ParsedAST> NewAST =
+ Invocation ? clang::clangd::buildAST(
+ FileName,
+ std::make_unique<CompilerInvocation>(*Invocation),
+ CompilerInvocationDiagConsumer.take(),
+ CurrentInputs, GetPreamble())
+ : None;
+ AST =
+ NewAST ? std::make_unique<ParsedAST>(std::move(*NewAST)) : nullptr;
+ }
+ // Make sure we put the AST back into the LRU cache.
+ auto _ = llvm::make_scope_exit(
+ [&AST, this]() { IdleASTs.put(this, std::move(*AST)); });
+ // Run the user-provided action.
+ if (!*AST)
+ return Action(llvm::make_error<llvm::StringError>(
+ "invalid AST", llvm::errc::invalid_argument));
+ vlog("ASTWorker running {0} on version {2} of {1}", Name, FileName,
+ CurrentInputs.Version);
+ Action(InputsAndAST{CurrentInputs, **AST});
+ };
+ addToQueue(Name, std::move(Task), /*UpdateType=*/llvm::None, Invalidation);
+ }
+
+private:
+ /// These two methods should only be accessed inside a request, otherwise
+ /// there will be a race.
+ const ParseInputs &getCurrentInputs() const { return CurrentInputs; }
+ void setCurrentInputs(const ParseInputs &PI) { CurrentInputs = PI; }
+
+ /// Puts a request to the end of the queue. Can invalidate existing requests
+ /// depending on their policy.
+ void addToQueue(llvm::StringRef Name, llvm::unique_function<void()> Task,
+ llvm::Optional<WantDiagnostics> UpdateType,
+ TUScheduler::ASTActionInvalidation Invalidation) {
+ assert(!isDone() && "running a task after stop()");
+ if (RunSync) {
+ trace::Span Tracer(Name + ":" + llvm::sys::path::filename(FileName));
+ Task();
+ return;
}
- // Make sure we put the AST back into the LRU cache.
- auto _ = llvm::make_scope_exit(
- [&AST, this]() { IdleASTs.put(this, std::move(*AST)); });
- // Run the user-provided action.
- if (!*AST)
- return Action(llvm::make_error<llvm::StringError>(
- "invalid AST", llvm::errc::invalid_argument));
- vlog("ASTWorker running {0} on version {2} of {1}", Name, FileName,
- CurrentInputs->Version);
- Action(InputsAndAST{*CurrentInputs, **AST});
- };
- startTask(Name, std::move(Task), /*UpdateType=*/None, Invalidation);
-}
-std::shared_ptr<const PreambleData>
-ASTWorker::getPossiblyStalePreamble() const {
- std::lock_guard<std::mutex> Lock(Mutex);
- return LastBuiltPreamble;
-}
+ {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ // Cancel any requests invalidated by this request.
+ if (UpdateType) {
+ for (auto &R : llvm::reverse(Requests)) {
+ if (R.InvalidationPolicy == TUScheduler::InvalidateOnUpdate)
+ R.Invalidate();
+ // Older requests were already invalidated by the older update.
+ if (R.UpdateType)
+ break;
+ }
+ }
-void ASTWorker::getCurrentPreamble(
- llvm::unique_function<void(std::shared_ptr<const PreambleData>)> Callback) {
- // We could just call startTask() to throw the read on the queue, knowing
- // it will run after any updates. But we know this task is cheap, so to
- // improve latency we cheat: insert it on the queue after the last update.
- std::unique_lock<std::mutex> Lock(Mutex);
- auto LastUpdate =
- std::find_if(Requests.rbegin(), Requests.rend(),
- [](const Request &R) { return R.UpdateType.hasValue(); });
- // If there were no writes in the queue, and CurrentRequest is not a write,
- // the preamble is ready now.
- if (LastUpdate == Requests.rend() &&
- (!CurrentRequest || CurrentRequest->UpdateType.hasValue())) {
- Lock.unlock();
- return Callback(getPossiblyStalePreamble());
+ Context Ctx =
+ Context::current().derive(kFileBeingProcessed, FileName.str());
+ // Allow this request to be cancelled if invalidated.
+ Canceler Invalidate = nullptr;
+ if (Invalidation) {
+ WithContext WC(std::move(Ctx));
+ std::tie(Ctx, Invalidate) = cancelableTask();
+ }
+ Requests.push_back({std::move(Task), std::string(Name),
+ steady_clock::now(), std::move(Ctx), UpdateType,
+ Invalidation, std::move(Invalidate)});
+ }
+ RequestsCV.notify_all();
}
- assert(!RunSync && "Running synchronously, but queue is non-empty!");
- Requests.insert(LastUpdate.base(),
- Request{[Callback = std::move(Callback), this]() mutable {
- Callback(getPossiblyStalePreamble());
- },
- "GetPreamble", steady_clock::now(),
- Context::current().clone(),
- /*UpdateType=*/None,
- /*InvalidationPolicy=*/TUScheduler::NoInvalidation,
- /*Invalidate=*/nullptr});
- Lock.unlock();
- RequestsCV.notify_all();
-}
-void ASTWorker::waitForFirstPreamble() const { PreambleWasBuilt.wait(); }
+ // Returns true if Requests.front() is a dead update that can be skipped.
+ bool shouldSkipHeadLocked() const {
+ assert(!Requests.empty());
+ auto Next = Requests.begin();
+ auto UpdateType = Next->UpdateType;
+ if (!UpdateType) // Only skip updates.
+ return false;
+ ++Next;
+ // An update is live if its AST might still be read.
+ // That is, if it's not immediately followed by another update.
+ if (Next == Requests.end() || !Next->UpdateType)
+ return false;
+ // The other way an update can be live is if its diagnostics might be used.
+ switch (*UpdateType) {
+ case WantDiagnostics::Yes:
+ return false; // Always used.
+ case WantDiagnostics::No:
+ return true; // Always dead.
+ case WantDiagnostics::Auto:
+ // Used unless followed by an update that generates diagnostics.
+ for (; Next != Requests.end(); ++Next)
+ if (Next->UpdateType == WantDiagnostics::Yes ||
+ Next->UpdateType == WantDiagnostics::Auto)
+ return true; // Prefer later diagnostics.
+ return false;
+ }
+ llvm_unreachable("Unknown WantDiagnostics");
+ }
-std::shared_ptr<const ParseInputs> ASTWorker::getCurrentFileInputs() const {
- std::unique_lock<std::mutex> Lock(Mutex);
- return FileInputs;
-}
+ /// Determines the next action to perform.
+ /// All actions that should never run are discarded.
+ /// Returns a deadline for the next action. If it's expired, run now.
+ /// scheduleLocked() is called again at the deadline, or if requests arrive.
+ Deadline scheduleLocked() {
+ if (Requests.empty())
+ return Deadline::infinity(); // Wait for new requests.
+ // Handle cancelled requests first so the rest of the scheduler doesn't.
+ for (auto I = Requests.begin(), E = Requests.end(); I != E; ++I) {
+ if (!isCancelled(I->Ctx)) {
+ // Cancellations after the first read don't affect current scheduling.
+ if (I->UpdateType == None)
+ break;
+ continue;
+ }
+ // Cancelled reads are moved to the front of the queue and run
+ // immediately.
+ if (I->UpdateType == None) {
+ Request R = std::move(*I);
+ Requests.erase(I);
+ Requests.push_front(std::move(R));
+ return Deadline::zero();
+ }
+ // Cancelled updates are downgraded to auto-diagnostics, and may be
+ // elided.
+ if (I->UpdateType == WantDiagnostics::Yes)
+ I->UpdateType = WantDiagnostics::Auto;
+ }
-tooling::CompileCommand ASTWorker::getCurrentCompileCommand() const {
- std::unique_lock<std::mutex> Lock(Mutex);
- return FileInputs->CompileCommand;
-}
+ while (shouldSkipHeadLocked()) {
+ vlog("ASTWorker skipping {0} for {1}", Requests.front().Name, FileName);
+ Requests.pop_front();
+ }
+ assert(!Requests.empty() && "skipped the whole queue");
+ // Some updates aren't dead yet, but never end up being used.
+ // e.g. the first keystroke is live until obsoleted by the second.
+ // We debounce "maybe-unused" writes, sleeping in case they become dead.
+ // But don't delay reads (including updates where diagnostics are needed).
+ for (const auto &R : Requests)
+ if (R.UpdateType == None || R.UpdateType == WantDiagnostics::Yes)
+ return Deadline::zero();
+ // Front request needs to be debounced, so determine when we're ready.
+ Deadline D(Requests.front().AddTime + UpdateDebounce.compute(RebuildTimes));
+ return D;
+ }
-std::size_t ASTWorker::getUsedBytes() const {
- // Note that we don't report the size of ASTs currently used for processing
- // the in-flight requests. We used this information for debugging purposes
- // only, so this should be fine.
- std::size_t Result = IdleASTs.getUsedBytes(this);
- if (auto Preamble = getPossiblyStalePreamble())
- Result += Preamble->Preamble.getSize();
- return Result;
-}
+ struct Request {
+ llvm::unique_function<void()> Action;
+ std::string Name;
+ steady_clock::time_point AddTime;
+ Context Ctx;
+ llvm::Optional<WantDiagnostics> UpdateType;
+ TUScheduler::ASTActionInvalidation InvalidationPolicy;
+ Canceler Invalidate;
+ };
+
+ // Used to limit total number of threads doing any work.
+ Semaphore &Barrier;
+
+ mutable std::mutex Mutex;
+ mutable std::condition_variable RequestsCV;
+ std::deque<Request> Requests;
+ llvm::Optional<Request> CurrentRequest;
+ // FIXME: Following might make sense for preamble worker as well, consider
+ // moving into StoppableTUWorker once we implement debouncing for preamble
+ // requests.
+ const DebouncePolicy UpdateDebounce;
+ llvm::SmallVector<DebouncePolicy::clock::duration, 8> RebuildTimes;
+
+ /// Following fields should only be accessed by worker thread.
+ /// Latest inputs used for building an AST.
+ ParseInputs CurrentInputs;
+ bool RanASTCallback = false;
+ /// The preamble that was used during latest AST build.
+ std::shared_ptr<const PreambleData> CurrentPreamble;
+
+ TUScheduler::ASTCache &IdleASTs;
+};
+
+class TUWorkerHandle;
+
+/// Owns one instance of the AST, schedules updates and reads of it.
+/// Also responsible for building and providing access to the preamble.
+/// Each TUWorker processes the async requests sent to it on a separate
+/// dedicated thread.
+/// The TUWorker that manages the AST is shared by both the processing thread
+/// and the TUScheduler. The TUScheduler should discard a TUWorker when
+/// remove() is called, but its thread may be busy and we don't want to block.
+/// So the workers are accessed via a TUWorkerHandle. Destroying the handle
+/// signals the worker to exit its run loop and gives up shared ownership of the
+/// worker.
+class TUWorker {
+ friend class TUWorkerHandle;
+ TUWorker(PathRef FileName, const GlobalCompilationDatabase &CDB,
+ TUScheduler::ASTCache &LRUCache, Semaphore &Barrier, bool RunSync,
+ DebouncePolicy UpdateDebounce, bool StorePreamblesInMemory,
+ ParsingCallbacks &Callbacks);
+
+public:
+ /// Create a new ASTWorker and return a handle to it.
+ /// The processing thread is spawned using \p Tasks. However, when \p Tasks
+ /// is null, all requests will be processed on the calling thread
+ /// synchronously instead. \p Barrier is acquired when processing each
+ /// request, it is used to limit the number of actively running threads.
+ static TUWorkerHandle
+ create(PathRef FileName, const GlobalCompilationDatabase &CDB,
+ TUScheduler::ASTCache &IdleASTs, AsyncTaskRunner *Tasks,
+ Semaphore &Barrier, DebouncePolicy UpdateDebounce,
+ bool StorePreamblesInMemory, ParsingCallbacks &Callbacks);
+
+ /// Issues build of a new preamble and ast with \p Inputs.
+ void update(ParseInputs Inputs, WantDiagnostics);
+ /// Issues a read that will happen after any preceding writes.
+ void
+ runWithAST(llvm::StringRef Name,
+ llvm::unique_function<void(llvm::Expected<InputsAndAST>)> Action,
+ TUScheduler::ASTActionInvalidation);
+ /// Blocks until worker thread goes idle or Timeout is reached, should only be
+ /// used by tests. Returns true for the former.
+ bool blockUntilIdle(Deadline Timeout) const;
+
+ /// Blocks until at least one preamble is built.
+ std::shared_ptr<const PreambleData> getLatestBuiltPreamble() const;
+ std::shared_ptr<const PreambleData> getPossiblyMissingPreamble() const;
+
+ tooling::CompileCommand getCurrentCompileCommand() const;
+ void setCurrentCompileCommand(tooling::CompileCommand Cmd);
+ /// Obtain a preamble reflecting all updates so far. Threadsafe.
+ void getCurrentPreamble(
+ llvm::unique_function<void(std::shared_ptr<const PreambleData>)>);
+
+ std::size_t getUsedBytes() const;
+ bool isASTCached() const;
+
+private:
+ /// Signal that workers should finish processing pending requests and exit.
+ void stop();
+
+ /// File that ASTWorker is responsible for.
+ const Path FileName;
+ const GlobalCompilationDatabase &CDB;
+
+ mutable std::mutex Mutex;
+ /// Latest compile command used by any request so far.
+ // FIXME: Only used by runWithPreamble, should rather come from the preamble
+ // itself.
+ tooling::CompileCommand CurrentCompileCmd;
-bool ASTWorker::isASTCached() const { return IdleASTs.getUsedBytes(this) != 0; }
+ PreambleWorker PW;
+ MainFileWorker MW;
+};
-void ASTWorker::stop() {
- {
- std::lock_guard<std::mutex> Lock(PublishMu);
- CanPublishResults = false;
+/// A smart-pointer-like class that points to an active ASTWorker.
+/// In destructor, signals to the underlying ASTWorker that no new requests will
+/// be sent and the processing loop may exit (after running all pending
+/// requests).
+class TUWorkerHandle {
+ friend class TUWorker;
+ TUWorkerHandle(std::shared_ptr<TUWorker> Worker) : Worker(std::move(Worker)) {
+ assert(this->Worker);
}
- {
- std::lock_guard<std::mutex> Lock(Mutex);
- assert(!Done && "stop() called twice");
- Done = true;
+
+public:
+ TUWorkerHandle(const TUWorkerHandle &) = delete;
+ TUWorkerHandle &operator=(const TUWorkerHandle &) = delete;
+ TUWorkerHandle(TUWorkerHandle &&) = default;
+ TUWorkerHandle &operator=(TUWorkerHandle &&) = default;
+
+ ~TUWorkerHandle() {
+ if (Worker)
+ Worker->stop();
}
- RequestsCV.notify_all();
-}
-void ASTWorker::startTask(llvm::StringRef Name,
- llvm::unique_function<void()> Task,
- llvm::Optional<WantDiagnostics> UpdateType,
- TUScheduler::ASTActionInvalidation Invalidation) {
- if (RunSync) {
- assert(!Done && "running a task after stop()");
- trace::Span Tracer(Name + ":" + llvm::sys::path::filename(FileName));
- Task();
- return;
+ TUWorker &operator*() {
+ assert(Worker && "Handle was moved from");
+ return *Worker;
}
- {
- std::lock_guard<std::mutex> Lock(Mutex);
- assert(!Done && "running a task after stop()");
- // Cancel any requests invalidated by this request.
- if (UpdateType) {
- for (auto &R : llvm::reverse(Requests)) {
- if (R.InvalidationPolicy == TUScheduler::InvalidateOnUpdate)
- R.Invalidate();
- if (R.UpdateType)
- break; // Older requests were already invalidated by the older update.
- }
- }
+ TUWorker *operator->() {
+ assert(Worker && "Handle was moved from");
+ return Worker.get();
+ }
- // Allow this request to be cancelled if invalidated.
- Context Ctx = Context::current().derive(kFileBeingProcessed, FileName);
- Canceler Invalidate = nullptr;
- if (Invalidation) {
- WithContext WC(std::move(Ctx));
- std::tie(Ctx, Invalidate) = cancelableTask();
- }
- Requests.push_back({std::move(Task), std::string(Name), steady_clock::now(),
- std::move(Ctx), UpdateType, Invalidation,
- std::move(Invalidate)});
+ /// Returns an owning reference to the underlying ASTWorker that can outlive
+ /// the ASTWorkerHandle. However, no new requests to an active ASTWorker can
+ /// be schedule via the returned reference, i.e. only reads of the preamble
+ /// are possible.
+ std::shared_ptr<const TUWorker> lock() { return Worker; }
+
+private:
+ std::shared_ptr<TUWorker> Worker;
+};
+
+TUWorkerHandle
+TUWorker::create(PathRef FileName, const GlobalCompilationDatabase &CDB,
+ TUScheduler::ASTCache &IdleASTs, AsyncTaskRunner *Tasks,
+ Semaphore &Barrier, DebouncePolicy UpdateDebounce,
+ bool StorePreamblesInMemory, ParsingCallbacks &Callbacks) {
+ std::shared_ptr<TUWorker> Worker(
+ new TUWorker(FileName, CDB, IdleASTs, Barrier, /*RunSync=*/!Tasks,
+ UpdateDebounce, StorePreamblesInMemory, Callbacks));
+ if (Tasks) {
+ Tasks->runAsync("workerMainFile:" + llvm::sys::path::filename(FileName),
+ [Worker]() { Worker->MW.run(); });
+ Tasks->runAsync("workerPreamble:" + llvm::sys::path::filename(FileName),
+ [Worker]() { Worker->PW.run(); });
}
- RequestsCV.notify_all();
+
+ return TUWorkerHandle(std::move(Worker));
}
-void ASTWorker::emitTUStatus(TUAction Action,
- const TUStatus::BuildDetails *Details) {
- Status.Action = std::move(Action);
- if (Details)
- Status.Details = *Details;
- std::lock_guard<std::mutex> Lock(PublishMu);
- // Do not emit TU statuses when the ASTWorker is shutting down.
- if (CanPublishResults) {
- Callbacks.onFileUpdated(FileName, Status);
- }
+void TUWorker::getCurrentPreamble(
+ llvm::unique_function<void(std::shared_ptr<const PreambleData>)> Callback) {
+ // FIXME: This will result in waiting for an extra AST built.
+ runWithAST(
+ "getCurrentPreamble",
+ [Callback = std::move(Callback),
+ this](llvm::Expected<InputsAndAST> AST) mutable {
+ if (!AST)
+ llvm::consumeError(AST.takeError());
+ Callback(getPossiblyMissingPreamble());
+ },
+ TUScheduler::NoInvalidation);
}
-void ASTWorker::run() {
- while (true) {
- {
- std::unique_lock<std::mutex> Lock(Mutex);
- assert(!CurrentRequest && "A task is already running, multiple workers?");
- for (auto Wait = scheduleLocked(); !Wait.expired();
- Wait = scheduleLocked()) {
- if (Done) {
- if (Requests.empty())
- return;
- else // Even though Done is set, finish pending requests.
- break; // However, skip delays to shutdown fast.
- }
+std::size_t TUWorker::getUsedBytes() const {
+ return PW.getUsedBytes() + MW.getUsedBytes();
+}
- // Tracing: we have a next request, attribute this sleep to it.
- llvm::Optional<WithContext> Ctx;
- llvm::Optional<trace::Span> Tracer;
- if (!Requests.empty()) {
- Ctx.emplace(Requests.front().Ctx.clone());
- Tracer.emplace("Debounce");
- SPAN_ATTACH(*Tracer, "next_request", Requests.front().Name);
- if (!(Wait == Deadline::infinity())) {
- emitTUStatus({TUAction::Queued, Requests.front().Name});
- SPAN_ATTACH(*Tracer, "sleep_ms",
- std::chrono::duration_cast<std::chrono::milliseconds>(
- Wait.time() - steady_clock::now())
- .count());
- }
- }
+bool TUWorker::isASTCached() const { return MW.isASTCached(); }
- wait(Lock, RequestsCV, Wait);
- }
- CurrentRequest = std::move(Requests.front());
- Requests.pop_front();
- } // unlock Mutex
+void TUWorker::setCurrentCompileCommand(tooling::CompileCommand Cmd) {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ CurrentCompileCmd = std::move(Cmd);
+}
- // It is safe to perform reads to CurrentRequest without holding the lock as
- // only writer is also this thread.
- {
- std::unique_lock<Semaphore> Lock(Barrier, std::try_to_lock);
- if (!Lock.owns_lock()) {
- emitTUStatus({TUAction::Queued, CurrentRequest->Name});
- Lock.lock();
- }
- WithContext Guard(std::move(CurrentRequest->Ctx));
- trace::Span Tracer(CurrentRequest->Name);
- emitTUStatus({TUAction::RunningAction, CurrentRequest->Name});
- CurrentRequest->Action();
- }
+TUWorker::TUWorker(PathRef FileName, const GlobalCompilationDatabase &CDB,
+ TUScheduler::ASTCache &LRUCache, Semaphore &Barrier,
+ bool RunSync, DebouncePolicy UpdateDebounce,
+ bool StorePreamblesInMemory, ParsingCallbacks &Callbacks)
+ : FileName(FileName), CDB(CDB),
+ PW(Callbacks, this->FileName, StorePreamblesInMemory, RunSync),
+ MW(this->FileName, LRUCache, Barrier, RunSync, UpdateDebounce,
+ Callbacks) {
+ // Set a fallback command because compile command can be accessed before
+ // `Inputs` is initialized. Other fields are only used after initialization
+ // from client inputs.
+ setCurrentCompileCommand(CDB.getFallbackCommand(FileName));
+}
- bool IsEmpty = false;
- {
- std::lock_guard<std::mutex> Lock(Mutex);
- CurrentRequest.reset();
- IsEmpty = Requests.empty();
- }
- if (IsEmpty)
- emitTUStatus({TUAction::Idle, /*Name*/ ""});
- RequestsCV.notify_all();
- }
+void TUWorker::update(ParseInputs Inputs, WantDiagnostics WantDiags) {
+ std::string Version = Inputs.Version;
+ auto Task = [Inputs = std::move(Inputs), WantDiags, this]() mutable {
+ // Get the actual command as `Inputs` does not have a command.
+ // FIXME: some build systems like Bazel will take time to preparing
+ // environment to build the file, it would be nice if we could emit a
+ // "PreparingBuild" status to inform users, it is non-trivial given the
+ // current implementation.
+ if (auto Cmd = CDB.getCompileCommand(FileName))
+ Inputs.CompileCommand = *Cmd;
+ else
+ // FIXME: consider using old command if it's not a fallback one.
+ Inputs.CompileCommand = CDB.getFallbackCommand(FileName);
+ setCurrentCompileCommand(Inputs.CompileCommand);
+ StoreDiags CompilerInvocationDiagConsumer;
+ std::vector<std::string> CC1Args;
+ std::unique_ptr<CompilerInvocation> Invocation = buildCompilerInvocation(
+ Inputs, CompilerInvocationDiagConsumer, &CC1Args);
+ // Log cc1 args even (especially!) if creating invocation failed.
+ if (!CC1Args.empty())
+ vlog("Driver produced command: cc1 {0}", llvm::join(CC1Args, " "));
+ std::vector<Diag> CompilerInvocationDiags =
+ CompilerInvocationDiagConsumer.take();
+ if (!Invocation)
+ elog("Could not build CompilerInvocation for file {0}", FileName);
+ PW.requestBuild(Invocation.get(), Inputs);
+ // FIXME: Enable out-of-order preamble/ast builds.
+ assert(PW.blockUntilIdle(timeoutSeconds(10)));
+ MW.buildAST(std::move(Invocation), std::move(CompilerInvocationDiags),
+ std::move(Inputs), std::move(WantDiags),
+ PW.getLatestBuiltPreamble());
+ };
+ MW.dispatchUpdate(std::move(Task), std::move(WantDiags), Version);
}
-Deadline ASTWorker::scheduleLocked() {
- if (Requests.empty())
- return Deadline::infinity(); // Wait for new requests.
- // Handle cancelled requests first so the rest of the scheduler doesn't.
- for (auto I = Requests.begin(), E = Requests.end(); I != E; ++I) {
- if (!isCancelled(I->Ctx)) {
- // Cancellations after the first read don't affect current scheduling.
- if (I->UpdateType == None)
- break;
- continue;
- }
- // Cancelled reads are moved to the front of the queue and run immediately.
- if (I->UpdateType == None) {
- Request R = std::move(*I);
- Requests.erase(I);
- Requests.push_front(std::move(R));
- return Deadline::zero();
- }
- // Cancelled updates are downgraded to auto-diagnostics, and may be elided.
- if (I->UpdateType == WantDiagnostics::Yes)
- I->UpdateType = WantDiagnostics::Auto;
- }
+void TUWorker::runWithAST(
+ llvm::StringRef Name,
+ llvm::unique_function<void(llvm::Expected<InputsAndAST>)> Action,
+ TUScheduler::ASTActionInvalidation Invalidation) {
+ MW.runAction(Name, std::move(Action), Invalidation,
+ [this] { return PW.getLatestBuiltPreamble(); });
+}
- while (shouldSkipHeadLocked()) {
- vlog("ASTWorker skipping {0} for {1}", Requests.front().Name, FileName);
- Requests.pop_front();
- }
- assert(!Requests.empty() && "skipped the whole queue");
- // Some updates aren't dead yet, but never end up being used.
- // e.g. the first keystroke is live until obsoleted by the second.
- // We debounce "maybe-unused" writes, sleeping in case they become dead.
- // But don't delay reads (including updates where diagnostics are needed).
- for (const auto &R : Requests)
- if (R.UpdateType == None || R.UpdateType == WantDiagnostics::Yes)
- return Deadline::zero();
- // Front request needs to be debounced, so determine when we're ready.
- Deadline D(Requests.front().AddTime + UpdateDebounce.compute(RebuildTimes));
- return D;
+bool TUWorker::blockUntilIdle(Deadline Timeout) const {
+ return MW.blockUntilIdle(Timeout) && PW.blockUntilIdle(Timeout);
}
-// Returns true if Requests.front() is a dead update that can be skipped.
-bool ASTWorker::shouldSkipHeadLocked() const {
- assert(!Requests.empty());
- auto Next = Requests.begin();
- auto UpdateType = Next->UpdateType;
- if (!UpdateType) // Only skip updates.
- return false;
- ++Next;
- // An update is live if its AST might still be read.
- // That is, if it's not immediately followed by another update.
- if (Next == Requests.end() || !Next->UpdateType)
- return false;
- // The other way an update can be live is if its diagnostics might be used.
- switch (*UpdateType) {
- case WantDiagnostics::Yes:
- return false; // Always used.
- case WantDiagnostics::No:
- return true; // Always dead.
- case WantDiagnostics::Auto:
- // Used unless followed by an update that generates diagnostics.
- for (; Next != Requests.end(); ++Next)
- if (Next->UpdateType == WantDiagnostics::Yes ||
- Next->UpdateType == WantDiagnostics::Auto)
- return true; // Prefer later diagnostics.
- return false;
- }
- llvm_unreachable("Unknown WantDiagnostics");
+std::shared_ptr<const PreambleData> TUWorker::getLatestBuiltPreamble() const {
+ PW.waitForFirstPreamble();
+ return PW.getLatestBuiltPreamble();
+}
+
+std::shared_ptr<const PreambleData>
+TUWorker::getPossiblyMissingPreamble() const {
+ return PW.getLatestBuiltPreamble();
+}
+
+tooling::CompileCommand TUWorker::getCurrentCompileCommand() const {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ return CurrentCompileCmd;
}
-bool ASTWorker::blockUntilIdle(Deadline Timeout) const {
- std::unique_lock<std::mutex> Lock(Mutex);
- return wait(Lock, RequestsCV, Timeout,
- [&] { return Requests.empty() && !CurrentRequest; });
+void TUWorker::stop() {
+ // We want PW to be stopped after MW finishes.
+ MW.dispatchUpdate([this] { PW.stop(); }, WantDiagnostics::No, "");
+ MW.stop();
}
// Render a TUAction to a user-facing string representation.
@@ -896,7 +1104,7 @@
struct TUScheduler::FileData {
/// Latest inputs, passed to TUScheduler::update().
std::string Contents;
- ASTWorkerHandle Worker;
+ TUWorkerHandle Worker;
};
TUScheduler::TUScheduler(const GlobalCompilationDatabase &CDB,
@@ -920,19 +1128,27 @@
Files.clear();
// Wait for all in-flight tasks to finish.
- if (PreambleTasks)
+ if (PreambleTasks) {
+ vlog("waiting for preamble tasks");
PreambleTasks->wait();
- if (WorkerThreads)
+ }
+ if (WorkerThreads) {
+ vlog("waiting for worker tasks");
WorkerThreads->wait();
+ }
}
bool TUScheduler::blockUntilIdle(Deadline D) const {
for (auto &File : Files)
- if (!File.getValue()->Worker->blockUntilIdle(D))
+ if (!File.getValue()->Worker->blockUntilIdle(D)) {
+ vlog("ASTWorker for {0} got stuck", File.first());
return false;
+ }
if (PreambleTasks)
- if (!PreambleTasks->wait(D))
+ if (!PreambleTasks->wait(D)) {
+ vlog("Preamble tasks got stuck");
return false;
+ }
return true;
}
@@ -942,7 +1158,7 @@
bool NewFile = FD == nullptr;
if (!FD) {
// Create a new worker to process the AST-related tasks.
- ASTWorkerHandle Worker = ASTWorker::create(
+ TUWorkerHandle Worker = TUWorker::create(
File, CDB, *IdleASTs,
WorkerThreads ? WorkerThreads.getPointer() : nullptr, Barrier,
UpdateDebounce, StorePreamblesInMemory, *Callbacks);
@@ -1006,14 +1222,14 @@
return;
}
+ std::shared_ptr<const TUWorker> Worker = It->second->Worker.lock();
+ tooling::CompileCommand Command = Worker->getCurrentCompileCommand();
if (!PreambleTasks) {
trace::Span Tracer(Name);
SPAN_ATTACH(Tracer, "file", File);
std::shared_ptr<const PreambleData> Preamble =
- It->second->Worker->getPossiblyStalePreamble();
- Action(InputsAndPreamble{It->second->Contents,
- It->second->Worker->getCurrentCompileCommand(),
- Preamble.get()});
+ It->second->Worker->getPossiblyMissingPreamble();
+ Action(InputsAndPreamble{It->second->Contents, Command, Preamble.get()});
return;
}
@@ -1029,25 +1245,24 @@
});
}
- std::shared_ptr<const ASTWorker> Worker = It->second->Worker.lock();
auto Task =
[Worker, Consistency, Name = Name.str(), File = File.str(),
- Contents = It->second->Contents,
- Command = Worker->getCurrentCompileCommand(),
+ Contents = It->second->Contents, Command = std::move(Command),
Ctx = Context::current().derive(kFileBeingProcessed, std::string(File)),
ConsistentPreamble = std::move(ConsistentPreamble),
Action = std::move(Action), this]() mutable {
std::shared_ptr<const PreambleData> Preamble;
if (ConsistentPreamble.valid()) {
+ assert(Consistency == Consistent);
Preamble = ConsistentPreamble.get();
+ } else if (Consistency == Stale) {
+ // Wait until the preamble is built for the first time, if preamble is
+ // required. This avoids extra work of processing the preamble headers
+ // in parallel multiple times.
+ Preamble = Worker->getLatestBuiltPreamble();
} else {
- if (Consistency != PreambleConsistency::StaleOrAbsent) {
- // Wait until the preamble is built for the first time, if preamble
- // is required. This avoids extra work of processing the preamble
- // headers in parallel multiple times.
- Worker->waitForFirstPreamble();
- }
- Preamble = Worker->getPossiblyStalePreamble();
+ assert(Consistency == StaleOrAbsent);
+ Preamble = Worker->getPossiblyMissingPreamble();
}
std::lock_guard<Semaphore> BarrierLock(Barrier);
Index: clang-tools-extra/clangd/Preamble.h
===================================================================
--- clang-tools-extra/clangd/Preamble.h
+++ clang-tools-extra/clangd/Preamble.h
@@ -78,13 +78,12 @@
/// building the preamble. Note that if the old preamble was reused, no AST is
/// built and, therefore, the callback will not be executed.
std::shared_ptr<const PreambleData>
-buildPreamble(PathRef FileName, CompilerInvocation &CI,
+buildPreamble(PathRef FileName, CompilerInvocation CI,
std::shared_ptr<const PreambleData> OldPreamble,
const tooling::CompileCommand &OldCompileCommand,
const ParseInputs &Inputs, bool StoreInMemory,
PreambleParsedCallback PreambleCallback);
-
} // namespace clangd
} // namespace clang
Index: clang-tools-extra/clangd/Preamble.cpp
===================================================================
--- clang-tools-extra/clangd/Preamble.cpp
+++ clang-tools-extra/clangd/Preamble.cpp
@@ -87,7 +87,7 @@
}
std::shared_ptr<const PreambleData>
-buildPreamble(PathRef FileName, CompilerInvocation &CI,
+buildPreamble(PathRef FileName, CompilerInvocation CI,
std::shared_ptr<const PreambleData> OldPreamble,
const tooling::CompileCommand &OldCompileCommand,
const ParseInputs &Inputs, bool StoreInMemory,
Index: clang-tools-extra/clangd/Compiler.cpp
===================================================================
--- clang-tools-extra/clangd/Compiler.cpp
+++ clang-tools-extra/clangd/Compiler.cpp
@@ -41,8 +41,7 @@
}
std::unique_ptr<CompilerInvocation>
-buildCompilerInvocation(const ParseInputs &Inputs,
- clang::DiagnosticConsumer &D,
+buildCompilerInvocation(const ParseInputs &Inputs, clang::DiagnosticConsumer &D,
std::vector<std::string> *CC1Args) {
std::vector<const char *> ArgStrs;
for (const auto &S : Inputs.CompileCommand.CommandLine)
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
