kadircet created this revision.
kadircet added reviewers: sammccall, nridge.
Herald added subscribers: usaxena95, arphaman.
Herald added a project: All.
kadircet requested review of this revision.
Herald added subscribers: cfe-commits, MaskRay, ilya-biryukov.
Herald added a project: clang-tools-extra.
Per LSP, multiline tokens should be handled as if they end at the end
of the line starting the token (there's also a capability to enable them, but
that's an adventure for a different day).
Fixes https://github.com/clangd/clangd/issues/1145
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D127856
Files:
clang-tools-extra/clangd/ClangdLSPServer.cpp
clang-tools-extra/clangd/SemanticHighlighting.cpp
clang-tools-extra/clangd/SemanticHighlighting.h
clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===================================================================
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -944,7 +944,7 @@
)");
Tokens.front().Modifiers |= unsigned(HighlightingModifier::Declaration);
Tokens.front().Modifiers |= unsigned(HighlightingModifier::Readonly);
- auto Results = toSemanticTokens(Tokens);
+ auto Results = toSemanticTokens(Tokens, /*Code=*/"");
ASSERT_THAT(Results, SizeIs(3));
EXPECT_EQ(Results[0].tokenType, unsigned(HighlightingKind::Variable));
@@ -972,13 +972,15 @@
auto Before = toSemanticTokens(tokens(R"(
[[foo]] [[bar]] [[baz]]
[[one]] [[two]] [[three]]
- )"));
+ )"),
+ /*Code=*/"");
EXPECT_THAT(diffTokens(Before, Before), IsEmpty());
auto After = toSemanticTokens(tokens(R"(
[[foo]] [[hello]] [[world]] [[baz]]
[[one]] [[two]] [[three]]
- )"));
+ )"),
+ /*Code=*/"");
// Replace [bar, baz] with [hello, world, baz]
auto Diff = diffTokens(Before, After);
@@ -1000,6 +1002,23 @@
EXPECT_EQ(3u, Diff.front().tokens[2].length);
}
+TEST(SemanticHighlighting, MultilineTokens) {
+ llvm::StringRef AnnotatedCode = R"cpp(
+[[fo
+o]] [[bar]])cpp";
+ auto Toks = toSemanticTokens(tokens(AnnotatedCode),
+ Annotations(AnnotatedCode).code());
+ ASSERT_THAT(Toks, SizeIs(2));
+ // foo
+ EXPECT_EQ(Toks[0].deltaLine, 1u);
+ EXPECT_EQ(Toks[0].deltaStart, 0u);
+ EXPECT_EQ(Toks[0].length, 2u);
+
+ // bar
+ EXPECT_EQ(Toks[1].deltaLine, 1u);
+ EXPECT_EQ(Toks[1].deltaStart, 2u);
+ EXPECT_EQ(Toks[1].length, 3u);
+}
} // namespace
} // namespace clangd
} // namespace clang
Index: clang-tools-extra/clangd/SemanticHighlighting.h
===================================================================
--- clang-tools-extra/clangd/SemanticHighlighting.h
+++ clang-tools-extra/clangd/SemanticHighlighting.h
@@ -21,6 +21,7 @@
#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICHIGHLIGHTING_H
#include "Protocol.h"
+#include "llvm/ADT/StringRef.h"
#include "llvm/Support/raw_ostream.h"
namespace clang {
@@ -101,7 +102,8 @@
// main AST.
std::vector<HighlightingToken> getSemanticHighlightings(ParsedAST &AST);
-std::vector<SemanticToken> toSemanticTokens(llvm::ArrayRef<HighlightingToken>);
+std::vector<SemanticToken> toSemanticTokens(llvm::ArrayRef<HighlightingToken>,
+ llvm::StringRef Code);
llvm::StringRef toSemanticTokenType(HighlightingKind Kind);
llvm::StringRef toSemanticTokenModifier(HighlightingModifier Modifier);
std::vector<SemanticTokensEdit> diffTokens(llvm::ArrayRef<SemanticToken> Before,
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===================================================================
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -30,7 +30,9 @@
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Casting.h"
+#include "llvm/Support/Error.h"
#include <algorithm>
namespace clang {
@@ -918,7 +920,8 @@
}
std::vector<SemanticToken>
-toSemanticTokens(llvm::ArrayRef<HighlightingToken> Tokens) {
+toSemanticTokens(llvm::ArrayRef<HighlightingToken> Tokens,
+ llvm::StringRef Code) {
assert(std::is_sorted(Tokens.begin(), Tokens.end()));
std::vector<SemanticToken> Result;
const HighlightingToken *Last = nullptr;
@@ -939,8 +942,22 @@
Out.deltaLine = Tok.R.start.line;
Out.deltaStart = Tok.R.start.character;
}
- assert(Tok.R.end.line == Tok.R.start.line);
- Out.length = Tok.R.end.character - Tok.R.start.character;
+ if (Tok.R.end.line == Tok.R.start.line) {
+ Out.length = Tok.R.end.character - Tok.R.start.character;
+ } else {
+ // If a tokens length is past the end of the line, it should be treated as
+ // if the token ends at the end of the line and will not wrap onto the
+ // next line.
+ // This is slow, but code rarely has multiline tokens.
+ // FIXME: There's a client capability for supporting multiline tokens,
+ // respect that.
+ auto TokStartOffset = llvm::cantFail(positionToOffset(Code, Tok.R.start));
+ auto LineEnd = Code.find('\n', TokStartOffset);
+ if (LineEnd == Code.npos)
+ Out.length = Code.size() - TokStartOffset;
+ else
+ Out.length = LineEnd - TokStartOffset;
+ }
Out.tokenType = static_cast<unsigned>(Tok.Kind);
Out.tokenModifiers = Tok.Modifiers;
Index: clang-tools-extra/clangd/ClangdLSPServer.cpp
===================================================================
--- clang-tools-extra/clangd/ClangdLSPServer.cpp
+++ clang-tools-extra/clangd/ClangdLSPServer.cpp
@@ -1397,14 +1397,15 @@
void ClangdLSPServer::onSemanticTokens(const SemanticTokensParams &Params,
Callback<SemanticTokens> CB) {
+ auto File = Params.textDocument.uri.file();
Server->semanticHighlights(
Params.textDocument.uri.file(),
- [this, File(Params.textDocument.uri.file().str()), CB(std::move(CB))](
+ [this, File(File.str()), CB(std::move(CB)), Code(Server->getDraft(File))](
llvm::Expected<std::vector<HighlightingToken>> HT) mutable {
if (!HT)
return CB(HT.takeError());
SemanticTokens Result;
- Result.tokens = toSemanticTokens(*HT);
+ Result.tokens = toSemanticTokens(*HT, *Code);
{
std::lock_guard<std::mutex> Lock(SemanticTokensMutex);
auto &Last = LastSemanticTokens[File];
@@ -1420,14 +1421,15 @@
void ClangdLSPServer::onSemanticTokensDelta(
const SemanticTokensDeltaParams &Params,
Callback<SemanticTokensOrDelta> CB) {
+ auto File = Params.textDocument.uri.file();
Server->semanticHighlights(
Params.textDocument.uri.file(),
- [this, PrevResultID(Params.previousResultId),
- File(Params.textDocument.uri.file().str()), CB(std::move(CB))](
+ [this, PrevResultID(Params.previousResultId), File(File.str()),
+ CB(std::move(CB)), Code(Server->getDraft(File))](
llvm::Expected<std::vector<HighlightingToken>> HT) mutable {
if (!HT)
return CB(HT.takeError());
- std::vector<SemanticToken> Toks = toSemanticTokens(*HT);
+ std::vector<SemanticToken> Toks = toSemanticTokens(*HT, *Code);
SemanticTokensOrDelta Result;
{
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
