sammccall updated this revision to Diff 319592.
sammccall retitled this revision from "[clangd] WIP playing with semantic
highlighting modifiers" to "[clangd] Implement semanticTokens modifiers".
sammccall edited the summary of this revision.
sammccall added a comment.
Update phab with commit message
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D77811/new/
https://reviews.llvm.org/D77811
Files:
clang-tools-extra/clangd/ClangdLSPServer.cpp
clang-tools-extra/clangd/SemanticHighlighting.cpp
clang-tools-extra/clangd/SemanticHighlighting.h
clang-tools-extra/clangd/refactor/tweaks/AnnotateHighlightings.cpp
clang-tools-extra/clangd/test/initialize-params.test
clang-tools-extra/clangd/test/semantic-tokens.test
clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
clang-tools-extra/clangd/unittests/tweaks/AnnotateHighlightingsTests.cpp
llvm/lib/Testing/Support/Annotations.cpp
Index: llvm/lib/Testing/Support/Annotations.cpp
===================================================================
--- llvm/lib/Testing/Support/Annotations.cpp
+++ llvm/lib/Testing/Support/Annotations.cpp
@@ -53,7 +53,8 @@
continue;
}
if (Text.consume_front("$")) {
- Name = Text.take_while(llvm::isAlnum);
+ Name =
+ Text.take_while([](char C) { return llvm::isAlnum(C) || C == '_'; });
Text = Text.drop_front(Name->size());
continue;
}
Index: clang-tools-extra/clangd/unittests/tweaks/AnnotateHighlightingsTests.cpp
===================================================================
--- clang-tools-extra/clangd/unittests/tweaks/AnnotateHighlightingsTests.cpp
+++ clang-tools-extra/clangd/unittests/tweaks/AnnotateHighlightingsTests.cpp
@@ -18,15 +18,17 @@
TEST_F(AnnotateHighlightingsTest, Test) {
EXPECT_AVAILABLE("^vo^id^ ^f(^) {^}^"); // available everywhere.
EXPECT_AVAILABLE("[[int a; int b;]]");
- EXPECT_EQ("void /* entity.name.function.cpp */f() {}", apply("void ^f() {}"));
+ EXPECT_EQ("void /* Function [decl] */f() {}", apply("void ^f() {}"));
- EXPECT_EQ(apply("[[void f1(); void f2();]]"),
- "void /* entity.name.function.cpp */f1(); "
- "void /* entity.name.function.cpp */f2();");
+ EXPECT_EQ(
+ apply("[[int f1(); const int x = f1();]]"),
+ "int /* Function [decl] */f1(); "
+ "const int /* Variable [decl] [readonly] */x = /* Function */f1();");
+ // Only the targeted range is annotated.
EXPECT_EQ(apply("void f1(); void f2() {^}"),
"void f1(); "
- "void /* entity.name.function.cpp */f2() {}");
+ "void /* Function [decl] */f2() {}");
}
} // namespace
Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===================================================================
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -18,6 +18,7 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/ScopedPrinter.h"
+#include "llvm/Support/raw_ostream.h"
#include "gmock/gmock.h"
#include <algorithm>
@@ -86,7 +87,8 @@
return L.R.start < R.R.start;
}));
- std::string Result;
+ std::string Buf;
+ llvm::raw_string_ostream OS(Buf);
unsigned NextChar = 0;
for (auto &T : Tokens) {
unsigned StartOffset = llvm::cantFail(positionToOffset(Input, T.R.start));
@@ -94,14 +96,18 @@
assert(StartOffset <= EndOffset);
assert(NextChar <= StartOffset);
- Result += Input.substr(NextChar, StartOffset - NextChar);
- Result += std::string(
- llvm::formatv("${0}[[{1}]]", T.Kind,
- Input.substr(StartOffset, EndOffset - StartOffset)));
+ OS << Input.substr(NextChar, StartOffset - NextChar);
+ OS << '$' << T.Kind;
+ for (unsigned I = 0;
+ I < static_cast<uint32_t>(HighlightingModifier::LastModifier); ++I) {
+ if (T.Modifiers & (1 << I))
+ OS << '_' << static_cast<HighlightingModifier>(I);
+ }
+ OS << "[[" << Input.substr(StartOffset, EndOffset - StartOffset) << "]]";
NextChar = EndOffset;
}
- Result += Input.substr(NextChar);
- return Result;
+ OS << Input.substr(NextChar);
+ return std::move(OS.str());
}
void checkHighlightings(llvm::StringRef Code,
@@ -160,337 +166,340 @@
TEST(SemanticHighlighting, GetsCorrectTokens) {
const char *TestCases[] = {
R"cpp(
- struct $Class[[AS]] {
- double $Field[[SomeMember]];
+ struct $Class_decl[[AS]] {
+ double $Field_decl[[SomeMember]];
};
struct {
- } $Variable[[S]];
- void $Function[[foo]](int $Parameter[[A]], $Class[[AS]] $Parameter[[As]]) {
- $Primitive[[auto]] $LocalVariable[[VeryLongVariableName]] = 12312;
- $Class[[AS]] $LocalVariable[[AA]];
- $Primitive[[auto]] $LocalVariable[[L]] = $LocalVariable[[AA]].$Field[[SomeMember]] + $Parameter[[A]];
- auto $LocalVariable[[FN]] = [ $LocalVariable[[AA]]](int $Parameter[[A]]) -> void {};
+ } $Variable_decl[[S]];
+ void $Function_decl[[foo]](int $Parameter_decl[[A]], $Class[[AS]] $Parameter_decl[[As]]) {
+ $Primitive_deduced[[auto]] $LocalVariable_decl[[VeryLongVariableName]] = 12312;
+ $Class[[AS]] $LocalVariable_decl[[AA]];
+ $Primitive_deduced[[auto]] $LocalVariable_decl[[L]] = $LocalVariable[[AA]].$Field[[SomeMember]] + $Parameter[[A]];
+ auto $LocalVariable_decl[[FN]] = [ $LocalVariable[[AA]]](int $Parameter_decl[[A]]) -> void {};
$LocalVariable[[FN]](12312);
}
)cpp",
R"cpp(
- void $Function[[foo]](int);
- void $Function[[Gah]]();
- void $Function[[foo]]() {
- auto $LocalVariable[[Bou]] = $Function[[Gah]];
+ void $Function_decl[[foo]](int);
+ void $Function_decl[[Gah]]();
+ void $Function_decl[[foo]]() {
+ auto $LocalVariable_decl[[Bou]] = $Function[[Gah]];
}
- struct $Class[[A]] {
- void $Method[[abc]]();
+ struct $Class_decl[[A]] {
+ void $Method_decl[[abc]]();
};
)cpp",
R"cpp(
- namespace $Namespace[[abc]] {
- template<typename $TemplateParameter[[T]]>
- struct $Class[[A]] {
- $TemplateParameter[[T]] $Field[[t]];
+ namespace $Namespace_decl[[abc]] {
+ template<typename $TemplateParameter_decl[[T]]>
+ struct $Class_decl[[A]] {
+ $TemplateParameter[[T]] $Field_decl[[t]];
};
}
- template<typename $TemplateParameter[[T]]>
- struct $Class[[C]] : $Namespace[[abc]]::$Class[[A]]<$TemplateParameter[[T]]> {
- typename $TemplateParameter[[T]]::$DependentType[[A]]* $Field[[D]];
- };
- $Namespace[[abc]]::$Class[[A]]<int> $Variable[[AA]];
- typedef $Namespace[[abc]]::$Class[[A]]<int> $Class[[AAA]];
- struct $Class[[B]] {
- $Class[[B]]();
- ~$Class[[B]]();
+ template<typename $TemplateParameter_decl[[T]]>
+ struct $Class_decl[[C]] : $Namespace[[abc]]::$Class[[A]]<$TemplateParameter[[T]]> {
+ typename $TemplateParameter[[T]]::$DependentType[[A]]* $Field_decl[[D]];
+ };
+ $Namespace[[abc]]::$Class[[A]]<int> $Variable_decl[[AA]];
+ typedef $Namespace[[abc]]::$Class[[A]]<int> $Class_decl[[AAA]];
+ struct $Class_decl[[B]] {
+ $Class_decl[[B]]();
+ ~$Class[[B]](); // FIXME: inconsistent with constructor
void operator<<($Class[[B]]);
- $Class[[AAA]] $Field[[AA]];
+ $Class[[AAA]] $Field_decl[[AA]];
};
- $Class[[B]]::$Class[[B]]() {}
- $Class[[B]]::~$Class[[B]]() {}
- void $Function[[f]] () {
- $Class[[B]] $LocalVariable[[BB]] = $Class[[B]]();
+ $Class[[B]]::$Class_decl[[B]]() {}
+ $Class[[B]]::~$Class[[B]]() {} // FIXME: inconsistent with constructor
+ void $Function_decl[[f]] () {
+ $Class[[B]] $LocalVariable_decl[[BB]] = $Class[[B]]();
$LocalVariable[[BB]].~$Class[[B]]();
$Class[[B]]();
}
)cpp",
R"cpp(
- enum class $Enum[[E]] {
- $EnumConstant[[A]],
- $EnumConstant[[B]],
+ enum class $Enum_decl[[E]] {
+ $EnumConstant_decl_readonly[[A]],
+ $EnumConstant_decl_readonly[[B]],
};
- enum $Enum[[EE]] {
- $EnumConstant[[Hi]],
+ enum $Enum_decl[[EE]] {
+ $EnumConstant_decl_readonly[[Hi]],
};
- struct $Class[[A]] {
- $Enum[[E]] $Field[[EEE]];
- $Enum[[EE]] $Field[[EEEE]];
+ struct $Class_decl[[A]] {
+ $Enum[[E]] $Field_decl[[EEE]];
+ $Enum[[EE]] $Field_decl[[EEEE]];
};
- int $Variable[[I]] = $EnumConstant[[Hi]];
- $Enum[[E]] $Variable[[L]] = $Enum[[E]]::$EnumConstant[[B]];
+ int $Variable_decl[[I]] = $EnumConstant_readonly[[Hi]];
+ $Enum[[E]] $Variable_decl[[L]] = $Enum[[E]]::$EnumConstant_readonly[[B]];
)cpp",
R"cpp(
- namespace $Namespace[[abc]] {
+ namespace $Namespace_decl[[abc]] {
namespace {}
- namespace $Namespace[[bcd]] {
- struct $Class[[A]] {};
- namespace $Namespace[[cde]] {
- struct $Class[[A]] {
- enum class $Enum[[B]] {
- $EnumConstant[[Hi]],
+ namespace $Namespace_decl[[bcd]] {
+ struct $Class_decl[[A]] {};
+ namespace $Namespace_decl[[cde]] {
+ struct $Class_decl[[A]] {
+ enum class $Enum_decl[[B]] {
+ $EnumConstant_decl_readonly[[Hi]],
};
};
}
}
}
using namespace $Namespace[[abc]]::$Namespace[[bcd]];
- namespace $Namespace[[vwz]] =
+ namespace $Namespace_decl[[vwz]] =
$Namespace[[abc]]::$Namespace[[bcd]]::$Namespace[[cde]];
- $Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[AA]];
- $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]] $Variable[[AAA]] =
- $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::$EnumConstant[[Hi]];
- ::$Namespace[[vwz]]::$Class[[A]] $Variable[[B]];
- ::$Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[BB]];
- )cpp",
- R"cpp(
- struct $Class[[D]] {
- double $Field[[C]];
- };
- struct $Class[[A]] {
- double $Field[[B]];
- $Class[[D]] $Field[[E]];
- static double $StaticField[[S]];
- static void $StaticMethod[[bar]]() {}
- void $Method[[foo]]() {
+ $Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable_decl[[AA]];
+ $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]] $Variable_decl[[AAA]] =
+ $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::$EnumConstant_readonly[[Hi]];
+ ::$Namespace[[vwz]]::$Class[[A]] $Variable_decl[[B]];
+ ::$Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable_decl[[BB]];
+ )cpp",
+ R"cpp(
+ struct $Class_decl[[D]] {
+ double $Field_decl[[C]];
+ };
+ struct $Class_decl[[A]] {
+ double $Field_decl[[B]];
+ $Class[[D]] $Field_decl[[E]];
+ static double $StaticField_decl_static[[S]];
+ static void $StaticMethod_decl_static[[bar]]() {}
+ void $Method_decl[[foo]]() {
$Field[[B]] = 123;
this->$Field[[B]] = 156;
this->$Method[[foo]]();
$Method[[foo]]();
- $StaticMethod[[bar]]();
- $StaticField[[S]] = 90.1;
+ $StaticMethod_static[[bar]]();
+ $StaticField_static[[S]] = 90.1;
}
};
- void $Function[[foo]]() {
- $Class[[A]] $LocalVariable[[AA]];
+ void $Function_decl[[foo]]() {
+ $Class[[A]] $LocalVariable_decl[[AA]];
$LocalVariable[[AA]].$Field[[B]] += 2;
$LocalVariable[[AA]].$Method[[foo]]();
$LocalVariable[[AA]].$Field[[E]].$Field[[C]];
- $Class[[A]]::$StaticField[[S]] = 90;
+ $Class[[A]]::$StaticField_static[[S]] = 90;
}
)cpp",
R"cpp(
- struct $Class[[AA]] {
- int $Field[[A]];
+ struct $Class_decl[[AA]] {
+ int $Field_decl[[A]];
};
- int $Variable[[B]];
- $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
+ int $Variable_decl[[B]];
+ $Class[[AA]] $Variable_decl[[A]]{$Variable[[B]]};
)cpp",
R"cpp(
- namespace $Namespace[[a]] {
- struct $Class[[A]] {};
- typedef char $Primitive[[C]];
+ namespace $Namespace_decl[[a]] {
+ struct $Class_decl[[A]] {};
+ typedef char $Primitive_decl[[C]];
}
- typedef $Namespace[[a]]::$Class[[A]] $Class[[B]];
- using $Class[[BB]] = $Namespace[[a]]::$Class[[A]];
- enum class $Enum[[E]] {};
- typedef $Enum[[E]] $Enum[[C]];
- typedef $Enum[[C]] $Enum[[CC]];
- using $Enum[[CD]] = $Enum[[CC]];
- $Enum[[CC]] $Function[[f]]($Class[[B]]);
- $Enum[[CD]] $Function[[f]]($Class[[BB]]);
- typedef $Namespace[[a]]::$Primitive[[C]] $Primitive[[PC]];
- typedef float $Primitive[[F]];
- )cpp",
- R"cpp(
- template<typename $TemplateParameter[[T]], typename = void>
- class $Class[[A]] {
- $TemplateParameter[[T]] $Field[[AA]];
- $TemplateParameter[[T]] $Method[[foo]]();
- };
- template<class $TemplateParameter[[TT]]>
- class $Class[[B]] {
- $Class[[A]]<$TemplateParameter[[TT]]> $Field[[AA]];
- };
- template<class $TemplateParameter[[TT]], class $TemplateParameter[[GG]]>
- class $Class[[BB]] {};
- template<class $TemplateParameter[[T]]>
- class $Class[[BB]]<$TemplateParameter[[T]], int> {};
- template<class $TemplateParameter[[T]]>
- class $Class[[BB]]<$TemplateParameter[[T]], $TemplateParameter[[T]]*> {};
-
- template<template<class> class $TemplateParameter[[T]], class $TemplateParameter[[C]]>
- $TemplateParameter[[T]]<$TemplateParameter[[C]]> $Function[[f]]();
+ typedef $Namespace[[a]]::$Class[[A]] $Class_decl[[B]];
+ using $Class_decl[[BB]] = $Namespace[[a]]::$Class[[A]];
+ enum class $Enum_decl[[E]] {};
+ typedef $Enum[[E]] $Enum_decl[[C]];
+ typedef $Enum[[C]] $Enum_decl[[CC]];
+ using $Enum_decl[[CD]] = $Enum[[CC]];
+ $Enum[[CC]] $Function_decl[[f]]($Class[[B]]);
+ $Enum[[CD]] $Function_decl[[f]]($Class[[BB]]);
+ typedef $Namespace[[a]]::$Primitive[[C]] $Primitive_decl[[PC]];
+ typedef float $Primitive_decl[[F]];
+ )cpp",
+ R"cpp(
+ template<typename $TemplateParameter_decl[[T]], typename = void>
+ class $Class_decl[[A]] {
+ $TemplateParameter[[T]] $Field_decl[[AA]];
+ $TemplateParameter[[T]] $Method_decl[[foo]]();
+ };
+ template<class $TemplateParameter_decl[[TT]]>
+ class $Class_decl[[B]] {
+ $Class[[A]]<$TemplateParameter[[TT]]> $Field_decl[[AA]];
+ };
+ template<class $TemplateParameter_decl[[TT]], class $TemplateParameter_decl[[GG]]>
+ class $Class_decl[[BB]] {};
+ template<class $TemplateParameter_decl[[T]]>
+ class $Class_decl[[BB]]<$TemplateParameter[[T]], int> {};
+ template<class $TemplateParameter_decl[[T]]>
+ class $Class_decl[[BB]]<$TemplateParameter[[T]], $TemplateParameter[[T]]*> {};
+
+ template<template<class> class $TemplateParameter_decl[[T]], class $TemplateParameter_decl[[C]]>
+ $TemplateParameter[[T]]<$TemplateParameter[[C]]> $Function_decl[[f]]();
template<typename>
- class $Class[[Foo]] {};
+ class $Class_decl[[Foo]] {};
- template<typename $TemplateParameter[[T]]>
- void $Function[[foo]]($TemplateParameter[[T]] ...);
+ template<typename $TemplateParameter_decl[[T]]>
+ void $Function_decl[[foo]]($TemplateParameter[[T]] ...);
)cpp",
R"cpp(
- template <class $TemplateParameter[[T]]>
- struct $Class[[Tmpl]] {$TemplateParameter[[T]] $Field[[x]] = 0;};
- extern template struct $Class[[Tmpl]]<float>;
- template struct $Class[[Tmpl]]<double>;
+ template <class $TemplateParameter_decl[[T]]>
+ struct $Class_decl[[Tmpl]] {$TemplateParameter[[T]] $Field_decl[[x]] = 0;};
+ // FIXME: highlights dropped due to conflicts.
+ // explicitReferenceTargets reports ClassTemplateSpecializationDecl twice
+ // at its location, calling it a declaration/non-declaration once each.
+ extern template struct Tmpl<float>;
+ template struct Tmpl<double>;
)cpp",
// This test is to guard against highlightings disappearing when using
// conversion operators as their behaviour in the clang AST differ from
// other CXXMethodDecls.
R"cpp(
- class $Class[[Foo]] {};
- struct $Class[[Bar]] {
+ class $Class_decl[[Foo]] {};
+ struct $Class_decl[[Bar]] {
explicit operator $Class[[Foo]]*() const;
explicit operator int() const;
operator $Class[[Foo]]();
};
- void $Function[[f]]() {
- $Class[[Bar]] $LocalVariable[[B]];
- $Class[[Foo]] $LocalVariable[[F]] = $LocalVariable[[B]];
- $Class[[Foo]] *$LocalVariable[[FP]] = ($Class[[Foo]]*)$LocalVariable[[B]];
- int $LocalVariable[[I]] = (int)$LocalVariable[[B]];
+ void $Function_decl[[f]]() {
+ $Class[[Bar]] $LocalVariable_decl[[B]];
+ $Class[[Foo]] $LocalVariable_decl[[F]] = $LocalVariable[[B]];
+ $Class[[Foo]] *$LocalVariable_decl[[FP]] = ($Class[[Foo]]*)$LocalVariable[[B]];
+ int $LocalVariable_decl[[I]] = (int)$LocalVariable[[B]];
}
)cpp",
R"cpp(
- struct $Class[[B]] {};
- struct $Class[[A]] {
- $Class[[B]] $Field[[BB]];
- $Class[[A]] &operator=($Class[[A]] &&$Parameter[[O]]);
+ struct $Class_decl[[B]] {};
+ struct $Class_decl[[A]] {
+ $Class[[B]] $Field_decl[[BB]];
+ $Class[[A]] &operator=($Class[[A]] &&$Parameter_decl[[O]]);
};
- $Class[[A]] &$Class[[A]]::operator=($Class[[A]] &&$Parameter[[O]]) = default;
+ $Class[[A]] &$Class[[A]]::operator=($Class[[A]] &&$Parameter_decl[[O]]) = default;
)cpp",
R"cpp(
- enum $Enum[[En]] {
- $EnumConstant[[EC]],
+ enum $Enum_decl[[En]] {
+ $EnumConstant_decl_readonly[[EC]],
};
- class $Class[[Foo]] {};
- class $Class[[Bar]] {
+ class $Class_decl[[Foo]] {};
+ class $Class_decl[[Bar]] {
public:
- $Class[[Foo]] $Field[[Fo]];
- $Enum[[En]] $Field[[E]];
- int $Field[[I]];
- $Class[[Bar]] ($Class[[Foo]] $Parameter[[F]],
- $Enum[[En]] $Parameter[[E]])
+ $Class[[Foo]] $Field_decl[[Fo]];
+ $Enum[[En]] $Field_decl[[E]];
+ int $Field_decl[[I]];
+ $Class_decl[[Bar]] ($Class[[Foo]] $Parameter_decl[[F]],
+ $Enum[[En]] $Parameter_decl[[E]])
: $Field[[Fo]] ($Parameter[[F]]), $Field[[E]] ($Parameter[[E]]),
$Field[[I]] (123) {}
};
- class $Class[[Bar2]] : public $Class[[Bar]] {
- $Class[[Bar2]]() : $Class[[Bar]]($Class[[Foo]](), $EnumConstant[[EC]]) {}
+ class $Class_decl[[Bar2]] : public $Class[[Bar]] {
+ $Class_decl[[Bar2]]() : $Class[[Bar]]($Class[[Foo]](), $EnumConstant_readonly[[EC]]) {}
};
)cpp",
R"cpp(
- enum $Enum[[E]] {
- $EnumConstant[[E]],
+ enum $Enum_decl[[E]] {
+ $EnumConstant_decl_readonly[[E]],
};
- class $Class[[Foo]] {};
- $Enum[[auto]] $Variable[[AE]] = $Enum[[E]]::$EnumConstant[[E]];
- $Class[[auto]] $Variable[[AF]] = $Class[[Foo]]();
- $Class[[decltype]](auto) $Variable[[AF2]] = $Class[[Foo]]();
- $Class[[auto]] *$Variable[[AFP]] = &$Variable[[AF]];
- $Enum[[auto]] &$Variable[[AER]] = $Variable[[AE]];
- $Primitive[[auto]] $Variable[[Form]] = 10.2 + 2 * 4;
- $Primitive[[decltype]]($Variable[[Form]]) $Variable[[F]] = 10;
- auto $Variable[[Fun]] = []()->void{};
+ class $Class_decl[[Foo]] {};
+ $Enum_deduced[[auto]] $Variable_decl[[AE]] = $Enum[[E]]::$EnumConstant_readonly[[E]];
+ $Class_deduced[[auto]] $Variable_decl[[AF]] = $Class[[Foo]]();
+ $Class_deduced[[decltype]](auto) $Variable_decl[[AF2]] = $Class[[Foo]]();
+ $Class_deduced[[auto]] *$Variable_decl[[AFP]] = &$Variable[[AF]];
+ $Enum_deduced[[auto]] &$Variable_decl[[AER]] = $Variable[[AE]];
+ $Primitive_deduced[[auto]] $Variable_decl[[Form]] = 10.2 + 2 * 4;
+ $Primitive_deduced[[decltype]]($Variable[[Form]]) $Variable_decl[[F]] = 10;
+ auto $Variable_decl[[Fun]] = []()->void{};
)cpp",
R"cpp(
- class $Class[[G]] {};
- template<$Class[[G]] *$TemplateParameter[[U]]>
- class $Class[[GP]] {};
- template<$Class[[G]] &$TemplateParameter[[U]]>
- class $Class[[GR]] {};
- template<int *$TemplateParameter[[U]]>
- class $Class[[IP]] {
- void $Method[[f]]() {
- *$TemplateParameter[[U]] += 5;
+ class $Class_decl[[G]] {};
+ template<$Class[[G]] *$TemplateParameter_decl_readonly[[U]]>
+ class $Class_decl[[GP]] {};
+ template<$Class[[G]] &$TemplateParameter_decl_readonly[[U]]>
+ class $Class_decl[[GR]] {};
+ template<int *$TemplateParameter_decl_readonly[[U]]>
+ class $Class_decl[[IP]] {
+ void $Method_decl[[f]]() {
+ *$TemplateParameter_readonly[[U]] += 5;
}
};
- template<unsigned $TemplateParameter[[U]] = 2>
- class $Class[[Foo]] {
- void $Method[[f]]() {
- for(int $LocalVariable[[I]] = 0;
- $LocalVariable[[I]] < $TemplateParameter[[U]];) {}
+ template<unsigned $TemplateParameter_decl_readonly[[U]] = 2>
+ class $Class_decl[[Foo]] {
+ void $Method_decl[[f]]() {
+ for(int $LocalVariable_decl[[I]] = 0;
+ $LocalVariable[[I]] < $TemplateParameter_readonly[[U]];) {}
}
};
- $Class[[G]] $Variable[[L]];
- void $Function[[f]]() {
- $Class[[Foo]]<123> $LocalVariable[[F]];
- $Class[[GP]]<&$Variable[[L]]> $LocalVariable[[LL]];
- $Class[[GR]]<$Variable[[L]]> $LocalVariable[[LLL]];
+ $Class[[G]] $Variable_decl[[L]];
+ void $Function_decl[[f]]() {
+ $Class[[Foo]]<123> $LocalVariable_decl[[F]];
+ $Class[[GP]]<&$Variable[[L]]> $LocalVariable_decl[[LL]];
+ $Class[[GR]]<$Variable[[L]]> $LocalVariable_decl[[LLL]];
}
)cpp",
R"cpp(
- template<typename $TemplateParameter[[T]],
- void ($TemplateParameter[[T]]::*$TemplateParameter[[method]])(int)>
- struct $Class[[G]] {
- void $Method[[foo]](
- $TemplateParameter[[T]] *$Parameter[[O]]) {
- ($Parameter[[O]]->*$TemplateParameter[[method]])(10);
+ template<typename $TemplateParameter_decl[[T]],
+ void ($TemplateParameter[[T]]::*$TemplateParameter_decl_readonly[[method]])(int)>
+ struct $Class_decl[[G]] {
+ void $Method_decl[[foo]](
+ $TemplateParameter[[T]] *$Parameter_decl[[O]]) {
+ ($Parameter[[O]]->*$TemplateParameter_readonly[[method]])(10);
}
};
- struct $Class[[F]] {
- void $Method[[f]](int);
+ struct $Class_decl[[F]] {
+ void $Method_decl[[f]](int);
};
- template<void (*$TemplateParameter[[Func]])()>
- struct $Class[[A]] {
- void $Method[[f]]() {
- (*$TemplateParameter[[Func]])();
+ template<void (*$TemplateParameter_decl_readonly[[Func]])()>
+ struct $Class_decl[[A]] {
+ void $Method_decl[[f]]() {
+ (*$TemplateParameter_readonly[[Func]])();
}
};
- void $Function[[foo]]() {
- $Class[[F]] $LocalVariable[[FF]];
- $Class[[G]]<$Class[[F]], &$Class[[F]]::$Method[[f]]> $LocalVariable[[GG]];
+ void $Function_decl[[foo]]() {
+ $Class[[F]] $LocalVariable_decl[[FF]];
+ $Class[[G]]<$Class[[F]], &$Class[[F]]::$Method[[f]]> $LocalVariable_decl[[GG]];
$LocalVariable[[GG]].$Method[[foo]](&$LocalVariable[[FF]]);
- $Class[[A]]<$Function[[foo]]> $LocalVariable[[AA]];
+ $Class[[A]]<$Function[[foo]]> $LocalVariable_decl[[AA]];
}
)cpp",
// Tokens that share a source range but have conflicting Kinds are not
// highlighted.
R"cpp(
- #define $Macro[[DEF_MULTIPLE]](X) namespace X { class X { int X; }; }
- #define $Macro[[DEF_CLASS]](T) class T {};
+ #define $Macro_decl[[DEF_MULTIPLE]](X) namespace X { class X { int X; }; }
+ #define $Macro_decl[[DEF_CLASS]](T) class T {};
// Preamble ends.
$Macro[[DEF_MULTIPLE]](XYZ);
$Macro[[DEF_MULTIPLE]](XYZW);
- $Macro[[DEF_CLASS]]($Class[[A]])
- #define $Macro[[MACRO_CONCAT]](X, V, T) T foo##X = V
- #define $Macro[[DEF_VAR]](X, V) int X = V
- #define $Macro[[DEF_VAR_T]](T, X, V) T X = V
- #define $Macro[[DEF_VAR_REV]](V, X) DEF_VAR(X, V)
- #define $Macro[[CPY]](X) X
- #define $Macro[[DEF_VAR_TYPE]](X, Y) X Y
- #define $Macro[[SOME_NAME]] variable
- #define $Macro[[SOME_NAME_SET]] variable2 = 123
- #define $Macro[[INC_VAR]](X) X += 2
- void $Function[[foo]]() {
- $Macro[[DEF_VAR]]($LocalVariable[[X]], 123);
- $Macro[[DEF_VAR_REV]](908, $LocalVariable[[XY]]);
- int $Macro[[CPY]]( $LocalVariable[[XX]] );
- $Macro[[DEF_VAR_TYPE]]($Class[[A]], $LocalVariable[[AA]]);
+ $Macro[[DEF_CLASS]]($Class_decl[[A]])
+ #define $Macro_decl[[MACRO_CONCAT]](X, V, T) T foo##X = V
+ #define $Macro_decl[[DEF_VAR]](X, V) int X = V
+ #define $Macro_decl[[DEF_VAR_T]](T, X, V) T X = V
+ #define $Macro_decl[[DEF_VAR_REV]](V, X) DEF_VAR(X, V)
+ #define $Macro_decl[[CPY]](X) X
+ #define $Macro_decl[[DEF_VAR_TYPE]](X, Y) X Y
+ #define $Macro_decl[[SOME_NAME]] variable
+ #define $Macro_decl[[SOME_NAME_SET]] variable2 = 123
+ #define $Macro_decl[[INC_VAR]](X) X += 2
+ void $Function_decl[[foo]]() {
+ $Macro[[DEF_VAR]]($LocalVariable_decl[[X]], 123);
+ $Macro[[DEF_VAR_REV]](908, $LocalVariable_decl[[XY]]);
+ int $Macro[[CPY]]( $LocalVariable_decl[[XX]] );
+ $Macro[[DEF_VAR_TYPE]]($Class[[A]], $LocalVariable_decl[[AA]]);
double $Macro[[SOME_NAME]];
int $Macro[[SOME_NAME_SET]];
$LocalVariable[[variable]] = 20.1;
$Macro[[MACRO_CONCAT]](var, 2, float);
$Macro[[DEF_VAR_T]]($Class[[A]], $Macro[[CPY]](
- $Macro[[CPY]]($LocalVariable[[Nested]])),
+ $Macro[[CPY]]($LocalVariable_decl[[Nested]])),
$Macro[[CPY]]($Class[[A]]()));
$Macro[[INC_VAR]]($LocalVariable[[variable]]);
}
void $Macro[[SOME_NAME]]();
- $Macro[[DEF_VAR]]($Variable[[MMMMM]], 567);
- $Macro[[DEF_VAR_REV]](756, $Variable[[AB]]);
+ $Macro[[DEF_VAR]]($Variable_decl[[MMMMM]], 567);
+ $Macro[[DEF_VAR_REV]](756, $Variable_decl[[AB]]);
- #define $Macro[[CALL_FN]](F) F();
- #define $Macro[[DEF_FN]](F) void F ()
- $Macro[[DEF_FN]]($Function[[g]]) {
+ #define $Macro_decl[[CALL_FN]](F) F();
+ #define $Macro_decl[[DEF_FN]](F) void F ()
+ $Macro[[DEF_FN]]($Function_decl[[g]]) {
$Macro[[CALL_FN]]($Function[[foo]]);
}
)cpp",
R"cpp(
- #define $Macro[[fail]](expr) expr
- #define $Macro[[assert]](COND) if (!(COND)) { fail("assertion failed" #COND); }
+ #define $Macro_decl[[fail]](expr) expr
+ #define $Macro_decl[[assert]](COND) if (!(COND)) { fail("assertion failed" #COND); }
// Preamble ends.
- int $Variable[[x]];
- int $Variable[[y]];
- int $Function[[f]]();
- void $Function[[foo]]() {
+ int $Variable_decl[[x]];
+ int $Variable_decl[[y]];
+ int $Function_decl[[f]]();
+ void $Function_decl[[foo]]() {
$Macro[[assert]]($Variable[[x]] != $Variable[[y]]);
$Macro[[assert]]($Variable[[x]] != $Function[[f]]());
}
@@ -498,10 +507,10 @@
// highlighting all macro references
R"cpp(
#ifndef $Macro[[name]]
- #define $Macro[[name]]
+ #define $Macro_decl[[name]]
#endif
- #define $Macro[[test]]
+ #define $Macro_decl[[test]]
#undef $Macro[[test]]
$InactiveCode[[#ifdef test]]
$InactiveCode[[#endif]]
@@ -510,104 +519,104 @@
$InactiveCode[[#endif]]
)cpp",
R"cpp(
- struct $Class[[S]] {
- float $Field[[Value]];
- $Class[[S]] *$Field[[Next]];
+ struct $Class_decl[[S]] {
+ float $Field_decl[[Value]];
+ $Class[[S]] *$Field_decl[[Next]];
};
- $Class[[S]] $Variable[[Global]][2] = {$Class[[S]](), $Class[[S]]()};
- auto [$Variable[[G1]], $Variable[[G2]]] = $Variable[[Global]];
- void $Function[[f]]($Class[[S]] $Parameter[[P]]) {
- int $LocalVariable[[A]][2] = {1,2};
- auto [$LocalVariable[[B1]], $LocalVariable[[B2]]] = $LocalVariable[[A]];
- auto [$LocalVariable[[G1]], $LocalVariable[[G2]]] = $Variable[[Global]];
- $Class[[auto]] [$LocalVariable[[P1]], $LocalVariable[[P2]]] = $Parameter[[P]];
+ $Class[[S]] $Variable_decl[[Global]][2] = {$Class[[S]](), $Class[[S]]()};
+ auto [$Variable_decl[[G1]], $Variable_decl[[G2]]] = $Variable[[Global]];
+ void $Function_decl[[f]]($Class[[S]] $Parameter_decl[[P]]) {
+ int $LocalVariable_decl[[A]][2] = {1,2};
+ auto [$LocalVariable_decl[[B1]], $LocalVariable_decl[[B2]]] = $LocalVariable[[A]];
+ auto [$LocalVariable_decl[[G1]], $LocalVariable_decl[[G2]]] = $Variable[[Global]];
+ $Class_deduced[[auto]] [$LocalVariable_decl[[P1]], $LocalVariable_decl[[P2]]] = $Parameter[[P]];
// Highlights references to BindingDecls.
$LocalVariable[[B1]]++;
}
)cpp",
R"cpp(
- template<class $TemplateParameter[[T]]>
- class $Class[[A]] {
- using $TemplateParameter[[TemplateParam1]] = $TemplateParameter[[T]];
- typedef $TemplateParameter[[T]] $TemplateParameter[[TemplateParam2]];
- using $Primitive[[IntType]] = int;
+ template<class $TemplateParameter_decl[[T]]>
+ class $Class_decl[[A]] {
+ using $TemplateParameter_decl[[TemplateParam1]] = $TemplateParameter[[T]];
+ typedef $TemplateParameter[[T]] $TemplateParameter_decl[[TemplateParam2]];
+ using $Primitive_decl[[IntType]] = int;
- using $Typedef[[Pointer]] = $TemplateParameter[[T]] *;
- using $Typedef[[LVReference]] = $TemplateParameter[[T]] &;
- using $Typedef[[RVReference]] = $TemplateParameter[[T]]&&;
- using $Typedef[[Array]] = $TemplateParameter[[T]]*[3];
- using $Typedef[[MemberPointer]] = int ($Class[[A]]::*)(int);
+ using $Typedef_decl[[Pointer]] = $TemplateParameter[[T]] *;
+ using $Typedef_decl[[LVReference]] = $TemplateParameter[[T]] &;
+ using $Typedef_decl[[RVReference]] = $TemplateParameter[[T]]&&;
+ using $Typedef_decl[[Array]] = $TemplateParameter[[T]]*[3];
+ using $Typedef_decl[[MemberPointer]] = int ($Class[[A]]::*)(int);
// Use various previously defined typedefs in a function type.
- void $Method[[func]](
+ void $Method_decl[[func]](
$Typedef[[Pointer]], $Typedef[[LVReference]], $Typedef[[RVReference]],
$Typedef[[Array]], $Typedef[[MemberPointer]]);
};
)cpp",
R"cpp(
- template <class $TemplateParameter[[T]]>
- void $Function[[phase1]]($TemplateParameter[[T]]);
- template <class $TemplateParameter[[T]]>
- void $Function[[foo]]($TemplateParameter[[T]] $Parameter[[P]]) {
+ template <class $TemplateParameter_decl[[T]]>
+ void $Function_decl[[phase1]]($TemplateParameter[[T]]);
+ template <class $TemplateParameter_decl[[T]]>
+ void $Function_decl[[foo]]($TemplateParameter[[T]] $Parameter_decl[[P]]) {
$Function[[phase1]]($Parameter[[P]]);
$DependentName[[phase2]]($Parameter[[P]]);
}
)cpp",
R"cpp(
- class $Class[[A]] {
- template <class $TemplateParameter[[T]]>
- void $Method[[bar]]($TemplateParameter[[T]]);
+ class $Class_decl[[A]] {
+ template <class $TemplateParameter_decl[[T]]>
+ void $Method_decl[[bar]]($TemplateParameter[[T]]);
};
- template <class $TemplateParameter[[U]]>
- void $Function[[foo]]($TemplateParameter[[U]] $Parameter[[P]]) {
+ template <class $TemplateParameter_decl[[U]]>
+ void $Function_decl[[foo]]($TemplateParameter[[U]] $Parameter_decl[[P]]) {
$Class[[A]]().$Method[[bar]]($Parameter[[P]]);
}
)cpp",
R"cpp(
- struct $Class[[A]] {
- template <class $TemplateParameter[[T]]>
- static void $StaticMethod[[foo]]($TemplateParameter[[T]]);
+ struct $Class_decl[[A]] {
+ template <class $TemplateParameter_decl[[T]]>
+ static void $StaticMethod_decl_static[[foo]]($TemplateParameter[[T]]);
};
- template <class $TemplateParameter[[T]]>
- struct $Class[[B]] {
- void $Method[[bar]]() {
- $Class[[A]]::$StaticMethod[[foo]]($TemplateParameter[[T]]());
+ template <class $TemplateParameter_decl[[T]]>
+ struct $Class_decl[[B]] {
+ void $Method_decl[[bar]]() {
+ $Class[[A]]::$StaticMethod_static[[foo]]($TemplateParameter[[T]]());
}
};
)cpp",
R"cpp(
- template <class $TemplateParameter[[T]]>
- void $Function[[foo]](typename $TemplateParameter[[T]]::$DependentType[[Type]]
+ template <class $TemplateParameter_decl[[T]]>
+ void $Function_decl[[foo]](typename $TemplateParameter[[T]]::$DependentType[[Type]]
= $TemplateParameter[[T]]::$DependentName[[val]]);
)cpp",
R"cpp(
- template <class $TemplateParameter[[T]]>
- void $Function[[foo]]($TemplateParameter[[T]] $Parameter[[P]]) {
+ template <class $TemplateParameter_decl[[T]]>
+ void $Function_decl[[foo]]($TemplateParameter[[T]] $Parameter_decl[[P]]) {
$Parameter[[P]].$DependentName[[Field]];
}
)cpp",
R"cpp(
- template <class $TemplateParameter[[T]]>
- class $Class[[A]] {
- int $Method[[foo]]() {
+ template <class $TemplateParameter_decl[[T]]>
+ class $Class_decl[[A]] {
+ int $Method_decl[[foo]]() {
return $TemplateParameter[[T]]::$DependentName[[Field]];
}
};
)cpp",
// Highlighting the using decl as the underlying using shadow decl.
R"cpp(
- void $Function[[foo]]();
+ void $Function_decl[[foo]]();
using ::$Function[[foo]];
)cpp",
// Highlighting of template template arguments.
R"cpp(
- template <template <class> class $TemplateParameter[[TT]],
- template <class> class ...$TemplateParameter[[TTs]]>
- struct $Class[[Foo]] {
+ template <template <class> class $TemplateParameter_decl[[TT]],
+ template <class> class ...$TemplateParameter_decl[[TTs]]>
+ struct $Class_decl[[Foo]] {
$Class[[Foo]]<$TemplateParameter[[TT]], $TemplateParameter[[TTs]]...>
- *$Field[[t]];
+ *$Field_decl[[t]];
};
)cpp",
// Inactive code highlighting
@@ -618,86 +627,86 @@
$InactiveCode[[#endif]]
// A declaration to cause the preamble to end.
- int $Variable[[EndPreamble]];
+ int $Variable_decl[[EndPreamble]];
// Code after the preamble.
// Code inside inactive blocks does not get regular highlightings
// because it's not part of the AST.
- #define $Macro[[test2]]
+ #define $Macro_decl[[test2]]
$InactiveCode[[#if defined(test)]]
$InactiveCode[[int Inactive2;]]
$InactiveCode[[#elif defined(test2)]]
- int $Variable[[Active1]];
+ int $Variable_decl[[Active1]];
$InactiveCode[[#else]]
$InactiveCode[[int Inactive3;]]
$InactiveCode[[#endif]]
#ifndef $Macro[[test]]
- int $Variable[[Active2]];
+ int $Variable_decl[[Active2]];
#endif
$InactiveCode[[#ifdef test]]
$InactiveCode[[int Inactive4;]]
$InactiveCode[[#else]]
- int $Variable[[Active3]];
+ int $Variable_decl[[Active3]];
#endif
)cpp",
// Argument to 'sizeof...'
R"cpp(
- template <typename... $TemplateParameter[[Elements]]>
- struct $Class[[TupleSize]] {
- static const int $StaticField[[size]] =
+ template <typename... $TemplateParameter_decl[[Elements]]>
+ struct $Class_decl[[TupleSize]] {
+ static const int $StaticField_decl_readonly_static[[size]] =
sizeof...($TemplateParameter[[Elements]]);
};
)cpp",
// More dependent types
R"cpp(
- template <typename $TemplateParameter[[T]]>
- struct $Class[[Waldo]] {
- using $Typedef[[Location1]] = typename $TemplateParameter[[T]]
+ template <typename $TemplateParameter_decl[[T]]>
+ struct $Class_decl[[Waldo]] {
+ using $Typedef_decl[[Location1]] = typename $TemplateParameter[[T]]
::$DependentType[[Resolver]]::$DependentType[[Location]];
- using $Typedef[[Location2]] = typename $TemplateParameter[[T]]
+ using $Typedef_decl[[Location2]] = typename $TemplateParameter[[T]]
::template $DependentType[[Resolver]]<$TemplateParameter[[T]]>
::$DependentType[[Location]];
- using $Typedef[[Location3]] = typename $TemplateParameter[[T]]
+ using $Typedef_decl[[Location3]] = typename $TemplateParameter[[T]]
::$DependentType[[Resolver]]
::template $DependentType[[Location]]<$TemplateParameter[[T]]>;
- static const int $StaticField[[Value]] = $TemplateParameter[[T]]
+ static const int $StaticField_decl_readonly_static[[Value]] = $TemplateParameter[[T]]
::$DependentType[[Resolver]]::$DependentName[[Value]];
};
)cpp",
// Dependent name with heuristic target
R"cpp(
template <typename>
- struct $Class[[Foo]] {
- int $Field[[Waldo]];
- void $Method[[bar]]() {
+ struct $Class_decl[[Foo]] {
+ int $Field_decl[[Waldo]];
+ void $Method_decl[[bar]]() {
$Class[[Foo]]().$Field[[Waldo]];
}
- template <typename $TemplateParameter[[U]]>
- void $Method[[bar1]]() {
+ template <typename $TemplateParameter_decl[[U]]>
+ void $Method_decl[[bar1]]() {
$Class[[Foo]]<$TemplateParameter[[U]]>().$Field[[Waldo]];
}
};
)cpp",
// Concepts
R"cpp(
- template <typename $TemplateParameter[[T]]>
- concept $Concept[[Fooable]] =
- requires($TemplateParameter[[T]] $Parameter[[F]]) {
+ template <typename $TemplateParameter_decl[[T]]>
+ concept $Concept_decl[[Fooable]] =
+ requires($TemplateParameter[[T]] $Parameter_decl[[F]]) {
$Parameter[[F]].$DependentName[[foo]]();
};
- template <typename $TemplateParameter[[T]]>
+ template <typename $TemplateParameter_decl[[T]]>
requires $Concept[[Fooable]]<$TemplateParameter[[T]]>
- void $Function[[bar]]($TemplateParameter[[T]] $Parameter[[F]]) {
+ void $Function_decl[[bar]]($TemplateParameter[[T]] $Parameter_decl[[F]]) {
$Parameter[[F]].$DependentName[[foo]]();
}
)cpp",
// Dependent template name
R"cpp(
- template <template <typename> class> struct $Class[[A]] {};
- template <typename $TemplateParameter[[T]]>
- using $Typedef[[W]] = $Class[[A]]<
+ template <template <typename> class> struct $Class_decl[[A]] {};
+ template <typename $TemplateParameter_decl[[T]]>
+ using $Typedef_decl[[W]] = $Class[[A]]<
$TemplateParameter[[T]]::template $DependentType[[Waldo]]
>;
)cpp"};
@@ -706,7 +715,7 @@
}
checkHighlightings(R"cpp(
- class $Class[[A]] {
+ class $Class_decl[[A]] {
#include "imp.h"
};
)cpp",
@@ -758,35 +767,43 @@
Annotations A(MarkedText);
std::vector<HighlightingToken> Results;
for (const Range& R : A.ranges())
- Results.push_back({HighlightingKind::Variable, R});
+ Results.push_back({HighlightingKind::Variable, 0, R});
for (unsigned I = 0; I < static_cast<unsigned>(HighlightingKind::LastKind); ++I) {
HighlightingKind Kind = static_cast<HighlightingKind>(I);
for (const Range& R : A.ranges(llvm::to_string(Kind)))
- Results.push_back({Kind, R});
+ Results.push_back({Kind, 0, R});
}
llvm::sort(Results);
return Results;
}
TEST(SemanticHighlighting, toSemanticTokens) {
- auto Results = toSemanticTokens(tokens(R"(
+ auto Tokens = tokens(R"(
[[blah]]
$Function[[big]] [[bang]]
- )"));
+ )");
+ Tokens.front().Modifiers |= unsigned(HighlightingModifier::Declaration);
+ Tokens.front().Modifiers |= unsigned(HighlightingModifier::Readonly);
+ auto Results = toSemanticTokens(Tokens);
ASSERT_THAT(Results, SizeIs(3));
EXPECT_EQ(Results[0].tokenType, unsigned(HighlightingKind::Variable));
+ EXPECT_EQ(Results[0].tokenModifiers,
+ unsigned(HighlightingModifier::Declaration) |
+ unsigned(HighlightingModifier::Readonly));
EXPECT_EQ(Results[0].deltaLine, 1u);
EXPECT_EQ(Results[0].deltaStart, 1u);
EXPECT_EQ(Results[0].length, 4u);
EXPECT_EQ(Results[1].tokenType, unsigned(HighlightingKind::Function));
+ EXPECT_EQ(Results[1].tokenModifiers, 0u);
EXPECT_EQ(Results[1].deltaLine, 2u);
EXPECT_EQ(Results[1].deltaStart, 4u);
EXPECT_EQ(Results[1].length, 3u);
EXPECT_EQ(Results[2].tokenType, unsigned(HighlightingKind::Variable));
+ EXPECT_EQ(Results[1].tokenModifiers, 0u);
EXPECT_EQ(Results[2].deltaLine, 0u);
EXPECT_EQ(Results[2].deltaStart, 4u);
EXPECT_EQ(Results[2].length, 4u);
@@ -834,13 +851,13 @@
std::vector<LineHighlightings> Tokens{
{3,
- {{HighlightingKind::Variable,
+ {{HighlightingKind::Variable, 0,
Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
- {HighlightingKind::Function,
+ {HighlightingKind::Function, 0,
Range{CreatePosition(3, 4), CreatePosition(3, 7)}}},
/* IsInactive = */ false},
{1,
- {{HighlightingKind::Variable,
+ {{HighlightingKind::Variable, 0,
Range{CreatePosition(1, 1), CreatePosition(1, 5)}}},
/* IsInactive = */ true}};
std::vector<TheiaSemanticHighlightingInformation> ActualResults =
Index: clang-tools-extra/clangd/test/semantic-tokens.test
===================================================================
--- clang-tools-extra/clangd/test/semantic-tokens.test
+++ clang-tools-extra/clangd/test/semantic-tokens.test
@@ -18,12 +18,12 @@
# CHECK-NEXT: "jsonrpc": "2.0",
# CHECK-NEXT: "result": {
# CHECK-NEXT: "data": [
-# First line, char 5, variable, no modifiers.
+# First line, char 5, variable, declaration
# CHECK-NEXT: 0,
# CHECK-NEXT: 4,
# CHECK-NEXT: 1,
# CHECK-NEXT: 0,
-# CHECK-NEXT: 0
+# CHECK-NEXT: 1
# CHECK-NEXT: ],
# CHECK-NEXT: "resultId": "1"
# CHECK-NEXT: }
@@ -44,12 +44,12 @@
# CHECK-NEXT: "edits": [
# CHECK-NEXT: {
# CHECK-NEXT: "data": [
-# Next line, char 5, variable, no modifiers
+# Next line, char 5, variable, declaration
# CHECK-NEXT: 1,
# CHECK-NEXT: 4,
# CHECK-NEXT: 1,
# CHECK-NEXT: 0,
-# CHECK-NEXT: 0
+# CHECK-NEXT: 1
# CHECK-NEXT: ],
# Inserted at position 1
# CHECK-NEXT: "deleteCount": 0,
@@ -72,12 +72,12 @@
# CHECK-NEXT: 4,
# CHECK-NEXT: 1,
# CHECK-NEXT: 0,
-# CHECK-NEXT: 0,
+# CHECK-NEXT: 1,
# CHECK-NEXT: 1,
# CHECK-NEXT: 4,
# CHECK-NEXT: 1,
# CHECK-NEXT: 0,
-# CHECK-NEXT: 0
+# CHECK-NEXT: 1
# CHECK-NEXT: ],
# CHECK-NEXT: "resultId": "3"
# CHECK-NEXT: }
Index: clang-tools-extra/clangd/test/initialize-params.test
===================================================================
--- clang-tools-extra/clangd/test/initialize-params.test
+++ clang-tools-extra/clangd/test/initialize-params.test
@@ -81,7 +81,14 @@
# CHECK-NEXT: "delta": true
# CHECK-NEXT: },
# CHECK-NEXT: "legend": {
-# CHECK-NEXT: "tokenModifiers": [],
+# CHECK-NEXT: "tokenModifiers": [
+# CHECK-NEXT: "declaration",
+# CHECK-NEXT: "deprecated",
+# CHECK-NEXT: "deduced",
+# CHECK-NEXT: "readonly",
+# CHECK-NEXT: "static",
+# CHECK-NEXT: "abstract"
+# CHECK-NEXT: ],
# CHECK-NEXT: "tokenTypes": [
# CHECK-NEXT: "variable",
# CHECK: ]
Index: clang-tools-extra/clangd/refactor/tweaks/AnnotateHighlightings.cpp
===================================================================
--- clang-tools-extra/clangd/refactor/tweaks/AnnotateHighlightings.cpp
+++ clang-tools-extra/clangd/refactor/tweaks/AnnotateHighlightings.cpp
@@ -8,6 +8,7 @@
#include "SemanticHighlighting.h"
#include "refactor/Tweak.h"
#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/ScopedPrinter.h"
namespace clang {
namespace clangd {
@@ -67,9 +68,19 @@
if (!InsertOffset)
return InsertOffset.takeError();
- auto InsertReplacement = tooling::Replacement(
- FilePath, *InsertOffset, 0,
- ("/* " + toTextMateScope(Token.Kind) + " */").str());
+ std::string Comment = "/* ";
+ Comment.append(llvm::to_string(Token.Kind));
+ for (unsigned I = 0;
+ I <= static_cast<unsigned>(HighlightingModifier::LastModifier); ++I) {
+ if (Token.Modifiers & (1 << I)) {
+ Comment.append(" [");
+ Comment.append(llvm::to_string(static_cast<HighlightingModifier>(I)));
+ Comment.push_back(']');
+ }
+ }
+ Comment.append(" */");
+ auto InsertReplacement =
+ tooling::Replacement(FilePath, *InsertOffset, 0, Comment);
if (auto Err = Result.add(InsertReplacement))
return std::move(Err);
}
Index: clang-tools-extra/clangd/SemanticHighlighting.h
===================================================================
--- clang-tools-extra/clangd/SemanticHighlighting.h
+++ clang-tools-extra/clangd/SemanticHighlighting.h
@@ -64,12 +64,34 @@
LastKind = InactiveCode
};
+
llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingKind K);
+enum class HighlightingModifier {
+ Declaration,
+ // FIXME: Definition (needs findExplicitReferences support)
+ Deprecated,
+ Deduced,
+ Readonly,
+ Static,
+ Abstract,
+
+ LastModifier = Abstract
+};
+static_assert(static_cast<unsigned>(HighlightingModifier::LastModifier) < 32,
+ "Increase width of modifiers bitfield!");
+llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingModifier K);
+
// Contains all information needed for the highlighting a token.
struct HighlightingToken {
HighlightingKind Kind;
+ uint32_t Modifiers = 0;
Range R;
+
+ HighlightingToken &addModifier(HighlightingModifier M) {
+ Modifiers |= 1 << static_cast<unsigned>(M);
+ return *this;
+ }
};
bool operator==(const HighlightingToken &L, const HighlightingToken &R);
@@ -89,9 +111,10 @@
std::vector<HighlightingToken> getSemanticHighlightings(ParsedAST &AST);
std::vector<SemanticToken> toSemanticTokens(llvm::ArrayRef<HighlightingToken>);
-llvm::StringRef toSemanticTokenType(HighlightingKind Kind);
std::vector<SemanticTokensEdit> diffTokens(llvm::ArrayRef<SemanticToken> Before,
llvm::ArrayRef<SemanticToken> After);
+llvm::StringRef toSemanticTokenType(HighlightingKind Kind);
+llvm::StringRef toSemanticTokenModifier(HighlightingModifier Modifier);
/// Converts a HighlightingKind to a corresponding TextMate scope
/// (https://manual.macromates.com/en/language_grammars).
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===================================================================
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -15,6 +15,8 @@
#include "clang/AST/ASTContext.h"
#include "clang/AST/Decl.h"
#include "clang/AST/DeclCXX.h"
+#include "clang/AST/DeclObjC.h"
+#include "clang/AST/DeclTemplate.h"
#include "clang/AST/DeclarationName.h"
#include "clang/AST/ExprCXX.h"
#include "clang/AST/RecursiveASTVisitor.h"
@@ -118,17 +120,60 @@
return llvm::None;
}
-llvm::Optional<HighlightingKind> kindForReference(const ReferenceLoc &R) {
- llvm::Optional<HighlightingKind> Result;
- for (const NamedDecl *Decl : R.Targets) {
- if (!canHighlightName(Decl->getDeclName()))
- return llvm::None;
- auto Kind = kindForDecl(Decl);
- if (!Kind || (Result && Kind != Result))
- return llvm::None;
- Result = Kind;
+// Whether T is const in a loose sense - is a variable with this type readonly?
+bool isConst(QualType T) {
+ if (T.isNull() || T->isDependentType())
+ return false;
+ T = T.getNonReferenceType();
+ if (T.isConstQualified())
+ return true;
+ if (const auto *AT = T->getAsArrayTypeUnsafe())
+ return isConst(AT->getElementType());
+ return isConst(T->getPointeeType());
+}
+
+// Whether D is const in a loose sense (should it be highlighted as such?)
+bool isConst(const Decl *D) {
+ if (llvm::isa<EnumConstantDecl>(D) || llvm::isa<NonTypeTemplateParmDecl>(D))
+ return true;
+ if (llvm::isa<FieldDecl>(D) || llvm::isa<VarDecl>(D) ||
+ llvm::isa<MSPropertyDecl>(D) || llvm::isa<BindingDecl>(D)) {
+ if (isConst(llvm::cast<ValueDecl>(D)->getType()))
+ return true;
}
- return Result;
+ if (const auto *OCPD = llvm::dyn_cast<ObjCPropertyDecl>(D)) {
+ if (OCPD->isReadOnly())
+ return true;
+ }
+ if (const auto *MPD = llvm::dyn_cast<MSPropertyDecl>(D)) {
+ if (!MPD->hasSetter())
+ return true;
+ }
+ if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D)) {
+ if (CMD->isConst())
+ return true;
+ }
+ return false;
+}
+
+bool isStatic(const Decl *D) {
+ if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D))
+ return CMD->isStatic();
+ if (const VarDecl *VD = llvm::dyn_cast<VarDecl>(D))
+ return VD->isStaticDataMember() || VD->isStaticLocal();
+ if (const auto *FD = llvm::dyn_cast<FieldDecl>(D))
+ return FD->isCXXClassMember() && !FD->isCXXInstanceMember();
+ if (const auto *OMD = llvm::dyn_cast<ObjCMethodDecl>(D))
+ return OMD->isClassMethod();
+ return false;
+}
+
+bool isAbstract(const Decl *D) {
+ if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D))
+ return CMD->isPure();
+ if (const auto *CRD = llvm::dyn_cast<CXXRecordDecl>(D))
+ return CRD->isAbstract();
+ return false;
}
// For a macro usage `DUMP(foo)`, we want:
@@ -182,18 +227,24 @@
: TB(AST.getTokens()), SourceMgr(AST.getSourceManager()),
LangOpts(AST.getLangOpts()) {}
- void addToken(HighlightingToken T) { Tokens.push_back(T); }
-
- void addToken(SourceLocation Loc, HighlightingKind Kind) {
+ HighlightingToken &addToken(SourceLocation Loc, HighlightingKind Kind) {
Loc = getHighlightableSpellingToken(Loc, SourceMgr);
if (Loc.isInvalid())
- return;
+ return Dummy;
const auto *Tok = TB.spelledTokenAt(Loc);
assert(Tok);
+ return addToken(
+ halfOpenToRange(SourceMgr,
+ Tok->range(SourceMgr).toCharRange(SourceMgr)),
+ Kind);
+ }
- auto Range = halfOpenToRange(SourceMgr,
- Tok->range(SourceMgr).toCharRange(SourceMgr));
- Tokens.push_back(HighlightingToken{Kind, std::move(Range)});
+ HighlightingToken &addToken(Range R, HighlightingKind Kind) {
+ HighlightingToken HT;
+ HT.R = std::move(R);
+ HT.Kind = Kind;
+ Tokens.push_back(std::move(HT));
+ return Tokens.back();
}
std::vector<HighlightingToken> collect(ParsedAST &AST) && {
@@ -248,10 +299,13 @@
MainCode.drop_front(*StartOfLine).take_until([](char C) {
return C == '\n';
});
- WithInactiveLines.push_back(
- {HighlightingKind::InactiveCode,
- {Position{Line, 0},
- Position{Line, static_cast<int>(lspLength(LineText))}}});
+ HighlightingToken HT;
+ WithInactiveLines.emplace_back();
+ WithInactiveLines.back().Kind = HighlightingKind::InactiveCode;
+ WithInactiveLines.back().R.start.line = Line;
+ WithInactiveLines.back().R.end.line = Line;
+ WithInactiveLines.back().R.end.character =
+ static_cast<int>(lspLength(LineText));
} else {
elog("Failed to convert position to offset: {0}",
StartOfLine.takeError());
@@ -277,6 +331,7 @@
const SourceManager &SourceMgr;
const LangOptions &LangOpts;
std::vector<HighlightingToken> Tokens;
+ HighlightingToken Dummy; // returned from addToken(InvalidLoc)
};
/// Produces highlightings, which are not captured by findExplicitReferences,
@@ -288,7 +343,8 @@
bool VisitDecltypeTypeLoc(DecltypeTypeLoc L) {
if (auto K = kindForType(L.getTypePtr()))
- H.addToken(L.getBeginLoc(), *K);
+ H.addToken(L.getBeginLoc(), *K)
+ .addModifier(HighlightingModifier::Deduced);
return true;
}
@@ -297,7 +353,8 @@
if (!AT)
return true;
if (auto K = kindForType(AT->getDeducedType().getTypePtrOrNull()))
- H.addToken(D->getTypeSpecStartLoc(), *K);
+ H.addToken(D->getTypeSpecStartLoc(), *K)
+ .addModifier(HighlightingModifier::Deduced);
return true;
}
@@ -391,16 +448,45 @@
CollectExtraHighlightings(Builder).TraverseAST(C);
// Highlight all decls and references coming from the AST.
findExplicitReferences(C, [&](ReferenceLoc R) {
- if (auto Kind = kindForReference(R))
- Builder.addToken(R.NameLoc, *Kind);
+ llvm::Optional<HighlightingKind> Result;
+ for (const NamedDecl *Decl : R.Targets) {
+ if (!canHighlightName(Decl->getDeclName()))
+ continue;
+ auto Kind = kindForDecl(Decl);
+ if (!Kind || (Result && Kind != Result))
+ continue;
+ Result = Kind;
+
+ auto &Tok = Builder.addToken(R.NameLoc, *Kind);
+
+ // The attribute tests don't want to look at the template.
+ if (auto *TD = dyn_cast<TemplateDecl>(Decl)) {
+ if (auto *Templated = TD->getTemplatedDecl())
+ Decl = Templated;
+ }
+ if (isConst(Decl))
+ Tok.addModifier(HighlightingModifier::Readonly);
+ if (isStatic(Decl))
+ Tok.addModifier(HighlightingModifier::Static);
+ if (isAbstract(Decl))
+ Tok.addModifier(HighlightingModifier::Abstract);
+ if (Decl->isDeprecated())
+ Tok.addModifier(HighlightingModifier::Deprecated);
+ if (R.IsDecl)
+ Tok.addModifier(HighlightingModifier::Declaration);
+ }
});
// Add highlightings for macro references.
- for (const auto &SIDToRefs : AST.getMacros().MacroRefs) {
+ auto AddMacro = [&](const MacroOccurrence &M) {
+ auto &T = Builder.addToken(M.Rng, HighlightingKind::Macro);
+ if (M.IsDefinition)
+ T.addModifier(HighlightingModifier::Declaration);
+ };
+ for (const auto &SIDToRefs : AST.getMacros().MacroRefs)
for (const auto &M : SIDToRefs.second)
- Builder.addToken({HighlightingKind::Macro, M.Rng});
- }
+ AddMacro(M);
for (const auto &M : AST.getMacros().UnknownMacros)
- Builder.addToken({HighlightingKind::Macro, M.Rng});
+ AddMacro(M);
return std::move(Builder).collect(AST);
}
@@ -450,6 +536,14 @@
}
llvm_unreachable("invalid HighlightingKind");
}
+llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingModifier K) {
+ switch (K) {
+ case HighlightingModifier::Declaration:
+ return OS << "decl"; // abbrevation for common case
+ default:
+ return OS << toSemanticTokenModifier(K);
+ }
+}
std::vector<LineHighlightings>
diffHighlightings(ArrayRef<HighlightingToken> New,
@@ -510,10 +604,12 @@
}
bool operator==(const HighlightingToken &L, const HighlightingToken &R) {
- return std::tie(L.R, L.Kind) == std::tie(R.R, R.Kind);
+ return std::tie(L.R, L.Kind, L.Modifiers) ==
+ std::tie(R.R, R.Kind, R.Modifiers);
}
bool operator<(const HighlightingToken &L, const HighlightingToken &R) {
- return std::tie(L.R, L.Kind) < std::tie(R.R, R.Kind);
+ return std::tie(L.R, L.Kind, R.Modifiers) <
+ std::tie(R.R, R.Kind, R.Modifiers);
}
bool operator==(const LineHighlightings &L, const LineHighlightings &R) {
return std::tie(L.Line, L.Tokens) == std::tie(R.Line, R.Tokens);
@@ -544,6 +640,7 @@
assert(Tok.R.end.line == Tok.R.start.line);
Out.length = Tok.R.end.character - Tok.R.start.character;
Out.tokenType = static_cast<unsigned>(Tok.Kind);
+ Out.tokenModifiers = Tok.Modifiers;
Last = &Tok;
}
@@ -575,7 +672,6 @@
case HighlightingKind::Typedef:
return "type";
case HighlightingKind::DependentType:
- return "dependent"; // nonstandard
case HighlightingKind::DependentName:
return "dependent"; // nonstandard
case HighlightingKind::Namespace:
@@ -594,6 +690,23 @@
llvm_unreachable("unhandled HighlightingKind");
}
+llvm::StringRef toSemanticTokenModifier(HighlightingModifier Modifier) {
+ switch (Modifier) {
+ case HighlightingModifier::Declaration:
+ return "declaration";
+ case HighlightingModifier::Deprecated:
+ return "deprecated";
+ case HighlightingModifier::Readonly:
+ return "readonly";
+ case HighlightingModifier::Static:
+ return "static";
+ case HighlightingModifier::Deduced:
+ return "deduced"; // nonstandard
+ case HighlightingModifier::Abstract:
+ return "abstract";
+ }
+}
+
std::vector<TheiaSemanticHighlightingInformation>
toTheiaSemanticHighlightingInformation(
llvm::ArrayRef<LineHighlightings> Tokens) {
Index: clang-tools-extra/clangd/ClangdLSPServer.cpp
===================================================================
--- clang-tools-extra/clangd/ClangdLSPServer.cpp
+++ clang-tools-extra/clangd/ClangdLSPServer.cpp
@@ -490,6 +490,15 @@
return Types;
}
+static std::vector<llvm::StringRef> semanticTokenModifiers() {
+ std::vector<llvm::StringRef> Modifiers;
+ for (unsigned I = 0;
+ I <= static_cast<unsigned>(HighlightingModifier::LastModifier); ++I)
+ Modifiers.push_back(
+ toSemanticTokenModifier(static_cast<HighlightingModifier>(I)));
+ return Modifiers;
+}
+
void ClangdLSPServer::onInitialize(const InitializeParams &Params,
Callback<llvm::json::Value> Reply) {
// Determine character encoding first as it affects constructed ClangdServer.
@@ -623,8 +632,9 @@
{"full", llvm::json::Object{{"delta", true}}},
{"range", false},
{"legend",
- llvm::json::Object{{"tokenTypes", semanticTokenTypes()},
- {"tokenModifiers", llvm::json::Array()}}},
+ llvm::json::Object{
+ {"tokenTypes", semanticTokenTypes()},
+ {"tokenModifiers", semanticTokenModifiers()}}},
}},
{"signatureHelpProvider",
llvm::json::Object{
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
