https://github.com/inbelic created https://github.com/llvm/llvm-project/pull/122981
- Define required tokens to parse a Descriptor Table in TokenKinds.def - Implements a Lexer to handle all of the defined tokens in ParseHLSLRootSignature This is an initial part of #120472 >From c1fc823abf07dfb8326b6190672d9881890bbb15 Mon Sep 17 00:00:00 2001 From: Finn Plummer <canadienf...@gmail.com> Date: Tue, 14 Jan 2025 22:22:45 +0000 Subject: [PATCH] [HLSL][RootSignature] Implement Lexing of DescriptorTables - Define required tokens to parse a Descriptor Table in TokenKinds.def - Implements a Lexer to handle all of the defined tokens in ParseHLSLRootSignature --- .../Parse/HLSLRootSignatureTokenKinds.def | 121 ++++++++++++++ .../clang/Parse/ParseHLSLRootSignature.h | 96 +++++++++++ clang/lib/Parse/CMakeLists.txt | 1 + clang/lib/Parse/ParseHLSLRootSignature.cpp | 151 ++++++++++++++++++ clang/unittests/CMakeLists.txt | 1 + clang/unittests/Parse/CMakeLists.txt | 26 +++ .../Parse/ParseHLSLRootSignatureTest.cpp | 130 +++++++++++++++ 7 files changed, 526 insertions(+) create mode 100644 clang/include/clang/Parse/HLSLRootSignatureTokenKinds.def create mode 100644 clang/include/clang/Parse/ParseHLSLRootSignature.h create mode 100644 clang/lib/Parse/ParseHLSLRootSignature.cpp create mode 100644 clang/unittests/Parse/CMakeLists.txt create mode 100644 clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp diff --git a/clang/include/clang/Parse/HLSLRootSignatureTokenKinds.def b/clang/include/clang/Parse/HLSLRootSignatureTokenKinds.def new file mode 100644 index 00000000000000..5e47af8d4b1364 --- /dev/null +++ b/clang/include/clang/Parse/HLSLRootSignatureTokenKinds.def @@ -0,0 +1,121 @@ +//===--- HLSLRootSignature.def - Tokens and Enum Database -------*- C++ -*-===// + +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the TokenKinds used in the Root Signature DSL. This +// includes keywords, enums and a small subset of punctuators. Users of this +// file must optionally #define the TOK, KEYWORD, ENUM or specific ENUM macros +// to make use of this file. +// +//===----------------------------------------------------------------------===// + +#ifndef TOK +#define TOK(X) +#endif +#ifndef PUNCTUATOR +#define PUNCTUATOR(X,Y) TOK(pu_ ## X) +#endif +#ifndef KEYWORD +#define KEYWORD(X) TOK(kw_ ## X) +#endif +#ifndef ENUM +#define ENUM(NAME, LIT) TOK(en_ ## NAME) +#endif + +// Defines the various types of enum +#ifndef DESCRIPTOR_RANGE_OFFSET_ENUM +#define DESCRIPTOR_RANGE_OFFSET_ENUM(NAME, LIT) ENUM(NAME, LIT) +#endif +#ifndef ROOT_DESCRIPTOR_FLAG_ENUM +#define ROOT_DESCRIPTOR_FLAG_ENUM(NAME, LIT) ENUM(NAME, LIT) +#endif +// Note: ON denotes that the flag is unique from the above Root Descriptor +// Flags. This is required to avoid token kind enum conflicts. +#ifndef DESCRIPTOR_RANGE_FLAG_ENUM_OFF +#define DESCRIPTOR_RANGE_FLAG_ENUM_OFF(NAME, LIT) +#endif +#ifndef DESCRIPTOR_RANGE_FLAG_ENUM_ON +#define DESCRIPTOR_RANGE_FLAG_ENUM_ON(NAME, LIT) ENUM(NAME, LIT) +#endif +#ifndef DESCRIPTOR_RANGE_FLAG_ENUM +#define DESCRIPTOR_RANGE_FLAG_ENUM(NAME, LIT, ON) DESCRIPTOR_RANGE_FLAG_ENUM_##ON(NAME, LIT) +#endif +#ifndef SHADER_VISIBILITY_ENUM +#define SHADER_VISIBILITY_ENUM(NAME, LIT) ENUM(NAME, LIT) +#endif + +// General Tokens: +TOK(invalid) +TOK(int_literal) + +// Register Tokens: +TOK(bReg) +TOK(tReg) +TOK(uReg) +TOK(sReg) + +// Punctuators: +PUNCTUATOR(l_paren, '(') +PUNCTUATOR(r_paren, ')') +PUNCTUATOR(comma, ',') +PUNCTUATOR(or, '|') +PUNCTUATOR(equal, '=') + +// RootElement Keywords: +KEYWORD(DescriptorTable) + +// DescriptorTable Keywords: +KEYWORD(CBV) +KEYWORD(SRV) +KEYWORD(UAV) +KEYWORD(Sampler) + +// General Parameter Keywords: +KEYWORD(space) +KEYWORD(visibility) +KEYWORD(flags) + +// View Parameter Keywords: +KEYWORD(numDescriptors) +KEYWORD(offset) + +// Descriptor Range Offset Enum: +DESCRIPTOR_RANGE_OFFSET_ENUM(DescriptorRangeOffsetAppend, "DESCRIPTOR_RANGE_OFFSET_APPEND") + +// Root Descriptor Flag Enums: +ROOT_DESCRIPTOR_FLAG_ENUM(DataVolatile, "DATA_VOLATILE") +ROOT_DESCRIPTOR_FLAG_ENUM(DataStaticWhileSetAtExecute, "DATA_STATIC_WHILE_SET_AT_EXECUTE") +ROOT_DESCRIPTOR_FLAG_ENUM(DataStatic, "DATA_STATIC") + +// Descriptor Range Flag Enums: +DESCRIPTOR_RANGE_FLAG_ENUM(DescriptorsVolatile, "DESCRIPTORS_VOLATILE", ON) +DESCRIPTOR_RANGE_FLAG_ENUM(DataVolatile, "DATA_VOLATILE", OFF) +DESCRIPTOR_RANGE_FLAG_ENUM(DataStaticWhileSetAtExecute, "DATA_STATIC_WHILE_SET_AT_EXECUTE", OFF) +DESCRIPTOR_RANGE_FLAG_ENUM(DataStatic, "DATA_STATIC", OFF) +DESCRIPTOR_RANGE_FLAG_ENUM(DescriptorsStaticKeepingBufferBoundsChecks, "DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS", ON) + +// Shader Visibiliy Enums: +SHADER_VISIBILITY_ENUM(All, "SHADER_VISIBILITY_ALL") +SHADER_VISIBILITY_ENUM(Vertex, "SHADER_VISIBILITY_VERTEX") +SHADER_VISIBILITY_ENUM(Hull, "SHADER_VISIBILITY_HULL") +SHADER_VISIBILITY_ENUM(Domain, "SHADER_VISIBILITY_DOMAIN") +SHADER_VISIBILITY_ENUM(Geometry, "SHADER_VISIBILITY_GEOMETRY") +SHADER_VISIBILITY_ENUM(Pixel, "SHADER_VISIBILITY_PIXEL") +SHADER_VISIBILITY_ENUM(Amplification, "SHADER_VISIBILITY_AMPLIFICATION") +SHADER_VISIBILITY_ENUM(Mesh, "SHADER_VISIBILITY_MESH") + +#undef SHADER_VISIBILITY_ENUM +#undef DESCRIPTOR_RANGE_FLAG_ENUM +#undef DESCRIPTOR_RANGE_FLAG_ENUM_OFF +#undef DESCRIPTOR_RANGE_FLAG_ENUM_ON +#undef ROOT_DESCRIPTOR_FLAG_ENUM +#undef DESCRIPTOR_RANGE_OFFSET_ENUM +#undef ENUM +#undef KEYWORD +#undef PUNCTUATOR +#undef TOK diff --git a/clang/include/clang/Parse/ParseHLSLRootSignature.h b/clang/include/clang/Parse/ParseHLSLRootSignature.h new file mode 100644 index 00000000000000..6c534411e754a0 --- /dev/null +++ b/clang/include/clang/Parse/ParseHLSLRootSignature.h @@ -0,0 +1,96 @@ +//===--- ParseHLSLRootSignature.h -------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the ParseHLSLRootSignature interface. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_PARSE_PARSEHLSLROOTSIGNATURE_H +#define LLVM_CLANG_PARSE_PARSEHLSLROOTSIGNATURE_H + +#include "clang/Lex/LiteralSupport.h" +#include "clang/Lex/Preprocessor.h" + +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSwitch.h" + +namespace llvm { +namespace hlsl { +namespace root_signature { + +struct RootSignatureToken { + enum Kind { +#define TOK(X) X, +#include "clang/Parse/HLSLRootSignatureTokenKinds.def" + }; + + Kind Kind = Kind::invalid; + + // Retain the SouceLocation of the token for diagnostics + clang::SourceLocation TokLoc; + + // Retain if the uint32_t bits represent a signed integer + bool Signed = false; + union { + uint32_t IntLiteral = 0; + float FloatLiteral; + }; + + // Constructors + RootSignatureToken() {} + RootSignatureToken(clang::SourceLocation TokLoc) : TokLoc(TokLoc) {} +}; +using TokenKind = enum RootSignatureToken::Kind; + +class RootSignatureLexer { +public: + RootSignatureLexer(StringRef Signature, clang::SourceLocation SourceLoc, + clang::Preprocessor &PP) + : Buffer(Signature), SourceLoc(SourceLoc), PP(PP) {} + + // Consumes the internal buffer as a list of tokens and will emplace them + // onto the given tokens. + // + // It will consume until it successfully reaches the end of the buffer, + // or, until the first error is encountered. The return value denotes if + // there was a failure. + bool Lex(SmallVector<RootSignatureToken> &Tokens); + + // Get the current source location of the lexer + clang::SourceLocation GetLocation() { return SourceLoc; }; + +private: + // Internal buffer to iterate over + StringRef Buffer; + + // Passed down parameters from Sema + clang::SourceLocation SourceLoc; + clang::Preprocessor &PP; + + bool LexNumber(RootSignatureToken &Result); + + // Consumes the internal buffer for a single token. + // + // The return value denotes if there was a failure. + bool LexToken(RootSignatureToken &Token); + + // Advance the buffer by the specified number of characters. Updates the + // SourceLocation appropriately. + void AdvanceBuffer(unsigned NumCharacters = 1) { + Buffer = Buffer.drop_front(NumCharacters); + SourceLoc = SourceLoc.getLocWithOffset(NumCharacters); + } +}; + +} // namespace root_signature +} // namespace hlsl +} // namespace llvm + +#endif // LLVM_CLANG_PARSE_PARSEHLSLROOTSIGNATURE_H diff --git a/clang/lib/Parse/CMakeLists.txt b/clang/lib/Parse/CMakeLists.txt index 22e902f7e1bc50..00fde537bb9c60 100644 --- a/clang/lib/Parse/CMakeLists.txt +++ b/clang/lib/Parse/CMakeLists.txt @@ -14,6 +14,7 @@ add_clang_library(clangParse ParseExpr.cpp ParseExprCXX.cpp ParseHLSL.cpp + ParseHLSLRootSignature.cpp ParseInit.cpp ParseObjc.cpp ParseOpenMP.cpp diff --git a/clang/lib/Parse/ParseHLSLRootSignature.cpp b/clang/lib/Parse/ParseHLSLRootSignature.cpp new file mode 100644 index 00000000000000..491e4cafa04b3c --- /dev/null +++ b/clang/lib/Parse/ParseHLSLRootSignature.cpp @@ -0,0 +1,151 @@ +#include "clang/Parse/ParseHLSLRootSignature.h" + +namespace llvm { +namespace hlsl { +namespace root_signature { + +// Lexer Definitions + +static bool IsPreprocessorNumberChar(char C) { + // TODO: extend for float support with or without hexadecimal/exponent + return isdigit(C); // integer support +} + +bool RootSignatureLexer::LexNumber(RootSignatureToken &Result) { + // NumericLiteralParser does not handle the sign so we will manually apply it + Result.Signed = Buffer.front() == '-'; + if (Result.Signed) + AdvanceBuffer(); + + // Retrieve the possible number + StringRef NumSpelling = Buffer.take_while(IsPreprocessorNumberChar); + + // Parse the numeric value and so semantic checks on its specification + clang::NumericLiteralParser Literal(NumSpelling, SourceLoc, + PP.getSourceManager(), PP.getLangOpts(), + PP.getTargetInfo(), PP.getDiagnostics()); + if (Literal.hadError) + return true; // Error has already been reported so just return + + // Retrieve the number value to store into the token + if (Literal.isIntegerLiteral()) { + Result.Kind = TokenKind::int_literal; + + APSInt X = APSInt(32, Result.Signed); + if (Literal.GetIntegerValue(X)) + return true; // TODO: Report overflow error + + X = Result.Signed ? -X : X; + Result.IntLiteral = (uint32_t)X.getZExtValue(); + } else { + return true; // TODO: report unsupported number literal specification + } + + AdvanceBuffer(NumSpelling.size()); + return false; +} + +bool RootSignatureLexer::Lex(SmallVector<RootSignatureToken> &Tokens) { + // Discard any leading whitespace + AdvanceBuffer(Buffer.take_while(isspace).size()); + + while (!Buffer.empty()) { + RootSignatureToken Result; + if (LexToken(Result)) + return true; + + // Successfully Lexed the token so we can store it + Tokens.push_back(Result); + + // Discard any trailing whitespace + AdvanceBuffer(Buffer.take_while(isspace).size()); + } + + return false; +} + +bool RootSignatureLexer::LexToken(RootSignatureToken &Result) { + // Record where this token is in the text for usage in parser diagnostics + Result.TokLoc = SourceLoc; + + char C = Buffer.front(); + + // Punctuators + switch (C) { +#define PUNCTUATOR(X, Y) \ + case Y: { \ + Result.Kind = TokenKind::pu_##X; \ + AdvanceBuffer(); \ + return false; \ + } +#include "clang/Parse/HLSLRootSignatureTokenKinds.def" + default: + break; + } + + // Numeric constant + if (isdigit(C) || C == '-') + return LexNumber(Result); + + // All following tokens require at least one additional character + if (Buffer.size() <= 1) + return true; // TODO: Report invalid token error + + // Peek at the next character to deteremine token type + char NextC = Buffer[1]; + + // Registers: [tsub][0-9+] + if ((C == 't' || C == 's' || C == 'u' || C == 'b') && isdigit(NextC)) { + AdvanceBuffer(); + + if (LexNumber(Result)) + return true; + + // Lex number could also parse a float so ensure it was an unsigned int + if (Result.Kind != TokenKind::int_literal || Result.Signed) + return true; // Return invalid number literal for register error + + // Convert character to the register type. + // This is done after LexNumber to override the TokenKind + switch (C) { + case 'b': + Result.Kind = TokenKind::bReg; + break; + case 't': + Result.Kind = TokenKind::tReg; + break; + case 'u': + Result.Kind = TokenKind::uReg; + break; + case 's': + Result.Kind = TokenKind::sReg; + break; + default: + llvm_unreachable("Switch for an expected token was not provided"); + return true; + } + return false; + } + + // Keywords and Enums: + StringRef TokSpelling = + Buffer.take_while([](char C) { return isalnum(C) || C == '_'; }); + + // Define a large string switch statement for all the keywords and enums + auto Switch = llvm::StringSwitch<TokenKind>(TokSpelling); +#define KEYWORD(NAME) Switch.Case(#NAME, TokenKind::kw_##NAME); +#define ENUM(NAME, LIT) Switch.CaseLower(LIT, TokenKind::en_##NAME); +#include "clang/Parse/HLSLRootSignatureTokenKinds.def" + + // Then attempt to retreive a string from it + auto Kind = Switch.Default(TokenKind::invalid); + if (Kind == TokenKind::invalid) + return true; // TODO: Report invalid identifier + + Result.Kind = Kind; + AdvanceBuffer(TokSpelling.size()); + return false; +} + +} // namespace root_signature +} // namespace hlsl diff --git a/clang/unittests/CMakeLists.txt b/clang/unittests/CMakeLists.txt index 85d265426ec80b..9b3ce8aa7de739 100644 --- a/clang/unittests/CMakeLists.txt +++ b/clang/unittests/CMakeLists.txt @@ -25,6 +25,7 @@ endfunction() add_subdirectory(Basic) add_subdirectory(Lex) +add_subdirectory(Parse) add_subdirectory(Driver) if(CLANG_ENABLE_STATIC_ANALYZER) add_subdirectory(Analysis) diff --git a/clang/unittests/Parse/CMakeLists.txt b/clang/unittests/Parse/CMakeLists.txt new file mode 100644 index 00000000000000..1b7eb4934a46c8 --- /dev/null +++ b/clang/unittests/Parse/CMakeLists.txt @@ -0,0 +1,26 @@ +set(LLVM_LINK_COMPONENTS + Support + ) + +add_clang_unittest(ParseTests + ParseHLSLRootSignatureTest.cpp + ) + +clang_target_link_libraries(ParseTests + PRIVATE + clangAST + clangASTMatchers + clangBasic + clangFrontend + clangParse + clangSema + clangSerialization + clangTooling + ) + +target_link_libraries(ParseTests + PRIVATE + LLVMTestingAnnotations + LLVMTestingSupport + clangTesting + ) diff --git a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp new file mode 100644 index 00000000000000..c430b0657dacf5 --- /dev/null +++ b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp @@ -0,0 +1,130 @@ +//=== ParseHLSLRootSignatureTest.cpp - Parse Root Signature tests ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/Basic/Diagnostic.h" +#include "clang/Basic/DiagnosticOptions.h" +#include "clang/Basic/FileManager.h" +#include "clang/Basic/LangOptions.h" +#include "clang/Basic/SourceLocation.h" +#include "clang/Basic/SourceManager.h" +#include "clang/Basic/TargetInfo.h" +#include "clang/Lex/HeaderSearch.h" +#include "clang/Lex/HeaderSearchOptions.h" +#include "clang/Lex/Lexer.h" +#include "clang/Lex/ModuleLoader.h" +#include "clang/Lex/Preprocessor.h" +#include "clang/Lex/PreprocessorOptions.h" + +#include "clang/Parse/ParseHLSLRootSignature.h" +#include "gtest/gtest.h" + +using namespace llvm::hlsl::root_signature; +using namespace clang; + +namespace { + +// The test fixture. +class ParseHLSLRootSignatureTest : public ::testing::Test { +protected: + ParseHLSLRootSignatureTest() + : FileMgr(FileMgrOpts), DiagID(new DiagnosticIDs()), + Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()), + SourceMgr(Diags, FileMgr), TargetOpts(new TargetOptions) { + TargetOpts->Triple = "x86_64-apple-darwin11.1.0"; + Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts); + } + + Preprocessor *CreatePP(StringRef Source, TrivialModuleLoader &ModLoader) { + std::unique_ptr<llvm::MemoryBuffer> Buf = + llvm::MemoryBuffer::getMemBuffer(Source); + SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf))); + + HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr, + Diags, LangOpts, Target.get()); + Preprocessor *PP = + new Preprocessor(std::make_shared<PreprocessorOptions>(), Diags, + LangOpts, SourceMgr, HeaderInfo, ModLoader, + /*IILookup =*/nullptr, + /*OwnsHeaderSearch =*/false); + PP->Initialize(*Target); + PP->EnterMainSourceFile(); + return PP; + } + + void CheckTokens(SmallVector<RootSignatureToken> &Computed, + SmallVector<TokenKind> &Expected) { + ASSERT_EQ(Computed.size(), Expected.size()); + for (unsigned I = 0, E = Expected.size(); I != E; ++I) { + ASSERT_EQ(Computed[I].Kind, Expected[I]); + } + } + + FileSystemOptions FileMgrOpts; + FileManager FileMgr; + IntrusiveRefCntPtr<DiagnosticIDs> DiagID; + DiagnosticsEngine Diags; + SourceManager SourceMgr; + LangOptions LangOpts; + std::shared_ptr<TargetOptions> TargetOpts; + IntrusiveRefCntPtr<TargetInfo> Target; +}; + +TEST_F(ParseHLSLRootSignatureTest, LexValidTokensTest) { + const llvm::StringLiteral Source = R"cc( + -42 + + b0 t43 u987 s234 + + (),|= + + DescriptorTable + + CBV SRV UAV Sampler + space visibility flags + numDescriptors offset + + DESCRIPTOR_RANGE_OFFSET_APPEND + + DATA_VOLATILE + DATA_STATIC_WHILE_SET_AT_EXECUTE + DATA_STATIC + DESCRIPTORS_VOLATILE + DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS + + shader_visibility_all + shader_visibility_vertex + shader_visibility_hull + shader_visibility_domain + shader_visibility_geometry + shader_visibility_pixel + shader_visibility_amplification + shader_visibility_mesh + )cc"; + + TrivialModuleLoader ModLoader; + Preprocessor *PP = CreatePP(Source, ModLoader); + auto TokLoc = SourceLocation(); + + RootSignatureLexer Lexer(Source, TokLoc, *PP); + + SmallVector<RootSignatureToken> Tokens = { + RootSignatureToken() // invalid token for completeness + }; + ASSERT_FALSE(Lexer.Lex(Tokens)); + + SmallVector<TokenKind> Expected = { +#define TOK(NAME) TokenKind::NAME, +#include "clang/Parse/HLSLRootSignatureTokenKinds.def" + }; + + CheckTokens(Tokens, Expected); + + delete PP; +} + +} // anonymous namespace _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
