https://github.com/inbelic updated https://github.com/llvm/llvm-project/pull/122981
>From c1fc823abf07dfb8326b6190672d9881890bbb15 Mon Sep 17 00:00:00 2001 From: Finn Plummer <canadienf...@gmail.com> Date: Tue, 14 Jan 2025 22:22:45 +0000 Subject: [PATCH 1/7] [HLSL][RootSignature] Implement Lexing of DescriptorTables - Define required tokens to parse a Descriptor Table in TokenKinds.def - Implements a Lexer to handle all of the defined tokens in ParseHLSLRootSignature --- .../Parse/HLSLRootSignatureTokenKinds.def | 121 ++++++++++++++ .../clang/Parse/ParseHLSLRootSignature.h | 96 +++++++++++ clang/lib/Parse/CMakeLists.txt | 1 + clang/lib/Parse/ParseHLSLRootSignature.cpp | 151 ++++++++++++++++++ clang/unittests/CMakeLists.txt | 1 + clang/unittests/Parse/CMakeLists.txt | 26 +++ .../Parse/ParseHLSLRootSignatureTest.cpp | 130 +++++++++++++++ 7 files changed, 526 insertions(+) create mode 100644 clang/include/clang/Parse/HLSLRootSignatureTokenKinds.def create mode 100644 clang/include/clang/Parse/ParseHLSLRootSignature.h create mode 100644 clang/lib/Parse/ParseHLSLRootSignature.cpp create mode 100644 clang/unittests/Parse/CMakeLists.txt create mode 100644 clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp diff --git a/clang/include/clang/Parse/HLSLRootSignatureTokenKinds.def b/clang/include/clang/Parse/HLSLRootSignatureTokenKinds.def new file mode 100644 index 00000000000000..5e47af8d4b1364 --- /dev/null +++ b/clang/include/clang/Parse/HLSLRootSignatureTokenKinds.def @@ -0,0 +1,121 @@ +//===--- HLSLRootSignature.def - Tokens and Enum Database -------*- C++ -*-===// + +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the TokenKinds used in the Root Signature DSL. This +// includes keywords, enums and a small subset of punctuators. Users of this +// file must optionally #define the TOK, KEYWORD, ENUM or specific ENUM macros +// to make use of this file. +// +//===----------------------------------------------------------------------===// + +#ifndef TOK +#define TOK(X) +#endif +#ifndef PUNCTUATOR +#define PUNCTUATOR(X,Y) TOK(pu_ ## X) +#endif +#ifndef KEYWORD +#define KEYWORD(X) TOK(kw_ ## X) +#endif +#ifndef ENUM +#define ENUM(NAME, LIT) TOK(en_ ## NAME) +#endif + +// Defines the various types of enum +#ifndef DESCRIPTOR_RANGE_OFFSET_ENUM +#define DESCRIPTOR_RANGE_OFFSET_ENUM(NAME, LIT) ENUM(NAME, LIT) +#endif +#ifndef ROOT_DESCRIPTOR_FLAG_ENUM +#define ROOT_DESCRIPTOR_FLAG_ENUM(NAME, LIT) ENUM(NAME, LIT) +#endif +// Note: ON denotes that the flag is unique from the above Root Descriptor +// Flags. This is required to avoid token kind enum conflicts. +#ifndef DESCRIPTOR_RANGE_FLAG_ENUM_OFF +#define DESCRIPTOR_RANGE_FLAG_ENUM_OFF(NAME, LIT) +#endif +#ifndef DESCRIPTOR_RANGE_FLAG_ENUM_ON +#define DESCRIPTOR_RANGE_FLAG_ENUM_ON(NAME, LIT) ENUM(NAME, LIT) +#endif +#ifndef DESCRIPTOR_RANGE_FLAG_ENUM +#define DESCRIPTOR_RANGE_FLAG_ENUM(NAME, LIT, ON) DESCRIPTOR_RANGE_FLAG_ENUM_##ON(NAME, LIT) +#endif +#ifndef SHADER_VISIBILITY_ENUM +#define SHADER_VISIBILITY_ENUM(NAME, LIT) ENUM(NAME, LIT) +#endif + +// General Tokens: +TOK(invalid) +TOK(int_literal) + +// Register Tokens: +TOK(bReg) +TOK(tReg) +TOK(uReg) +TOK(sReg) + +// Punctuators: +PUNCTUATOR(l_paren, '(') +PUNCTUATOR(r_paren, ')') +PUNCTUATOR(comma, ',') +PUNCTUATOR(or, '|') +PUNCTUATOR(equal, '=') + +// RootElement Keywords: +KEYWORD(DescriptorTable) + +// DescriptorTable Keywords: +KEYWORD(CBV) +KEYWORD(SRV) +KEYWORD(UAV) +KEYWORD(Sampler) + +// General Parameter Keywords: +KEYWORD(space) +KEYWORD(visibility) +KEYWORD(flags) + +// View Parameter Keywords: +KEYWORD(numDescriptors) +KEYWORD(offset) + +// Descriptor Range Offset Enum: +DESCRIPTOR_RANGE_OFFSET_ENUM(DescriptorRangeOffsetAppend, "DESCRIPTOR_RANGE_OFFSET_APPEND") + +// Root Descriptor Flag Enums: +ROOT_DESCRIPTOR_FLAG_ENUM(DataVolatile, "DATA_VOLATILE") +ROOT_DESCRIPTOR_FLAG_ENUM(DataStaticWhileSetAtExecute, "DATA_STATIC_WHILE_SET_AT_EXECUTE") +ROOT_DESCRIPTOR_FLAG_ENUM(DataStatic, "DATA_STATIC") + +// Descriptor Range Flag Enums: +DESCRIPTOR_RANGE_FLAG_ENUM(DescriptorsVolatile, "DESCRIPTORS_VOLATILE", ON) +DESCRIPTOR_RANGE_FLAG_ENUM(DataVolatile, "DATA_VOLATILE", OFF) +DESCRIPTOR_RANGE_FLAG_ENUM(DataStaticWhileSetAtExecute, "DATA_STATIC_WHILE_SET_AT_EXECUTE", OFF) +DESCRIPTOR_RANGE_FLAG_ENUM(DataStatic, "DATA_STATIC", OFF) +DESCRIPTOR_RANGE_FLAG_ENUM(DescriptorsStaticKeepingBufferBoundsChecks, "DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS", ON) + +// Shader Visibiliy Enums: +SHADER_VISIBILITY_ENUM(All, "SHADER_VISIBILITY_ALL") +SHADER_VISIBILITY_ENUM(Vertex, "SHADER_VISIBILITY_VERTEX") +SHADER_VISIBILITY_ENUM(Hull, "SHADER_VISIBILITY_HULL") +SHADER_VISIBILITY_ENUM(Domain, "SHADER_VISIBILITY_DOMAIN") +SHADER_VISIBILITY_ENUM(Geometry, "SHADER_VISIBILITY_GEOMETRY") +SHADER_VISIBILITY_ENUM(Pixel, "SHADER_VISIBILITY_PIXEL") +SHADER_VISIBILITY_ENUM(Amplification, "SHADER_VISIBILITY_AMPLIFICATION") +SHADER_VISIBILITY_ENUM(Mesh, "SHADER_VISIBILITY_MESH") + +#undef SHADER_VISIBILITY_ENUM +#undef DESCRIPTOR_RANGE_FLAG_ENUM +#undef DESCRIPTOR_RANGE_FLAG_ENUM_OFF +#undef DESCRIPTOR_RANGE_FLAG_ENUM_ON +#undef ROOT_DESCRIPTOR_FLAG_ENUM +#undef DESCRIPTOR_RANGE_OFFSET_ENUM +#undef ENUM +#undef KEYWORD +#undef PUNCTUATOR +#undef TOK diff --git a/clang/include/clang/Parse/ParseHLSLRootSignature.h b/clang/include/clang/Parse/ParseHLSLRootSignature.h new file mode 100644 index 00000000000000..6c534411e754a0 --- /dev/null +++ b/clang/include/clang/Parse/ParseHLSLRootSignature.h @@ -0,0 +1,96 @@ +//===--- ParseHLSLRootSignature.h -------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the ParseHLSLRootSignature interface. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_PARSE_PARSEHLSLROOTSIGNATURE_H +#define LLVM_CLANG_PARSE_PARSEHLSLROOTSIGNATURE_H + +#include "clang/Lex/LiteralSupport.h" +#include "clang/Lex/Preprocessor.h" + +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSwitch.h" + +namespace llvm { +namespace hlsl { +namespace root_signature { + +struct RootSignatureToken { + enum Kind { +#define TOK(X) X, +#include "clang/Parse/HLSLRootSignatureTokenKinds.def" + }; + + Kind Kind = Kind::invalid; + + // Retain the SouceLocation of the token for diagnostics + clang::SourceLocation TokLoc; + + // Retain if the uint32_t bits represent a signed integer + bool Signed = false; + union { + uint32_t IntLiteral = 0; + float FloatLiteral; + }; + + // Constructors + RootSignatureToken() {} + RootSignatureToken(clang::SourceLocation TokLoc) : TokLoc(TokLoc) {} +}; +using TokenKind = enum RootSignatureToken::Kind; + +class RootSignatureLexer { +public: + RootSignatureLexer(StringRef Signature, clang::SourceLocation SourceLoc, + clang::Preprocessor &PP) + : Buffer(Signature), SourceLoc(SourceLoc), PP(PP) {} + + // Consumes the internal buffer as a list of tokens and will emplace them + // onto the given tokens. + // + // It will consume until it successfully reaches the end of the buffer, + // or, until the first error is encountered. The return value denotes if + // there was a failure. + bool Lex(SmallVector<RootSignatureToken> &Tokens); + + // Get the current source location of the lexer + clang::SourceLocation GetLocation() { return SourceLoc; }; + +private: + // Internal buffer to iterate over + StringRef Buffer; + + // Passed down parameters from Sema + clang::SourceLocation SourceLoc; + clang::Preprocessor &PP; + + bool LexNumber(RootSignatureToken &Result); + + // Consumes the internal buffer for a single token. + // + // The return value denotes if there was a failure. + bool LexToken(RootSignatureToken &Token); + + // Advance the buffer by the specified number of characters. Updates the + // SourceLocation appropriately. + void AdvanceBuffer(unsigned NumCharacters = 1) { + Buffer = Buffer.drop_front(NumCharacters); + SourceLoc = SourceLoc.getLocWithOffset(NumCharacters); + } +}; + +} // namespace root_signature +} // namespace hlsl +} // namespace llvm + +#endif // LLVM_CLANG_PARSE_PARSEHLSLROOTSIGNATURE_H diff --git a/clang/lib/Parse/CMakeLists.txt b/clang/lib/Parse/CMakeLists.txt index 22e902f7e1bc50..00fde537bb9c60 100644 --- a/clang/lib/Parse/CMakeLists.txt +++ b/clang/lib/Parse/CMakeLists.txt @@ -14,6 +14,7 @@ add_clang_library(clangParse ParseExpr.cpp ParseExprCXX.cpp ParseHLSL.cpp + ParseHLSLRootSignature.cpp ParseInit.cpp ParseObjc.cpp ParseOpenMP.cpp diff --git a/clang/lib/Parse/ParseHLSLRootSignature.cpp b/clang/lib/Parse/ParseHLSLRootSignature.cpp new file mode 100644 index 00000000000000..491e4cafa04b3c --- /dev/null +++ b/clang/lib/Parse/ParseHLSLRootSignature.cpp @@ -0,0 +1,151 @@ +#include "clang/Parse/ParseHLSLRootSignature.h" + +namespace llvm { +namespace hlsl { +namespace root_signature { + +// Lexer Definitions + +static bool IsPreprocessorNumberChar(char C) { + // TODO: extend for float support with or without hexadecimal/exponent + return isdigit(C); // integer support +} + +bool RootSignatureLexer::LexNumber(RootSignatureToken &Result) { + // NumericLiteralParser does not handle the sign so we will manually apply it + Result.Signed = Buffer.front() == '-'; + if (Result.Signed) + AdvanceBuffer(); + + // Retrieve the possible number + StringRef NumSpelling = Buffer.take_while(IsPreprocessorNumberChar); + + // Parse the numeric value and so semantic checks on its specification + clang::NumericLiteralParser Literal(NumSpelling, SourceLoc, + PP.getSourceManager(), PP.getLangOpts(), + PP.getTargetInfo(), PP.getDiagnostics()); + if (Literal.hadError) + return true; // Error has already been reported so just return + + // Retrieve the number value to store into the token + if (Literal.isIntegerLiteral()) { + Result.Kind = TokenKind::int_literal; + + APSInt X = APSInt(32, Result.Signed); + if (Literal.GetIntegerValue(X)) + return true; // TODO: Report overflow error + + X = Result.Signed ? -X : X; + Result.IntLiteral = (uint32_t)X.getZExtValue(); + } else { + return true; // TODO: report unsupported number literal specification + } + + AdvanceBuffer(NumSpelling.size()); + return false; +} + +bool RootSignatureLexer::Lex(SmallVector<RootSignatureToken> &Tokens) { + // Discard any leading whitespace + AdvanceBuffer(Buffer.take_while(isspace).size()); + + while (!Buffer.empty()) { + RootSignatureToken Result; + if (LexToken(Result)) + return true; + + // Successfully Lexed the token so we can store it + Tokens.push_back(Result); + + // Discard any trailing whitespace + AdvanceBuffer(Buffer.take_while(isspace).size()); + } + + return false; +} + +bool RootSignatureLexer::LexToken(RootSignatureToken &Result) { + // Record where this token is in the text for usage in parser diagnostics + Result.TokLoc = SourceLoc; + + char C = Buffer.front(); + + // Punctuators + switch (C) { +#define PUNCTUATOR(X, Y) \ + case Y: { \ + Result.Kind = TokenKind::pu_##X; \ + AdvanceBuffer(); \ + return false; \ + } +#include "clang/Parse/HLSLRootSignatureTokenKinds.def" + default: + break; + } + + // Numeric constant + if (isdigit(C) || C == '-') + return LexNumber(Result); + + // All following tokens require at least one additional character + if (Buffer.size() <= 1) + return true; // TODO: Report invalid token error + + // Peek at the next character to deteremine token type + char NextC = Buffer[1]; + + // Registers: [tsub][0-9+] + if ((C == 't' || C == 's' || C == 'u' || C == 'b') && isdigit(NextC)) { + AdvanceBuffer(); + + if (LexNumber(Result)) + return true; + + // Lex number could also parse a float so ensure it was an unsigned int + if (Result.Kind != TokenKind::int_literal || Result.Signed) + return true; // Return invalid number literal for register error + + // Convert character to the register type. + // This is done after LexNumber to override the TokenKind + switch (C) { + case 'b': + Result.Kind = TokenKind::bReg; + break; + case 't': + Result.Kind = TokenKind::tReg; + break; + case 'u': + Result.Kind = TokenKind::uReg; + break; + case 's': + Result.Kind = TokenKind::sReg; + break; + default: + llvm_unreachable("Switch for an expected token was not provided"); + return true; + } + return false; + } + + // Keywords and Enums: + StringRef TokSpelling = + Buffer.take_while([](char C) { return isalnum(C) || C == '_'; }); + + // Define a large string switch statement for all the keywords and enums + auto Switch = llvm::StringSwitch<TokenKind>(TokSpelling); +#define KEYWORD(NAME) Switch.Case(#NAME, TokenKind::kw_##NAME); +#define ENUM(NAME, LIT) Switch.CaseLower(LIT, TokenKind::en_##NAME); +#include "clang/Parse/HLSLRootSignatureTokenKinds.def" + + // Then attempt to retreive a string from it + auto Kind = Switch.Default(TokenKind::invalid); + if (Kind == TokenKind::invalid) + return true; // TODO: Report invalid identifier + + Result.Kind = Kind; + AdvanceBuffer(TokSpelling.size()); + return false; +} + +} // namespace root_signature +} // namespace hlsl diff --git a/clang/unittests/CMakeLists.txt b/clang/unittests/CMakeLists.txt index 85d265426ec80b..9b3ce8aa7de739 100644 --- a/clang/unittests/CMakeLists.txt +++ b/clang/unittests/CMakeLists.txt @@ -25,6 +25,7 @@ endfunction() add_subdirectory(Basic) add_subdirectory(Lex) +add_subdirectory(Parse) add_subdirectory(Driver) if(CLANG_ENABLE_STATIC_ANALYZER) add_subdirectory(Analysis) diff --git a/clang/unittests/Parse/CMakeLists.txt b/clang/unittests/Parse/CMakeLists.txt new file mode 100644 index 00000000000000..1b7eb4934a46c8 --- /dev/null +++ b/clang/unittests/Parse/CMakeLists.txt @@ -0,0 +1,26 @@ +set(LLVM_LINK_COMPONENTS + Support + ) + +add_clang_unittest(ParseTests + ParseHLSLRootSignatureTest.cpp + ) + +clang_target_link_libraries(ParseTests + PRIVATE + clangAST + clangASTMatchers + clangBasic + clangFrontend + clangParse + clangSema + clangSerialization + clangTooling + ) + +target_link_libraries(ParseTests + PRIVATE + LLVMTestingAnnotations + LLVMTestingSupport + clangTesting + ) diff --git a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp new file mode 100644 index 00000000000000..c430b0657dacf5 --- /dev/null +++ b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp @@ -0,0 +1,130 @@ +//=== ParseHLSLRootSignatureTest.cpp - Parse Root Signature tests ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/Basic/Diagnostic.h" +#include "clang/Basic/DiagnosticOptions.h" +#include "clang/Basic/FileManager.h" +#include "clang/Basic/LangOptions.h" +#include "clang/Basic/SourceLocation.h" +#include "clang/Basic/SourceManager.h" +#include "clang/Basic/TargetInfo.h" +#include "clang/Lex/HeaderSearch.h" +#include "clang/Lex/HeaderSearchOptions.h" +#include "clang/Lex/Lexer.h" +#include "clang/Lex/ModuleLoader.h" +#include "clang/Lex/Preprocessor.h" +#include "clang/Lex/PreprocessorOptions.h" + +#include "clang/Parse/ParseHLSLRootSignature.h" +#include "gtest/gtest.h" + +using namespace llvm::hlsl::root_signature; +using namespace clang; + +namespace { + +// The test fixture. +class ParseHLSLRootSignatureTest : public ::testing::Test { +protected: + ParseHLSLRootSignatureTest() + : FileMgr(FileMgrOpts), DiagID(new DiagnosticIDs()), + Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()), + SourceMgr(Diags, FileMgr), TargetOpts(new TargetOptions) { + TargetOpts->Triple = "x86_64-apple-darwin11.1.0"; + Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts); + } + + Preprocessor *CreatePP(StringRef Source, TrivialModuleLoader &ModLoader) { + std::unique_ptr<llvm::MemoryBuffer> Buf = + llvm::MemoryBuffer::getMemBuffer(Source); + SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf))); + + HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr, + Diags, LangOpts, Target.get()); + Preprocessor *PP = + new Preprocessor(std::make_shared<PreprocessorOptions>(), Diags, + LangOpts, SourceMgr, HeaderInfo, ModLoader, + /*IILookup =*/nullptr, + /*OwnsHeaderSearch =*/false); + PP->Initialize(*Target); + PP->EnterMainSourceFile(); + return PP; + } + + void CheckTokens(SmallVector<RootSignatureToken> &Computed, + SmallVector<TokenKind> &Expected) { + ASSERT_EQ(Computed.size(), Expected.size()); + for (unsigned I = 0, E = Expected.size(); I != E; ++I) { + ASSERT_EQ(Computed[I].Kind, Expected[I]); + } + } + + FileSystemOptions FileMgrOpts; + FileManager FileMgr; + IntrusiveRefCntPtr<DiagnosticIDs> DiagID; + DiagnosticsEngine Diags; + SourceManager SourceMgr; + LangOptions LangOpts; + std::shared_ptr<TargetOptions> TargetOpts; + IntrusiveRefCntPtr<TargetInfo> Target; +}; + +TEST_F(ParseHLSLRootSignatureTest, LexValidTokensTest) { + const llvm::StringLiteral Source = R"cc( + -42 + + b0 t43 u987 s234 + + (),|= + + DescriptorTable + + CBV SRV UAV Sampler + space visibility flags + numDescriptors offset + + DESCRIPTOR_RANGE_OFFSET_APPEND + + DATA_VOLATILE + DATA_STATIC_WHILE_SET_AT_EXECUTE + DATA_STATIC + DESCRIPTORS_VOLATILE + DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS + + shader_visibility_all + shader_visibility_vertex + shader_visibility_hull + shader_visibility_domain + shader_visibility_geometry + shader_visibility_pixel + shader_visibility_amplification + shader_visibility_mesh + )cc"; + + TrivialModuleLoader ModLoader; + Preprocessor *PP = CreatePP(Source, ModLoader); + auto TokLoc = SourceLocation(); + + RootSignatureLexer Lexer(Source, TokLoc, *PP); + + SmallVector<RootSignatureToken> Tokens = { + RootSignatureToken() // invalid token for completeness + }; + ASSERT_FALSE(Lexer.Lex(Tokens)); + + SmallVector<TokenKind> Expected = { +#define TOK(NAME) TokenKind::NAME, +#include "clang/Parse/HLSLRootSignatureTokenKinds.def" + }; + + CheckTokens(Tokens, Expected); + + delete PP; +} + +} // anonymous namespace >From 949b50af4c60eae588595260d9384d01324245e5 Mon Sep 17 00:00:00 2001 From: Finn Plummer <canadienf...@gmail.com> Date: Tue, 14 Jan 2025 23:07:46 +0000 Subject: [PATCH 2/7] fix typo when breaking up commits --- clang/lib/Parse/ParseHLSLRootSignature.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/clang/lib/Parse/ParseHLSLRootSignature.cpp b/clang/lib/Parse/ParseHLSLRootSignature.cpp index 491e4cafa04b3c..baa9950163a5c6 100644 --- a/clang/lib/Parse/ParseHLSLRootSignature.cpp +++ b/clang/lib/Parse/ParseHLSLRootSignature.cpp @@ -149,3 +149,4 @@ bool RootSignatureLexer::LexToken(RootSignatureToken &Result) { } // namespace root_signature } // namespace hlsl +} // namespace llvm >From a3286192694fd8599b74690b79f333482289f548 Mon Sep 17 00:00:00 2001 From: Finn Plummer <canadienf...@gmail.com> Date: Thu, 16 Jan 2025 17:10:05 +0000 Subject: [PATCH 3/7] review comments: - fix typo - use std::unique_ptr for testing mem management --- clang/lib/Parse/ParseHLSLRootSignature.cpp | 2 +- .../Parse/ParseHLSLRootSignatureTest.cpp | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/clang/lib/Parse/ParseHLSLRootSignature.cpp b/clang/lib/Parse/ParseHLSLRootSignature.cpp index baa9950163a5c6..9f2154471ec3f1 100644 --- a/clang/lib/Parse/ParseHLSLRootSignature.cpp +++ b/clang/lib/Parse/ParseHLSLRootSignature.cpp @@ -20,7 +20,7 @@ bool RootSignatureLexer::LexNumber(RootSignatureToken &Result) { // Retrieve the possible number StringRef NumSpelling = Buffer.take_while(IsPreprocessorNumberChar); - // Parse the numeric value and so semantic checks on its specification + // Parse the numeric value and do semantic checks on its specification clang::NumericLiteralParser Literal(NumSpelling, SourceLoc, PP.getSourceManager(), PP.getLangOpts(), PP.getTargetInfo(), PP.getDiagnostics()); diff --git a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp index c430b0657dacf5..4fd60488c6ff20 100644 --- a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp +++ b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp @@ -39,18 +39,19 @@ class ParseHLSLRootSignatureTest : public ::testing::Test { Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts); } - Preprocessor *CreatePP(StringRef Source, TrivialModuleLoader &ModLoader) { + std::unique_ptr<Preprocessor> CreatePP(StringRef Source, + TrivialModuleLoader &ModLoader) { std::unique_ptr<llvm::MemoryBuffer> Buf = llvm::MemoryBuffer::getMemBuffer(Source); SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf))); HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr, Diags, LangOpts, Target.get()); - Preprocessor *PP = - new Preprocessor(std::make_shared<PreprocessorOptions>(), Diags, - LangOpts, SourceMgr, HeaderInfo, ModLoader, - /*IILookup =*/nullptr, - /*OwnsHeaderSearch =*/false); + std::unique_ptr<Preprocessor> PP = std::make_unique<Preprocessor>( + std::make_shared<PreprocessorOptions>(), Diags, LangOpts, SourceMgr, + HeaderInfo, ModLoader, + /*IILookup =*/nullptr, + /*OwnsHeaderSearch =*/false); PP->Initialize(*Target); PP->EnterMainSourceFile(); return PP; @@ -107,7 +108,7 @@ TEST_F(ParseHLSLRootSignatureTest, LexValidTokensTest) { )cc"; TrivialModuleLoader ModLoader; - Preprocessor *PP = CreatePP(Source, ModLoader); + auto PP = CreatePP(Source, ModLoader); auto TokLoc = SourceLocation(); RootSignatureLexer Lexer(Source, TokLoc, *PP); @@ -123,8 +124,6 @@ TEST_F(ParseHLSLRootSignatureTest, LexValidTokensTest) { }; CheckTokens(Tokens, Expected); - - delete PP; } } // anonymous namespace >From 8e88fa70156ae5a5d3d0103d65ea99028fbfdf0b Mon Sep 17 00:00:00 2001 From: Finn Plummer <canadienf...@gmail.com> Date: Thu, 16 Jan 2025 17:23:47 +0000 Subject: [PATCH 4/7] review comments: fix handling of signed ints - update Lexer to allow '+' prefix to denote a signed positive integer - add missing errors when we expect an unsigned int for parameters - add tests to show lexing of positive signed integer and unsigned integer --- clang/lib/Parse/ParseHLSLRootSignature.cpp | 14 ++++++++------ .../unittests/Parse/ParseHLSLRootSignatureTest.cpp | 4 ++++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/clang/lib/Parse/ParseHLSLRootSignature.cpp b/clang/lib/Parse/ParseHLSLRootSignature.cpp index 9f2154471ec3f1..0cdd2273489406 100644 --- a/clang/lib/Parse/ParseHLSLRootSignature.cpp +++ b/clang/lib/Parse/ParseHLSLRootSignature.cpp @@ -6,19 +6,20 @@ namespace root_signature { // Lexer Definitions -static bool IsPreprocessorNumberChar(char C) { +static bool IsNumberChar(char C) { // TODO: extend for float support with or without hexadecimal/exponent return isdigit(C); // integer support } bool RootSignatureLexer::LexNumber(RootSignatureToken &Result) { // NumericLiteralParser does not handle the sign so we will manually apply it - Result.Signed = Buffer.front() == '-'; + bool Negative = Buffer.front() == '-'; + Result.Signed = Negative || Buffer.front() == '+'; if (Result.Signed) AdvanceBuffer(); // Retrieve the possible number - StringRef NumSpelling = Buffer.take_while(IsPreprocessorNumberChar); + StringRef NumSpelling = Buffer.take_while(IsNumberChar); // Parse the numeric value and do semantic checks on its specification clang::NumericLiteralParser Literal(NumSpelling, SourceLoc, @@ -35,7 +36,7 @@ bool RootSignatureLexer::LexNumber(RootSignatureToken &Result) { if (Literal.GetIntegerValue(X)) return true; // TODO: Report overflow error - X = Result.Signed ? -X : X; + X = Negative ? -X : X; Result.IntLiteral = (uint32_t)X.getZExtValue(); } else { return true; // TODO: report unsupported number literal specification @@ -84,7 +85,7 @@ bool RootSignatureLexer::LexToken(RootSignatureToken &Result) { } // Numeric constant - if (isdigit(C) || C == '-') + if (isdigit(C) || C == '-' || C == '+') return LexNumber(Result); // All following tokens require at least one additional character @@ -101,7 +102,8 @@ bool RootSignatureLexer::LexToken(RootSignatureToken &Result) { if (LexNumber(Result)) return true; - // Lex number could also parse a float so ensure it was an unsigned int + // Lex number could also parse a signed int/float so ensure it was an + // unsigned int if (Result.Kind != TokenKind::int_literal || Result.Signed) return true; // Return invalid number literal for register error diff --git a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp index 4fd60488c6ff20..42c57731ddb804 100644 --- a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp +++ b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp @@ -105,6 +105,8 @@ TEST_F(ParseHLSLRootSignatureTest, LexValidTokensTest) { shader_visibility_pixel shader_visibility_amplification shader_visibility_mesh + + 42 +42 )cc"; TrivialModuleLoader ModLoader; @@ -121,6 +123,8 @@ TEST_F(ParseHLSLRootSignatureTest, LexValidTokensTest) { SmallVector<TokenKind> Expected = { #define TOK(NAME) TokenKind::NAME, #include "clang/Parse/HLSLRootSignatureTokenKinds.def" + TokenKind::int_literal, + TokenKind::int_literal, }; CheckTokens(Tokens, Expected); >From a76b907053462405287a2a5b09454765ff1bf73c Mon Sep 17 00:00:00 2001 From: Finn Plummer <canadienf...@gmail.com> Date: Thu, 16 Jan 2025 17:25:20 +0000 Subject: [PATCH 5/7] review comments: use early exit code convention --- clang/lib/Parse/ParseHLSLRootSignature.cpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/clang/lib/Parse/ParseHLSLRootSignature.cpp b/clang/lib/Parse/ParseHLSLRootSignature.cpp index 0cdd2273489406..fac4a92f1920be 100644 --- a/clang/lib/Parse/ParseHLSLRootSignature.cpp +++ b/clang/lib/Parse/ParseHLSLRootSignature.cpp @@ -28,19 +28,18 @@ bool RootSignatureLexer::LexNumber(RootSignatureToken &Result) { if (Literal.hadError) return true; // Error has already been reported so just return + if (!Literal.isIntegerLiteral()) + return true; // TODO: report unsupported number literal specification + // Retrieve the number value to store into the token - if (Literal.isIntegerLiteral()) { - Result.Kind = TokenKind::int_literal; + Result.Kind = TokenKind::int_literal; - APSInt X = APSInt(32, Result.Signed); - if (Literal.GetIntegerValue(X)) - return true; // TODO: Report overflow error + APSInt X = APSInt(32, Result.Signed); + if (Literal.GetIntegerValue(X)) + return true; // TODO: Report overflow error - X = Negative ? -X : X; - Result.IntLiteral = (uint32_t)X.getZExtValue(); - } else { - return true; // TODO: report unsupported number literal specification - } + X = Negative ? -X : X; + Result.IntLiteral = (uint32_t)X.getZExtValue(); AdvanceBuffer(NumSpelling.size()); return false; >From 7a228d3028cff5d1082f1d68d3932a8a8dcd0bed Mon Sep 17 00:00:00 2001 From: Finn Plummer <canadienf...@gmail.com> Date: Wed, 22 Jan 2025 19:28:23 +0000 Subject: [PATCH 6/7] self-review: update namespacing - parser moved to clang dir so namespace is changed to clang - objects are prefixed with RootSignature[Parser] etc so remove the root_signature namespace --- .../clang/Parse/ParseHLSLRootSignature.h | 6 ++---- clang/lib/Parse/ParseHLSLRootSignature.cpp | 8 +++----- .../Parse/ParseHLSLRootSignatureTest.cpp | 19 +++++++++---------- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/clang/include/clang/Parse/ParseHLSLRootSignature.h b/clang/include/clang/Parse/ParseHLSLRootSignature.h index 6c534411e754a0..40e0c79ca5768f 100644 --- a/clang/include/clang/Parse/ParseHLSLRootSignature.h +++ b/clang/include/clang/Parse/ParseHLSLRootSignature.h @@ -21,9 +21,8 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSwitch.h" -namespace llvm { +namespace clang { namespace hlsl { -namespace root_signature { struct RootSignatureToken { enum Kind { @@ -89,8 +88,7 @@ class RootSignatureLexer { } }; -} // namespace root_signature } // namespace hlsl -} // namespace llvm +} // namespace clang #endif // LLVM_CLANG_PARSE_PARSEHLSLROOTSIGNATURE_H diff --git a/clang/lib/Parse/ParseHLSLRootSignature.cpp b/clang/lib/Parse/ParseHLSLRootSignature.cpp index fac4a92f1920be..1ea7f94b3ec1f8 100644 --- a/clang/lib/Parse/ParseHLSLRootSignature.cpp +++ b/clang/lib/Parse/ParseHLSLRootSignature.cpp @@ -1,8 +1,7 @@ #include "clang/Parse/ParseHLSLRootSignature.h" -namespace llvm { +namespace clang { namespace hlsl { -namespace root_signature { // Lexer Definitions @@ -34,7 +33,7 @@ bool RootSignatureLexer::LexNumber(RootSignatureToken &Result) { // Retrieve the number value to store into the token Result.Kind = TokenKind::int_literal; - APSInt X = APSInt(32, Result.Signed); + llvm::APSInt X = llvm::APSInt(32, Result.Signed); if (Literal.GetIntegerValue(X)) return true; // TODO: Report overflow error @@ -148,6 +147,5 @@ bool RootSignatureLexer::LexToken(RootSignatureToken &Result) { return false; } -} // namespace root_signature } // namespace hlsl -} // namespace llvm +} // namespace clang diff --git a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp index 42c57731ddb804..84c582b41d0d04 100644 --- a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp +++ b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp @@ -23,7 +23,6 @@ #include "clang/Parse/ParseHLSLRootSignature.h" #include "gtest/gtest.h" -using namespace llvm::hlsl::root_signature; using namespace clang; namespace { @@ -57,8 +56,8 @@ class ParseHLSLRootSignatureTest : public ::testing::Test { return PP; } - void CheckTokens(SmallVector<RootSignatureToken> &Computed, - SmallVector<TokenKind> &Expected) { + void CheckTokens(SmallVector<hlsl::RootSignatureToken> &Computed, + SmallVector<hlsl::TokenKind> &Expected) { ASSERT_EQ(Computed.size(), Expected.size()); for (unsigned I = 0, E = Expected.size(); I != E; ++I) { ASSERT_EQ(Computed[I].Kind, Expected[I]); @@ -113,18 +112,18 @@ TEST_F(ParseHLSLRootSignatureTest, LexValidTokensTest) { auto PP = CreatePP(Source, ModLoader); auto TokLoc = SourceLocation(); - RootSignatureLexer Lexer(Source, TokLoc, *PP); + hlsl::RootSignatureLexer Lexer(Source, TokLoc, *PP); - SmallVector<RootSignatureToken> Tokens = { - RootSignatureToken() // invalid token for completeness + SmallVector<hlsl::RootSignatureToken> Tokens = { + hlsl::RootSignatureToken() // invalid token for completeness }; ASSERT_FALSE(Lexer.Lex(Tokens)); - SmallVector<TokenKind> Expected = { -#define TOK(NAME) TokenKind::NAME, + SmallVector<hlsl::TokenKind> Expected = { +#define TOK(NAME) hlsl::TokenKind::NAME, #include "clang/Parse/HLSLRootSignatureTokenKinds.def" - TokenKind::int_literal, - TokenKind::int_literal, + hlsl::TokenKind::int_literal, + hlsl::TokenKind::int_literal, }; CheckTokens(Tokens, Expected); >From 0a1df31e105557e99f4cc8d4858f2f01286b2bb6 Mon Sep 17 00:00:00 2001 From: Finn Plummer <canadienf...@gmail.com> Date: Wed, 22 Jan 2025 19:29:45 +0000 Subject: [PATCH 7/7] review: add testcase description --- clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp index 84c582b41d0d04..62b0a213cc0d22 100644 --- a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp +++ b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp @@ -75,6 +75,8 @@ class ParseHLSLRootSignatureTest : public ::testing::Test { }; TEST_F(ParseHLSLRootSignatureTest, LexValidTokensTest) { + // This test will check that we can lex all defined tokens as defined in + // HLSLRootSignatureTokenKinds.def, plus some additional integer variations const llvm::StringLiteral Source = R"cc( -42 _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
