sunho created this revision.
Herald added a project: All.
sunho retitled this revision from "Add multiline support in clang-repl" to
"[clang-repl] Add basic multiline support".
sunho edited the summary of this revision.
sunho updated this revision to Diff 494983.
sunho added a comment.
sunho updated this revision to Diff 494985.
sunho added a comment.
sunho updated this revision to Diff 495041.
sunho added a comment.
sunho updated this revision to Diff 495043.
sunho updated this revision to Diff 495044.
sunho updated this revision to Diff 495054.
sunho retitled this revision from "[clang-repl] Add basic multiline support" to
"[clang-repl] Add basic multiline input support".
sunho edited the summary of this revision.
sunho published this revision for review.
Herald added a project: clang.
Herald added a subscriber: cfe-commits.
Update
sunho added a comment.
Update
sunho added a comment.
Update
sunho added a comment.
Update
Add basic multi-line input support to clang-repl by using newly available
SourceFileGrower interface of Preprocessor.
Compared to cling, this change uses fully fledged clang preprocessor (not
MetaLexer) so it supports very complicated macro expressions.
NOTE: this is the related to
https://discourse.llvm.org/t/rfc-flexible-lexer-buffering-for-handling-incomplete-input-in-interactive-c-c/64180
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D143148
Files:
clang/include/clang/Interpreter/Interpreter.h
clang/lib/Interpreter/IncrementalParser.cpp
clang/lib/Interpreter/IncrementalParser.h
clang/lib/Interpreter/Interpreter.cpp
clang/test/Interpreter/multiline-func-macro-brace-error.cpp
clang/test/Interpreter/multiline-func-macro-brace.cpp
clang/test/Interpreter/multiline-func.cpp
clang/test/Interpreter/multiline-ifdef.cpp
clang/tools/clang-repl/ClangRepl.cpp
Index: clang/tools/clang-repl/ClangRepl.cpp
===================================================================
--- clang/tools/clang-repl/ClangRepl.cpp
+++ clang/tools/clang-repl/ClangRepl.cpp
@@ -124,7 +124,8 @@
continue;
}
- if (auto Err = Interp->ParseAndExecute(*Line)) {
+ if (auto Err =
+ Interp->ParseAndExecute(*Line, [&]() { return LE.readLine(); })) {
llvm::logAllUnhandledErrors(std::move(Err), llvm::errs(), "error: ");
HasError = true;
}
Index: clang/test/Interpreter/multiline-ifdef.cpp
===================================================================
--- /dev/null
+++ clang/test/Interpreter/multiline-ifdef.cpp
@@ -0,0 +1,43 @@
+// REQUIRES: host-supports-jit
+// UNSUPPORTED: system-aix
+// RUN: cat %s | clang-repl 2>&1 | FileCheck %s
+// Check if complex multiline ifdef directive is working.
+extern "C" int printf(const char *, ...);
+
+#define ___DEFINED_SYMBOL
+
+#ifdef ___DEFINED_SYMBOL
+int test_ifdef_one() {
+ printf("Simple multiline ifdef\n");
+}
+#endif
+// CHECK: Simple multiline ifdef
+
+#ifdef ___DEFINED_SYMBOL
+#ifndef ___DEFINED_SYMBOL
+int test_ifdef_two() {
+ printf("Wrong\n");
+}
+#endif
+#ifdef ___DEFINED_SYMBOL
+int test_ifdef_two() {
+ printf("Multiline ifdef two stack\n");
+}
+#endif
+#endif
+// CHECK-NEXT: Multiline ifdef two stack
+
+#ifdef ___DEFINED_SYMBOL
+#ifdef ___DEFINED_SYMBOL
+#ifdef ___DEFINED_SYMBOL
+#ifdef ___DEFINED_SYMBOL
+int test_ifdef_four() {
+ printf("Multiline ifdef four stack\n");
+}
+#endif
+#endif
+#endif
+#endif
+// CHECK-NEXT: Multiline ifdef four stack
+
+%quit
\ No newline at end of file
Index: clang/test/Interpreter/multiline-func.cpp
===================================================================
--- /dev/null
+++ clang/test/Interpreter/multiline-func.cpp
@@ -0,0 +1,16 @@
+// REQUIRES: host-supports-jit
+// UNSUPPORTED: system-aix
+// RUN: cat %s | clang-repl | FileCheck %s
+// Check a multiline function is parsed and executed correctly.
+extern "C" int printf(const char *, ...);
+int test_multiline_function() {
+ printf("Multiline\n");
+ printf("Function\n");
+ return 0;
+}
+
+auto r1 = test_multiline_function();
+// CHECK: Multiline
+// CHECK-NEXT: Function
+
+% quit
\ No newline at end of file
Index: clang/test/Interpreter/multiline-func-macro-brace.cpp
===================================================================
--- /dev/null
+++ clang/test/Interpreter/multiline-func-macro-brace.cpp
@@ -0,0 +1,53 @@
+// REQUIRES: host-supports-jit
+// UNSUPPORTED: system-aix
+// RUN: cat %s | clang-repl | FileCheck %s
+// Check a brace counting is not broken by introduction of macro directives.
+// Brace counting is done to support multiline functions.
+extern "C" int printf(const char *, ...);
+
+#define ___DEFINED_SYMBOL
+
+int test_multiline_brace_count_simple() {
+#ifdef ___UNDEFINED_SYMBOL
+ {
+#else
+ printf("Simple multiline brace count\n");
+#endif
+ return 0;
+ }
+ auto r1 = test_multiline_brace_count_simple();
+ // CHECK: Simple multiline brace count
+
+ int test_multiline_brace_count_defined() {
+#ifdef ___DEFINED_SYMBOL
+ printf("Defined multiline brace count\n");
+#else
+}
+}
+}
+}
+}
+}
+#endif
+ return 0;
+ }
+ auto r2 = test_multiline_brace_count_defined();
+ // CHECK-NEXT: Defined multiline brace count
+
+#define A 1
+#define B 3
+#define C (A + B)
+#define D C
+
+ int test_multiline_brace_count_complex() {
+#if A + B - 2 * C + D == 0
+ printf("Complex multiline brace count\n");
+#else
+ {
+#endif
+ return 0;
+ }
+ auto r3 = test_multiline_brace_count_complex();
+ // CHECK-NEXT: Complex multiline brace count
+
+% quit
\ No newline at end of file
Index: clang/test/Interpreter/multiline-func-macro-brace-error.cpp
===================================================================
--- /dev/null
+++ clang/test/Interpreter/multiline-func-macro-brace-error.cpp
@@ -0,0 +1,33 @@
+// REQUIRES: host-supports-jit
+// UNSUPPORTED: system-aix
+// RUN: cat %s | clang-repl 2>&1 | FileCheck %s
+// Check invalid brace counts are detected.
+// Brace counting is done to support multiline functions.
+extern "C" int printf(const char *, ...);
+
+#define ___DEFINED_SYMBOL
+
+int test_multiline_brace_count_error() {
+#ifdef ___UNDEFINED_SYMBOL
+ printf("Simple multiline brace count\n");
+#else
+}
+#endif
+}
+// CHECK: error: Parsing failed. Unmathced braces.
+
+#define A 1
+#define B 3
+#define C (A + B)
+#define D C
+
+int test_multiline_brace_count_complex_error() {
+#if A + B - 2 * C + D == 0
+}
+#else
+ printf("Complex multiline brace count\n");
+#endif
+}
+// CHECK-NEXT: error: Parsing failed. Unmathced braces.
+
+% quit
\ No newline at end of file
Index: clang/lib/Interpreter/Interpreter.cpp
===================================================================
--- clang/lib/Interpreter/Interpreter.cpp
+++ clang/lib/Interpreter/Interpreter.cpp
@@ -210,8 +210,8 @@
}
llvm::Expected<PartialTranslationUnit &>
-Interpreter::Parse(llvm::StringRef Code) {
- return IncrParser->Parse(Code);
+Interpreter::Parse(llvm::StringRef Code, ReceiveAdditionalLine RecvLine) {
+ return IncrParser->Parse(Code, RecvLine);
}
llvm::Error Interpreter::Execute(PartialTranslationUnit &T) {
Index: clang/lib/Interpreter/IncrementalParser.h
===================================================================
--- clang/lib/Interpreter/IncrementalParser.h
+++ clang/lib/Interpreter/IncrementalParser.h
@@ -57,16 +57,19 @@
std::list<PartialTranslationUnit> PTUs;
public:
+ using ReceiveAdditionalLine = std::function<std::optional<std::string>()>;
IncrementalParser(std::unique_ptr<CompilerInstance> Instance,
llvm::LLVMContext &LLVMCtx, llvm::Error &Err);
~IncrementalParser();
const CompilerInstance *getCI() const { return CI.get(); }
- /// Parses incremental input by creating an in-memory file.
- ///\returns a \c PartialTranslationUnit which holds information about the
+ ///\param RecvLine a callback that will be called to get additional lines
+ /// required to finish parsing. \returns a \c PartialTranslationUnit which
+ /// holds information about the
/// \c TranslationUnitDecl and \c llvm::Module corresponding to the input.
- llvm::Expected<PartialTranslationUnit &> Parse(llvm::StringRef Input);
+ llvm::Expected<PartialTranslationUnit &>
+ Parse(llvm::StringRef Input, ReceiveAdditionalLine RecvLine = nullptr);
/// Uses the CodeGenModule mangled name cache and avoids recomputing.
///\returns the mangled name of a \c GD.
@@ -77,6 +80,9 @@
std::list<PartialTranslationUnit> &getPTUs() { return PTUs; }
private:
+ llvm::Expected<FileID>
+ ReceiveCompleteSourceInput(ReceiveAdditionalLine RecvLine,
+ StringRef InitialCode, StringRef SourceName);
llvm::Expected<PartialTranslationUnit &> ParseOrWrapTopLevelDecl();
};
} // end namespace clang
Index: clang/lib/Interpreter/IncrementalParser.cpp
===================================================================
--- clang/lib/Interpreter/IncrementalParser.cpp
+++ clang/lib/Interpreter/IncrementalParser.cpp
@@ -13,18 +13,24 @@
#include "IncrementalParser.h"
#include "clang/AST/DeclContextInternals.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/TokenKinds.h"
#include "clang/CodeGen/BackendUtil.h"
#include "clang/CodeGen/CodeGenAction.h"
#include "clang/CodeGen/ModuleBuilder.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/FrontendAction.h"
#include "clang/FrontendTool/Utils.h"
+#include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/PreprocessorLexer.h"
#include "clang/Parse/Parser.h"
#include "clang/Sema/Sema.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Support/CrashRecoveryContext.h"
#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBufferRef.h"
#include "llvm/Support/Timer.h"
#include <sstream>
@@ -213,40 +219,167 @@
return static_cast<CodeGenAction *>(WrappedAct)->getCodeGenerator();
}
-llvm::Expected<PartialTranslationUnit &>
-IncrementalParser::Parse(llvm::StringRef input) {
- Preprocessor &PP = CI->getPreprocessor();
- assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
-
- std::ostringstream SourceName;
- SourceName << "input_line_" << InputCount++;
-
+static std::unique_ptr<llvm::MemoryBuffer>
+CreateMemoryBuffer(llvm::StringRef SourceName, llvm::StringRef Str) {
// Create an uninitialized memory buffer, copy code in and append "\n"
- size_t InputSize = input.size(); // don't include trailing 0
+ size_t StrSize = Str.size(); // don't include trailing 0
// MemBuffer size should *not* include terminating zero
std::unique_ptr<llvm::MemoryBuffer> MB(
- llvm::WritableMemoryBuffer::getNewUninitMemBuffer(InputSize + 1,
- SourceName.str()));
+ llvm::WritableMemoryBuffer::getNewUninitMemBuffer(StrSize + 1,
+ SourceName));
char *MBStart = const_cast<char *>(MB->getBufferStart());
- memcpy(MBStart, input.data(), InputSize);
- MBStart[InputSize] = '\n';
+ memcpy(MBStart, Str.data(), StrSize);
+ MBStart[StrSize] = '\n';
+ return MB;
+}
- SourceManager &SM = CI->getSourceManager();
+class IncrementalInputReceiver : public SourceFileGrower {
+public:
+ IncrementalInputReceiver(Preprocessor &PP, SourceManager &SM, IncrementalParser::ReceiveAdditionalLine RecvLine):
+ PP(PP), SM(SM), RecvLine(RecvLine) {
+ }
+
+ llvm::Expected<FileID>
+ Receive(StringRef InitialCode,
+ StringRef SourceName) {
+ // Buffer holding collected source code
+ CodeBuffer = InitialCode.str();
+ CodeBuffer += '\n';
+ const clang::FileEntry* FE
+ = SM.getFileManager().getVirtualFile(SourceName, CodeBuffer.size(),
+ 0 /* mod time*/);
+ CurFileID = SM.createFileID(FE, SourceLocation(), SrcMgr::C_User);
+ PP.setSourceFileGrower(this);
+ SM.overrideFileContents(FE, llvm::MemoryBufferRef(CodeBuffer, ""));
+
+ SourceLocation NewLoc = SM.getLocForStartOfFile(CurFileID);
+ if (PP.EnterSourceFile(CurFileID, /*DirLookup=*/nullptr, NewLoc))
+ return llvm::make_error<llvm::StringError>("Parsing failed. "
+ "Cannot enter source file.",
+ std::error_code());
+ Token Tok;
+ do {
+ if (Err) {
+ PP.EndSourceFile();
+ PP.setSourceFileGrower(nullptr);
+ return std::move(Err);
+ }
+ PP.Lex(Tok);
+ switch(Tok.getKind()) {
+ case tok::l_brace:
+ case tok::l_paren:
+ case tok::l_square: {
+ BraceLevel[Tok.getKind()]++;
+ break;
+ }
+ case tok::r_brace:
+ case tok::r_paren:
+ case tok::r_square: {
+ if (BraceLevel[ToLBracket(Tok.getKind())] == 0) {
+ PP.EndSourceFile();
+ PP.setSourceFileGrower(nullptr);
+ return llvm::make_error<llvm::StringError>("Parsing failed. "
+ "Unmathced braces.",
+ std::error_code());
+ }
+ BraceLevel[ToLBracket(Tok.getKind())]--;
+ break;
+ }
+ default:
+ break;
+ }
+ } while (Tok.isNot(tok::eof));
+ PP.EndSourceFile();
+ PP.setSourceFileGrower(nullptr);
- // FIXME: Create SourceLocation, which will allow clang to order the overload
- // candidates for example
- SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID());
+ return SM.createFileID(CreateMemoryBuffer(SourceName, CodeBuffer),
+ SrcMgr::C_User, /*LoadedID=*/0,
+ /*LoadedOffset=*/0);
+ }
- // Create FileID for the current buffer.
- FileID FID = SM.createFileID(std::move(MB), SrcMgr::C_User, /*LoadedID=*/0,
- /*LoadedOffset=*/0, NewLoc);
+ tok::TokenKind ToLBracket(tok::TokenKind Tok) {
+ switch (Tok) {
+ case tok::r_brace:
+ return tok::l_brace;
+ case tok::r_paren:
+ return tok::l_paren;
+ case tok::r_square:
+ return tok::l_square;
+ default:
+ return tok::unknown;
+ }
+ }
- // NewLoc only used for diags.
- if (PP.EnterSourceFile(FID, /*DirLookup=*/nullptr, NewLoc))
- return llvm::make_error<llvm::StringError>("Parsing failed. "
- "Cannot enter source file.",
- std::error_code());
+ ~IncrementalInputReceiver() = default;
+
+ bool TryGrowFile(FileID FileID) override {
+ if (FileID != CurFileID)
+ return false;
+ if (IsBraceLevelZero())
+ return false;
+ if (auto Line = RecvLine()) {
+ CodeBuffer += *Line;
+ CodeBuffer += '\n';
+ const FileEntry* Entry = SM.getFileEntryForID(CurFileID);
+ assert(Entry);
+ SM.overrideFileContents(Entry, llvm::MemoryBufferRef(CodeBuffer, ""));
+ return true;
+ }
+ Err = llvm::make_error<llvm::StringError>("Parsing failed. "
+ "Truncated code.",
+ std::error_code());
+ return false;
+ }
+
+ bool IsBraceLevelZero() {
+ PreprocessorLexer* CurLexer = PP.getCurrentLexer();
+ if (CurLexer && CurLexer->conditional_begin() != CurLexer->conditional_end())
+ return false;
+ for (auto& It : BraceLevel) {
+ if (It.second)
+ return false;
+ }
+ return true;
+ }
+private:
+ Preprocessor &PP;
+ SourceManager &SM;
+ llvm::Error Err = llvm::Error::success();
+ FileID CurFileID;
+ std::string CodeBuffer;
+ IncrementalParser::ReceiveAdditionalLine RecvLine;
+ llvm::DenseMap<unsigned short, unsigned> BraceLevel;
+};
+// If RecvLine is not null, this will repeatedly call RecvLine function to fetch
+// the additional lines required to finish a cut-off multiline function
+// definition.
+llvm::Expected<FileID>
+IncrementalParser::ReceiveCompleteSourceInput(ReceiveAdditionalLine RecvLine,
+ StringRef InitialCode,
+ StringRef SourceName) {
+ IncrementalInputReceiver InputReciever(CI->getPreprocessor(), CI->getSourceManager(), RecvLine);
+ return InputReciever.Receive(InitialCode, SourceName);
+}
+
+llvm::Expected<PartialTranslationUnit &>
+ IncrementalParser::Parse(llvm::StringRef Input,
+ ReceiveAdditionalLine RecvLine) {
+ Preprocessor &PP = CI->getPreprocessor();
+ assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
+
+ auto SourceName = ("input_line_" + llvm::Twine(InputCount++)).str();
+
+ SourceManager &SM = CI->getSourceManager();
+ auto FID = ReceiveCompleteSourceInput(RecvLine, Input, SourceName);
+ if (!FID)
+ return FID.takeError();
+
+ SourceLocation NewLoc = SM.getLocForStartOfFile(*FID);
+ if (PP.EnterSourceFile(*FID, /*DirLookup=*/nullptr, NewLoc))
+ return llvm::make_error<llvm::StringError>("Parsing failed. "
+ "Cannot enter source file.",
+ std::error_code());
auto PTU = ParseOrWrapTopLevelDecl();
if (!PTU)
return PTU.takeError();
Index: clang/include/clang/Interpreter/Interpreter.h
===================================================================
--- clang/include/clang/Interpreter/Interpreter.h
+++ clang/include/clang/Interpreter/Interpreter.h
@@ -53,15 +53,22 @@
Interpreter(std::unique_ptr<CompilerInstance> CI, llvm::Error &Err);
public:
+ using ReceiveAdditionalLine = std::function<std::optional<std::string>()>;
~Interpreter();
static llvm::Expected<std::unique_ptr<Interpreter>>
create(std::unique_ptr<CompilerInstance> CI);
const CompilerInstance *getCompilerInstance() const;
const llvm::orc::LLJIT *getExecutionEngine() const;
- llvm::Expected<PartialTranslationUnit &> Parse(llvm::StringRef Code);
+ ///\param RecvLine a callback that will be called to get additional lines
+ /// required to finish parsing.
+ llvm::Expected<PartialTranslationUnit &>
+ Parse(llvm::StringRef Code, ReceiveAdditionalLine RecvLine = nullptr);
llvm::Error Execute(PartialTranslationUnit &T);
- llvm::Error ParseAndExecute(llvm::StringRef Code) {
- auto PTU = Parse(Code);
+ ///\param RecvLine a callback that will be called to get additional lines
+ /// required to finish parsing.
+ llvm::Error ParseAndExecute(llvm::StringRef Code,
+ ReceiveAdditionalLine RecvLine = nullptr) {
+ auto PTU = Parse(Code, RecvLine);
if (!PTU)
return PTU.takeError();
if (PTU->TheModule)
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
