https://github.com/azhan92 updated https://github.com/llvm/llvm-project/pull/206519
>From 41ef2c9d4b6d0a36682ed67dfa853897115d9ddd Mon Sep 17 00:00:00 2001 From: alisonzhang <[email protected]> Date: Tue, 14 Jul 2026 13:14:46 -0400 Subject: [PATCH 1/7] Get canonical names for encodings --- llvm/include/llvm/Support/TextEncoding.h | 11 +++++++++++ llvm/lib/Support/TextEncoding.cpp | 14 +++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/llvm/include/llvm/Support/TextEncoding.h b/llvm/include/llvm/Support/TextEncoding.h index 6e7eb95464eff..f1889e1dfd04d 100644 --- a/llvm/include/llvm/Support/TextEncoding.h +++ b/llvm/include/llvm/Support/TextEncoding.h @@ -104,6 +104,17 @@ class TextEncodingConverter { LLVM_ABI static ErrorOr<TextEncodingConverter> create(StringRef From, StringRef To); + /// Maps the encoding name to enum constant if possible. + /// Uses normalized charset name matching. + /// \param[in] Name the character encoding name + /// \return the TextEncoding enum value if known, std::nullopt otherwise + LLVM_ABI static std::optional<TextEncoding> getKnownEncoding(StringRef Name); + + /// Returns the canonical name for a known encoding. + /// \param[in] Encoding the TextEncoding enum value + /// \return the canonical name for the encoding (e.g., "UTF-8" or "IBM-1047") + LLVM_ABI static StringRef getKnownEncodingName(TextEncoding Encoding); + TextEncodingConverter(const TextEncodingConverter &) = delete; TextEncodingConverter &operator=(const TextEncodingConverter &) = delete; diff --git a/llvm/lib/Support/TextEncoding.cpp b/llvm/lib/Support/TextEncoding.cpp index c44bf78ea9fba..838956b9636f5 100644 --- a/llvm/lib/Support/TextEncoding.cpp +++ b/llvm/lib/Support/TextEncoding.cpp @@ -49,7 +49,8 @@ static void normalizeCharSetName(StringRef CSName, } // Maps the encoding name to enum constant if possible. -static std::optional<TextEncoding> getKnownEncoding(StringRef Name) { +std::optional<TextEncoding> +TextEncodingConverter::getKnownEncoding(StringRef Name) { SmallString<16> Normalized; normalizeCharSetName(Name, Normalized); if (Normalized.equals("utf8")) @@ -59,6 +60,17 @@ static std::optional<TextEncoding> getKnownEncoding(StringRef Name) { return std::nullopt; } +// Returns the canonical name for a known encoding. +StringRef TextEncodingConverter::getKnownEncodingName(TextEncoding Encoding) { + switch (Encoding) { + case TextEncoding::UTF8: + return "UTF-8"; + case TextEncoding::IBM1047: + return "IBM-1047"; + } + llvm_unreachable("Invalid TextEncoding value"); +} + [[maybe_unused]] static void HandleOverflow(size_t &Capacity, char *&Output, size_t &OutputLength, SmallVectorImpl<char> &Result) { >From 280a4c9970103740005ed6149ad862ddae560601 Mon Sep 17 00:00:00 2001 From: alisonzhang <[email protected]> Date: Tue, 14 Jul 2026 13:15:41 -0400 Subject: [PATCH 2/7] Text mode mismatch checking --- clang/lib/Basic/FileManager.cpp | 17 +++++++++----- llvm/include/llvm/Support/VirtualFileSystem.h | 9 ++++++++ llvm/lib/Support/VirtualFileSystem.cpp | 22 ++++++++++++++++--- 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/clang/lib/Basic/FileManager.cpp b/clang/lib/Basic/FileManager.cpp index 8fb3ba0a27aad..94fc4c15f51d2 100644 --- a/clang/lib/Basic/FileManager.cpp +++ b/clang/lib/Basic/FileManager.cpp @@ -539,15 +539,22 @@ FileManager::getBufferForFile(FileEntryRef FE, bool isVolatile, FileSize = -1; StringRef Filename = FE.getName(); - // If the file is already open, use the open file descriptor. + // If the file is already open, check if the mode matches. if (Entry->File) { - auto Result = Entry->File->getBuffer(Filename, FileSize, - RequiresNullTerminator, isVolatile); + // Check if the cached file's mode matches the requested mode + // Only perform mismatch recovery for real files + if (!Entry->File->realFileCheckTextModeMismatch(IsText)) { + // Mode matches, use the cached file descriptor + auto Result = Entry->File->getBuffer(Filename, FileSize, + RequiresNullTerminator, isVolatile); + Entry->closeFile(); + return Result; + } + // Mode mismatch - close the cached file and reopen with correct mode Entry->closeFile(); - return Result; } - // Otherwise, open the file. + // Open the file with the requested mode. return getBufferForFileImpl(Filename, FileSize, isVolatile, RequiresNullTerminator, IsText); } diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h index d22c534228331..274886f920fc6 100644 --- a/llvm/include/llvm/Support/VirtualFileSystem.h +++ b/llvm/include/llvm/Support/VirtualFileSystem.h @@ -137,6 +137,15 @@ class LLVM_ABI File { /// Closes the file. virtual std::error_code close() = 0; + /// Checks if this is a real file and the requested text mode differs + /// from the current mode. For real files with a text mode mismatch where + /// the buffer was previously requested, this will call + /// llvm::report_fatal_error. Always returns false for non-real files. Default + /// implementation returns false for non-real files. + virtual bool realFileCheckTextModeMismatch(bool RequestedIsText) const { + return false; + } + // Get the same file with a different path. static ErrorOr<std::unique_ptr<File>> getWithPath(ErrorOr<std::unique_ptr<File>> Result, const Twine &P); diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp index 42e8bb4f9958e..1c1f9e9dd15b6 100644 --- a/llvm/lib/Support/VirtualFileSystem.cpp +++ b/llvm/lib/Support/VirtualFileSystem.cpp @@ -194,11 +194,15 @@ class RealFile : public File { file_t FD; Status S; std::string RealName; + bool IsTextMode; + bool BufferWasRequested; - RealFile(file_t RawFD, StringRef NewName, StringRef NewRealPathName) + RealFile(file_t RawFD, StringRef NewName, StringRef NewRealPathName, + bool IsText) : FD(RawFD), S(NewName, {}, {}, {}, {}, {}, llvm::sys::fs::file_type::status_error, {}), - RealName(NewRealPathName.str()) { + RealName(NewRealPathName.str()), IsTextMode(IsText), + BufferWasRequested(false) { assert(FD != kInvalidFile && "Invalid or inactive file descriptor"); } @@ -213,6 +217,16 @@ class RealFile : public File { bool IsVolatile) override; std::error_code close() override; void setPath(const Twine &Path) override; + bool realFileCheckTextModeMismatch(bool RequestedIsText) const override { + bool HasMismatch = IsTextMode != RequestedIsText; + if (HasMismatch && BufferWasRequested) { + llvm::report_fatal_error( + "Text mode mismatch: file was previously opened with " + + Twine(IsTextMode ? "text" : "binary") + " mode, now requested with " + + Twine(RequestedIsText ? "text" : "binary") + " mode"); + } + return HasMismatch; + } }; } // namespace @@ -242,6 +256,7 @@ RealFile::getBuffer(const Twine &Name, int64_t FileSize, auto BypassSandbox = sys::sandbox::scopedDisable(); assert(FD != kInvalidFile && "cannot get buffer for closed file"); + BufferWasRequested = true; return MemoryBuffer::getOpenFile(FD, Name, FileSize, RequiresNullTerminator, IsVolatile); } @@ -320,8 +335,9 @@ class RealFileSystem : public FileSystem { adjustPath(Name, Storage), Flags, &RealName); if (!FDOrErr) return errorToErrorCode(FDOrErr.takeError()); + bool IsText = (Flags & sys::fs::OF_Text) != sys::fs::OF_None; return std::unique_ptr<File>( - new RealFile(*FDOrErr, Name.str(), RealName.str())); + new RealFile(*FDOrErr, Name.str(), RealName.str(), IsText)); } struct WorkingDirectory { >From 55c2f56ea77865e05d70376898c715595af6bcaf Mon Sep 17 00:00:00 2001 From: alisonzhang <[email protected]> Date: Sun, 28 Jun 2026 15:51:57 -0400 Subject: [PATCH 3/7] getEncodingNameFromFileTag --- llvm/include/llvm/Support/AutoConvert.h | 26 +++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/llvm/include/llvm/Support/AutoConvert.h b/llvm/include/llvm/Support/AutoConvert.h index 3c025cb605f2c..10a296c81a7a7 100644 --- a/llvm/include/llvm/Support/AutoConvert.h +++ b/llvm/include/llvm/Support/AutoConvert.h @@ -18,6 +18,7 @@ #include <_Ccsid.h> #endif #ifdef __cplusplus +#include "llvm/ADT/SmallString.h" #include "llvm/ADT/Twine.h" #include "llvm/Support/Error.h" #include <system_error> @@ -114,6 +115,31 @@ inline ErrorOr<bool> needConversion(const Twine &FileName, const int FD = -1) { return false; } +inline ErrorOr<SmallString<32>> +getEncodingNameFromFileTag(const Twine &FileName, const int FD = -1) { +#ifdef __MVS__ + ErrorOr<__ccsid_t> TagOrErr = getzOSFileTag(FileName, FD); + if (!TagOrErr) + return TagOrErr.getError(); + + __ccsid_t Tag = *TagOrErr; + if (Tag == 0) + return SmallString<32>(); // Return empty string for no tag + + if (Tag == 1208) + return SmallString<32>("utf-8"); + + if (Tag == 1047) + return SmallString<32>("ibm-1047"); + + SmallString<32> Result; + raw_svector_ostream(Result) << Tag; + return Result; +#else + return SmallString<32>(); // Return empty string for non-MVS platforms +#endif +} + } /* namespace llvm */ #endif /* __cplusplus */ >From 1b21117d5f620db589ba6badcd922914fd4c3922 Mon Sep 17 00:00:00 2001 From: alisonzhang <[email protected]> Date: Tue, 14 Jul 2026 13:17:22 -0400 Subject: [PATCH 4/7] Add cache for text encoding converters --- clang/include/clang/Basic/SourceManager.h | 15 ++++++++++ clang/lib/Basic/SourceManager.cpp | 35 +++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h index 1939d1aa4915e..2b5cf0b1e084e 100644 --- a/clang/include/clang/Basic/SourceManager.h +++ b/clang/include/clang/Basic/SourceManager.h @@ -50,6 +50,7 @@ #include "llvm/Support/Allocator.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/TextEncoding.h" #include <cassert> #include <cstddef> #include <map> @@ -846,6 +847,11 @@ class SourceManager : public RefCountedBase<SourceManager> { /// we can add a cc1-level option to do so. SmallVector<std::pair<std::string, FullSourceLoc>, 2> StoredModuleBuildStack; + /// Cache of all text encoding converters used by this SourceManager. + /// This includes both the input charset converter and file tag converters. + /// Maps from "source_encoding:target_encoding" to the converter. + llvm::StringMap<std::unique_ptr<llvm::TextEncodingConverter>> ConverterCache; + public: SourceManager(DiagnosticsEngine &Diag, FileManager &FileMgr, bool UserFilesAreVolatile = false); @@ -863,6 +869,15 @@ class SourceManager : public RefCountedBase<SourceManager> { FileManager &getFileManager() const { return FileMgr; } + /// Get or create a text encoding converter from the cache. + /// This method manages all converters (input charset and file tag converters) + /// in a single cache owned by SourceManager. + /// \param SourceEncoding the source character encoding name + /// \return pointer to the converter or an error code + /// The target encoding is always UTF-8. + llvm::ErrorOr<llvm::TextEncodingConverter *> + getOrCreateConverter(llvm::StringRef SourceEncoding); + /// Set true if the SourceManager should report the original file name /// for contents of files that were overridden by other files. Defaults to /// true. diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp index 5540aade05ef5..6175d09881f56 100644 --- a/clang/lib/Basic/SourceManager.cpp +++ b/clang/lib/Basic/SourceManager.cpp @@ -448,6 +448,41 @@ ContentCache &SourceManager::createMemBufferContentCache( return *Entry; } +llvm::ErrorOr<llvm::TextEncodingConverter *> +SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding) { + // Use getKnownEncoding to get normalized encoding names + std::optional<llvm::TextEncoding> SourceKnown = + llvm::TextEncodingConverter::getKnownEncoding(SourceEncoding); + + if (SourceKnown && *SourceKnown == llvm::TextEncoding::UTF8) + return nullptr; + + // Create a cache key using canonical encoding name + llvm::StringRef CacheKey = + SourceKnown + ? llvm::TextEncodingConverter::getKnownEncodingName(*SourceKnown) + : SourceEncoding; + + // Check if converter already exists in cache + auto It = ConverterCache.find(CacheKey); + if (It != ConverterCache.end()) + return It->second.get(); + + // Create a new converter + llvm::ErrorOr<llvm::TextEncodingConverter> NewConverter = + llvm::TextEncodingConverter::create(SourceEncoding, "UTF-8"); + + if (!NewConverter) + return NewConverter.getError(); + + // Store the converter in the cache + auto Inserted = ConverterCache.insert(std::make_pair( + CacheKey, + std::make_unique<llvm::TextEncodingConverter>(std::move(*NewConverter)))); + + return Inserted.first->second.get(); +} + const SrcMgr::SLocEntry &SourceManager::loadSLocEntry(unsigned Index, bool *Invalid) const { return const_cast<SourceManager *>(this)->loadSLocEntry(Index, Invalid); >From ba0b089c93f230790a0ceca0869e6178dc3480bf Mon Sep 17 00:00:00 2001 From: alisonzhang <[email protected]> Date: Tue, 14 Jul 2026 15:50:48 -0400 Subject: [PATCH 5/7] Add CLANG_DEFAULT_INPUT_ENCODING_IBM1047 flag --- clang/CMakeLists.txt | 3 +++ clang/include/clang/Config/config.h.cmake | 3 +++ clang/lib/Driver/ToolChains/Clang.cpp | 17 ++++++++++++++++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt index 89f584c5dceaa..e8c2c8ac65c67 100644 --- a/clang/CMakeLists.txt +++ b/clang/CMakeLists.txt @@ -283,6 +283,9 @@ set(ENABLE_X86_RELAX_RELOCATIONS ON CACHE BOOL set(PPC_LINUX_DEFAULT_IEEELONGDOUBLE OFF CACHE BOOL "Enable IEEE binary128 as default long double format on PowerPC Linux.") +set(CLANG_DEFAULT_INPUT_ENCODING_IBM1047 OFF CACHE BOOL + "Set IBM-1047 as the default input encoding") + set(CLANG_SPAWN_CC1 OFF CACHE BOOL "Whether clang should use a new process for the CC1 invocation") diff --git a/clang/include/clang/Config/config.h.cmake b/clang/include/clang/Config/config.h.cmake index 11b4096726f67..fbafafc710afe 100644 --- a/clang/include/clang/Config/config.h.cmake +++ b/clang/include/clang/Config/config.h.cmake @@ -75,6 +75,9 @@ /* Enable IEEE binary128 as default long double format on PowerPC Linux. */ #cmakedefine01 PPC_LINUX_DEFAULT_IEEELONGDOUBLE +/* Set IBM-1047 as the default input encoding */ +#cmakedefine01 CLANG_DEFAULT_INPUT_ENCODING_IBM1047 + /* Enable each functionality of modules */ #cmakedefine01 CLANG_ENABLE_OBJC_REWRITER #cmakedefine01 CLANG_ENABLE_STATIC_ANALYZER diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index 1d6877ffbd8a7..799ef0b6c36c4 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -7987,10 +7987,25 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, // -finput_charset=UTF-8 is default. Reject others if (Arg *inputCharset = Args.getLastArg(options::OPT_finput_charset_EQ)) { StringRef value = inputCharset->getValue(); - if (!value.equals_insensitive("utf-8")) +#if CLANG_DEFAULT_INPUT_ENCODING_IBM1047 + // -finput_charset=IBM-1047 is default + bool isValid = value.equals_insensitive("ibm-1047"); + CmdArgs.push_back("-finput-charset"); + CmdArgs.push_back("IBM-1047"); +#else + bool isValid = value.equals_insensitive("utf-8"); +#endif + if (!isValid) D.Diag(diag::err_drv_invalid_value) << inputCharset->getAsString(Args) << value; } +#if CLANG_DEFAULT_INPUT_ENCODING_IBM1047 + else { + // No explicit -finput_charset provided, use default IBM-1047 + CmdArgs.push_back("-finput-charset"); + CmdArgs.push_back("IBM-1047"); + } +#endif // -fexec_charset=UTF-8 is default. Reject others if (Arg *execCharset = Args.getLastArg(options::OPT_fexec_charset_EQ)) { >From d821d5c2b252ccfe42cbf448ce79d50b391b2b81 Mon Sep 17 00:00:00 2001 From: alisonzhang <[email protected]> Date: Tue, 14 Jul 2026 13:36:17 -0400 Subject: [PATCH 6/7] Changes related to finput-charset support --- .../clang/Basic/DiagnosticCommonKinds.td | 5 + clang/include/clang/Basic/LangOptions.h | 3 + clang/include/clang/Basic/SourceManager.h | 38 ++++- .../include/clang/Frontend/CompilerInstance.h | 1 + clang/lib/Basic/SourceManager.cpp | 149 +++++++++++++++--- clang/lib/Frontend/CompilerInstance.cpp | 20 ++- .../lib/Frontend/VerifyDiagnosticConsumer.cpp | 4 +- clang/lib/Lex/ModuleMap.cpp | 8 +- clang/lib/Lex/PPDirectives.cpp | 6 +- clang/lib/Lex/Preprocessor.cpp | 6 +- clang/lib/Serialization/ASTReader.cpp | 5 +- 11 files changed, 208 insertions(+), 37 deletions(-) diff --git a/clang/include/clang/Basic/DiagnosticCommonKinds.td b/clang/include/clang/Basic/DiagnosticCommonKinds.td index f2ed2f4698b8d..55e71020c9c91 100644 --- a/clang/include/clang/Basic/DiagnosticCommonKinds.td +++ b/clang/include/clang/Basic/DiagnosticCommonKinds.td @@ -417,6 +417,11 @@ def note_file_sloc_usage : Note< "%plural{0:|: plus %2B (%human2B) for macro expansions}2">; def note_file_misc_sloc_usage : Note< "%0 additional files entered using a total of %1B (%human1B) of space">; +def warn_encoding_conversion_failed : Warning< + "conversion from source encoding failed for '%0': %1; interpreting as %2">, + InGroup<DiagGroup<"encoding-conversion-failed">>; +def err_encoding_conversion_failed : Error< + "conversion from source encoding failed for '%0': %1">; // Modules def err_module_format_unhandled : Error< diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h index f21131622d03d..989069fd0b98b 100644 --- a/clang/include/clang/Basic/LangOptions.h +++ b/clang/include/clang/Basic/LangOptions.h @@ -623,6 +623,9 @@ class LangOptions : public LangOptionsBase { /// Name of the literal encoding to convert the internal encoding to. std::string LiteralEncoding; + + /// Name of the input encoding to convert to the internal encoding. + std::string InputEncoding; LangOptions(); diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h index 2b5cf0b1e084e..16dd446371540 100644 --- a/clang/include/clang/Basic/SourceManager.h +++ b/clang/include/clang/Basic/SourceManager.h @@ -157,6 +157,13 @@ class alignas(8) ContentCache { /// FIXME: Remove this once OrigEntry is a FileEntryRef with a stable name. StringRef Filename; + /// Information on whether this is associated with a FileID for a file (as + /// opposed to a buffer) and, if so, what conversion (if any) was requested. + /// The integer part uses 2 bits: bit 0 indicates if used by FileID, + /// bit 1 indicates if the file was tagged. + llvm::PointerIntPair<llvm::TextEncodingConverter *, 2u, unsigned> + FileIDConverterInfo; + /// A bump pointer allocated array of offsets for each source line. /// /// This is lazily computed. The lines are owned by the SourceManager @@ -273,6 +280,32 @@ class alignas(8) ContentCache { // If BufStr has an invalid BOM, returns the BOM name; otherwise, returns // nullptr static const char *getInvalidBOM(StringRef BufStr); + + /// Helper methods for FileIDConverterInfo bit manipulation. + /// Bit 0: Used by FileID flag + /// Bit 1: File tagged flag + + bool isUsedByFileID() const { return FileIDConverterInfo.getInt() & 0x1; } + + void setUsedByFileID(bool Used) { + unsigned Flags = FileIDConverterInfo.getInt(); + if (Used) + Flags |= 0x1; + else + Flags &= ~0x1; + FileIDConverterInfo.setInt(Flags); + } + + bool isFileTagged() const { return FileIDConverterInfo.getInt() & 0x2; } + + void setFileTagged(bool Tagged) { + unsigned Flags = FileIDConverterInfo.getInt(); + if (Tagged) + Flags |= 0x2; + else + Flags &= ~0x2; + FileIDConverterInfo.setInt(Flags); + } }; // Assert that the \c ContentCache objects will always be 8-byte aligned so @@ -939,7 +972,7 @@ class SourceManager : public RefCountedBase<SourceManager> { /// being \#included from the specified IncludePosition. FileID createFileID(FileEntryRef SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, - int LoadedID = 0, + llvm::StringRef InputEncodingName = {}, int LoadedID = 0, SourceLocation::UIntTy LoadedOffset = 0); /// Create a new FileID that represents the specified memory buffer. @@ -963,7 +996,8 @@ class SourceManager : public RefCountedBase<SourceManager> { /// Get the FileID for \p SourceFile if it exists. Otherwise, create a /// new FileID for the \p SourceFile. FileID getOrCreateFileID(FileEntryRef SourceFile, - SrcMgr::CharacteristicKind FileCharacter); + SrcMgr::CharacteristicKind FileCharacter, + llvm::StringRef InputEncodingName = {}); /// Creates an expansion SLocEntry for the substitution of an argument into a /// function-like macro's body. Returns the start of the expansion. diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h index 24488e053c628..0ebc50d99b86d 100644 --- a/clang/include/clang/Frontend/CompilerInstance.h +++ b/clang/include/clang/Frontend/CompilerInstance.h @@ -869,6 +869,7 @@ class CompilerInstance : public ModuleLoader { /// /// \return True on success. static bool InitializeSourceManager(const FrontendInputFile &Input, + llvm::StringRef InputEncodingName, DiagnosticsEngine &Diags, FileManager &FileMgr, SourceManager &SourceMgr); diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp index 6175d09881f56..5b12c6177de6c 100644 --- a/clang/lib/Basic/SourceManager.cpp +++ b/clang/lib/Basic/SourceManager.cpp @@ -16,6 +16,7 @@ #include "clang/Basic/LLVM.h" #include "clang/Basic/SourceLocation.h" #include "clang/Basic/SourceManagerInternals.h" +#include "clang/Config/config.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/STLExtras.h" @@ -31,6 +32,7 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" +#include "llvm/Support/SmallVectorMemoryBuffer.h" #include "llvm/Support/raw_ostream.h" #include <algorithm> #include <cassert> @@ -44,6 +46,7 @@ #include <vector> using namespace clang; + using namespace SrcMgr; using llvm::MemoryBuffer; @@ -152,6 +155,83 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM, Buffer = std::move(*BufferOrError); + // Unless this is a named pipe (in which case we can handle a mismatch), + // check that the file's size is the same as in the file entry (which may + // have come from a stat cache). + assert(Buffer->getBufferSize() >= (size_t)ContentsEntry->getSize()); + if (!ContentsEntry->isNamedPipe() && + Buffer->getBufferSize() < (size_t)ContentsEntry->getSize()) { + Diag.Report(Loc, diag::err_file_modified) << ContentsEntry->getName(); + + return std::nullopt; + } + + // Convert source from the input charset to UTF-8 if necessary. + llvm::TextEncodingConverter *Converter = FileIDConverterInfo.getPointer(); + if (Converter) { + StringRef OriginalBuf = Buffer->getBuffer(); + llvm::SmallString<0> UTF8Buf; + UTF8Buf.reserve(OriginalBuf.size() + 1); + + std::error_code EC = Converter->convert(OriginalBuf, UTF8Buf); + if (EC) { + // For tagged files, conversion failure is an error and we don't fall back + if (isFileTagged()) { + Diag.Report(Loc, diag::err_encoding_conversion_failed) + << ContentsEntry->getName() << EC.message(); + return std::nullopt; + } + + // Only fall back to the default encoding if the file is untagged. + // If conversion fails, emit a warning and attempt to fall back to the + // default encoding. + // + // This allows the compiler to accept system or third-party headers that + // are encoded in the default encoding even if conversion to the + // option-specified input encoding failed. + // + // TODO: Add input byte offset information. + // FIXME: Need to fallback to IBM-1047 if that is the default even if the + // file is being read as UTF-8. + const char *FallbackEncoding = + CLANG_DEFAULT_INPUT_ENCODING_IBM1047 ? "IBM-1047" : "UTF-8"; + Diag.Report(Loc, diag::warn_encoding_conversion_failed) + << ContentsEntry->getName() << EC.message() << FallbackEncoding; + + // If the default fallback encoding is IBM-1047, attempt conversion using + // that converter + if (CLANG_DEFAULT_INPUT_ENCODING_IBM1047) { + // Create fallback converter in place + llvm::ErrorOr<llvm::TextEncodingConverter> FallbackConverter = + llvm::TextEncodingConverter::create("IBM-1047", "UTF-8"); + // Only attempt conversion if the fallback converter was created + // successfully + if (FallbackConverter) { + UTF8Buf.clear(); + UTF8Buf.reserve(OriginalBuf.size() + 1); + std::error_code FallbackEC = + FallbackConverter->convert(OriginalBuf, UTF8Buf); + if (!FallbackEC) { + // Conversion with fallback encoding succeeded + auto NewBuf = std::make_unique<llvm::SmallVectorMemoryBuffer>( + std::move(UTF8Buf), Buffer->getBufferIdentifier()); + Buffer = std::move(NewBuf); + } else { + // Fallback conversion also failed, emit an error + Diag.Report(Loc, diag::err_encoding_conversion_failed) + << ContentsEntry->getName() << FallbackEC.message(); + return std::nullopt; + } + } + } + } else { + // TODO: Reclaim memory if the buffer size exceeds the content. + auto NewBuf = std::make_unique<llvm::SmallVectorMemoryBuffer>( + std::move(UTF8Buf), Buffer->getBufferIdentifier()); + Buffer = std::move(NewBuf); + } + } + // Check that the file's size fits in an 'unsigned' (with room for a // past-the-end value). This is deeply regrettable, but various parts of // Clang (including elsewhere in this file!) use 'unsigned' to represent file @@ -167,22 +247,15 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM, return std::nullopt; } - // Unless this is a named pipe (in which case we can handle a mismatch), - // check that the file's size is the same as in the file entry (which may - // have come from a stat cache). - // The buffer will always be larger than the file size on z/OS in the presence - // of characters outside the base character set. - assert(Buffer->getBufferSize() >= (size_t)ContentsEntry->getSize()); - if (!ContentsEntry->isNamedPipe() && - Buffer->getBufferSize() < (size_t)ContentsEntry->getSize()) { - Diag.Report(Loc, diag::err_file_modified) << ContentsEntry->getName(); - - return std::nullopt; - } - - // If the buffer is valid, check to see if it has a UTF Byte Order Mark - // (BOM). We only support UTF-8 with and without a BOM right now. See - // http://en.wikipedia.org/wiki/Byte_order_mark for more information. + // If the buffer is valid, check to see if it has a UTF Byte Order Mark (BOM) + // Note that any conversion requested using `-finput-charset` (if successful) + // has already occurred, so we are expecting UTF-8 with or without a BOM. + // + // In theory, if we see a non-UTF-8 BOM, we can assume that an appropriate + // conversion was not supplied via `-finput-charset` and we could try to + // convert based on the BOM. + // + // See http://en.wikipedia.org/wiki/Byte_order_mark for more information. StringRef BufStr = Buffer->getBuffer(); const char *InvalidBOM = getInvalidBOM(BufStr); @@ -592,6 +665,7 @@ FileID SourceManager::getNextFileID(FileID FID) const { FileID SourceManager::createFileID(FileEntryRef SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, + llvm::StringRef InputEncodingName, int LoadedID, SourceLocation::UIntTy LoadedOffset) { SrcMgr::ContentCache &IR = getOrCreateContentCache(SourceFile, @@ -605,9 +679,39 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile, FileIDContentCaches.push_back(Cache); } + llvm::ErrorOr<llvm::TextEncodingConverter *> Converter = nullptr; + if (!InputEncodingName.empty()) { + // No file tag but -finput-charset conversion is desired. + // Get the converter from the cache using the input encoding name. + Converter = getOrCreateConverter(InputEncodingName); + if (!Converter) { + llvm::report_fatal_error("Cannot create converter for file '" + + SourceFile.getName() + + "': " + Converter.getError().message()); + } + } + +#ifndef NDEBUG + // Either the content cache has never been used for a FileID (and, if we are + // being asked to use a converter, there should be no valid buffer set up for + // it) or the conversion (or lack thereof) should be the same as that used + // previously. + llvm::TextEncodingConverter *CacheConverter = + IR.FileIDConverterInfo.getPointer(); + bool CacheUsedByFileID = IR.isUsedByFileID(); + llvm::TextEncodingConverter *ConverterPtr = Converter ? *Converter : nullptr; + if (CacheUsedByFileID) + assert(CacheConverter == ConverterPtr); + else + assert(!ConverterPtr || IR.IsBufferInvalid || !IR.getBufferIfLoaded()); +#endif + IR.setUsedByFileID(true); + IR.FileIDConverterInfo.setPointer(Converter ? *Converter : nullptr); + // If this is a named pipe, immediately load the buffer to ensure subsequent // calls to ContentCache::getSize() are accurate. - if (Cache->ContentsEntry->isNamedPipe()) + // Do the same if character-encoding conversion was requested. + if (Cache->ContentsEntry->isNamedPipe() || Converter) (void)Cache->getBufferOrNone(Diag, getFileManager(), SourceLocation()); return createFileIDImpl(*Cache, Filename, IncludePos, FileCharacter, LoadedID, @@ -645,10 +749,12 @@ FileID SourceManager::createFileID(const llvm::MemoryBufferRef &Buffer, /// new FileID for the \p SourceFile. FileID SourceManager::getOrCreateFileID(FileEntryRef SourceFile, - SrcMgr::CharacteristicKind FileCharacter) { + SrcMgr::CharacteristicKind FileCharacter, + llvm::StringRef InputEncodingName) { FileID ID = translateFile(SourceFile); - return ID.isValid() ? ID : createFileID(SourceFile, SourceLocation(), - FileCharacter); + return ID.isValid() ? ID + : createFileID(SourceFile, SourceLocation(), + FileCharacter, InputEncodingName); } /// createFileID - Create a new FileID for the specified ContentCache and @@ -2404,7 +2510,8 @@ SourceManagerForFile::SourceManagerForFile(StringRef FileName, SourceMgr = std::make_unique<SourceManager>(*Diagnostics, *FileMgr); FileEntryRef FE = llvm::cantFail(FileMgr->getFileRef(FileName)); FileID ID = - SourceMgr->createFileID(FE, SourceLocation(), clang::SrcMgr::C_User); + SourceMgr->createFileID(FE, SourceLocation(), clang::SrcMgr::C_User, + /*InputEncodingName=*/{}); assert(ID.isValid()); SourceMgr->setMainFileID(ID); } diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index 941c9e40f0c1d..70839e8fa6607 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -918,15 +918,19 @@ CompilerInstance::createOutputFileImpl(StringRef OutputPath, bool Binary, // Initialization Utilities bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input){ - return InitializeSourceManager(Input, getDiagnostics(), getFileManager(), - getSourceManager()); + // Only get InputEncoding from LangOpts if we have a preprocessor, because + // LangOpts is only initialized when preprocessing is enabled. + StringRef InputEncodingName = + hasPreprocessor() ? llvm::StringRef(getLangOpts().InputEncoding) + : llvm::StringRef(); + return InitializeSourceManager(Input, InputEncodingName, getDiagnostics(), + getFileManager(), getSourceManager()); } // static -bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input, - DiagnosticsEngine &Diags, - FileManager &FileMgr, - SourceManager &SourceMgr) { +bool CompilerInstance::InitializeSourceManager( + const FrontendInputFile &Input, llvm::StringRef InputEncodingName, + DiagnosticsEngine &Diags, FileManager &FileMgr, SourceManager &SourceMgr) { SrcMgr::CharacteristicKind Kind = Input.getKind().getFormat() == InputKind::ModuleMap ? Input.isSystem() ? SrcMgr::C_System_ModuleMap @@ -955,8 +959,8 @@ bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input, return false; } - SourceMgr.setMainFileID( - SourceMgr.createFileID(*FileOrErr, SourceLocation(), Kind)); + SourceMgr.setMainFileID(SourceMgr.createFileID(*FileOrErr, SourceLocation(), + Kind, InputEncodingName)); assert(SourceMgr.getMainFileID().isValid() && "Couldn't establish MainFileID!"); diff --git a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp index 1bfe644b2525a..1ae2337adbb1f 100644 --- a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp +++ b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp @@ -610,8 +610,10 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM, } FileID FID = SM.translateFile(*File); + // FIXME: Figure out character-encoding converter treatment. if (FID.isInvalid()) - FID = SM.createFileID(*File, Pos, SrcMgr::C_User); + FID = SM.createFileID(*File, Pos, SrcMgr::C_User, + /*InputEncodingName=*/{}); if (PH.Next(Line) && Line > 0) ExpectedLoc = SM.translateLineCol(FID, Line, 1); diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp index 6c07386f89010..b9750e7a8c197 100644 --- a/clang/lib/Lex/ModuleMap.cpp +++ b/clang/lib/Lex/ModuleMap.cpp @@ -1473,7 +1473,13 @@ bool ModuleMap::parseModuleMapFile(FileEntryRef File, bool IsSystem, if (LocalFID.isInvalid()) { auto FileCharacter = IsSystem ? SrcMgr::C_System_ModuleMap : SrcMgr::C_User_ModuleMap; - LocalFID = SourceMgr.createFileID(File, ExternModuleLoc, FileCharacter); + // Module map files are textual "source files". Use input charset + // converter if available, and file tag converters are handled by + // SourceManager's cache. Get input encoding from LangOptions for charset + // conversion + llvm::StringRef InputEncodingName = LangOpts.InputEncoding; + LocalFID = SourceMgr.createFileID(File, ExternModuleLoc, FileCharacter, + InputEncodingName); } ID = LocalFID; } diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp index c161f6a03593e..a96873059f610 100644 --- a/clang/lib/Lex/PPDirectives.cpp +++ b/clang/lib/Lex/PPDirectives.cpp @@ -2800,7 +2800,11 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport( // position on the file where it will be included and after the expansions. if (IncludePos.isMacroID()) IncludePos = SourceMgr.getExpansionRange(IncludePos).getEnd(); - FileID FID = SourceMgr.createFileID(*File, IncludePos, FileCharacter); + // Use the SourceManager's input charset converter for non-tagged files + // by passing the input encoding name + llvm::StringRef InputEncodingName = getLangOpts().InputEncoding; + FileID FID = SourceMgr.createFileID(*File, IncludePos, FileCharacter, + InputEncodingName); if (!FID.isValid()) { TheModuleLoader.HadFatalFailure = true; return ImportAction::Failure; diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp index 427a3826b299a..c1ac6252f93fc 100644 --- a/clang/lib/Lex/Preprocessor.cpp +++ b/clang/lib/Lex/Preprocessor.cpp @@ -655,8 +655,10 @@ void Preprocessor::EnterMainSourceFile() { << PPOpts.PCHThroughHeader; return; } - setPCHThroughHeaderFileID( - SourceMgr.createFileID(*File, SourceLocation(), SrcMgr::C_User)); + // FIXME: Figure out character-encoding converter treatment. + setPCHThroughHeaderFileID(SourceMgr.createFileID(*File, SourceLocation(), + SrcMgr::C_User, + /*InputEncodingName=*/{})); } // Skip tokens from the Predefines and if needed the main file. diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index 080cb7df04406..575fad0e029e3 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -2005,7 +2005,10 @@ bool ASTReader::ReadSLocEntry(int ID) { } SrcMgr::CharacteristicKind FileCharacter = (SrcMgr::CharacteristicKind)Record[2]; - FileID FID = SourceMgr.createFileID(*File, IncludeLoc, FileCharacter, ID, + // Note: If conversion was originally necessary, OverriddenBuffer should be + // true and the associated handling will trigger. + FileID FID = SourceMgr.createFileID(*File, IncludeLoc, FileCharacter, + /*InputEncodingName=*/{}, ID, BaseOffset + Record[0]); SrcMgr::FileInfo &FileInfo = SourceMgr.getSLocEntry(FID).getFile(); FileInfo.NumCreatedFIDs = Record[5]; >From fb2f19d7113761dad00add1fc7dfd21db29aea8e Mon Sep 17 00:00:00 2001 From: alisonzhang <[email protected]> Date: Tue, 14 Jul 2026 13:28:50 -0400 Subject: [PATCH 7/7] Use explicit converters --- clang/lib/Basic/SourceManager.cpp | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp index 5b12c6177de6c..ea389ad56bc4d 100644 --- a/clang/lib/Basic/SourceManager.cpp +++ b/clang/lib/Basic/SourceManager.cpp @@ -139,7 +139,16 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM, // return paths. IsBufferInvalid = true; - auto BufferOrError = FM.getBufferForFile(*ContentsEntry, IsFileVolatile); + // If a converter is set, open the file in binary mode to get raw bytes + // and avoid platform-specific auto-conversion (e.g., EBCDIC->ASCII on z/OS, + // CRLF->LF on Windows). The explicit converter will handle all + // transformations. + bool NeedsExplicitConversion = FileIDConverterInfo.getPointer() != nullptr; + bool IsText = !NeedsExplicitConversion; + + auto BufferOrError = FM.getBufferForFile(*ContentsEntry, IsFileVolatile, + /*RequiresNullTerminator=*/true, + /*MaybeLimit=*/std::nullopt, IsText); // If we were unable to open the file, then we are in an inconsistent // situation where the content cache referenced a file which no longer @@ -680,7 +689,25 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile, } llvm::ErrorOr<llvm::TextEncodingConverter *> Converter = nullptr; - if (!InputEncodingName.empty()) { + llvm::ErrorOr<llvm::SmallString<32>> Ccsid = + llvm::getEncodingNameFromFileTag(SourceFile.getName()); + if (!Ccsid) { + Diag.Report(SourceLocation(), diag::err_cannot_open_file) + << SourceFile.getName() << Ccsid.getError().message(); + return FileID(); + } + if (!Ccsid->empty()) { + // File has a tag, use the converter from SourceManager's cache + Converter = getOrCreateConverter(*Ccsid); + if (!Converter) { + Diag.Report(SourceLocation(), diag::err_cannot_open_file) + << SourceFile.getName() + << (llvm::Twine("cannot create converter from encoding '") + *Ccsid + + "'"); + return FileID(); + } + IR.setFileTagged(true); + } else if (!InputEncodingName.empty()) { // No file tag but -finput-charset conversion is desired. // Get the converter from the cache using the input encoding name. Converter = getOrCreateConverter(InputEncodingName); _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
