MarcusJohnson91 updated this revision to Diff 363216.
MarcusJohnson91 added a comment.
It seems like this diff keeps getting reverted?
I've fixed all the issues mentioned, and the tests work now, everything is
formatted correctly too.
I've set git up to do full context diffs, but it's not working?
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D106753/new/
https://reviews.llvm.org/D106753
Files:
llvm/include/llvm/Support/ConvertUTF.h
llvm/lib/Support/ConvertUTFWrapper.cpp
llvm/unittests/Support/ConvertUTFTest.cpp
Index: llvm/unittests/Support/ConvertUTFTest.cpp
===================================================================
--- llvm/unittests/Support/ConvertUTFTest.cpp
+++ llvm/unittests/Support/ConvertUTFTest.cpp
@@ -25,6 +25,17 @@
EXPECT_EQ(Expected, Result);
}
+TEST(ConvertUTFTest, ConvertUTF32LittleEndianToUTF8String) {
+ // Src is a crystal ball.
+ alignas(UTF32) static const char Src[] = "\x2E\xF5\x01\x00";
+ ArrayRef<char> Ref(Src, sizeof(Src) - 1);
+ std::string Result;
+ bool Success = convertUTF32ToUTF8String(Ref, Result);
+ EXPECT_TRUE(Success);
+ std::string Expected("\xF0\x9F\x94\xAE");
+ EXPECT_EQ(Expected, Result);
+}
+
TEST(ConvertUTFTest, ConvertUTF16BigEndianToUTF8String) {
// Src is the look of disapproval.
alignas(UTF16) static const char Src[] = "\xfe\xff\x0c\xa0\x00_\x0c\xa0";
@@ -36,6 +47,17 @@
EXPECT_EQ(Expected, Result);
}
+TEST(ConvertUTFTest, ConvertUTF32BigEndianToUTF8String) {
+ // Src is a crystal ball.
+ alignas(UTF32) static const char Src[] = "\x00\x00\xfe\xff\x00\x01\xF5\x2E";
+ ArrayRef<char> Ref(Src, sizeof(Src) - 1);
+ std::string Result;
+ bool Success = convertUTF32ToUTF8String(Ref, Result);
+ EXPECT_TRUE(Success);
+ std::string Expected("\xF0\x9F\x94\xAE");
+ EXPECT_EQ(Expected, Result);
+}
+
TEST(ConvertUTFTest, ConvertUTF8ToUTF16String) {
// Src is the look of disapproval.
static const char Src[] = "\xe0\xb2\xa0_\xe0\xb2\xa0";
Index: llvm/lib/Support/ConvertUTFWrapper.cpp
===================================================================
--- llvm/lib/Support/ConvertUTFWrapper.cpp
+++ llvm/lib/Support/ConvertUTFWrapper.cpp
@@ -141,6 +141,60 @@
Src.size() * sizeof(UTF16)), Out);
}
+bool convertUTF32ToUTF8String(ArrayRef<char> SrcBytes, std::string &Out) {
+ assert(Out.empty());
+
+ // Avoid OOB by returning early on empty input.
+ if (SrcBytes.empty())
+ return true;
+
+ const UTF32 *Src = reinterpret_cast<const UTF32 *>(SrcBytes.begin());
+ const UTF32 *SrcEnd = reinterpret_cast<const UTF32 *>(SrcBytes.end());
+
+ assert((uintptr_t)Src % sizeof(UTF32) == 0);
+
+ // Byteswap if necessary.
+ std::vector<UTF32> ByteSwapped;
+ if (Src[0] == UNI_UTF32_BYTE_ORDER_MARK_SWAPPED) {
+ ByteSwapped.insert(ByteSwapped.end(), Src, SrcEnd);
+ for (unsigned I = 0, E = ByteSwapped.size(); I != E; ++I)
+ ByteSwapped[I] = llvm::ByteSwap_32(ByteSwapped[I]);
+ Src = &ByteSwapped[0];
+ SrcEnd = &ByteSwapped[ByteSwapped.size() - 1] + 1;
+ }
+
+ // Skip the BOM for conversion.
+ if (Src[0] == UNI_UTF32_BYTE_ORDER_MARK_NATIVE)
+ Src++;
+
+ // Just allocate enough space up front. We'll shrink it later. Allocate
+ // enough that we can fit a null terminator without reallocating.
+ Out.resize(SrcBytes.size() * UNI_MAX_UTF8_BYTES_PER_CODE_POINT + 1);
+ UTF8 *Dst = reinterpret_cast<UTF8 *>(&Out[0]);
+ UTF8 *DstEnd = Dst + Out.size();
+
+ ConversionResult CR =
+ ConvertUTF32toUTF8(&Src, SrcEnd, &Dst, DstEnd, strictConversion);
+ assert(CR != targetExhausted);
+
+ if (CR != conversionOK) {
+ Out.clear();
+ return false;
+ }
+
+ Out.resize(reinterpret_cast<char *>(Dst) - &Out[0]);
+ Out.push_back(0);
+ Out.pop_back();
+ return true;
+}
+
+bool convertUTF32ToUTF8String(ArrayRef<UTF32> Src, std::string &Out) {
+ return convertUTF32ToUTF8String(
+ llvm::ArrayRef<char>(reinterpret_cast<const char *>(Src.data()),
+ Src.size() * sizeof(UTF32)),
+ Out);
+}
+
bool convertUTF8ToUTF16String(StringRef SrcUTF8,
SmallVectorImpl<UTF16> &DstUTF16) {
assert(DstUTF16.empty());
Index: llvm/include/llvm/Support/ConvertUTF.h
===================================================================
--- llvm/include/llvm/Support/ConvertUTF.h
+++ llvm/include/llvm/Support/ConvertUTF.h
@@ -123,6 +123,9 @@
#define UNI_UTF16_BYTE_ORDER_MARK_NATIVE 0xFEFF
#define UNI_UTF16_BYTE_ORDER_MARK_SWAPPED 0xFFFE
+#define UNI_UTF32_BYTE_ORDER_MARK_NATIVE 0x0000FEFF
+#define UNI_UTF32_BYTE_ORDER_MARK_SWAPPED 0xFFFE0000
+
typedef enum {
conversionOK, /* conversion successful */
sourceExhausted, /* partial character in source, but hit end */
@@ -278,6 +281,24 @@
*/
bool convertUTF16ToUTF8String(ArrayRef<UTF16> Src, std::string &Out);
+/**
+ * Converts a stream of raw bytes assumed to be UTF32 into a UTF8 std::string.
+ *
+ * \param [in] SrcBytes A buffer of what is assumed to be UTF-32 encoded text.
+ * \param [out] Out Converted UTF-8 is stored here on success.
+ * \returns true on success
+ */
+bool convertUTF32ToUTF8String(ArrayRef<char> SrcBytes, std::string &Out);
+
+/**
+ * Converts a UTF32 string into a UTF8 std::string.
+ *
+ * \param [in] Src A buffer of UTF-32 encoded text.
+ * \param [out] Out Converted UTF-8 is stored here on success.
+ * \returns true on success
+ */
+bool convertUTF32ToUTF8String(ArrayRef<UTF32> Src, std::string &Out);
+
/**
* Converts a UTF-8 string into a UTF-16 string with native endianness.
*
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
