On 2026-10-04 Su 12:42 PM, Rui Zhao wrote:
Hi Andrew,
Thanks for working on this. I tried two further optimizations on top
of v4. They can be applied independently:
1. unicode_assigned() still calls pg_mbstrlen_with_len() on the
remaining input, then walks it again to check each code point. 0001
uses the byte length to walk to the end directly. This also avoids
scanning the whole suffix before returning false for an unassigned
code point near the beginning.
2. unicode_normalize_func() and unicode_is_normalized() allocate a C
string for the normalization form on every call, even when the input
is all ASCII. 0002 compares the form directly in the text value,
preserving case-insensitive matching and rejecting extra bytes.
It only builds a C string when reporting an invalid form.
0003 replaces the constant-folded VALUES test with stored short and
compressed text values.
Core regression and the Unicode normalization checks passed with these
patches.
OK, that all looks good. The attached combines all this.
cheers
andrew
--
Andrew Dunstan
EDB: https://www.enterprisedb.com
From f5ca9ffe4f35ea0b8880426ce6ffa61e265cbd6f Mon Sep 17 00:00:00 2001
From: Andrew Dunstan <[email protected]>
Date: Tue, 6 Oct 2026 17:10:09 -0400
Subject: [PATCH v5] Add ASCII fast path to Unicode normalization functions
unicode_is_normalized(), unicode_assigned(), and unicode_normalize_func()
each decoded its whole input to an array of char32_t codepoints, one
utf8_to_unicode()/pg_utf_mblen() call at a time, before doing any real
work. ASCII characters need none of that: they have no canonical or
compatibility decomposition, a combining class of zero, and are all
assigned. So leading ASCII in a string is unchanged by normalization in
any form, except that its last character might compose with a following
combining mark.
Find the leading ASCII with a byte scan using the existing SIMD-vectorized
is_valid_ascii() (already used by pg_utf8_verifystr()). If the whole
string is ASCII, return at once. Otherwise decode and process only the
rest of the string, starting with the last ASCII character so that
composition with a following combining mark still happens.
unicode_assigned() now walks the rest of the string directly instead of
counting its codepoints with pg_mbstrlen_with_len() first. Also, match
the normalization form argument against the text value instead of
converting it to a C string on every call; that allocation is a
noticeable part of the cost once the ASCII fast path is taken.
Add regression coverage for the all-ASCII case in all three functions,
including stored short-header and compressed values, sweeps over ASCII
prefixes of varying length, and invalid and mixed-case form names.
Author: Andrew Dunstan <[email protected]>
Author: Nazir Bilal Yavuz <[email protected]>
Author: Rui Zhao <[email protected]>
Reviewed-by: Tristan Partin <[email protected]>
Reviewed-by: Chao Li <[email protected]>
Discussion: https://postgr.es/m/[email protected]
---
src/backend/utils/adt/varlena.c | 125 +++++++++++++++++++++-----
src/test/regress/expected/unicode.out | 112 +++++++++++++++++++++++
src/test/regress/sql/unicode.sql | 38 ++++++++
3 files changed, 252 insertions(+), 23 deletions(-)
diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c
index 6c95fcb24b7..2ab05488c78 100644
--- a/src/backend/utils/adt/varlena.c
+++ b/src/backend/utils/adt/varlena.c
@@ -35,6 +35,7 @@
#include "parser/scansup.h"
#include "port/pg_bswap.h"
#include "regex/regex.h"
+#include "utils/ascii.h"
#include "utils/builtins.h"
#include "utils/guc.h"
#include "utils/lsyscache.h"
@@ -167,6 +168,8 @@ static void text_format_string_conversion(StringInfo buf, char conversion,
int flags, int width);
static void text_format_append_string(StringInfo buf, const char *str,
int flags, int width);
+static UnicodeNormalizationForm unicode_norm_form_from_text(text *formtxt);
+static int valid_ascii_prefix_len(const unsigned char *s, int len);
/*****************************************************************************
@@ -5419,8 +5422,10 @@ getClosestMatch(ClosestMatchState *state)
*/
static UnicodeNormalizationForm
-unicode_norm_form_from_string(const char *formstr)
+unicode_norm_form_from_text(text *formtxt)
{
+ const char *formstr = VARDATA_ANY(formtxt);
+ int formlen = VARSIZE_ANY_EXHDR(formtxt);
UnicodeNormalizationForm form = -1;
/*
@@ -5431,18 +5436,18 @@ unicode_norm_form_from_string(const char *formstr)
(errcode(ERRCODE_SYNTAX_ERROR),
errmsg("Unicode normalization can only be performed if server encoding is UTF8")));
- if (pg_strcasecmp(formstr, "NFC") == 0)
+ if (formlen == 3 && pg_strncasecmp(formstr, "NFC", 3) == 0)
form = UNICODE_NFC;
- else if (pg_strcasecmp(formstr, "NFD") == 0)
+ else if (formlen == 3 && pg_strncasecmp(formstr, "NFD", 3) == 0)
form = UNICODE_NFD;
- else if (pg_strcasecmp(formstr, "NFKC") == 0)
+ else if (formlen == 4 && pg_strncasecmp(formstr, "NFKC", 4) == 0)
form = UNICODE_NFKC;
- else if (pg_strcasecmp(formstr, "NFKD") == 0)
+ else if (formlen == 4 && pg_strncasecmp(formstr, "NFKD", 4) == 0)
form = UNICODE_NFKD;
else
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("invalid normalization form: %s", formstr)));
+ errmsg("invalid normalization form: %s", text_to_cstring(formtxt))));
return form;
}
@@ -5475,6 +5480,33 @@ icu_unicode_version(PG_FUNCTION_ARGS)
PG_RETURN_NULL();
}
+/*
+ * Return the length of an initial portion of s[0..len) that is valid ASCII,
+ * that is, contains no zero bytes and no bytes with the high bit set, or
+ * len if all of it is.
+ *
+ * is_valid_ascii() requires a length that is a multiple of sizeof(Vector8),
+ * so check one chunk at a time with it and then the remainder byte by byte.
+ * A failure inside a chunk is reported at the start of that chunk, so the
+ * result can be up to sizeof(Vector8) - 1 bytes short of the full valid
+ * ASCII prefix.
+ */
+static int
+valid_ascii_prefix_len(const unsigned char *s, int len)
+{
+ int chunk_len = len - (len % sizeof(Vector8));
+
+ for (int i = 0; i < chunk_len; i += sizeof(Vector8))
+ if (!is_valid_ascii(s + i, sizeof(Vector8)))
+ return i;
+
+ for (int i = chunk_len; i < len; i++)
+ if (s[i] == 0 || IS_HIGHBIT_SET(s[i]))
+ return i;
+
+ return len;
+}
+
/*
* Check whether the string contains only assigned Unicode code
* points. Requires that the database encoding is UTF-8.
@@ -5484,16 +5516,27 @@ unicode_assigned(PG_FUNCTION_ARGS)
{
text *input = PG_GETARG_TEXT_PP(0);
unsigned char *p;
- int size;
+ unsigned char *end;
+ int len;
+ int start;
if (GetDatabaseEncoding() != PG_UTF8)
ereport(ERROR,
(errmsg("Unicode categorization can only be performed if server encoding is UTF8")));
- /* convert to char32_t */
- size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input));
- p = (unsigned char *) VARDATA_ANY(input);
- for (int i = 0; i < size; i++)
+ /* ASCII code points are always assigned, so only check the rest */
+ len = VARSIZE_ANY_EXHDR(input);
+ start = valid_ascii_prefix_len((unsigned char *) VARDATA_ANY(input), len);
+ if (start == len)
+ PG_RETURN_BOOL(true);
+
+ /*
+ * Check the remaining code points without first counting them, stopping
+ * at a NUL as pg_mbstrlen_with_len() would.
+ */
+ p = (unsigned char *) VARDATA_ANY(input) + start;
+ end = (unsigned char *) VARDATA_ANY(input) + len;
+ while (p < end && *p)
{
char32_t uchar = utf8_to_unicode(p);
int category = unicode_category(uchar);
@@ -5511,7 +5554,7 @@ Datum
unicode_normalize_func(PG_FUNCTION_ARGS)
{
text *input = PG_GETARG_TEXT_PP(0);
- char *formstr = text_to_cstring(PG_GETARG_TEXT_PP(1));
+ text *formtxt = PG_GETARG_TEXT_PP(1);
UnicodeNormalizationForm form;
size_t size;
char32_t *input_chars;
@@ -5519,26 +5562,44 @@ unicode_normalize_func(PG_FUNCTION_ARGS)
unsigned char *p;
text *result;
size_t i;
+ int len;
+ int start;
- form = unicode_norm_form_from_string(formstr);
+ form = unicode_norm_form_from_text(formtxt);
+
+ /*
+ * ASCII characters have no decomposition and a combining class of zero,
+ * so leading ASCII is unchanged by normalization in any form, except that
+ * its last character might compose with a following combining mark. No
+ * canonical composition has an ASCII character as its second element, so
+ * nothing before that last character can be affected. So return an
+ * all-ASCII string as is, and otherwise normalize only the rest of the
+ * string, starting with that last ASCII character.
+ */
+ len = VARSIZE_ANY_EXHDR(input);
+ start = valid_ascii_prefix_len((unsigned char *) VARDATA_ANY(input), len);
+ if (start == len)
+ PG_RETURN_TEXT_P(input);
+ if (start > 0)
+ start--;
/* convert to char32_t */
- size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input));
+ size = pg_mbstrlen_with_len(VARDATA_ANY(input) + start, len - start);
input_chars = palloc_array(char32_t, size + 1);
- p = (unsigned char *) VARDATA_ANY(input);
+ p = (unsigned char *) VARDATA_ANY(input) + start;
for (i = 0; i < size; i++)
{
input_chars[i] = utf8_to_unicode(p);
p += pg_utf_mblen(p);
}
input_chars[i] = (char32_t) '\0';
- Assert((char *) p == VARDATA_ANY(input) + VARSIZE_ANY_EXHDR(input));
+ Assert((char *) p == VARDATA_ANY(input) + len);
/* action */
output_chars = unicode_normalize(form, input_chars);
- /* convert back to UTF-8 string */
- size = 0;
+ /* convert back to UTF-8 string, after the unchanged ASCII */
+ size = start;
for (char32_t *wp = output_chars; *wp; wp++)
{
unsigned char buf[4];
@@ -5551,6 +5612,8 @@ unicode_normalize_func(PG_FUNCTION_ARGS)
SET_VARSIZE(result, size + VARHDRSZ);
p = (unsigned char *) VARDATA_ANY(result);
+ memcpy(p, VARDATA_ANY(input), start);
+ p += start;
for (char32_t *wp = output_chars; *wp; wp++)
{
unicode_to_utf8(*wp, p);
@@ -5577,7 +5640,7 @@ Datum
unicode_is_normalized(PG_FUNCTION_ARGS)
{
text *input = PG_GETARG_TEXT_PP(0);
- char *formstr = text_to_cstring(PG_GETARG_TEXT_PP(1));
+ text *formtxt = PG_GETARG_TEXT_PP(1);
UnicodeNormalizationForm form;
size_t size;
char32_t *input_chars;
@@ -5587,20 +5650,36 @@ unicode_is_normalized(PG_FUNCTION_ARGS)
UnicodeNormalizationQC quickcheck;
size_t output_size;
bool result;
+ int len;
+ int start;
- form = unicode_norm_form_from_string(formstr);
+ form = unicode_norm_form_from_text(formtxt);
+
+ /*
+ * Leading ASCII is unchanged by normalization, except that its last
+ * character might compose with a following combining mark (see
+ * unicode_normalize_func()). So an all-ASCII string is normalized in
+ * every form, and otherwise only the rest of the string, starting with
+ * that last ASCII character, needs to be checked.
+ */
+ len = VARSIZE_ANY_EXHDR(input);
+ start = valid_ascii_prefix_len((unsigned char *) VARDATA_ANY(input), len);
+ if (start == len)
+ PG_RETURN_BOOL(true);
+ if (start > 0)
+ start--;
/* convert to char32_t */
- size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input));
+ size = pg_mbstrlen_with_len(VARDATA_ANY(input) + start, len - start);
input_chars = palloc_array(char32_t, size + 1);
- p = (unsigned char *) VARDATA_ANY(input);
+ p = (unsigned char *) VARDATA_ANY(input) + start;
for (i = 0; i < size; i++)
{
input_chars[i] = utf8_to_unicode(p);
p += pg_utf_mblen(p);
}
input_chars[i] = (char32_t) '\0';
- Assert((char *) p == VARDATA_ANY(input) + VARSIZE_ANY_EXHDR(input));
+ Assert((char *) p == VARDATA_ANY(input) + len);
/* quick check (see UAX #15) */
quickcheck = unicode_is_normalized_quickcheck(form, input_chars);
diff --git a/src/test/regress/expected/unicode.out b/src/test/regress/expected/unicode.out
index 63e48d3a961..496ee4e7ae1 100644
--- a/src/test/regress/expected/unicode.out
+++ b/src/test/regress/expected/unicode.out
@@ -26,6 +26,12 @@ SELECT unicode_assigned(U&'abc\+10FFFF');
f
(1 row)
+SELECT unicode_assigned('abc');
+ unicode_assigned
+------------------
+ t
+(1 row)
+
SELECT normalize('');
normalize
-----------
@@ -68,8 +74,35 @@ SELECT normalize(U&'\00E4\24D1c', NFKD) = U&'\0061\0308bc' COLLATE "C" AS test_n
t
(1 row)
+SELECT normalize('abc') = 'abc' COLLATE "C" AS test_ascii_idem;
+ test_ascii_idem
+-----------------
+ t
+(1 row)
+
+-- Use stored values so this check is not folded to a constant.
+CREATE TABLE unicode_ascii_input (val text);
+INSERT INTO unicode_ascii_input VALUES
+ (''), ('abc'), (repeat('a', 64)), (repeat('b', 5000));
+SELECT bool_and(normalize(val) = val COLLATE "C") AS test_ascii_idem_column,
+ count(pg_column_compression(val)) AS num_compressed
+FROM unicode_ascii_input;
+ test_ascii_idem_column | num_compressed
+------------------------+----------------
+ t | 1
+(1 row)
+
+DROP TABLE unicode_ascii_input;
SELECT "normalize"('abc', 'def'); -- run-time error
ERROR: invalid normalization form: def
+SELECT "normalize"('abc', 'NFCx'); -- run-time error
+ERROR: invalid normalization form: NFCx
+SELECT "normalize"(U&'\0061\0308', 'nFc') = U&'\00E4' COLLATE "C" AS test_form_case;
+ test_form_case
+----------------
+ t
+(1 row)
+
SELECT U&'\00E4\24D1c' IS NORMALIZED AS test_default;
test_default
--------------
@@ -82,6 +115,12 @@ SELECT U&'\00E4\24D1c' IS NFC NORMALIZED AS test_nfc;
t
(1 row)
+SELECT 'abc' IS NORMALIZED AS test_ascii;
+ test_ascii
+------------
+ t
+(1 row)
+
SELECT num, val,
val IS NFC NORMALIZED AS NFC,
val IS NFD NORMALIZED AS NFD,
@@ -105,6 +144,79 @@ ORDER BY num;
SELECT is_normalized('abc', 'def'); -- run-time error
ERROR: invalid normalization form: def
+SELECT is_normalized('abc', 'NFKCx'); -- run-time error
+ERROR: invalid normalization form: NFKCx
+-- Exercise the ASCII fast-path's chunk/remainder boundary handling: a
+-- non-NFC-normalized codepoint sequence must still be detected as such
+-- regardless of how much pure-ASCII padding surrounds it.
+SELECT len,
+ (repeat('a', len) || U&'\0061\0308') IS NORMALIZED AS non_nfc_at_end,
+ (U&'\0061\0308' || repeat('a', len)) IS NORMALIZED AS non_nfc_at_start
+FROM generate_series(0, 40) AS len
+ORDER BY len;
+ len | non_nfc_at_end | non_nfc_at_start
+-----+----------------+------------------
+ 0 | f | f
+ 1 | f | f
+ 2 | f | f
+ 3 | f | f
+ 4 | f | f
+ 5 | f | f
+ 6 | f | f
+ 7 | f | f
+ 8 | f | f
+ 9 | f | f
+ 10 | f | f
+ 11 | f | f
+ 12 | f | f
+ 13 | f | f
+ 14 | f | f
+ 15 | f | f
+ 16 | f | f
+ 17 | f | f
+ 18 | f | f
+ 19 | f | f
+ 20 | f | f
+ 21 | f | f
+ 22 | f | f
+ 23 | f | f
+ 24 | f | f
+ 25 | f | f
+ 26 | f | f
+ 27 | f | f
+ 28 | f | f
+ 29 | f | f
+ 30 | f | f
+ 31 | f | f
+ 32 | f | f
+ 33 | f | f
+ 34 | f | f
+ 35 | f | f
+ 36 | f | f
+ 37 | f | f
+ 38 | f | f
+ 39 | f | f
+ 40 | f | f
+(41 rows)
+
+-- normalize() and unicode_assigned() skip leading ASCII too, so check that
+-- they give the right answer whatever its length. A combining mark right
+-- after the ASCII must still compose with its last character. This lists
+-- only the lengths that give a wrong answer.
+SELECT len
+FROM generate_series(0, 40) AS len
+WHERE normalize(repeat('a', len) || U&'\0061\0308', NFC)
+ <> (repeat('a', len) || U&'\00E4') COLLATE "C"
+ OR normalize(repeat('a', len) || U&'\00E4' || repeat('b', len), NFD)
+ <> (repeat('a', len) || U&'\0061\0308' || repeat('b', len)) COLLATE "C"
+ OR normalize(repeat('a', len) || U&'\00E4' || repeat('b', len), NFC)
+ <> (repeat('a', len) || U&'\00E4' || repeat('b', len)) COLLATE "C"
+ OR unicode_assigned(repeat('a', len) || U&'\+10FFFF')
+ OR NOT unicode_assigned(repeat('a', len) || U&'\00E4' || repeat('b', len));
+ len
+-----
+(0 rows)
+
-- Hangul NFC recomposition tests
-- L+V -> LV composition (first and last)
SELECT normalize(U&'\1100\1161', NFC) = U&'\AC00' COLLATE "C" AS hangul_lv_first;
diff --git a/src/test/regress/sql/unicode.sql b/src/test/regress/sql/unicode.sql
index 951f86a336e..b21353d44f7 100644
--- a/src/test/regress/sql/unicode.sql
+++ b/src/test/regress/sql/unicode.sql
@@ -8,6 +8,7 @@ SELECT U&'\0061\0308bc' <> U&'\00E4bc' COLLATE "C" AS sanity_check;
SELECT unicode_version() IS NOT NULL;
SELECT unicode_assigned(U&'abc');
SELECT unicode_assigned(U&'abc\+10FFFF');
+SELECT unicode_assigned('abc');
SELECT normalize('');
SELECT normalize(U&'\0061\0308\24D1c') = U&'\00E4\24D1c' COLLATE "C" AS test_default;
@@ -16,11 +17,23 @@ SELECT normalize(U&'\00E4bc', NFC) = U&'\00E4bc' COLLATE "C" AS test_nfc_idem;
SELECT normalize(U&'\00E4\24D1c', NFD) = U&'\0061\0308\24D1c' COLLATE "C" AS test_nfd;
SELECT normalize(U&'\0061\0308\24D1c', NFKC) = U&'\00E4bc' COLLATE "C" AS test_nfkc;
SELECT normalize(U&'\00E4\24D1c', NFKD) = U&'\0061\0308bc' COLLATE "C" AS test_nfkd;
+SELECT normalize('abc') = 'abc' COLLATE "C" AS test_ascii_idem;
+-- Use stored values so this check is not folded to a constant.
+CREATE TABLE unicode_ascii_input (val text);
+INSERT INTO unicode_ascii_input VALUES
+ (''), ('abc'), (repeat('a', 64)), (repeat('b', 5000));
+SELECT bool_and(normalize(val) = val COLLATE "C") AS test_ascii_idem_column,
+ count(pg_column_compression(val)) AS num_compressed
+FROM unicode_ascii_input;
+DROP TABLE unicode_ascii_input;
SELECT "normalize"('abc', 'def'); -- run-time error
+SELECT "normalize"('abc', 'NFCx'); -- run-time error
+SELECT "normalize"(U&'\0061\0308', 'nFc') = U&'\00E4' COLLATE "C" AS test_form_case;
SELECT U&'\00E4\24D1c' IS NORMALIZED AS test_default;
SELECT U&'\00E4\24D1c' IS NFC NORMALIZED AS test_nfc;
+SELECT 'abc' IS NORMALIZED AS test_ascii;
SELECT num, val,
val IS NFC NORMALIZED AS NFC,
@@ -36,6 +49,31 @@ FROM
ORDER BY num;
SELECT is_normalized('abc', 'def'); -- run-time error
+SELECT is_normalized('abc', 'NFKCx'); -- run-time error
+
+-- Exercise the ASCII fast-path's chunk/remainder boundary handling: a
+-- non-NFC-normalized codepoint sequence must still be detected as such
+-- regardless of how much pure-ASCII padding surrounds it.
+SELECT len,
+ (repeat('a', len) || U&'\0061\0308') IS NORMALIZED AS non_nfc_at_end,
+ (U&'\0061\0308' || repeat('a', len)) IS NORMALIZED AS non_nfc_at_start
+FROM generate_series(0, 40) AS len
+ORDER BY len;
+
+-- normalize() and unicode_assigned() skip leading ASCII too, so check that
+-- they give the right answer whatever its length. A combining mark right
+-- after the ASCII must still compose with its last character. This lists
+-- only the lengths that give a wrong answer.
+SELECT len
+FROM generate_series(0, 40) AS len
+WHERE normalize(repeat('a', len) || U&'\0061\0308', NFC)
+ <> (repeat('a', len) || U&'\00E4') COLLATE "C"
+ OR normalize(repeat('a', len) || U&'\00E4' || repeat('b', len), NFD)
+ <> (repeat('a', len) || U&'\0061\0308' || repeat('b', len)) COLLATE "C"
+ OR normalize(repeat('a', len) || U&'\00E4' || repeat('b', len), NFC)
+ <> (repeat('a', len) || U&'\00E4' || repeat('b', len)) COLLATE "C"
+ OR unicode_assigned(repeat('a', len) || U&'\+10FFFF')
+ OR NOT unicode_assigned(repeat('a', len) || U&'\00E4' || repeat('b', len));
-- Hangul NFC recomposition tests
-- L+V -> LV composition (first and last)
--
2.43.0
