Serhiy Storchaka added the comment:
Here is a set of patches for all versions (patch for 3.4 updated).
----------
Added file:
http://bugs.python.org/file28833/unicode_escape_decode_error_handling-2.7.patch
Added file:
http://bugs.python.org/file28834/unicode_escape_decode_error_handling-3.2.patch
Added file:
http://bugs.python.org/file28835/unicode_escape_decode_error_handling-3.3.patch
Added file:
http://bugs.python.org/file28836/unicode_escape_decode_error_handling-3.4.patch
_______________________________________
Python tracker <rep...@bugs.python.org>
<http://bugs.python.org/issue16979>
_______________________________________
diff -r 5970c90dd8d1 Lib/test/test_codeccallbacks.py
--- a/Lib/test/test_codeccallbacks.py Fri Jan 25 23:30:50 2013 +0200
+++ b/Lib/test/test_codeccallbacks.py Sat Jan 26 00:51:30 2013 +0200
@@ -262,12 +262,12 @@
self.assertEqual(
"\\u3042\u3xxx".decode("unicode-escape", "test.handler1"),
- u"\u3042[<92><117><51><120>]xx"
+ u"\u3042[<92><117><51>]xxx"
)
self.assertEqual(
"\\u3042\u3xx".decode("unicode-escape", "test.handler1"),
- u"\u3042[<92><117><51><120><120>]"
+ u"\u3042[<92><117><51>]xx"
)
self.assertEqual(
diff -r 5970c90dd8d1 Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py Fri Jan 25 23:30:50 2013 +0200
+++ b/Lib/test/test_codecs.py Sat Jan 26 00:51:30 2013 +0200
@@ -1786,6 +1786,84 @@
self.assertEqual(srw.read(), u"\xfc")
+class UnicodeEscapeTest(unittest.TestCase):
+ def test_empty(self):
+ self.assertEqual(codecs.unicode_escape_encode(u""), ("", 0))
+ self.assertEqual(codecs.unicode_escape_decode(""), (u"", 0))
+
+ def test_raw_encode(self):
+ encode = codecs.unicode_escape_encode
+ for b in range(32, 127):
+ if b != ord('\\'):
+ self.assertEqual(encode(unichr(b)), (chr(b), 1))
+
+ def test_raw_decode(self):
+ decode = codecs.unicode_escape_decode
+ for b in range(256):
+ if b != ord('\\'):
+ self.assertEqual(decode(chr(b) + '0'), (unichr(b) + u'0', 2))
+
+ def test_escape_encode(self):
+ encode = codecs.unicode_escape_encode
+ self.assertEqual(encode(u'\t'), (r'\t', 1))
+ self.assertEqual(encode(u'\n'), (r'\n', 1))
+ self.assertEqual(encode(u'\r'), (r'\r', 1))
+ self.assertEqual(encode(u'\\'), (r'\\', 1))
+ for b in range(32):
+ if chr(b) not in '\t\n\r':
+ self.assertEqual(encode(unichr(b)), ('\\x%02x' % b, 1))
+ for b in range(127, 256):
+ self.assertEqual(encode(unichr(b)), ('\\x%02x' % b, 1))
+ self.assertEqual(encode(u'\u20ac'), (r'\u20ac', 1))
+ self.assertEqual(encode(u'\U0001d120'),
+ (r'\U0001d120', len(u'\U0001d120')))
+
+ def test_escape_decode(self):
+ decode = codecs.unicode_escape_decode
+ self.assertEqual(decode("[\\\n]"), (u"[]", 4))
+ self.assertEqual(decode(r'[\"]'), (u'["]', 4))
+ self.assertEqual(decode(r"[\']"), (u"[']", 4))
+ self.assertEqual(decode(r"[\\]"), (ur"[\]", 4))
+ self.assertEqual(decode(r"[\a]"), (u"[\x07]", 4))
+ self.assertEqual(decode(r"[\b]"), (u"[\x08]", 4))
+ self.assertEqual(decode(r"[\t]"), (u"[\x09]", 4))
+ self.assertEqual(decode(r"[\n]"), (u"[\x0a]", 4))
+ self.assertEqual(decode(r"[\v]"), (u"[\x0b]", 4))
+ self.assertEqual(decode(r"[\f]"), (u"[\x0c]", 4))
+ self.assertEqual(decode(r"[\r]"), (u"[\x0d]", 4))
+ self.assertEqual(decode(r"[\7]"), (u"[\x07]", 4))
+ self.assertEqual(decode(r"[\8]"), (ur"[\8]", 4))
+ self.assertEqual(decode(r"[\78]"), (u"[\x078]", 5))
+ self.assertEqual(decode(r"[\41]"), (u"[!]", 5))
+ self.assertEqual(decode(r"[\418]"), (u"[!8]", 6))
+ self.assertEqual(decode(r"[\101]"), (u"[A]", 6))
+ self.assertEqual(decode(r"[\1010]"), (u"[A0]", 7))
+ self.assertEqual(decode(r"[\x41]"), (u"[A]", 6))
+ self.assertEqual(decode(r"[\x410]"), (u"[A0]", 7))
+ self.assertEqual(decode(r"\u20ac"), (u"\u20ac", 6))
+ self.assertEqual(decode(r"\U0001d120"), (u"\U0001d120", 10))
+ for b in range(256):
+ if chr(b) not in '\n"\'\\abtnvfr01234567xuUN':
+ self.assertEqual(decode('\\' + chr(b)),
+ (u'\\' + unichr(b), 2))
+
+ def test_decode_errors(self):
+ decode = codecs.unicode_escape_decode
+ for c, d in ('x', 2), ('u', 4), ('U', 4):
+ for i in range(d):
+ self.assertRaises(UnicodeDecodeError, decode,
+ "\\" + c + "0"*i)
+ self.assertRaises(UnicodeDecodeError, decode,
+ "[\\" + c + "0"*i + "]")
+ data = "[\\" + c + "0"*i + "]\\" + c + "0"*i
+ self.assertEqual(decode(data, "ignore"), (u"[]", len(data)))
+ self.assertEqual(decode(data, "replace"),
+ (u"[\ufffd]\ufffd", len(data)))
+ self.assertRaises(UnicodeDecodeError, decode, r"\U00110000")
+ self.assertEqual(decode(r"\U00110000", "ignore"), (u"", 10))
+ self.assertEqual(decode(r"\U00110000", "replace"), (u"\ufffd", 10))
+
+
class BomTest(unittest.TestCase):
def test_seek0(self):
data = u"1234567890"
@@ -1871,6 +1949,7 @@
BasicStrTest,
CharmapTest,
WithStmtTest,
+ UnicodeEscapeTest,
BomTest,
)
diff -r 5970c90dd8d1 Objects/unicodeobject.c
--- a/Objects/unicodeobject.c Fri Jan 25 23:30:50 2013 +0200
+++ b/Objects/unicodeobject.c Sat Jan 26 00:51:30 2013 +0200
@@ -2732,7 +2732,6 @@
Py_ssize_t startinpos;
Py_ssize_t endinpos;
Py_ssize_t outpos;
- int i;
PyUnicodeObject *v;
Py_UNICODE *p;
const char *end;
@@ -2818,29 +2817,19 @@
message = "truncated \\UXXXXXXXX escape";
hexescape:
chr = 0;
- outpos = p-PyUnicode_AS_UNICODE(v);
- if (s+digits>end) {
- endinpos = size;
- if (unicode_decode_call_errorhandler(
- errors, &errorHandler,
- "unicodeescape", "end of string in escape sequence",
- starts, size, &startinpos, &endinpos, &exc, &s,
- &v, &outpos, &p))
- goto onError;
- goto nextByte;
+ if (end - s < digits) {
+ /* count only hex digits */
+ for (; s < end; ++s) {
+ c = (unsigned char)*s;
+ if (!Py_ISXDIGIT(c))
+ goto error;
+ }
+ goto error;
}
- for (i = 0; i < digits; ++i) {
- c = (unsigned char) s[i];
- if (!isxdigit(c)) {
- endinpos = (s+i+1)-starts;
- if (unicode_decode_call_errorhandler(
- errors, &errorHandler,
- "unicodeescape", message,
- starts, size, &startinpos, &endinpos, &exc, &s,
- &v, &outpos, &p))
- goto onError;
- goto nextByte;
- }
+ for (; digits--; ++s) {
+ c = (unsigned char)*s;
+ if (!Py_ISXDIGIT(c))
+ goto error;
chr = (chr<<4) & ~0xF;
if (c >= '0' && c <= '9')
chr += c - '0';
@@ -2849,7 +2838,6 @@
else
chr += 10 + c - 'A';
}
- s += i;
if (chr == 0xffffffff && PyErr_Occurred())
/* _decoding_error will have already written into the
target buffer. */
@@ -2870,14 +2858,8 @@
*p++ = 0xDC00 + (Py_UNICODE) (chr & 0x03FF);
#endif
} else {
- endinpos = s-starts;
- outpos = p-PyUnicode_AS_UNICODE(v);
- if (unicode_decode_call_errorhandler(
- errors, &errorHandler,
- "unicodeescape", "illegal Unicode character",
- starts, size, &startinpos, &endinpos, &exc, &s,
- &v, &outpos, &p))
- goto onError;
+ message = "illegal Unicode character";
+ goto error;
}
break;
@@ -2904,28 +2886,13 @@
goto store;
}
}
- endinpos = s-starts;
- outpos = p-PyUnicode_AS_UNICODE(v);
- if (unicode_decode_call_errorhandler(
- errors, &errorHandler,
- "unicodeescape", message,
- starts, size, &startinpos, &endinpos, &exc, &s,
- &v, &outpos, &p))
- goto onError;
- break;
+ goto error;
default:
if (s > end) {
message = "\\ at end of string";
s--;
- endinpos = s-starts;
- outpos = p-PyUnicode_AS_UNICODE(v);
- if (unicode_decode_call_errorhandler(
- errors, &errorHandler,
- "unicodeescape", message,
- starts, size, &startinpos, &endinpos, &exc, &s,
- &v, &outpos, &p))
- goto onError;
+ goto error;
}
else {
*p++ = '\\';
@@ -2933,8 +2900,18 @@
}
break;
}
- nextByte:
- ;
+ continue;
+
+ error:
+ endinpos = s-starts;
+ outpos = p-PyUnicode_AS_UNICODE(v);
+ if (unicode_decode_call_errorhandler(
+ errors, &errorHandler,
+ "unicodeescape", message,
+ starts, size, &startinpos, &endinpos, &exc, &s,
+ &v, &outpos, &p))
+ goto onError;
+ continue;
}
if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0)
goto onError;
diff -r 22594c5060eb Lib/test/test_codeccallbacks.py
--- a/Lib/test/test_codeccallbacks.py Fri Jan 25 23:31:43 2013 +0200
+++ b/Lib/test/test_codeccallbacks.py Sat Jan 26 00:51:39 2013 +0200
@@ -262,12 +262,12 @@
self.assertEqual(
b"\\u3042\u3xxx".decode("unicode-escape", "test.handler1"),
- "\u3042[<92><117><51><120>]xx"
+ "\u3042[<92><117><51>]xxx"
)
self.assertEqual(
b"\\u3042\u3xx".decode("unicode-escape", "test.handler1"),
- "\u3042[<92><117><51><120><120>]"
+ "\u3042[<92><117><51>]xx"
)
self.assertEqual(
diff -r 22594c5060eb Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py Fri Jan 25 23:31:43 2013 +0200
+++ b/Lib/test/test_codecs.py Sat Jan 26 00:51:39 2013 +0200
@@ -1846,6 +1846,85 @@
self.assertEqual(codecs.raw_unicode_escape_decode(r"\u1234"),
("\u1234", 6))
self.assertEqual(codecs.raw_unicode_escape_decode(br"\u1234"),
("\u1234", 6))
+
+class UnicodeEscapeTest(unittest.TestCase):
+ def test_empty(self):
+ self.assertEqual(codecs.unicode_escape_encode(""), (b"", 0))
+ self.assertEqual(codecs.unicode_escape_decode(b""), ("", 0))
+
+ def test_raw_encode(self):
+ encode = codecs.unicode_escape_encode
+ for b in range(32, 127):
+ if b != b'\\'[0]:
+ self.assertEqual(encode(chr(b)), (bytes([b]), 1))
+
+ def test_raw_decode(self):
+ decode = codecs.unicode_escape_decode
+ for b in range(256):
+ if b != b'\\'[0]:
+ self.assertEqual(decode(bytes([b]) + b'0'), (chr(b) + '0', 2))
+
+ def test_escape_encode(self):
+ encode = codecs.unicode_escape_encode
+ self.assertEqual(encode('\t'), (br'\t', 1))
+ self.assertEqual(encode('\n'), (br'\n', 1))
+ self.assertEqual(encode('\r'), (br'\r', 1))
+ self.assertEqual(encode('\\'), (br'\\', 1))
+ for b in range(32):
+ if chr(b) not in '\t\n\r':
+ self.assertEqual(encode(chr(b)), (('\\x%02x' % b).encode(), 1))
+ for b in range(127, 256):
+ self.assertEqual(encode(chr(b)), (('\\x%02x' % b).encode(), 1))
+ self.assertEqual(encode('\u20ac'), (br'\u20ac', 1))
+ self.assertEqual(encode('\U0001d120'),
+ (br'\U0001d120', len('\U0001d120')))
+
+ def test_escape_decode(self):
+ decode = codecs.unicode_escape_decode
+ self.assertEqual(decode(b"[\\\n]"), ("[]", 4))
+ self.assertEqual(decode(br'[\"]'), ('["]', 4))
+ self.assertEqual(decode(br"[\']"), ("[']", 4))
+ self.assertEqual(decode(br"[\\]"), (r"[\]", 4))
+ self.assertEqual(decode(br"[\a]"), ("[\x07]", 4))
+ self.assertEqual(decode(br"[\b]"), ("[\x08]", 4))
+ self.assertEqual(decode(br"[\t]"), ("[\x09]", 4))
+ self.assertEqual(decode(br"[\n]"), ("[\x0a]", 4))
+ self.assertEqual(decode(br"[\v]"), ("[\x0b]", 4))
+ self.assertEqual(decode(br"[\f]"), ("[\x0c]", 4))
+ self.assertEqual(decode(br"[\r]"), ("[\x0d]", 4))
+ self.assertEqual(decode(br"[\7]"), ("[\x07]", 4))
+ self.assertEqual(decode(br"[\8]"), (r"[\8]", 4))
+ self.assertEqual(decode(br"[\78]"), ("[\x078]", 5))
+ self.assertEqual(decode(br"[\41]"), ("[!]", 5))
+ self.assertEqual(decode(br"[\418]"), ("[!8]", 6))
+ self.assertEqual(decode(br"[\101]"), ("[A]", 6))
+ self.assertEqual(decode(br"[\1010]"), ("[A0]", 7))
+ self.assertEqual(decode(br"[\x41]"), ("[A]", 6))
+ self.assertEqual(decode(br"[\x410]"), ("[A0]", 7))
+ self.assertEqual(decode(br"\u20ac"), ("\u20ac", 6))
+ self.assertEqual(decode(br"\U0001d120"), ("\U0001d120", 10))
+ for b in range(256):
+ if b not in b'\n"\'\\abtnvfr01234567xuUN':
+ self.assertEqual(decode(b'\\' + bytes([b])),
+ ('\\' + chr(b), 2))
+
+ def test_decode_errors(self):
+ decode = codecs.unicode_escape_decode
+ for c, d in (b'x', 2), (b'u', 4), (b'U', 4):
+ for i in range(d):
+ self.assertRaises(UnicodeDecodeError, decode,
+ b"\\" + c + b"0"*i)
+ self.assertRaises(UnicodeDecodeError, decode,
+ b"[\\" + c + b"0"*i + b"]")
+ data = b"[\\" + c + b"0"*i + b"]\\" + c + b"0"*i
+ self.assertEqual(decode(data, "ignore"), ("[]", len(data)))
+ self.assertEqual(decode(data, "replace"),
+ ("[\ufffd]\ufffd", len(data)))
+ self.assertRaises(UnicodeDecodeError, decode, br"\U00110000")
+ self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10))
+ self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10))
+
+
class SurrogateEscapeTest(unittest.TestCase):
def test_utf8(self):
@@ -2011,6 +2090,7 @@
CharmapTest,
WithStmtTest,
TypesTest,
+ UnicodeEscapeTest,
SurrogateEscapeTest,
BomTest,
TransformCodecTest,
diff -r 22594c5060eb Objects/unicodeobject.c
--- a/Objects/unicodeobject.c Fri Jan 25 23:31:43 2013 +0200
+++ b/Objects/unicodeobject.c Sat Jan 26 00:51:39 2013 +0200
@@ -3756,7 +3756,6 @@
Py_ssize_t startinpos;
Py_ssize_t endinpos;
Py_ssize_t outpos;
- int i;
PyUnicodeObject *v;
Py_UNICODE *p;
const char *end;
@@ -3842,29 +3841,19 @@
message = "truncated \\UXXXXXXXX escape";
hexescape:
chr = 0;
- outpos = p-PyUnicode_AS_UNICODE(v);
- if (s+digits>end) {
- endinpos = size;
- if (unicode_decode_call_errorhandler(
- errors, &errorHandler,
- "unicodeescape", "end of string in escape sequence",
- &starts, &end, &startinpos, &endinpos, &exc, &s,
- &v, &outpos, &p))
- goto onError;
- goto nextByte;
- }
- for (i = 0; i < digits; ++i) {
- c = (unsigned char) s[i];
- if (!Py_ISXDIGIT(c)) {
- endinpos = (s+i+1)-starts;
- if (unicode_decode_call_errorhandler(
- errors, &errorHandler,
- "unicodeescape", message,
- &starts, &end, &startinpos, &endinpos, &exc, &s,
- &v, &outpos, &p))
- goto onError;
- goto nextByte;
+ if (end - s < digits) {
+ /* count only hex digits */
+ for (; s < end; ++s) {
+ c = (unsigned char)*s;
+ if (!Py_ISXDIGIT(c))
+ goto error;
}
+ goto error;
+ }
+ for (; digits--; ++s) {
+ c = (unsigned char)*s;
+ if (!Py_ISXDIGIT(c))
+ goto error;
chr = (chr<<4) & ~0xF;
if (c >= '0' && c <= '9')
chr += c - '0';
@@ -3873,7 +3862,6 @@
else
chr += 10 + c - 'A';
}
- s += i;
if (chr == 0xffffffff && PyErr_Occurred())
/* _decoding_error will have already written into the
target buffer. */
@@ -3894,14 +3882,8 @@
*p++ = 0xDC00 + (Py_UNICODE) (chr & 0x03FF);
#endif
} else {
- endinpos = s-starts;
- outpos = p-PyUnicode_AS_UNICODE(v);
- if (unicode_decode_call_errorhandler(
- errors, &errorHandler,
- "unicodeescape", "illegal Unicode character",
- &starts, &end, &startinpos, &endinpos, &exc, &s,
- &v, &outpos, &p))
- goto onError;
+ message = "illegal Unicode character";
+ goto error;
}
break;
@@ -3928,28 +3910,13 @@
goto store;
}
}
- endinpos = s-starts;
- outpos = p-PyUnicode_AS_UNICODE(v);
- if (unicode_decode_call_errorhandler(
- errors, &errorHandler,
- "unicodeescape", message,
- &starts, &end, &startinpos, &endinpos, &exc, &s,
- &v, &outpos, &p))
- goto onError;
- break;
+ goto error;
default:
if (s > end) {
message = "\\ at end of string";
s--;
- endinpos = s-starts;
- outpos = p-PyUnicode_AS_UNICODE(v);
- if (unicode_decode_call_errorhandler(
- errors, &errorHandler,
- "unicodeescape", message,
- &starts, &end, &startinpos, &endinpos, &exc, &s,
- &v, &outpos, &p))
- goto onError;
+ goto error;
}
else {
*p++ = '\\';
@@ -3957,8 +3924,18 @@
}
break;
}
- nextByte:
- ;
+ continue;
+
+ error:
+ endinpos = s-starts;
+ outpos = p-PyUnicode_AS_UNICODE(v);
+ if (unicode_decode_call_errorhandler(
+ errors, &errorHandler,
+ "unicodeescape", message,
+ &starts, &end, &startinpos, &endinpos, &exc, &s,
+ &v, &outpos, &p))
+ goto onError;
+ continue;
}
if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0)
goto onError;
diff -r dcbc16afd6bc Lib/test/test_codeccallbacks.py
--- a/Lib/test/test_codeccallbacks.py Fri Jan 25 23:52:21 2013 +0200
+++ b/Lib/test/test_codeccallbacks.py Sat Jan 26 00:51:50 2013 +0200
@@ -271,12 +271,12 @@
self.assertEqual(
b"\\u3042\u3xxx".decode("unicode-escape", "test.handler1"),
- "\u3042[<92><117><51><120>]xx"
+ "\u3042[<92><117><51>]xxx"
)
self.assertEqual(
b"\\u3042\u3xx".decode("unicode-escape", "test.handler1"),
- "\u3042[<92><117><51><120><120>]"
+ "\u3042[<92><117><51>]xx"
)
self.assertEqual(
diff -r dcbc16afd6bc Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py Fri Jan 25 23:52:21 2013 +0200
+++ b/Lib/test/test_codecs.py Sat Jan 26 00:51:50 2013 +0200
@@ -2009,6 +2009,84 @@
self.assertRaises(UnicodeDecodeError,
codecs.raw_unicode_escape_decode, br"\U00110000")
self.assertEqual(codecs.raw_unicode_escape_decode(r"\U00110000",
"replace"), ("\ufffd", 10))
+
+class UnicodeEscapeTest(unittest.TestCase):
+ def test_empty(self):
+ self.assertEqual(codecs.unicode_escape_encode(""), (b"", 0))
+ self.assertEqual(codecs.unicode_escape_decode(b""), ("", 0))
+
+ def test_raw_encode(self):
+ encode = codecs.unicode_escape_encode
+ for b in range(32, 127):
+ if b != b'\\'[0]:
+ self.assertEqual(encode(chr(b)), (bytes([b]), 1))
+
+ def test_raw_decode(self):
+ decode = codecs.unicode_escape_decode
+ for b in range(256):
+ if b != b'\\'[0]:
+ self.assertEqual(decode(bytes([b]) + b'0'), (chr(b) + '0', 2))
+
+ def test_escape_encode(self):
+ encode = codecs.unicode_escape_encode
+ self.assertEqual(encode('\t'), (br'\t', 1))
+ self.assertEqual(encode('\n'), (br'\n', 1))
+ self.assertEqual(encode('\r'), (br'\r', 1))
+ self.assertEqual(encode('\\'), (br'\\', 1))
+ for b in range(32):
+ if chr(b) not in '\t\n\r':
+ self.assertEqual(encode(chr(b)), (('\\x%02x' % b).encode(), 1))
+ for b in range(127, 256):
+ self.assertEqual(encode(chr(b)), (('\\x%02x' % b).encode(), 1))
+ self.assertEqual(encode('\u20ac'), (br'\u20ac', 1))
+ self.assertEqual(encode('\U0001d120'), (br'\U0001d120', 1))
+
+ def test_escape_decode(self):
+ decode = codecs.unicode_escape_decode
+ self.assertEqual(decode(b"[\\\n]"), ("[]", 4))
+ self.assertEqual(decode(br'[\"]'), ('["]', 4))
+ self.assertEqual(decode(br"[\']"), ("[']", 4))
+ self.assertEqual(decode(br"[\\]"), (r"[\]", 4))
+ self.assertEqual(decode(br"[\a]"), ("[\x07]", 4))
+ self.assertEqual(decode(br"[\b]"), ("[\x08]", 4))
+ self.assertEqual(decode(br"[\t]"), ("[\x09]", 4))
+ self.assertEqual(decode(br"[\n]"), ("[\x0a]", 4))
+ self.assertEqual(decode(br"[\v]"), ("[\x0b]", 4))
+ self.assertEqual(decode(br"[\f]"), ("[\x0c]", 4))
+ self.assertEqual(decode(br"[\r]"), ("[\x0d]", 4))
+ self.assertEqual(decode(br"[\7]"), ("[\x07]", 4))
+ self.assertEqual(decode(br"[\8]"), (r"[\8]", 4))
+ self.assertEqual(decode(br"[\78]"), ("[\x078]", 5))
+ self.assertEqual(decode(br"[\41]"), ("[!]", 5))
+ self.assertEqual(decode(br"[\418]"), ("[!8]", 6))
+ self.assertEqual(decode(br"[\101]"), ("[A]", 6))
+ self.assertEqual(decode(br"[\1010]"), ("[A0]", 7))
+ self.assertEqual(decode(br"[\x41]"), ("[A]", 6))
+ self.assertEqual(decode(br"[\x410]"), ("[A0]", 7))
+ self.assertEqual(decode(br"\u20ac"), ("\u20ac", 6))
+ self.assertEqual(decode(br"\U0001d120"), ("\U0001d120", 10))
+ for b in range(256):
+ if b not in b'\n"\'\\abtnvfr01234567xuUN':
+ self.assertEqual(decode(b'\\' + bytes([b])),
+ ('\\' + chr(b), 2))
+
+ def test_decode_errors(self):
+ decode = codecs.unicode_escape_decode
+ for c, d in (b'x', 2), (b'u', 4), (b'U', 4):
+ for i in range(d):
+ self.assertRaises(UnicodeDecodeError, decode,
+ b"\\" + c + b"0"*i)
+ self.assertRaises(UnicodeDecodeError, decode,
+ b"[\\" + c + b"0"*i + b"]")
+ data = b"[\\" + c + b"0"*i + b"]\\" + c + b"0"*i
+ self.assertEqual(decode(data, "ignore"), ("[]", len(data)))
+ self.assertEqual(decode(data, "replace"),
+ ("[\ufffd]\ufffd", len(data)))
+ self.assertRaises(UnicodeDecodeError, decode, br"\U00110000")
+ self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10))
+ self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10))
+
+
class SurrogateEscapeTest(unittest.TestCase):
def test_utf8(self):
diff -r dcbc16afd6bc Objects/unicodeobject.c
--- a/Objects/unicodeobject.c Fri Jan 25 23:52:21 2013 +0200
+++ b/Objects/unicodeobject.c Sat Jan 26 00:51:50 2013 +0200
@@ -5504,7 +5504,6 @@
const char *starts = s;
Py_ssize_t startinpos;
Py_ssize_t endinpos;
- int j;
PyObject *v;
const char *end;
char* message;
@@ -5626,29 +5625,19 @@
message = "truncated \\UXXXXXXXX escape";
hexescape:
chr = 0;
- if (s+digits>end) {
- endinpos = size;
- if (unicode_decode_call_errorhandler(
- errors, &errorHandler,
- "unicodeescape", "end of string in escape sequence",
- &starts, &end, &startinpos, &endinpos, &exc, &s,
- &v, &i))
- goto onError;
- goto nextByte;
- }
- for (j = 0; j < digits; ++j) {
- c = (unsigned char) s[j];
- if (!Py_ISXDIGIT(c)) {
- endinpos = (s+j+1)-starts;
- if (unicode_decode_call_errorhandler(
- errors, &errorHandler,
- "unicodeescape", message,
- &starts, &end, &startinpos, &endinpos, &exc, &s,
- &v, &i))
- goto onError;
- len = PyUnicode_GET_LENGTH(v);
- goto nextByte;
- }
+ if (end - s < digits) {
+ /* count only hex digits */
+ for (; s < end; ++s) {
+ c = (unsigned char)*s;
+ if (!Py_ISXDIGIT(c))
+ goto error;
+ }
+ goto error;
+ }
+ for (; digits--; ++s) {
+ c = (unsigned char)*s;
+ if (!Py_ISXDIGIT(c))
+ goto error;
chr = (chr<<4) & ~0xF;
if (c >= '0' && c <= '9')
chr += c - '0';
@@ -5657,24 +5646,16 @@
else
chr += 10 + c - 'A';
}
- s += j;
if (chr == 0xffffffff && PyErr_Occurred())
/* _decoding_error will have already written into the
target buffer. */
break;
store:
/* when we get here, chr is a 32-bit unicode character */
- if (chr <= MAX_UNICODE) {
- WRITECHAR(chr);
- } else {
- endinpos = s-starts;
- if (unicode_decode_call_errorhandler(
- errors, &errorHandler,
- "unicodeescape", "illegal Unicode character",
- &starts, &end, &startinpos, &endinpos, &exc, &s,
- &v, &i))
- goto onError;
- }
+ message = "illegal Unicode character";
+ if (chr > MAX_UNICODE)
+ goto error;
+ WRITECHAR(chr);
break;
/* \N{name} */
@@ -5702,26 +5683,13 @@
goto store;
}
}
- endinpos = s-starts;
- if (unicode_decode_call_errorhandler(
- errors, &errorHandler,
- "unicodeescape", message,
- &starts, &end, &startinpos, &endinpos, &exc, &s,
- &v, &i))
- goto onError;
- break;
+ goto error;
default:
if (s > end) {
message = "\\ at end of string";
s--;
- endinpos = s-starts;
- if (unicode_decode_call_errorhandler(
- errors, &errorHandler,
- "unicodeescape", message,
- &starts, &end, &startinpos, &endinpos, &exc, &s,
- &v, &i))
- goto onError;
+ goto error;
}
else {
WRITECHAR('\\');
@@ -5729,8 +5697,18 @@
}
break;
}
- nextByte:
- ;
+ continue;
+
+ error:
+ endinpos = s-starts;
+ if (unicode_decode_call_errorhandler(
+ errors, &errorHandler,
+ "unicodeescape", message,
+ &starts, &end, &startinpos, &endinpos, &exc, &s,
+ &v, &i))
+ goto onError;
+ len = PyUnicode_GET_LENGTH(v);
+ continue;
}
#undef WRITECHAR
diff -r d8c2ce63f5a4 Lib/test/test_codeccallbacks.py
--- a/Lib/test/test_codeccallbacks.py Fri Jan 25 23:53:29 2013 +0200
+++ b/Lib/test/test_codeccallbacks.py Sat Jan 26 00:52:02 2013 +0200
@@ -271,12 +271,12 @@
self.assertEqual(
b"\\u3042\u3xxx".decode("unicode-escape", "test.handler1"),
- "\u3042[<92><117><51><120>]xx"
+ "\u3042[<92><117><51>]xxx"
)
self.assertEqual(
b"\\u3042\u3xx".decode("unicode-escape", "test.handler1"),
- "\u3042[<92><117><51><120><120>]"
+ "\u3042[<92><117><51>]xx"
)
self.assertEqual(
diff -r d8c2ce63f5a4 Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py Fri Jan 25 23:53:29 2013 +0200
+++ b/Lib/test/test_codecs.py Sat Jan 26 00:52:02 2013 +0200
@@ -2009,6 +2009,84 @@
self.assertRaises(UnicodeDecodeError,
codecs.raw_unicode_escape_decode, br"\U00110000")
self.assertEqual(codecs.raw_unicode_escape_decode(r"\U00110000",
"replace"), ("\ufffd", 10))
+
+class UnicodeEscapeTest(unittest.TestCase):
+ def test_empty(self):
+ self.assertEqual(codecs.unicode_escape_encode(""), (b"", 0))
+ self.assertEqual(codecs.unicode_escape_decode(b""), ("", 0))
+
+ def test_raw_encode(self):
+ encode = codecs.unicode_escape_encode
+ for b in range(32, 127):
+ if b != b'\\'[0]:
+ self.assertEqual(encode(chr(b)), (bytes([b]), 1))
+
+ def test_raw_decode(self):
+ decode = codecs.unicode_escape_decode
+ for b in range(256):
+ if b != b'\\'[0]:
+ self.assertEqual(decode(bytes([b]) + b'0'), (chr(b) + '0', 2))
+
+ def test_escape_encode(self):
+ encode = codecs.unicode_escape_encode
+ self.assertEqual(encode('\t'), (br'\t', 1))
+ self.assertEqual(encode('\n'), (br'\n', 1))
+ self.assertEqual(encode('\r'), (br'\r', 1))
+ self.assertEqual(encode('\\'), (br'\\', 1))
+ for b in range(32):
+ if chr(b) not in '\t\n\r':
+ self.assertEqual(encode(chr(b)), (('\\x%02x' % b).encode(), 1))
+ for b in range(127, 256):
+ self.assertEqual(encode(chr(b)), (('\\x%02x' % b).encode(), 1))
+ self.assertEqual(encode('\u20ac'), (br'\u20ac', 1))
+ self.assertEqual(encode('\U0001d120'), (br'\U0001d120', 1))
+
+ def test_escape_decode(self):
+ decode = codecs.unicode_escape_decode
+ self.assertEqual(decode(b"[\\\n]"), ("[]", 4))
+ self.assertEqual(decode(br'[\"]'), ('["]', 4))
+ self.assertEqual(decode(br"[\']"), ("[']", 4))
+ self.assertEqual(decode(br"[\\]"), (r"[\]", 4))
+ self.assertEqual(decode(br"[\a]"), ("[\x07]", 4))
+ self.assertEqual(decode(br"[\b]"), ("[\x08]", 4))
+ self.assertEqual(decode(br"[\t]"), ("[\x09]", 4))
+ self.assertEqual(decode(br"[\n]"), ("[\x0a]", 4))
+ self.assertEqual(decode(br"[\v]"), ("[\x0b]", 4))
+ self.assertEqual(decode(br"[\f]"), ("[\x0c]", 4))
+ self.assertEqual(decode(br"[\r]"), ("[\x0d]", 4))
+ self.assertEqual(decode(br"[\7]"), ("[\x07]", 4))
+ self.assertEqual(decode(br"[\8]"), (r"[\8]", 4))
+ self.assertEqual(decode(br"[\78]"), ("[\x078]", 5))
+ self.assertEqual(decode(br"[\41]"), ("[!]", 5))
+ self.assertEqual(decode(br"[\418]"), ("[!8]", 6))
+ self.assertEqual(decode(br"[\101]"), ("[A]", 6))
+ self.assertEqual(decode(br"[\1010]"), ("[A0]", 7))
+ self.assertEqual(decode(br"[\x41]"), ("[A]", 6))
+ self.assertEqual(decode(br"[\x410]"), ("[A0]", 7))
+ self.assertEqual(decode(br"\u20ac"), ("\u20ac", 6))
+ self.assertEqual(decode(br"\U0001d120"), ("\U0001d120", 10))
+ for b in range(256):
+ if b not in b'\n"\'\\abtnvfr01234567xuUN':
+ self.assertEqual(decode(b'\\' + bytes([b])),
+ ('\\' + chr(b), 2))
+
+ def test_decode_errors(self):
+ decode = codecs.unicode_escape_decode
+ for c, d in (b'x', 2), (b'u', 4), (b'U', 4):
+ for i in range(d):
+ self.assertRaises(UnicodeDecodeError, decode,
+ b"\\" + c + b"0"*i)
+ self.assertRaises(UnicodeDecodeError, decode,
+ b"[\\" + c + b"0"*i + b"]")
+ data = b"[\\" + c + b"0"*i + b"]\\" + c + b"0"*i
+ self.assertEqual(decode(data, "ignore"), ("[]", len(data)))
+ self.assertEqual(decode(data, "replace"),
+ ("[\ufffd]\ufffd", len(data)))
+ self.assertRaises(UnicodeDecodeError, decode, br"\U00110000")
+ self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10))
+ self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10))
+
+
class SurrogateEscapeTest(unittest.TestCase):
def test_utf8(self):
diff -r d8c2ce63f5a4 Objects/unicodeobject.c
--- a/Objects/unicodeobject.c Fri Jan 25 23:53:29 2013 +0200
+++ b/Objects/unicodeobject.c Sat Jan 26 00:52:02 2013 +0200
@@ -5376,7 +5376,6 @@
const char *starts = s;
Py_ssize_t startinpos;
Py_ssize_t endinpos;
- int j;
_PyUnicodeWriter writer;
const char *end;
char* message;
@@ -5500,28 +5499,19 @@
message = "truncated \\UXXXXXXXX escape";
hexescape:
chr = 0;
- if (s+digits>end) {
- endinpos = size;
- if (unicode_decode_call_errorhandler_writer(
- errors, &errorHandler,
- "unicodeescape", "end of string in escape sequence",
- &starts, &end, &startinpos, &endinpos, &exc, &s,
- &writer))
- goto onError;
- goto nextByte;
- }
- for (j = 0; j < digits; ++j) {
- c = (unsigned char) s[j];
- if (!Py_ISXDIGIT(c)) {
- endinpos = (s+j+1)-starts;
- if (unicode_decode_call_errorhandler_writer(
- errors, &errorHandler,
- "unicodeescape", message,
- &starts, &end, &startinpos, &endinpos, &exc, &s,
- &writer))
- goto onError;
- goto nextByte;
+ if (end - s < digits) {
+ /* count only hex digits */
+ for (; s < end; ++s) {
+ c = (unsigned char)*s;
+ if (!Py_ISXDIGIT(c))
+ goto error;
}
+ goto error;
+ }
+ for (; digits--; ++s) {
+ c = (unsigned char)*s;
+ if (!Py_ISXDIGIT(c))
+ goto error;
chr = (chr<<4) & ~0xF;
if (c >= '0' && c <= '9')
chr += c - '0';
@@ -5530,24 +5520,16 @@
else
chr += 10 + c - 'A';
}
- s += j;
if (chr == 0xffffffff && PyErr_Occurred())
/* _decoding_error will have already written into the
target buffer. */
break;
store:
/* when we get here, chr is a 32-bit unicode character */
- if (chr <= MAX_UNICODE) {
- WRITECHAR(chr);
- } else {
- endinpos = s-starts;
- if (unicode_decode_call_errorhandler_writer(
- errors, &errorHandler,
- "unicodeescape", "illegal Unicode character",
- &starts, &end, &startinpos, &endinpos, &exc, &s,
- &writer))
- goto onError;
- }
+ message = "illegal Unicode character";
+ if (chr > MAX_UNICODE)
+ goto error;
+ WRITECHAR(chr);
break;
/* \N{name} */
@@ -5575,26 +5557,13 @@
goto store;
}
}
- endinpos = s-starts;
- if (unicode_decode_call_errorhandler_writer(
- errors, &errorHandler,
- "unicodeescape", message,
- &starts, &end, &startinpos, &endinpos, &exc, &s,
- &writer))
- goto onError;
- break;
+ goto error;
default:
if (s > end) {
message = "\\ at end of string";
s--;
- endinpos = s-starts;
- if (unicode_decode_call_errorhandler_writer(
- errors, &errorHandler,
- "unicodeescape", message,
- &starts, &end, &startinpos, &endinpos, &exc, &s,
- &writer))
- goto onError;
+ goto error;
}
else {
WRITECHAR('\\');
@@ -5602,8 +5571,17 @@
}
break;
}
- nextByte:
- ;
+ continue;
+
+ error:
+ endinpos = s-starts;
+ if (unicode_decode_call_errorhandler_writer(
+ errors, &errorHandler,
+ "unicodeescape", message,
+ &starts, &end, &startinpos, &endinpos, &exc, &s,
+ &writer))
+ goto onError;
+ continue;
}
#undef WRITECHAR
_______________________________________________
Python-bugs-list mailing list
Unsubscribe:
http://mail.python.org/mailman/options/python-bugs-list/archive%40mail-archive.com
