gh-148821: Always reject known multi-byte encodings in pyexpat

Include/codecs.h

@@ -170,6 +170,12 @@ PyAPI_FUNC(PyObject *) PyCodec_NameReplaceErrors(PyObject *exc);
 PyAPI_DATA(const char *) Py_hexdigits;
 #endif
 
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject*) _PyCodec_LookupTextEncoding(
+   const char *encoding,
+   const char *alternate_command);
+#endif
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_codecs.h

@@ -45,7 +45,7 @@ extern int _PyCodec_UnregisterError(const char *name);
    in Python 3.5+?
 
  */
-extern PyObject* _PyCodec_LookupTextEncoding(
+PyAPI_FUNC(PyObject*) _PyCodec_LookupTextEncoding(
    const char *encoding,
    const char *alternate_command);
 

Lib/codecs.py

@@ -93,7 +93,8 @@ class CodecInfo(tuple):
 
     def __new__(cls, encode, decode, streamreader=None, streamwriter=None,
         incrementalencoder=None, incrementaldecoder=None, name=None,
-        *, _is_text_encoding=None):
+        *, _is_text_encoding=None,
+        _is_single_byte=None):
         self = tuple.__new__(cls, (encode, decode, streamreader, streamwriter))
         self.name = name
         self.encode = encode
@@ -104,6 +105,8 @@ def __new__(cls, encode, decode, streamreader=None, streamwriter=None,
         self.streamreader = streamreader
         if _is_text_encoding is not None:
             self._is_text_encoding = _is_text_encoding
+        if _is_single_byte is not None:
+            self._is_single_byte = _is_single_byte
         return self
 
     def __repr__(self):

Lib/encodings/big5.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/big5hkscs.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/cp932.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/cp949.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/cp950.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/euc_jis_2004.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/euc_jisx0213.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/euc_jp.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/euc_kr.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/gb18030.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/gb2312.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/gbk.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/hz.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/idna.py

@@ -385,4 +385,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamwriter=StreamWriter,
         streamreader=StreamReader,
+        _is_single_byte=False,
     )

Lib/encodings/iso2022_jp.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/iso2022_jp_1.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/iso2022_jp_2.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/iso2022_jp_2004.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/iso2022_jp_3.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/iso2022_jp_ext.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/iso2022_kr.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/johab.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/punycode.py

@@ -250,4 +250,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamwriter=StreamWriter,
         streamreader=StreamReader,
+        _is_single_byte=False,
     )

Lib/encodings/raw_unicode_escape.py

@@ -43,4 +43,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamwriter=StreamWriter,
         streamreader=StreamReader,
+        _is_single_byte=False,
     )

Lib/encodings/shift_jis.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/shift_jis_2004.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/shift_jisx0213.py

@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/unicode_escape.py

@@ -43,4 +43,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamwriter=StreamWriter,
         streamreader=StreamReader,
+        _is_single_byte=False,
     )

Lib/encodings/utf_16.py

@@ -152,4 +152,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/utf_16_be.py

@@ -39,4 +39,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/utf_16_le.py

@@ -39,4 +39,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/utf_32.py

@@ -147,4 +147,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/utf_32_be.py

@@ -34,4 +34,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/utf_32_le.py

@@ -34,4 +34,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/utf_7.py

@@ -35,4 +35,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/utf_8.py

@@ -39,4 +39,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/encodings/utf_8_sig.py

@@ -127,4 +127,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )

Lib/test/test_codecs.py

@@ -1892,6 +1892,7 @@ def test_copy(self):
         self.assertIsNot(dup, orig)
         self.assertEqual(dup, orig)
         self.assertTrue(orig._is_text_encoding)
+        self.assertFalse(orig._is_single_byte)
         self.assertEqual(dup.encode, orig.encode)
         self.assertEqual(dup.name, orig.name)
         self.assertEqual(dup.incrementalencoder, orig.incrementalencoder)
@@ -1912,6 +1913,7 @@ def test_deepcopy(self):
         self.assertIsNot(dup, orig)
         self.assertEqual(dup, orig)
         self.assertTrue(orig._is_text_encoding)
+        self.assertFalse(orig._is_single_byte)
         self.assertEqual(dup.encode, orig.encode)
         self.assertEqual(dup.name, orig.name)
         self.assertEqual(dup.incrementalencoder, orig.incrementalencoder)
@@ -1940,6 +1942,7 @@ def test_pickle(self):
                      unpickled_codec_info.incrementalencoder
                 )
                 self.assertTrue(unpickled_codec_info._is_text_encoding)
+                self.assertFalse(unpickled_codec_info._is_single_byte)
 
         # Test a CodecInfo with _is_text_encoding equal to false.
         codec_info = codecs.lookup('base64')

Lib/test/test_pyexpat.py

@@ -227,7 +227,7 @@ def _verify_parse_output(self, operations):
             "Character data: '\xb5'",
             "End element: 'root'",
         ]
-        for operation, expected_operation in zip(operations, expected_operations):
+        for operation, expected_operation in zip(operations, expected_operations, strict=True):
             self.assertEqual(operation, expected_operation)
 
     def test_parse_bytes(self):
@@ -276,6 +276,51 @@ def test_parse_again(self):
         self.assertEqual(expat.ErrorString(cm.exception.code),
                           expat.errors.XML_ERROR_FINISHED)
 
+    @support.subTests('enc', ['UTF-8', 'utf-8', 'utf-16', 'koi8-u',
+                              'cp1125', 'cp1251', 'iso8859-5',
+                              'mac_cyrillic'])
+    def test_supportes_ecodings(self, enc):
+        out = self.Outputter()
+        parser = expat.ParserCreate()
+        self._hookup_callbacks(parser, out)
+        data = (f'<?xml version="1.0" encoding="{enc}"?>\n'
+                '<корінь атрибут="значення">зміст</корінь>').encode(enc)
+        parser.Parse(data, True)
+        self.assertEqual(out.out, [
+            ('XML declaration', ('1.0', enc, -1)),
+            "Start element: 'корінь' {'атрибут': 'значення'}",
+            "Character data: 'зміст'",
+            "End element: 'корінь'",
+        ])
+
+    @support.subTests('enc', [
+        'UTF8', 'UTF-7',
+        "unicode-escape", "raw-unicode-escape",
+        "Big5-HKSCS", "Big5",
+        "cp932", "cp949", "cp950",
+        "EUC_JIS-2004", "EUC_JISX0213", "EUC-JP", "EUC-KR",
+        "GB18030", "GB2312", "GBK",
+        "HZ-GB-2312",
+        "ISO-2022-JP", "ISO-2022-JP-1", "ISO-2022-JP-2004",
+        "ISO-2022-JP-2", "ISO-2022-JP-3", "ISO-2022-JP-EXT",
+        "ISO-2022-KR",
+        "johab",
+        "Shift_JIS", "Shift_JIS-2004", "Shift_JISX0213",
+    ])
+    def test_unsupportes_ecodings(self, enc):
+        parser = expat.ParserCreate()
+        data = (f'<?xml version="1.0" encoding="{enc}"?>\n'
+                '<root></root>').encode(enc)
+        with self.assertRaises(ValueError):
+            parser.Parse(data, True)
+
+    def test_unknown_ecoding(self):
+        parser = expat.ParserCreate()
+        data = b'<?xml version="1.0" encoding="xyz"?>\n<root></root>'
+        with self.assertRaises(LookupError):
+            parser.Parse(data, True)
+
+
 class NamespaceSeparatorTest(unittest.TestCase):
     def test_legal(self):
         # Tests that make sure we get errors when the namespace_separator value