2016-02-12 23:41:21 +01:00
|
|
|
import codecs
|
|
|
|
import locale
|
2016-03-07 11:08:59 +01:00
|
|
|
import re
|
2016-02-12 23:41:21 +01:00
|
|
|
|
|
|
|
|
|
|
|
BOMS = [
|
|
|
|
(codecs.BOM_UTF8, 'utf8'),
|
|
|
|
(codecs.BOM_UTF16, 'utf16'),
|
|
|
|
(codecs.BOM_UTF16_BE, 'utf16-be'),
|
|
|
|
(codecs.BOM_UTF16_LE, 'utf16-le'),
|
|
|
|
(codecs.BOM_UTF32, 'utf32'),
|
|
|
|
(codecs.BOM_UTF32_BE, 'utf32-be'),
|
|
|
|
(codecs.BOM_UTF32_LE, 'utf32-le'),
|
|
|
|
]
|
|
|
|
|
2016-03-07 11:08:59 +01:00
|
|
|
ENCODING_RE = re.compile('coding[:=]\s*([-\w.]+)')
|
|
|
|
|
2016-02-12 23:41:21 +01:00
|
|
|
|
|
|
|
def auto_decode(data):
|
|
|
|
"""Check a bytes string for a BOM to correctly detect the encoding
|
|
|
|
|
|
|
|
Fallback to locale.getpreferredencoding(False) like open() on Python3"""
|
|
|
|
for bom, encoding in BOMS:
|
|
|
|
if data.startswith(bom):
|
|
|
|
return data[len(bom):].decode(encoding)
|
2016-03-07 11:08:59 +01:00
|
|
|
# Lets check the first two lines as in PEP263
|
|
|
|
for line in data.splitlines()[:2]:
|
|
|
|
if line.startswith('#') and ENCODING_RE.search(line):
|
|
|
|
encoding = ENCODING_RE.search(line).groups()[0]
|
|
|
|
return data.decode(encoding)
|
2016-02-12 23:41:21 +01:00
|
|
|
return data.decode(locale.getpreferredencoding(False))
|