bazarr/libs/commonmark/normalize_reference.py

166 lines
6.2 KiB
Python

"""Case-folding and whitespace normalization"""
# Unicode Case Folding table has been derived from the following work:
#
# CaseFolding-12.0.0.txt
# Date: 2019-01-22, 08:18:22 GMT
# (c) 2019 Unicode(R) Inc.
# Unicode and the Unicode Logo are registered trademarks
# of Unicode, Inc. in the U.S. and other countries.
# For terms of use, see http://www.unicode.org/terms_of_use.html
#
# Unicode Character Database
# For documentation, see http://www.unicode.org/reports/tr44/
import re
import sys
from builtins import str, chr
__all__ = ["normalize_reference"]
if sys.version_info < (3,) and sys.maxunicode <= 0xffff:
# shim for Python 2.x UCS2 build
_unichr = chr
def chr(cdp):
if 0x10000 <= cdp < 0x110000:
cdp -= 0x10000
return (_unichr(0xd800 | (cdp >> 10)) +
_unichr(0xdc00 | (cdp & 0x3ff)))
return _unichr(cdp)
def _parse_table(tbl):
xlat = {}
cur_i, cur_j = -1, 0
for entry in tbl.split(';'):
arr = entry.split(',')
info = [int(x, 36) if x else 0 for x in arr[0].split(':')]
arr = [int(x, 36) for x in arr[1:]]
assert not any(x in xlat for x in arr)
sfx = ''.join(map(chr, arr))
streak, stride = 0, 1
if len(info) == 2:
fdt, delta = info
elif len(info) == 3:
fdt, streak, delta = info
else:
fdt, streak, delta, stride = info
assert streak >= 0 and stride >= 1
cur_i += fdt + 1
cur_j -= delta
assert cur_j != 0
i = cur_i
last = cur_i + streak
while i <= last:
# uniqueness and idempotency
assert i not in xlat and i + cur_j not in xlat
assert i not in arr
xlat[i] = chr(i + cur_j) + sfx
i += stride
return xlat
XLAT = _parse_table(
# ===== Start of Unicode Case Folding table =====
'1t:p:-w;37:-kn;a:m:kn;n:6:;6:3w,37;w:1a:-31:2;1b:5k,lj;1:4:-5k:2;6:e::'
'2;f:-aa,32;:18:aa:2;19:3e;:4:-3e:2;5:7h;1:-da;:2:5t:2;3:-5p;:5p;1:1:-5'
'o;1:5o;2:-26;:-3f;:-1;:5m;1:-5o;:-2;1:-4;:2;:5s;3:-5u;:-2;1:-1;:4:5x:2'
';5:-61;:61;1:-61;2:61;1:-61;:61;1:1:-60;1:2:60:2;3:-62;:4:62:4;b:-1;:1'
';1:-1;:1;1:-1;:g:1:2;i:g::2;h:av,lo;:-aw;:2:1:2;3:2q;:-15;:12:-1l:2;13'
':3n;1:g:-3n:2;n:-8bu;:8bu;1:4k;:-8gb;2:8br;1:5g;:-7c;:-2;:8:1y:2;72:-3'
'7;16:2:37:2;5:;8:-37;6:26;1:2:1;3:-r;1:1:1;1:m,lk,ld;:g:9;h:8:;c:b,lk,'
'ld;h:k;c:-7;:12;:-5;3:-a;:7;1:m:-n:2;n:1j;:-6;2:c;:4;1:-1t;1:8;:-8;2:2'
':3n;2:f:-5u;f:v:1c;27:w:v:2;15:1g::2;1h:-e;:c:e:2;e:2m::2;2o:11:-1b;2d'
':2a,136;26w:11:-5mq;12:6::6;mo:5:5m0;1on:4sm;:-1;:-9;:1:-2;1:1;:-7;:-o'
';:-vzb;7:16:tj7;18:2:;8y:44:-2bl:2;45:5yn,mp;:-b,lk;:-2,lm;:-1,lm;:p,j'
'i;:-5xb;2:5wx,37;1:2m:-5yk:2;2v:7:9;f:5:;f:7:;f:7:;f:5:;7:5fn,lv;1:2,l'
'v,lc;1:2,lv,ld;1:2,lv,n6;2:6:-5ft:2;e:7:;n:7:3c,qh;7:7:8,qh;7:7:-o,qh;'
'7:7:8,qh;7:7:-1k,qh;7:7:8,qh;9:-6,qh;:5hc,qh;:6,qh;1:-3,n6;:1,n6,qh;:1'
':-5j2;1:1:1u;1:5hd,qh;1:-6;3:-5h3,qh;:5ha,qh;:a,qh;1:-7,n6;:1,n6,qh;:3'
':-5h6;3:5hb,qh;5:4,lk,lc;:1,lk,ld;2:3,n6;:1,lk,n6;:1:-5jq;1:1:2k;7:5h5'
',lk,lc;:1,lk,ld;:5,lv;1:-2,n6;:1,lk,n6;:1:-5ju;1:1:2w;1:-2x;5:33,qh;:5'
'h0,qh;:-4,qh;1:7,n6;:1,n6,qh;:1:-5gu;1:1:-2;1:5h1,qh;89:8a;3:o2;:-3d;6'
':-6ea;19:f:c;y:f;mq:p:-p;1ft:1a:-m;2n:1b;1:8ag;:-5ch;:5c1;2:4:-8a0:2;5'
':8bh;:-v;:y;:-1;1:3:-8bj:3;b:1:8cg;1:2q:-8cg:2;2y:2::2;6:nym::nym;nyn:'
'16::2;1p:q::2;4h:c::2;f:1o::2;1y:2::2;3:r9h;:8:-r9h:2;c:;1:wmh;2:2:-wm'
'h:2;5:i::2;j:wn9;:b;:-4;:-a;:3;1:-1e;:o;:-l;:-xbp;:a:pr:2;d:;1:1d;:wlv'
';:-5cb;q1:27:2oo;fpr:jii,2u;:1,2x;:1,30;:1,2u,2x;:1,2u,30;:-c,38;:1,38'
';c:-z8,12u;:1,12d;:1,12j;:-9,12u;:b,12l;sp:p:-1cjn;ym:13:-8;4v:z:;1jj:'
'1e:-o;2e7:v:w;gwv:v:;o8v:x:-2'
# ===== End of Unicode Case Folding table =====
)
def _check_native(tbl):
"""
Determine if Python's own native implementation
subsumes the supplied case folding table
"""
try:
for i in tbl:
stv = chr(i)
if stv.casefold() == stv:
return False
except AttributeError:
return False
return True
# Hoist version check out of function for performance
SPACE_RE = re.compile(r'[ \t\r\n]+')
if _check_native(XLAT):
def normalize_reference(string):
"""
Normalize reference label: collapse internal whitespace
to single space, remove leading/trailing whitespace, case fold.
"""
return SPACE_RE.sub(' ', string[1:-1].strip()).casefold()
elif sys.version_info >= (3,) or sys.maxunicode > 0xffff:
def normalize_reference(string):
"""
Normalize reference label: collapse internal whitespace
to single space, remove leading/trailing whitespace, case fold.
"""
return SPACE_RE.sub(' ', string[1:-1].strip()).translate(XLAT)
else:
def _get_smp_regex():
xls = sorted(x - 0x10000 for x in XLAT if x >= 0x10000)
xls.append(-1)
fmt, (dsh, opn, pip, cse) = str('\\u%04x'), str('-[|]')
rga, srk, erk = [str(r'[ \t\r\n]+')], 0, -2
for k in xls:
new_hir = (erk ^ k) >> 10 != 0
if new_hir or erk + 1 != k:
if erk >= 0 and srk != erk:
if srk + 1 != erk:
rga.append(dsh)
rga.append(fmt % (0xdc00 + (erk & 0x3ff)))
if new_hir:
if erk >= 0:
rga.append(cse)
if k < 0:
break
rga.append(pip)
rga.append(fmt % (0xd800 + (k >> 10)))
rga.append(opn)
srk = k
rga.append(fmt % (0xdc00 + (srk & 0x3ff)))
erk = k
return re.compile(str().join(rga))
def _subst_handler(matchobj):
src = matchobj.group(0)
hiv = ord(src[0])
if hiv < 0xd800:
return ' '
return XLAT[0x10000 + ((hiv & 0x3ff) << 10) | (ord(src[1]) & 0x3ff)]
SMP_RE = _get_smp_regex()
def normalize_reference(string):
"""
Normalize reference label: collapse internal whitespace
to single space, remove leading/trailing whitespace, case fold.
"""
return SMP_RE.sub(_subst_handler, string[1:-1].strip()).translate(XLAT)