bazarr/libs/commonmark/normalize_reference.py

"""Case-folding and whitespace normalization"""
# Unicode Case Folding table has been derived from the following work:
#
#   CaseFolding-12.0.0.txt
#   Date: 2019-01-22, 08:18:22 GMT
#   (c) 2019 Unicode(R) Inc.
#   Unicode and the Unicode Logo are registered trademarks
#   of Unicode, Inc. in the U.S. and other countries.
#   For terms of use, see http://www.unicode.org/terms_of_use.html
#
#   Unicode Character Database
#     For documentation, see http://www.unicode.org/reports/tr44/

import re
import sys
from builtins import str, chr

__all__ = ["normalize_reference"]

if sys.version_info < (3,) and sys.maxunicode <= 0xffff:
    # shim for Python 2.x UCS2 build
    _unichr = chr

    def chr(cdp):
        if 0x10000 <= cdp < 0x110000:
            cdp -= 0x10000
            return (_unichr(0xd800 | (cdp >> 10)) +
                    _unichr(0xdc00 | (cdp & 0x3ff)))
        return _unichr(cdp)


def _parse_table(tbl):
    xlat = {}
    cur_i, cur_j = -1, 0
    for entry in tbl.split(';'):
        arr = entry.split(',')
        info = [int(x, 36) if x else 0 for x in arr[0].split(':')]
        arr = [int(x, 36) for x in arr[1:]]
        assert not any(x in xlat for x in arr)
        sfx = ''.join(map(chr, arr))
        streak, stride = 0, 1
        if len(info) == 2:
            fdt, delta = info
        elif len(info) == 3:
            fdt, streak, delta = info
        else:
            fdt, streak, delta, stride = info
        assert streak >= 0 and stride >= 1
        cur_i += fdt + 1
        cur_j -= delta
        assert cur_j != 0
        i = cur_i
        last = cur_i + streak
        while i <= last:
            # uniqueness and idempotency
            assert i not in xlat and i + cur_j not in xlat
            assert i not in arr
            xlat[i] = chr(i + cur_j) + sfx
            i += stride
    return xlat


XLAT = _parse_table(
    # ===== Start of Unicode Case Folding table =====
    '1t:p:-w;37:-kn;a:m:kn;n:6:;6:3w,37;w:1a:-31:2;1b:5k,lj;1:4:-5k:2;6:e::'
    '2;f:-aa,32;:18:aa:2;19:3e;:4:-3e:2;5:7h;1:-da;:2:5t:2;3:-5p;:5p;1:1:-5'
    'o;1:5o;2:-26;:-3f;:-1;:5m;1:-5o;:-2;1:-4;:2;:5s;3:-5u;:-2;1:-1;:4:5x:2'
    ';5:-61;:61;1:-61;2:61;1:-61;:61;1:1:-60;1:2:60:2;3:-62;:4:62:4;b:-1;:1'
    ';1:-1;:1;1:-1;:g:1:2;i:g::2;h:av,lo;:-aw;:2:1:2;3:2q;:-15;:12:-1l:2;13'
    ':3n;1:g:-3n:2;n:-8bu;:8bu;1:4k;:-8gb;2:8br;1:5g;:-7c;:-2;:8:1y:2;72:-3'
    '7;16:2:37:2;5:;8:-37;6:26;1:2:1;3:-r;1:1:1;1:m,lk,ld;:g:9;h:8:;c:b,lk,'
    'ld;h:k;c:-7;:12;:-5;3:-a;:7;1:m:-n:2;n:1j;:-6;2:c;:4;1:-1t;1:8;:-8;2:2'
    ':3n;2:f:-5u;f:v:1c;27:w:v:2;15:1g::2;1h:-e;:c:e:2;e:2m::2;2o:11:-1b;2d'
    ':2a,136;26w:11:-5mq;12:6::6;mo:5:5m0;1on:4sm;:-1;:-9;:1:-2;1:1;:-7;:-o'
    ';:-vzb;7:16:tj7;18:2:;8y:44:-2bl:2;45:5yn,mp;:-b,lk;:-2,lm;:-1,lm;:p,j'
    'i;:-5xb;2:5wx,37;1:2m:-5yk:2;2v:7:9;f:5:;f:7:;f:7:;f:5:;7:5fn,lv;1:2,l'
    'v,lc;1:2,lv,ld;1:2,lv,n6;2:6:-5ft:2;e:7:;n:7:3c,qh;7:7:8,qh;7:7:-o,qh;'
    '7:7:8,qh;7:7:-1k,qh;7:7:8,qh;9:-6,qh;:5hc,qh;:6,qh;1:-3,n6;:1,n6,qh;:1'
    ':-5j2;1:1:1u;1:5hd,qh;1:-6;3:-5h3,qh;:5ha,qh;:a,qh;1:-7,n6;:1,n6,qh;:3'
    ':-5h6;3:5hb,qh;5:4,lk,lc;:1,lk,ld;2:3,n6;:1,lk,n6;:1:-5jq;1:1:2k;7:5h5'
    ',lk,lc;:1,lk,ld;:5,lv;1:-2,n6;:1,lk,n6;:1:-5ju;1:1:2w;1:-2x;5:33,qh;:5'
    'h0,qh;:-4,qh;1:7,n6;:1,n6,qh;:1:-5gu;1:1:-2;1:5h1,qh;89:8a;3:o2;:-3d;6'
    ':-6ea;19:f:c;y:f;mq:p:-p;1ft:1a:-m;2n:1b;1:8ag;:-5ch;:5c1;2:4:-8a0:2;5'
    ':8bh;:-v;:y;:-1;1:3:-8bj:3;b:1:8cg;1:2q:-8cg:2;2y:2::2;6:nym::nym;nyn:'
    '16::2;1p:q::2;4h:c::2;f:1o::2;1y:2::2;3:r9h;:8:-r9h:2;c:;1:wmh;2:2:-wm'
    'h:2;5:i::2;j:wn9;:b;:-4;:-a;:3;1:-1e;:o;:-l;:-xbp;:a:pr:2;d:;1:1d;:wlv'
    ';:-5cb;q1:27:2oo;fpr:jii,2u;:1,2x;:1,30;:1,2u,2x;:1,2u,30;:-c,38;:1,38'
    ';c:-z8,12u;:1,12d;:1,12j;:-9,12u;:b,12l;sp:p:-1cjn;ym:13:-8;4v:z:;1jj:'
    '1e:-o;2e7:v:w;gwv:v:;o8v:x:-2'
    # ===== End of Unicode Case Folding table =====
)


def _check_native(tbl):
    """
    Determine if Python's own native implementation
    subsumes the supplied case folding table
    """
    try:
        for i in tbl:
            stv = chr(i)
            if stv.casefold() == stv:
                return False
    except AttributeError:
        return False
    return True


# Hoist version check out of function for performance
SPACE_RE = re.compile(r'[ \t\r\n]+')
if _check_native(XLAT):
    def normalize_reference(string):
        """
        Normalize reference label: collapse internal whitespace
        to single space, remove leading/trailing whitespace, case fold.
        """
        return SPACE_RE.sub(' ', string[1:-1].strip()).casefold()
elif sys.version_info >= (3,) or sys.maxunicode > 0xffff:
    def normalize_reference(string):
        """
        Normalize reference label: collapse internal whitespace
        to single space, remove leading/trailing whitespace, case fold.
        """
        return SPACE_RE.sub(' ', string[1:-1].strip()).translate(XLAT)
else:
    def _get_smp_regex():
        xls = sorted(x - 0x10000 for x in XLAT if x >= 0x10000)
        xls.append(-1)
        fmt, (dsh, opn, pip, cse) = str('\\u%04x'), str('-[|]')
        rga, srk, erk = [str(r'[ \t\r\n]+')], 0, -2
        for k in xls:
            new_hir = (erk ^ k) >> 10 != 0
            if new_hir or erk + 1 != k:
                if erk >= 0 and srk != erk:
                    if srk + 1 != erk:
                        rga.append(dsh)
                    rga.append(fmt % (0xdc00 + (erk & 0x3ff)))
                if new_hir:
                    if erk >= 0:
                        rga.append(cse)
                    if k < 0:
                        break
                    rga.append(pip)
                    rga.append(fmt % (0xd800 + (k >> 10)))
                    rga.append(opn)
                srk = k
                rga.append(fmt % (0xdc00 + (srk & 0x3ff)))
            erk = k
        return re.compile(str().join(rga))

    def _subst_handler(matchobj):
        src = matchobj.group(0)
        hiv = ord(src[0])
        if hiv < 0xd800:
            return ' '
        return XLAT[0x10000 + ((hiv & 0x3ff) << 10) | (ord(src[1]) & 0x3ff)]

    SMP_RE = _get_smp_regex()

    def normalize_reference(string):
        """
        Normalize reference label: collapse internal whitespace
        to single space, remove leading/trailing whitespace, case fold.
        """
        return SMP_RE.sub(_subst_handler, string[1:-1].strip()).translate(XLAT)