mirror of
https://github.com/morpheus65535/bazarr
synced 2024-12-23 00:03:33 +00:00
115 lines
4.7 KiB
Python
115 lines
4.7 KiB
Python
######################## BEGIN LICENSE BLOCK ########################
|
|
# This library is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU Lesser General Public
|
|
# License as published by the Free Software Foundation; either
|
|
# version 2.1 of the License, or (at your option) any later version.
|
|
#
|
|
# This library is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
# Lesser General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Lesser General Public
|
|
# License along with this library; if not, write to the Free Software
|
|
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
|
|
# 02110-1301 USA
|
|
######################### END LICENSE BLOCK #########################
|
|
|
|
from typing import List, Union
|
|
|
|
from .charsetgroupprober import CharSetGroupProber
|
|
from .charsetprober import CharSetProber
|
|
from .enums import InputState
|
|
from .resultdict import ResultDict
|
|
from .universaldetector import UniversalDetector
|
|
from .version import VERSION, __version__
|
|
|
|
__all__ = ["UniversalDetector", "detect", "detect_all", "__version__", "VERSION"]
|
|
|
|
|
|
def detect(
|
|
byte_str: Union[bytes, bytearray], should_rename_legacy: bool = False
|
|
) -> ResultDict:
|
|
"""
|
|
Detect the encoding of the given byte string.
|
|
|
|
:param byte_str: The byte sequence to examine.
|
|
:type byte_str: ``bytes`` or ``bytearray``
|
|
:param should_rename_legacy: Should we rename legacy encodings
|
|
to their more modern equivalents?
|
|
:type should_rename_legacy: ``bool``
|
|
"""
|
|
if not isinstance(byte_str, bytearray):
|
|
if not isinstance(byte_str, bytes):
|
|
raise TypeError(
|
|
f"Expected object of type bytes or bytearray, got: {type(byte_str)}"
|
|
)
|
|
byte_str = bytearray(byte_str)
|
|
detector = UniversalDetector(should_rename_legacy=should_rename_legacy)
|
|
detector.feed(byte_str)
|
|
return detector.close()
|
|
|
|
|
|
def detect_all(
|
|
byte_str: Union[bytes, bytearray],
|
|
ignore_threshold: bool = False,
|
|
should_rename_legacy: bool = False,
|
|
) -> List[ResultDict]:
|
|
"""
|
|
Detect all the possible encodings of the given byte string.
|
|
|
|
:param byte_str: The byte sequence to examine.
|
|
:type byte_str: ``bytes`` or ``bytearray``
|
|
:param ignore_threshold: Include encodings that are below
|
|
``UniversalDetector.MINIMUM_THRESHOLD``
|
|
in results.
|
|
:type ignore_threshold: ``bool``
|
|
:param should_rename_legacy: Should we rename legacy encodings
|
|
to their more modern equivalents?
|
|
:type should_rename_legacy: ``bool``
|
|
"""
|
|
if not isinstance(byte_str, bytearray):
|
|
if not isinstance(byte_str, bytes):
|
|
raise TypeError(
|
|
f"Expected object of type bytes or bytearray, got: {type(byte_str)}"
|
|
)
|
|
byte_str = bytearray(byte_str)
|
|
|
|
detector = UniversalDetector(should_rename_legacy=should_rename_legacy)
|
|
detector.feed(byte_str)
|
|
detector.close()
|
|
|
|
if detector.input_state == InputState.HIGH_BYTE:
|
|
results: List[ResultDict] = []
|
|
probers: List[CharSetProber] = []
|
|
for prober in detector.charset_probers:
|
|
if isinstance(prober, CharSetGroupProber):
|
|
probers.extend(p for p in prober.probers)
|
|
else:
|
|
probers.append(prober)
|
|
for prober in probers:
|
|
if ignore_threshold or prober.get_confidence() > detector.MINIMUM_THRESHOLD:
|
|
charset_name = prober.charset_name or ""
|
|
lower_charset_name = charset_name.lower()
|
|
# Use Windows encoding name instead of ISO-8859 if we saw any
|
|
# extra Windows-specific bytes
|
|
if lower_charset_name.startswith("iso-8859") and detector.has_win_bytes:
|
|
charset_name = detector.ISO_WIN_MAP.get(
|
|
lower_charset_name, charset_name
|
|
)
|
|
# Rename legacy encodings with superset encodings if asked
|
|
if should_rename_legacy:
|
|
charset_name = detector.LEGACY_MAP.get(
|
|
charset_name.lower(), charset_name
|
|
)
|
|
results.append(
|
|
{
|
|
"encoding": charset_name,
|
|
"confidence": prober.get_confidence(),
|
|
"language": prober.language,
|
|
}
|
|
)
|
|
if len(results) > 0:
|
|
return sorted(results, key=lambda result: -result["confidence"])
|
|
|
|
return [detector.result]
|