mirror of https://github.com/morpheus65535/bazarr
144 lines
4.0 KiB
Python
144 lines
4.0 KiB
Python
"""
|
|
A command-line utility for fixing text found in a file.
|
|
"""
|
|
import os
|
|
import sys
|
|
|
|
from ftfy import __version__, fix_file, TextFixerConfig
|
|
|
|
ENCODE_ERROR_TEXT_UNIX = """ftfy error:
|
|
Unfortunately, this output stream does not support Unicode.
|
|
|
|
Your system locale may be very old or misconfigured. You should use a locale
|
|
that supports UTF-8. One way to do this is to `export LANG=C.UTF-8`.
|
|
"""
|
|
|
|
ENCODE_ERROR_TEXT_WINDOWS = """ftfy error:
|
|
Unfortunately, this output stream does not support Unicode.
|
|
|
|
You might be trying to output to the Windows Command Prompt (cmd.exe), which
|
|
does not fully support Unicode for historical reasons. In general, we recommend
|
|
finding a way to run Python without using cmd.exe.
|
|
|
|
You can work around this problem by using the '-o filename' option in ftfy to
|
|
output to a file instead.
|
|
"""
|
|
|
|
DECODE_ERROR_TEXT = """ftfy error:
|
|
This input couldn't be decoded as %r. We got the following error:
|
|
|
|
%s
|
|
|
|
ftfy works best when its input is in a known encoding. You can use `ftfy -g`
|
|
to guess, if you're desperate. Otherwise, give the encoding name with the
|
|
`-e` option, such as `ftfy -e latin-1`.
|
|
"""
|
|
|
|
SAME_FILE_ERROR_TEXT = """ftfy error:
|
|
Can't read and write the same file. Please output to a new file instead.
|
|
"""
|
|
|
|
|
|
def main():
|
|
"""
|
|
Run ftfy as a command-line utility.
|
|
"""
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description="ftfy (fixes text for you), version %s" % __version__
|
|
)
|
|
parser.add_argument(
|
|
"filename",
|
|
default="-",
|
|
nargs="?",
|
|
help="The file whose Unicode is to be fixed. Defaults "
|
|
"to -, meaning standard input.",
|
|
)
|
|
parser.add_argument(
|
|
"-o",
|
|
"--output",
|
|
type=str,
|
|
default="-",
|
|
help="The file to output to. Defaults to -, meaning " "standard output.",
|
|
)
|
|
parser.add_argument(
|
|
"-g",
|
|
"--guess",
|
|
action="store_true",
|
|
help="Ask ftfy to guess the encoding of your input. "
|
|
"This is risky. Overrides -e.",
|
|
)
|
|
parser.add_argument(
|
|
"-e",
|
|
"--encoding",
|
|
type=str,
|
|
default="utf-8",
|
|
help="The encoding of the input. Defaults to UTF-8.",
|
|
)
|
|
parser.add_argument(
|
|
"-n",
|
|
"--normalization",
|
|
type=str,
|
|
default="NFC",
|
|
help="The normalization of Unicode to apply. "
|
|
'Defaults to NFC. Can be "none".',
|
|
)
|
|
parser.add_argument(
|
|
"--preserve-entities",
|
|
action="store_true",
|
|
help="Leave HTML entities as they are. The default "
|
|
"is to decode them, as long as no HTML tags "
|
|
"have appeared in the file.",
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
encoding = args.encoding
|
|
if args.guess:
|
|
encoding = None
|
|
|
|
if args.filename == "-":
|
|
# Get a standard input stream made of bytes, so we can decode it as
|
|
# whatever encoding is necessary.
|
|
file = sys.stdin.buffer
|
|
else:
|
|
file = open(args.filename, "rb")
|
|
|
|
if args.output == "-":
|
|
outfile = sys.stdout
|
|
else:
|
|
if os.path.realpath(args.output) == os.path.realpath(args.filename):
|
|
sys.stderr.write(SAME_FILE_ERROR_TEXT)
|
|
sys.exit(1)
|
|
outfile = open(args.output, "w", encoding="utf-8")
|
|
|
|
normalization = args.normalization
|
|
if normalization.lower() == "none":
|
|
normalization = None
|
|
|
|
if args.preserve_entities:
|
|
unescape_html = False
|
|
else:
|
|
unescape_html = "auto"
|
|
|
|
config = TextFixerConfig(unescape_html=unescape_html, normalization=normalization)
|
|
|
|
try:
|
|
for line in fix_file(file, encoding=encoding, config=config):
|
|
try:
|
|
outfile.write(line)
|
|
except UnicodeEncodeError:
|
|
if sys.platform == "win32":
|
|
sys.stderr.write(ENCODE_ERROR_TEXT_WINDOWS)
|
|
else:
|
|
sys.stderr.write(ENCODE_ERROR_TEXT_UNIX)
|
|
sys.exit(1)
|
|
except UnicodeDecodeError as err:
|
|
sys.stderr.write(DECODE_ERROR_TEXT % (encoding, err))
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|