1
0
Fork 0
mirror of https://github.com/morpheus65535/bazarr synced 2024-12-23 08:13:14 +00:00
bazarr/custom_libs/subliminal_patch/providers/opensubtitlescom.py

606 lines
24 KiB
Python

# -*- coding: utf-8 -*-
import logging
import os
import time
import datetime
import json
from requests import Session, ConnectionError, Timeout, ReadTimeout, RequestException
from requests.exceptions import JSONDecodeError
from subzero.language import Language
from babelfish import language_converters
from subliminal import Episode, Movie
from subliminal.score import get_equivalent_release_groups
from subliminal.utils import sanitize_release_group, sanitize
from subliminal_patch.exceptions import TooManyRequests, APIThrottled
from subliminal.exceptions import DownloadLimitExceeded, AuthenticationError, ConfigurationError, ServiceUnavailable, \
ProviderError
from .mixins import ProviderRetryMixin
from subliminal_patch.subtitle import Subtitle
from subliminal.subtitle import fix_line_ending, SUBTITLE_EXTENSIONS
from subliminal_patch.providers import Provider
from subliminal_patch.subtitle import guess_matches
from subliminal_patch.utils import fix_inconsistent_naming
from subliminal.cache import region
from dogpile.cache.api import NO_VALUE
from guessit import guessit
logger = logging.getLogger(__name__)
SHOW_EXPIRATION_TIME = datetime.timedelta(weeks=1).total_seconds()
TOKEN_EXPIRATION_TIME = datetime.timedelta(hours=12).total_seconds()
retry_amount = 3
def fix_tv_naming(title):
"""Fix TV show titles with inconsistent naming using dictionary, but do not sanitize them.
:param str title: original title.
:return: new title.
:rtype: str
"""
return fix_inconsistent_naming(title, {"Superman & Lois": "Superman and Lois",
}, True)
def fix_movie_naming(title):
return fix_inconsistent_naming(title, {
}, True)
custom_languages = {
'pt': 'pt-PT',
'zh': 'zh-CN',
}
def to_opensubtitlescom(lang):
if lang in custom_languages.keys():
return custom_languages[lang]
else:
return lang
def from_opensubtitlescom(lang):
from_custom_languages = {v: k for k, v in custom_languages.items()}
if lang in from_custom_languages.keys():
return from_custom_languages[lang]
else:
return lang
class OpenSubtitlesComSubtitle(Subtitle):
provider_name = 'opensubtitlescom'
hash_verifiable = True
hearing_impaired_verifiable = True
def __init__(self, language, forced, hearing_impaired, page_link, file_id, releases, uploader, title, year,
hash_matched, file_hash=None, season=None, episode=None, imdb_match=False):
super().__init__(language, hearing_impaired, page_link)
language = Language.rebuild(language, hi=hearing_impaired, forced=forced)
self.title = title
self.year = year
self.season = season
self.episode = episode
self.releases = releases
self.release_info = releases
self.language = language
self.hearing_impaired = hearing_impaired
self.forced = forced
self.file_id = file_id
self.page_link = page_link
self.download_link = None
self.uploader = uploader
self.matches = None
self.hash = file_hash
self.encoding = 'utf-8'
self.hash_matched = hash_matched
self.imdb_match = imdb_match
@property
def id(self):
return self.file_id
def get_matches(self, video):
matches = set()
type_ = "movie" if isinstance(video, Movie) else "episode"
# handle movies and series separately
if type_ == "episode":
# series
matches.add('series')
# season
if video.season == self.season:
matches.add('season')
# episode
if video.episode == self.episode:
matches.add('episode')
# imdb
if self.imdb_match:
matches.add('series_imdb_id')
else:
# title
matches.add('title')
# imdb
if self.imdb_match:
matches.add('imdb_id')
# rest is same for both groups
# year
if video.year == self.year:
matches.add('year')
# release_group
if (video.release_group and self.releases and
any(r in sanitize_release_group(self.releases)
for r in get_equivalent_release_groups(sanitize_release_group(video.release_group)))):
matches.add('release_group')
if self.hash_matched:
matches.add('hash')
# other properties
matches |= guess_matches(video, guessit(self.releases, {"type": type_}))
self.matches = matches
return matches
class OpenSubtitlesComProvider(ProviderRetryMixin, Provider):
"""OpenSubtitlesCom Provider"""
server_hostname = 'api.opensubtitles.com'
languages = {Language.fromopensubtitles(lang) for lang in language_converters['szopensubtitles'].codes}
languages.update(set(Language.rebuild(lang, forced=True) for lang in languages))
languages.update(set(Language.rebuild(l, hi=True) for l in languages))
video_types = (Episode, Movie)
def __init__(self, username=None, password=None, use_hash=True, include_ai_translated=False, api_key=None):
if not all((username, password)):
raise ConfigurationError('Username and password must be specified')
if not api_key:
raise ConfigurationError('Api_key must be specified')
if not all((username, password)):
raise ConfigurationError('Username and password must be specified')
self.session = Session()
self.session.headers = {'User-Agent': os.environ.get("SZ_USER_AGENT", "Sub-Zero/2"),
'Api-Key': api_key,
'Content-Type': 'application/json'}
self.token = None
self.username = username
self.password = password
self.video = None
self.use_hash = use_hash
self.include_ai_translated = include_ai_translated
self._started = None
def initialize(self):
self._started = time.time()
if region.get("oscom_token", expiration_time=TOKEN_EXPIRATION_TIME) is NO_VALUE:
logger.debug("No cached token, we'll try to login again.")
self.login()
else:
self.token = region.get("oscom_token", expiration_time=TOKEN_EXPIRATION_TIME)
if region.get("oscom_server", expiration_time=TOKEN_EXPIRATION_TIME) is NO_VALUE:
logger.debug("No cached server, we'll try to login again.")
self.login()
else:
self.server_hostname = region.get("oscom_server", expiration_time=TOKEN_EXPIRATION_TIME)
def terminate(self):
self.session.close()
def ping(self):
return self._started and (time.time() - self._started) < TOKEN_EXPIRATION_TIME
def server_url(self):
return f'https://{self.server_hostname}/api/v1/'
def login(self, is_retry=False):
r = self.checked(
lambda: self.session.post(self.server_url() + 'login',
json={"username": self.username, "password": self.password},
allow_redirects=False,
timeout=30),
is_retry=is_retry)
try:
self.token = r.json()['token']
except (ValueError, JSONDecodeError, AttributeError):
log_request_response(r)
raise ProviderError("Cannot get token from provider login response")
else:
log_request_response(r, non_standard=False)
region.set("oscom_token", self.token)
try:
self.server_hostname = r.json()['base_url']
except (ValueError, JSONDecodeError):
log_request_response(r)
raise ProviderError("Cannot get server from provider login response")
else:
log_request_response(r, non_standard=False)
region.set("oscom_server", self.server_hostname)
finally:
if self.server_hostname.startswith('vip'):
self.session.headers.update({'Authorization': 'Bearer ' + self.token})
else:
self.session.headers.pop('Authorization', None)
@staticmethod
def sanitize_external_ids(external_id):
if isinstance(external_id, str):
external_id = external_id.lower().lstrip('tt').lstrip('0')
sanitized_id = external_id[:-1].lstrip('0') + external_id[-1]
return int(sanitized_id)
@region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
def search_titles(self, title):
title_id = None
parameters = {'query': title.lower()}
logger.debug(f'Searching using this title: {title}')
results = self.retry(
lambda: self.checked(
lambda: self.session.get(self.server_url() + 'features', params=parameters, timeout=30),
validate_json=True,
json_key_name='data'
),
amount=retry_amount
)
# deserialize results
results_dict = results.json()['data']
# loop over results
for result in results_dict:
if 'title' in result['attributes']:
if isinstance(self.video, Episode):
if fix_tv_naming(title).lower() == result['attributes']['title'].lower() and \
(not self.video.year or self.video.year == int(result['attributes']['year'])):
title_id = result['id']
break
else:
if fix_movie_naming(title).lower() == result['attributes']['title'].lower() and \
(not self.video.year or self.video.year == int(result['attributes']['year'])):
title_id = result['id']
break
else:
continue
if title_id:
logger.debug(f'Found this title ID: {title_id}')
return self.sanitize_external_ids(title_id)
if not title_id:
logger.debug(f'No match found for {title}')
def query(self, languages, video):
self.video = video
if self.use_hash:
file_hash = self.video.hashes.get('opensubtitlescom')
logger.debug(f'Searching using this hash: {file_hash}')
else:
file_hash = None
if isinstance(self.video, Episode):
title = self.video.series
else:
title = self.video.title
imdb_id = None
if isinstance(self.video, Episode) and self.video.series_imdb_id:
imdb_id = self.sanitize_external_ids(self.video.series_imdb_id)
elif isinstance(self.video, Movie) and self.video.imdb_id:
imdb_id = self.sanitize_external_ids(self.video.imdb_id)
title_id = None
if not imdb_id:
title_id = self.search_titles(title)
if not title_id:
return []
# be sure to remove duplicates using list(set())
langs_list = sorted(list(set([to_opensubtitlescom(lang.basename).lower() for lang in languages])))
langs = ','.join(langs_list)
logger.debug(f'Searching for those languages: {langs}')
# query the server
if isinstance(self.video, Episode):
res = self.retry(
lambda: self.checked(
lambda: self.session.get(self.server_url() + 'subtitles',
params=(('ai_translated', 'exclude' if not self.include_ai_translated
else 'include'),
('episode_number', self.video.episode),
('imdb_id', imdb_id if not title_id else None),
('languages', langs),
('moviehash', file_hash),
('parent_feature_id', title_id if title_id else None),
('season_number', self.video.season)),
timeout=30),
validate_json=True,
json_key_name='data'
),
amount=retry_amount
)
else:
res = self.retry(
lambda: self.checked(
lambda: self.session.get(self.server_url() + 'subtitles',
params=(('ai_translated', 'exclude' if not self.include_ai_translated
else 'include'),
('id', title_id if title_id else None),
('imdb_id', imdb_id if not title_id else None),
('languages', langs),
('moviehash', file_hash)),
timeout=30),
validate_json=True,
json_key_name='data'
),
amount=retry_amount
)
subtitles = []
result = res.json()
# filter out forced subtitles or not depending on the required languages
if all([lang.forced for lang in languages]): # only forced
result['data'] = [x for x in result['data'] if x['attributes']['foreign_parts_only']]
elif any([lang.forced for lang in languages]): # also forced
pass
else: # not forced
result['data'] = [x for x in result['data'] if not x['attributes']['foreign_parts_only']]
logger.debug(f"Query returned {len(result['data'])} subtitles")
if len(result['data']):
for item in result['data']:
# ignore AI translated subtitles
if not self.include_ai_translated:
if 'ai_translated' in item['attributes'] and item['attributes']['ai_translated']:
logger.debug("Skipping AI translated subtitles")
continue
# ignore machine translated subtitles
if 'machine_translated' in item['attributes'] and item['attributes']['machine_translated']:
logger.debug("Skipping machine translated subtitles")
continue
if 'season_number' in item['attributes']['feature_details']:
season_number = item['attributes']['feature_details']['season_number']
else:
season_number = None
if 'episode_number' in item['attributes']['feature_details']:
episode_number = item['attributes']['feature_details']['episode_number']
else:
episode_number = None
if 'moviehash_match' in item['attributes']:
moviehash_match = item['attributes']['moviehash_match']
else:
moviehash_match = False
try:
year = int(item['attributes']['feature_details']['year'])
except TypeError:
year = item['attributes']['feature_details']['year']
if len(item['attributes']['files']):
subtitle = OpenSubtitlesComSubtitle(
language=Language.fromietf(from_opensubtitlescom(item['attributes']['language'])),
forced=item['attributes']['foreign_parts_only'],
hearing_impaired=item['attributes']['hearing_impaired'],
page_link=item['attributes']['url'],
file_id=item['attributes']['files'][0]['file_id'],
releases=item['attributes']['release'],
uploader=item['attributes']['uploader']['name'],
title=item['attributes']['feature_details']['movie_name'],
year=year,
season=season_number,
episode=episode_number,
hash_matched=moviehash_match,
imdb_match=True if imdb_id else False
)
subtitle.get_matches(self.video)
subtitles.append(subtitle)
return subtitles
def list_subtitles(self, video, languages):
return self.query(languages, video)
def download_subtitle(self, subtitle):
logger.info('Downloading subtitle %r', subtitle)
headers = {'Accept': 'application/json', 'Content-Type': 'application/json',
'Authorization': 'Bearer ' + self.token}
res = self.retry(
lambda: self.checked(
lambda: self.session.post(self.server_url() + 'download',
json={'file_id': subtitle.file_id, 'sub_format': 'srt'},
headers=headers,
timeout=30),
validate_json=True,
json_key_name='link'
),
amount=retry_amount
)
download_data = res.json()
subtitle.download_link = download_data['link']
r = self.retry(
lambda: self.checked(
lambda: self.session.get(subtitle.download_link, timeout=30),
validate_content=True
),
amount=retry_amount
)
if not r:
logger.debug(f'Could not download subtitle from {subtitle.download_link}')
subtitle.content = None
return
else:
subtitle_content = r.content
subtitle.content = fix_line_ending(subtitle_content)
@staticmethod
def reset_token():
logger.debug('Authentication failed: clearing cache and attempting to login.')
region.delete("oscom_token")
region.delete("oscom_server")
return
def checked(self, fn, raise_api_limit=False, validate_json=False, json_key_name=None, validate_content=False,
is_retry=False):
"""Run :fn: and check the response status before returning it.
:param fn: the function to make an API call to OpenSubtitles.com.
:param raise_api_limit: if True we wait a little bit longer before running the call again.
:param validate_json: test if response is valid json.
:param json_key_name: test if returned json contain a specific key.
:param validate_content: test if response have a content (used with download).
:param is_retry: prevent additional retries with login endpoint.
:return: the response.
"""
response = None
try:
try:
response = fn()
except APIThrottled:
if not raise_api_limit:
logger.info("API request limit hit, waiting and trying again once.")
time.sleep(15)
return self.checked(fn, raise_api_limit=True)
raise
except (ConnectionError, Timeout, ReadTimeout):
raise ServiceUnavailable(f'Unknown Error, empty response: {response.status_code}: {response}')
except Exception:
logger.exception('Unhandled exception raised.')
raise ProviderError('Unhandled exception raised. Check log.')
else:
status_code = response.status_code
except Exception:
status_code = None
else:
if status_code == 400:
try:
json_response = response.json()
message = json_response['message']
except JSONDecodeError:
raise ProviderError('Invalid JSON returned by provider')
else:
log_request_response(response)
raise ConfigurationError(message)
elif status_code == 401:
log_request_response(response)
self.reset_token()
if is_retry:
raise AuthenticationError('Login failed')
else:
time.sleep(1)
self.login(is_retry=True)
self.checked(fn, raise_api_limit=raise_api_limit, validate_json=validate_json,
json_key_name=json_key_name, validate_content=validate_content, is_retry=True)
elif status_code == 403:
log_request_response(response)
raise ProviderError("Bazarr API key seems to be in problem")
elif status_code == 406:
try:
json_response = response.json()
download_count = json_response['requests']
remaining_download = json_response['remaining']
quota_reset_time = json_response['reset_time']
except JSONDecodeError:
raise ProviderError('Invalid JSON returned by provider')
else:
log_request_response(response)
raise DownloadLimitExceeded(f"Daily download limit reached. {download_count} subtitles have been "
f"downloaded and {remaining_download} remaining subtitles can be "
f"downloaded. Quota will be reset in {quota_reset_time}.")
elif status_code == 410:
log_request_response(response)
raise ProviderError("Download as expired")
elif status_code == 429:
log_request_response(response)
raise TooManyRequests()
elif status_code == 500:
logger.debug("Server side exception raised while downloading from opensubtitles.com website. They "
"should mitigate this soon.")
return None
elif status_code == 502:
# this one should deal with Bad Gateway issue on their side.
raise APIThrottled()
elif 500 <= status_code <= 599:
raise ProviderError(response.reason)
if status_code != 200:
log_request_response(response)
raise ProviderError(f'Bad status code: {response.status_code}')
if validate_json:
try:
json_test = response.json()
except JSONDecodeError:
raise ProviderError('Invalid JSON returned by provider')
else:
if json_key_name not in json_test:
raise ProviderError(f'Invalid JSON returned by provider: no {json_key_name} key in returned json.')
if validate_content:
if not hasattr(response, 'content'):
logger.error('Download link returned no content attribute.')
return False
elif not response.content:
logger.error(f'This download link returned empty content: {response.url}')
return False
return response
def log_request_response(response, non_standard=True):
redacted_request_headers = response.request.headers
if 'Authorization' in redacted_request_headers and isinstance(redacted_request_headers['Authorization'], str):
redacted_request_headers['Authorization'] = redacted_request_headers['Authorization'][:-8]+8*'x'
if response.request.body:
redacted_request_body = json.loads(response.request.body)
if 'password' in redacted_request_body:
redacted_request_body['password'] = 'redacted'
else:
redacted_request_body = None
redacted_response_body = json.loads(response.text)
if 'token' in redacted_response_body and isinstance(redacted_response_body['token'], str):
redacted_response_body['token'] = redacted_response_body['token'][:-8] + 8 * 'x'
if non_standard:
logger.debug("opensubtitlescom returned a non standard response. Logging request/response for debugging "
"purpose.")
else:
logger.debug("opensubtitlescom returned a standard response. Logging request/response for debugging purpose.")
logger.debug(f"Request URL: {response.request.url}")
logger.debug(f"Request Headers: {redacted_request_headers}")
logger.debug(f"Request Body: {json.dumps(redacted_request_body)}")
logger.debug(f"Response Status Code: {response.status_code}")
logger.debug(f"Response Headers: {response.headers}")
logger.debug(f"Response Body: {json.dumps(redacted_response_body)}")