From 333c6b23f54024ddf0055825cb5e0877e4c98b6f Mon Sep 17 00:00:00 2001 From: morpheus65535 Date: Sun, 16 Jul 2023 23:30:37 -0400 Subject: [PATCH 1/3] Fixed opensubtitles.com issue with looping over failed login. --- .../providers/opensubtitlescom.py | 65 +++++++++---------- 1 file changed, 31 insertions(+), 34 deletions(-) diff --git a/libs/subliminal_patch/providers/opensubtitlescom.py b/libs/subliminal_patch/providers/opensubtitlescom.py index e0e751c81..1bb1120b7 100644 --- a/libs/subliminal_patch/providers/opensubtitlescom.py +++ b/libs/subliminal_patch/providers/opensubtitlescom.py @@ -183,7 +183,12 @@ class OpenSubtitlesComProvider(ProviderRetryMixin, Provider): def initialize(self): self._started = time.time() - self.login() + + if region.get("oscom_token", expiration_time=TOKEN_EXPIRATION_TIME) is NO_VALUE: + logger.debug("No cached token, we'll try to login again.") + self.login() + else: + self.token = region.get("oscom_token", expiration_time=TOKEN_EXPIRATION_TIME) def terminate(self): self.session.close() @@ -192,25 +197,25 @@ class OpenSubtitlesComProvider(ProviderRetryMixin, Provider): return self._started and (time.time() - self._started) < TOKEN_EXPIRATION_TIME def login(self): - r = self.retry( - lambda: self.checked( - lambda: self.session.post(self.server_url + 'login', - json={"username": self.username, "password": self.password}, - allow_redirects=False, - timeout=30), - validate_json=True, - json_key_name='token' - ), - amount=retry_amount - ) + r = self.session.post(self.server_url + 'login', + json={"username": self.username, "password": self.password}, + allow_redirects=False, + timeout=30) + + if r.status_code == 400: + raise ConfigurationError('Do not use email but username') + elif r.status_code == 401: + raise AuthenticationError('Login failed') + + r.raise_for_status() try: self.token = r.json()['token'] except (ValueError, JSONDecodeError): - return False + log_request_response(r) + raise ProviderError("Cannot get token from provider login response") else: region.set("oscom_token", self.token) - return True @staticmethod def sanitize_external_ids(external_id): @@ -289,9 +294,11 @@ class OpenSubtitlesComProvider(ProviderRetryMixin, Provider): if not title_id: return [] - lang_strings = [to_opensubtitlescom(lang.basename) for lang in languages] + # be sure to remove duplicates + lang_strings = list(set([to_opensubtitlescom(lang.basename) for lang in languages])) + langs = ','.join(lang_strings) - logging.debug(f'Searching for this languages: {lang_strings}') + logging.debug(f'Searching for those languages: {lang_strings}') # query the server if isinstance(self.video, Episode): @@ -400,13 +407,6 @@ class OpenSubtitlesComProvider(ProviderRetryMixin, Provider): return self.query(languages, video) def download_subtitle(self, subtitle): - if region.get("oscom_token", expiration_time=TOKEN_EXPIRATION_TIME) is NO_VALUE: - logger.debug("No cached token, we'll try to login again.") - self.login() - if self.token is NO_VALUE: - logger.debug("Unable to obtain an authentication token right now, we'll try again later.") - raise ProviderError("Unable to obtain an authentication token") - logger.info('Downloading subtitle %r', subtitle) headers = {'Accept': 'application/json', 'Content-Type': 'application/json', @@ -442,10 +442,11 @@ class OpenSubtitlesComProvider(ProviderRetryMixin, Provider): subtitle_content = r.content subtitle.content = fix_line_ending(subtitle_content) - def reset_token(self): + @staticmethod + def reset_token(): logging.debug('Authentication failed: clearing cache and attempting to login.') region.delete("oscom_token") - return self.login() + return def checked(self, fn, raise_api_limit=False, validate_json=False, json_key_name=None, validate_content=False): """Run :fn: and check the response status before returning it. @@ -478,17 +479,13 @@ class OpenSubtitlesComProvider(ProviderRetryMixin, Provider): except Exception: status_code = None else: - if status_code == 400: - raise ConfigurationError('Do not use email but username') - elif status_code == 401: + if status_code == 401: time.sleep(1) log_request_response(response) - logged_in = self.reset_token() - if logged_in: - return self.checked(fn, raise_api_limit=raise_api_limit, validate_json=validate_json, - json_key_name=json_key_name, validate_content=validate_content) - else: - raise AuthenticationError('Login failed') + self.reset_token() + self.login() + self.checked(fn, raise_api_limit=raise_api_limit, validate_json=validate_json, + json_key_name=json_key_name, validate_content=validate_content) elif status_code == 403: log_request_response(response) raise ProviderError("Bazarr API key seems to be in problem") From faa9566431860873aebbac542990b07a6cba47e8 Mon Sep 17 00:00:00 2001 From: morpheus65535 Date: Sun, 16 Jul 2023 23:32:46 -0400 Subject: [PATCH 2/3] Improved subtitles providers settings to reset Authentication or Configuration error throttling while saving. --- bazarr/app/config.py | 17 +++++++++++++++++ bazarr/app/get_providers.py | 13 ++++++++++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/bazarr/app/config.py b/bazarr/app/config.py index 240d67fc0..baef13993 100644 --- a/bazarr/app/config.py +++ b/bazarr/app/config.py @@ -391,6 +391,7 @@ def save_settings(settings_items): undefined_audio_track_default_changed = False undefined_subtitles_track_default_changed = False audio_tracks_parsing_changed = False + reset_providers = False # Subzero Mods update_subzero = False @@ -491,46 +492,62 @@ def save_settings(settings_items): if key == 'settings-addic7ed-username': if key != settings.addic7ed.username: + reset_providers = True region.delete('addic7ed_data') elif key == 'settings-addic7ed-password': if key != settings.addic7ed.password: + reset_providers = True region.delete('addic7ed_data') if key == 'settings-legendasdivx-username': if key != settings.legendasdivx.username: + reset_providers = True region.delete('legendasdivx_cookies2') elif key == 'settings-legendasdivx-password': if key != settings.legendasdivx.password: + reset_providers = True region.delete('legendasdivx_cookies2') if key == 'settings-opensubtitles-username': if key != settings.opensubtitles.username: + reset_providers = True region.delete('os_token') elif key == 'settings-opensubtitles-password': if key != settings.opensubtitles.password: + reset_providers = True region.delete('os_token') if key == 'settings-opensubtitlescom-username': if key != settings.opensubtitlescom.username: + reset_providers = True region.delete('oscom_token') elif key == 'settings-opensubtitlescom-password': if key != settings.opensubtitlescom.password: + reset_providers = True region.delete('oscom_token') if key == 'settings-subscene-username': if key != settings.subscene.username: + reset_providers = True region.delete('subscene_cookies2') elif key == 'settings-subscene-password': if key != settings.subscene.password: + reset_providers = True region.delete('subscene_cookies2') if key == 'settings-titlovi-username': if key != settings.titlovi.username: + reset_providers = True region.delete('titlovi_token') elif key == 'settings-titlovi-password': if key != settings.titlovi.password: + reset_providers = True region.delete('titlovi_token') + if reset_providers: + from .get_providers import reset_throttled_providers + reset_throttled_providers(only_auth_or_conf_error=True) + if settings_keys[0] == 'settings': settings[settings_keys[1]][settings_keys[2]] = str(value) diff --git a/bazarr/app/get_providers.py b/bazarr/app/get_providers.py index d4ee7fe2e..f1fef1d33 100644 --- a/bazarr/app/get_providers.py +++ b/bazarr/app/get_providers.py @@ -14,7 +14,7 @@ import requests from subliminal_patch.exceptions import TooManyRequests, APIThrottled, ParseResponseError, IPAddressBlocked, \ MustGetBlacklisted, SearchLimitReached from subliminal.providers.opensubtitles import DownloadLimitReached -from subliminal.exceptions import DownloadLimitExceeded, ServiceUnavailable +from subliminal.exceptions import DownloadLimitExceeded, ServiceUnavailable, AuthenticationError, ConfigurationError from subliminal import region as subliminal_cache_region from subliminal_patch.extensions import provider_registry @@ -76,6 +76,8 @@ def provider_throttle_map(): APIThrottled: (datetime.timedelta(seconds=15), "15 seconds"), }, "opensubtitlescom": { + AuthenticationError: (datetime.timedelta(hours=12), "12 hours"), + ConfigurationError: (datetime.timedelta(hours=12), "12 hours"), TooManyRequests: (datetime.timedelta(minutes=1), "1 minute"), DownloadLimitExceeded: (datetime.timedelta(hours=24), "24 hours"), }, @@ -413,12 +415,17 @@ def list_throttled_providers(): return throttled_providers -def reset_throttled_providers(): +def reset_throttled_providers(only_auth_or_conf_error=False): for provider in list(tp): + if only_auth_or_conf_error and tp[provider][0] not in ['AuthenticationError', 'ConfigurationError']: + continue del tp[provider] set_throttled_providers(str(tp)) update_throttled_provider() - logging.info('BAZARR throttled providers have been reset.') + if only_auth_or_conf_error: + logging.info('BAZARR throttled providers have been reset (only AuthenticationError and ConfigurationError).') + else: + logging.info('BAZARR throttled providers have been reset.') def get_throttled_providers(): From 6e7858fc7ce0905ba1f241885c386d70507f6b56 Mon Sep 17 00:00:00 2001 From: morpheus65535 Date: Wed, 19 Jul 2023 16:01:29 -0400 Subject: [PATCH 3/3] Fixed more stuff related to opensubtitles.com --- .../providers/opensubtitlescom.py | 39 +++++++++---------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/libs/subliminal_patch/providers/opensubtitlescom.py b/libs/subliminal_patch/providers/opensubtitlescom.py index 1bb1120b7..6a21ddf5f 100644 --- a/libs/subliminal_patch/providers/opensubtitlescom.py +++ b/libs/subliminal_patch/providers/opensubtitlescom.py @@ -196,18 +196,13 @@ class OpenSubtitlesComProvider(ProviderRetryMixin, Provider): def ping(self): return self._started and (time.time() - self._started) < TOKEN_EXPIRATION_TIME - def login(self): - r = self.session.post(self.server_url + 'login', - json={"username": self.username, "password": self.password}, - allow_redirects=False, - timeout=30) - - if r.status_code == 400: - raise ConfigurationError('Do not use email but username') - elif r.status_code == 401: - raise AuthenticationError('Login failed') - - r.raise_for_status() + def login(self, is_retry=False): + r = self.checked( + lambda: self.session.post(self.server_url + 'login', + json={"username": self.username, "password": self.password}, + allow_redirects=False, + timeout=30), + is_retry=is_retry) try: self.token = r.json()['token'] @@ -267,9 +262,6 @@ class OpenSubtitlesComProvider(ProviderRetryMixin, Provider): logger.debug(f'No match found for {title}') def query(self, languages, video): - if region.get("oscom_token", expiration_time=TOKEN_EXPIRATION_TIME) is NO_VALUE: - logger.debug("No cached token, we'll try to login again.") - self.login() self.video = video if self.use_hash: file_hash = self.video.hashes.get('opensubtitlescom') @@ -448,7 +440,8 @@ class OpenSubtitlesComProvider(ProviderRetryMixin, Provider): region.delete("oscom_token") return - def checked(self, fn, raise_api_limit=False, validate_json=False, json_key_name=None, validate_content=False): + def checked(self, fn, raise_api_limit=False, validate_json=False, json_key_name=None, validate_content=False, + is_retry=False): """Run :fn: and check the response status before returning it. :param fn: the function to make an API call to OpenSubtitles.com. @@ -456,6 +449,7 @@ class OpenSubtitlesComProvider(ProviderRetryMixin, Provider): :param validate_json: test if response is valid json. :param json_key_name: test if returned json contain a specific key. :param validate_content: test if response have a content (used with download). + :param is_retry: prevent additional retries with login endpoint. :return: the response. """ @@ -466,7 +460,7 @@ class OpenSubtitlesComProvider(ProviderRetryMixin, Provider): except APIThrottled: if not raise_api_limit: logger.info("API request limit hit, waiting and trying again once.") - time.sleep(2) + time.sleep(15) return self.checked(fn, raise_api_limit=True) raise except (ConnectionError, Timeout, ReadTimeout): @@ -480,12 +474,15 @@ class OpenSubtitlesComProvider(ProviderRetryMixin, Provider): status_code = None else: if status_code == 401: - time.sleep(1) log_request_response(response) self.reset_token() - self.login() - self.checked(fn, raise_api_limit=raise_api_limit, validate_json=validate_json, - json_key_name=json_key_name, validate_content=validate_content) + if is_retry: + raise AuthenticationError('Login failed') + else: + time.sleep(1) + self.login(is_retry=True) + self.checked(fn, raise_api_limit=raise_api_limit, validate_json=validate_json, + json_key_name=json_key_name, validate_content=validate_content, is_retry=True) elif status_code == 403: log_request_response(response) raise ProviderError("Bazarr API key seems to be in problem")