From c605d6fa381f9a989da571f84591d8a934d80f55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Morav=C4=8D=C3=ADk=2C=20Marian?= Date: Sun, 31 Jan 2021 10:03:40 +0100 Subject: [PATCH] Change time limit for DownloadLimitExceeded in titulky provider --- bazarr/get_providers.py | 207 +++++++++++++++++++++------------------- 1 file changed, 110 insertions(+), 97 deletions(-) diff --git a/bazarr/get_providers.py b/bazarr/get_providers.py index f0754c1b9..15775eba2 100644 --- a/bazarr/get_providers.py +++ b/bazarr/get_providers.py @@ -16,6 +16,7 @@ from subliminal.providers.opensubtitles import DownloadLimitReached from subliminal.exceptions import DownloadLimitExceeded, ServiceUnavailable from subliminal import region as subliminal_cache_region + def time_until_end_of_day(dt=None): # type: (datetime.datetime) -> datetime.timedelta """ @@ -26,6 +27,7 @@ def time_until_end_of_day(dt=None): tomorrow = dt + datetime.timedelta(days=1) return datetime.datetime.combine(tomorrow, datetime.time.min) - dt + hours_until_end_of_day = time_until_end_of_day().seconds // 3600 + 1 VALID_THROTTLE_EXCEPTIONS = (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled, @@ -34,44 +36,48 @@ VALID_COUNT_EXCEPTIONS = ('TooManyRequests', 'ServiceUnavailable', 'APIThrottled requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout, socket.timeout) PROVIDER_THROTTLE_MAP = { - "default": { - TooManyRequests: (datetime.timedelta(hours=1), "1 hour"), - DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"), - ServiceUnavailable: (datetime.timedelta(minutes=20), "20 minutes"), - APIThrottled: (datetime.timedelta(minutes=10), "10 minutes"), - ParseResponseError: (datetime.timedelta(hours=6), "6 hours"), - requests.exceptions.Timeout: (datetime.timedelta(hours=1), "1 hour"), - socket.timeout: (datetime.timedelta(hours=1), "1 hour"), + "default" : { + TooManyRequests : (datetime.timedelta(hours=1), "1 hour"), + DownloadLimitExceeded : (datetime.timedelta(hours=3), "3 hours"), + ServiceUnavailable : (datetime.timedelta(minutes=20), "20 minutes"), + APIThrottled : (datetime.timedelta(minutes=10), "10 minutes"), + ParseResponseError : (datetime.timedelta(hours=6), "6 hours"), + requests.exceptions.Timeout : (datetime.timedelta(hours=1), "1 hour"), + socket.timeout : (datetime.timedelta(hours=1), "1 hour"), requests.exceptions.ConnectTimeout: (datetime.timedelta(hours=1), "1 hour"), - requests.exceptions.ReadTimeout: (datetime.timedelta(hours=1), "1 hour"), + requests.exceptions.ReadTimeout : (datetime.timedelta(hours=1), "1 hour"), }, - "opensubtitles": { - TooManyRequests: (datetime.timedelta(hours=3), "3 hours"), + "opensubtitles" : { + TooManyRequests : (datetime.timedelta(hours=3), "3 hours"), DownloadLimitExceeded: (datetime.timedelta(hours=6), "6 hours"), - DownloadLimitReached: (datetime.timedelta(hours=6), "6 hours"), - APIThrottled: (datetime.timedelta(seconds=15), "15 seconds"), + DownloadLimitReached : (datetime.timedelta(hours=6), "6 hours"), + APIThrottled : (datetime.timedelta(seconds=15), "15 seconds"), }, "opensubtitlescom": { - TooManyRequests: (datetime.timedelta(minutes=1), "1 minute"), - DownloadLimitExceeded: (datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))), + TooManyRequests : (datetime.timedelta(minutes=1), "1 minute"), + DownloadLimitExceeded: ( + datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))), }, - "addic7ed": { + "addic7ed" : { DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"), - TooManyRequests: (datetime.timedelta(minutes=5), "5 minutes"), - IPAddressBlocked: (datetime.timedelta(hours=1), "1 hours"), + TooManyRequests : (datetime.timedelta(minutes=5), "5 minutes"), + IPAddressBlocked : (datetime.timedelta(hours=1), "1 hours"), }, - "titulky": { - DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours") + "titulky" : { + DownloadLimitExceeded: ( + datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))) }, - "legendasdivx": { - TooManyRequests: (datetime.timedelta(hours=3), "3 hours"), - DownloadLimitExceeded: (datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))), - IPAddressBlocked: (datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))), + "legendasdivx" : { + TooManyRequests : (datetime.timedelta(hours=3), "3 hours"), + DownloadLimitExceeded: ( + datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))), + IPAddressBlocked : ( + datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))), } } -PROVIDERS_FORCED_OFF = ["addic7ed", "tvsubtitles", "legendasdivx", "legendastv", "napiprojekt", "shooter", "hosszupuska", - "supersubtitles", "titlovi", "argenteam", "assrt", "subscene"] +PROVIDERS_FORCED_OFF = ["addic7ed", "tvsubtitles", "legendasdivx", "legendastv", "napiprojekt", "shooter", + "hosszupuska", "supersubtitles", "titlovi", "argenteam", "assrt", "subscene"] throttle_count = {} @@ -94,7 +100,7 @@ def get_providers(): for provider in settings.general.enabled_providers.lower().split(','): reason, until, throttle_desc = tp.get(provider, (None, None, None)) providers_list.append(provider) - + if reason: now = datetime.datetime.now() if now < until: @@ -106,74 +112,89 @@ def get_providers(): del tp[provider] settings.general.throtteled_providers = str(tp) changed = True - + if changed: with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle: settings.write(handle) - + # if forced only is enabled: # fixme: Prepared for forced only implementation to remove providers with don't support forced only subtitles # for provider in providers_list: # if provider in PROVIDERS_FORCED_OFF: # providers_list.remove(provider) - + if not providers_list: providers_list = None - + return providers_list def get_providers_auth(): - providers_auth = { - 'addic7ed': {'username': settings.addic7ed.username, - 'password': settings.addic7ed.password, - }, - 'opensubtitles': {'username': settings.opensubtitles.username, - 'password': settings.opensubtitles.password, - 'use_tag_search': settings.opensubtitles.getboolean('use_tag_search'), - 'only_foreign': False, # fixme - 'also_foreign': False, # fixme - 'is_vip': settings.opensubtitles.getboolean('vip'), - 'use_ssl': settings.opensubtitles.getboolean('ssl'), - 'timeout': int(settings.opensubtitles.timeout) or 15, - 'skip_wrong_fps': settings.opensubtitles.getboolean('skip_wrong_fps'), - }, + return { + 'addic7ed' : { + 'username': settings.addic7ed.username, + 'password': settings.addic7ed.password, + }, + 'opensubtitles' : { + 'username' : settings.opensubtitles.username, + 'password' : settings.opensubtitles.password, + 'use_tag_search': settings.opensubtitles.getboolean( + 'use_tag_search' + ), + 'only_foreign' : False, # fixme + 'also_foreign' : False, # fixme + 'is_vip' : settings.opensubtitles.getboolean('vip'), + 'use_ssl' : settings.opensubtitles.getboolean('ssl'), + 'timeout' : int(settings.opensubtitles.timeout) or 15, + 'skip_wrong_fps': settings.opensubtitles.getboolean( + 'skip_wrong_fps' + ), + }, 'opensubtitlescom': {'username': settings.opensubtitlescom.username, 'password': settings.opensubtitlescom.password, 'use_hash': settings.opensubtitlescom.getboolean('use_hash'), - 'api_key': 's38zmzVlW7IlYruWi7mHwDYl2SfMQoC1' + 'api_key' : 's38zmzVlW7IlYruWi7mHwDYl2SfMQoC1' }, - 'podnapisi': { + 'podnapisi' : { 'only_foreign': False, # fixme 'also_foreign': False, # fixme }, - 'subscene': {'username': settings.subscene.username, - 'password': settings.subscene.password, - 'only_foreign': False, # fixme - }, - 'legendasdivx': {'username': settings.legendasdivx.username, - 'password': settings.legendasdivx.password, - 'skip_wrong_fps': settings.legendasdivx.getboolean('skip_wrong_fps'), - }, - 'legendastv': {'username': settings.legendastv.username, - 'password': settings.legendastv.password, - }, - 'xsubs': {'username': settings.xsubs.username, - 'password': settings.xsubs.password, - }, - 'assrt': {'token': settings.assrt.token, }, - 'napisy24': {'username': settings.napisy24.username, - 'password': settings.napisy24.password, - }, - 'betaseries': {'token': settings.betaseries.token}, - 'titulky': {'username': settings.titulky.username, - 'password': settings.titulky.password, - }, - 'titlovi': {'username': settings.titlovi.username, - 'password': settings.titlovi.password, - }, + 'subscene' : { + 'username' : settings.subscene.username, + 'password' : settings.subscene.password, + 'only_foreign': False, # fixme + }, + 'legendasdivx' : { + 'username' : settings.legendasdivx.username, + 'password' : settings.legendasdivx.password, + 'skip_wrong_fps': settings.legendasdivx.getboolean( + 'skip_wrong_fps' + ), + }, + 'legendastv' : { + 'username': settings.legendastv.username, + 'password': settings.legendastv.password, + }, + 'xsubs' : { + 'username': settings.xsubs.username, + 'password': settings.xsubs.password, + }, + 'assrt' : { + 'token': settings.assrt.token, + }, + 'napisy24' : { + 'username': settings.napisy24.username, + 'password': settings.napisy24.password, + }, + 'betaseries' : {'token': settings.betaseries.token}, + 'titulky' : { + 'username': settings.titulky.username, + 'password': settings.titulky.password, + }, + 'titlovi' : { + 'username': settings.titlovi.username, + 'password': settings.titlovi.password, + }, } - - return providers_auth def provider_throttle(name, exception): @@ -183,17 +204,17 @@ def provider_throttle(name, exception): for valid_cls in VALID_THROTTLE_EXCEPTIONS: if isinstance(cls, valid_cls): cls = valid_cls - + throttle_data = PROVIDER_THROTTLE_MAP.get(name, PROVIDER_THROTTLE_MAP["default"]).get(cls, None) or \ PROVIDER_THROTTLE_MAP["default"].get(cls, None) - + if throttle_data: throttle_delta, throttle_description = throttle_data else: throttle_delta, throttle_description = datetime.timedelta(minutes=10), "10 minutes" - + throttle_until = datetime.datetime.now() + throttle_delta - + if cls_name not in VALID_COUNT_EXCEPTIONS or throttled_count(name): if cls_name == 'ValueError' and exception.args[0].startswith('unsupported pickle protocol'): for fn in subliminal_cache_region.backend.all_filenames: @@ -223,24 +244,18 @@ def throttled_count(name): throttle_count[name]['count'] = value else: throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))} - + else: throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))} - - if throttle_count[name]['count'] < 5: - if throttle_count[name]['time'] > datetime.datetime.now(): - logging.info("Provider %s throttle count %s of 5, waiting 5sec and trying again", name, - throttle_count[name]['count']) - time.sleep(5) - return False - else: - throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))} - logging.info("Provider %s throttle count %s of 5, waiting 5sec and trying again", name, - throttle_count[name]['count']) - time.sleep(5) - return False - else: + + if throttle_count[name]['count'] >= 5: return True + if throttle_count[name]['time'] <= datetime.datetime.now(): + throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))} + logging.info("Provider %s throttle count %s of 5, waiting 5sec and trying again", name, + throttle_count[name]['count']) + time.sleep(5) + return False def update_throttled_provider(): @@ -256,14 +271,12 @@ def update_throttled_provider(): if reason: now = datetime.datetime.now() - if now < until: - pass - else: + if now >= until: logging.info("Using %s again after %s, (disabled because: %s)", provider, throttle_desc, reason) del tp[provider] settings.general.throtteled_providers = str(tp) changed = True - + if changed: with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle: settings.write(handle)