mirror of https://github.com/morpheus65535/bazarr
Merge remote-tracking branch 'origin/development' into development
# Conflicts: # bazarr/get_providers.py
This commit is contained in:
morpheus65535
commit
f93af2f3e0
|
|
@ -16,6 +16,7 @@ from subliminal.providers.opensubtitles import DownloadLimitReached
|
|||
from subliminal.exceptions import DownloadLimitExceeded, ServiceUnavailable
|
||||
from subliminal import region as subliminal_cache_region
|
||||
|
||||
|
||||
def time_until_end_of_day(dt=None):
|
||||
# type: (datetime.datetime) -> datetime.timedelta
|
||||
"""
|
||||
|
|
@ -26,6 +27,7 @@ def time_until_end_of_day(dt=None):
|
|||
tomorrow = dt + datetime.timedelta(days=1)
|
||||
return datetime.datetime.combine(tomorrow, datetime.time.min) - dt
|
||||
|
||||
|
||||
hours_until_end_of_day = time_until_end_of_day().seconds // 3600 + 1
|
||||
|
||||
VALID_THROTTLE_EXCEPTIONS = (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled,
|
||||
|
|
@ -34,44 +36,48 @@ VALID_COUNT_EXCEPTIONS = ('TooManyRequests', 'ServiceUnavailable', 'APIThrottled
|
|||
requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout, socket.timeout)
|
||||
|
||||
PROVIDER_THROTTLE_MAP = {
|
||||
"default": {
|
||||
TooManyRequests: (datetime.timedelta(hours=1), "1 hour"),
|
||||
DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"),
|
||||
ServiceUnavailable: (datetime.timedelta(minutes=20), "20 minutes"),
|
||||
APIThrottled: (datetime.timedelta(minutes=10), "10 minutes"),
|
||||
ParseResponseError: (datetime.timedelta(hours=6), "6 hours"),
|
||||
requests.exceptions.Timeout: (datetime.timedelta(hours=1), "1 hour"),
|
||||
socket.timeout: (datetime.timedelta(hours=1), "1 hour"),
|
||||
"default" : {
|
||||
TooManyRequests : (datetime.timedelta(hours=1), "1 hour"),
|
||||
DownloadLimitExceeded : (datetime.timedelta(hours=3), "3 hours"),
|
||||
ServiceUnavailable : (datetime.timedelta(minutes=20), "20 minutes"),
|
||||
APIThrottled : (datetime.timedelta(minutes=10), "10 minutes"),
|
||||
ParseResponseError : (datetime.timedelta(hours=6), "6 hours"),
|
||||
requests.exceptions.Timeout : (datetime.timedelta(hours=1), "1 hour"),
|
||||
socket.timeout : (datetime.timedelta(hours=1), "1 hour"),
|
||||
requests.exceptions.ConnectTimeout: (datetime.timedelta(hours=1), "1 hour"),
|
||||
requests.exceptions.ReadTimeout: (datetime.timedelta(hours=1), "1 hour"),
|
||||
requests.exceptions.ReadTimeout : (datetime.timedelta(hours=1), "1 hour"),
|
||||
},
|
||||
"opensubtitles": {
|
||||
TooManyRequests: (datetime.timedelta(hours=3), "3 hours"),
|
||||
"opensubtitles" : {
|
||||
TooManyRequests : (datetime.timedelta(hours=3), "3 hours"),
|
||||
DownloadLimitExceeded: (datetime.timedelta(hours=6), "6 hours"),
|
||||
DownloadLimitReached: (datetime.timedelta(hours=6), "6 hours"),
|
||||
APIThrottled: (datetime.timedelta(seconds=15), "15 seconds"),
|
||||
DownloadLimitReached : (datetime.timedelta(hours=6), "6 hours"),
|
||||
APIThrottled : (datetime.timedelta(seconds=15), "15 seconds"),
|
||||
},
|
||||
"opensubtitlescom": {
|
||||
TooManyRequests: (datetime.timedelta(minutes=1), "1 minute"),
|
||||
DownloadLimitExceeded: (datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))),
|
||||
TooManyRequests : (datetime.timedelta(minutes=1), "1 minute"),
|
||||
DownloadLimitExceeded: (
|
||||
datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))),
|
||||
},
|
||||
"addic7ed": {
|
||||
"addic7ed" : {
|
||||
DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"),
|
||||
TooManyRequests: (datetime.timedelta(minutes=5), "5 minutes"),
|
||||
IPAddressBlocked: (datetime.timedelta(hours=1), "1 hours"),
|
||||
TooManyRequests : (datetime.timedelta(minutes=5), "5 minutes"),
|
||||
IPAddressBlocked : (datetime.timedelta(hours=1), "1 hours"),
|
||||
},
|
||||
"titulky": {
|
||||
DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours")
|
||||
"titulky" : {
|
||||
DownloadLimitExceeded: (
|
||||
datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day)))
|
||||
},
|
||||
"legendasdivx": {
|
||||
TooManyRequests: (datetime.timedelta(hours=3), "3 hours"),
|
||||
DownloadLimitExceeded: (datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))),
|
||||
IPAddressBlocked: (datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))),
|
||||
"legendasdivx" : {
|
||||
TooManyRequests : (datetime.timedelta(hours=3), "3 hours"),
|
||||
DownloadLimitExceeded: (
|
||||
datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))),
|
||||
IPAddressBlocked : (
|
||||
datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))),
|
||||
}
|
||||
}
|
||||
|
||||
PROVIDERS_FORCED_OFF = ["addic7ed", "tvsubtitles", "legendasdivx", "legendastv", "napiprojekt", "shooter", "hosszupuska",
|
||||
"supersubtitles", "titlovi", "argenteam", "assrt", "subscene"]
|
||||
PROVIDERS_FORCED_OFF = ["addic7ed", "tvsubtitles", "legendasdivx", "legendastv", "napiprojekt", "shooter",
|
||||
"hosszupuska", "supersubtitles", "titlovi", "argenteam", "assrt", "subscene"]
|
||||
|
||||
throttle_count = {}
|
||||
|
||||
|
|
@ -88,7 +94,7 @@ def get_providers():
|
|||
for provider in settings.general.enabled_providers.lower().split(','):
|
||||
reason, until, throttle_desc = tp.get(provider, (None, None, None))
|
||||
providers_list.append(provider)
|
||||
|
||||
|
||||
if reason:
|
||||
now = datetime.datetime.now()
|
||||
if now < until:
|
||||
|
|
@ -104,65 +110,80 @@ def get_providers():
|
|||
# for provider in providers_list:
|
||||
# if provider in PROVIDERS_FORCED_OFF:
|
||||
# providers_list.remove(provider)
|
||||
|
||||
|
||||
if not providers_list:
|
||||
providers_list = None
|
||||
|
||||
|
||||
return providers_list
|
||||
|
||||
|
||||
def get_providers_auth():
|
||||
providers_auth = {
|
||||
'addic7ed': {'username': settings.addic7ed.username,
|
||||
'password': settings.addic7ed.password,
|
||||
},
|
||||
'opensubtitles': {'username': settings.opensubtitles.username,
|
||||
'password': settings.opensubtitles.password,
|
||||
'use_tag_search': settings.opensubtitles.getboolean('use_tag_search'),
|
||||
'only_foreign': False, # fixme
|
||||
'also_foreign': False, # fixme
|
||||
'is_vip': settings.opensubtitles.getboolean('vip'),
|
||||
'use_ssl': settings.opensubtitles.getboolean('ssl'),
|
||||
'timeout': int(settings.opensubtitles.timeout) or 15,
|
||||
'skip_wrong_fps': settings.opensubtitles.getboolean('skip_wrong_fps'),
|
||||
},
|
||||
return {
|
||||
'addic7ed' : {
|
||||
'username': settings.addic7ed.username,
|
||||
'password': settings.addic7ed.password,
|
||||
},
|
||||
'opensubtitles' : {
|
||||
'username' : settings.opensubtitles.username,
|
||||
'password' : settings.opensubtitles.password,
|
||||
'use_tag_search': settings.opensubtitles.getboolean(
|
||||
'use_tag_search'
|
||||
),
|
||||
'only_foreign' : False, # fixme
|
||||
'also_foreign' : False, # fixme
|
||||
'is_vip' : settings.opensubtitles.getboolean('vip'),
|
||||
'use_ssl' : settings.opensubtitles.getboolean('ssl'),
|
||||
'timeout' : int(settings.opensubtitles.timeout) or 15,
|
||||
'skip_wrong_fps': settings.opensubtitles.getboolean(
|
||||
'skip_wrong_fps'
|
||||
),
|
||||
},
|
||||
'opensubtitlescom': {'username': settings.opensubtitlescom.username,
|
||||
'password': settings.opensubtitlescom.password,
|
||||
'use_hash': settings.opensubtitlescom.getboolean('use_hash'),
|
||||
'api_key': 's38zmzVlW7IlYruWi7mHwDYl2SfMQoC1'
|
||||
'api_key' : 's38zmzVlW7IlYruWi7mHwDYl2SfMQoC1'
|
||||
},
|
||||
'podnapisi': {
|
||||
'podnapisi' : {
|
||||
'only_foreign': False, # fixme
|
||||
'also_foreign': False, # fixme
|
||||
},
|
||||
'subscene': {'username': settings.subscene.username,
|
||||
'password': settings.subscene.password,
|
||||
'only_foreign': False, # fixme
|
||||
},
|
||||
'legendasdivx': {'username': settings.legendasdivx.username,
|
||||
'password': settings.legendasdivx.password,
|
||||
'skip_wrong_fps': settings.legendasdivx.getboolean('skip_wrong_fps'),
|
||||
},
|
||||
'legendastv': {'username': settings.legendastv.username,
|
||||
'password': settings.legendastv.password,
|
||||
},
|
||||
'xsubs': {'username': settings.xsubs.username,
|
||||
'password': settings.xsubs.password,
|
||||
},
|
||||
'assrt': {'token': settings.assrt.token, },
|
||||
'napisy24': {'username': settings.napisy24.username,
|
||||
'password': settings.napisy24.password,
|
||||
},
|
||||
'betaseries': {'token': settings.betaseries.token},
|
||||
'titulky': {'username': settings.titulky.username,
|
||||
'password': settings.titulky.password,
|
||||
},
|
||||
'titlovi': {'username': settings.titlovi.username,
|
||||
'password': settings.titlovi.password,
|
||||
},
|
||||
'subscene' : {
|
||||
'username' : settings.subscene.username,
|
||||
'password' : settings.subscene.password,
|
||||
'only_foreign': False, # fixme
|
||||
},
|
||||
'legendasdivx' : {
|
||||
'username' : settings.legendasdivx.username,
|
||||
'password' : settings.legendasdivx.password,
|
||||
'skip_wrong_fps': settings.legendasdivx.getboolean(
|
||||
'skip_wrong_fps'
|
||||
),
|
||||
},
|
||||
'legendastv' : {
|
||||
'username': settings.legendastv.username,
|
||||
'password': settings.legendastv.password,
|
||||
},
|
||||
'xsubs' : {
|
||||
'username': settings.xsubs.username,
|
||||
'password': settings.xsubs.password,
|
||||
},
|
||||
'assrt' : {
|
||||
'token': settings.assrt.token,
|
||||
},
|
||||
'napisy24' : {
|
||||
'username': settings.napisy24.username,
|
||||
'password': settings.napisy24.password,
|
||||
},
|
||||
'betaseries' : {'token': settings.betaseries.token},
|
||||
'titulky' : {
|
||||
'username': settings.titulky.username,
|
||||
'password': settings.titulky.password,
|
||||
},
|
||||
'titlovi' : {
|
||||
'username': settings.titlovi.username,
|
||||
'password': settings.titlovi.password,
|
||||
},
|
||||
}
|
||||
|
||||
return providers_auth
|
||||
|
||||
|
||||
def provider_throttle(name, exception):
|
||||
|
|
@ -172,17 +193,17 @@ def provider_throttle(name, exception):
|
|||
for valid_cls in VALID_THROTTLE_EXCEPTIONS:
|
||||
if isinstance(cls, valid_cls):
|
||||
cls = valid_cls
|
||||
|
||||
|
||||
throttle_data = PROVIDER_THROTTLE_MAP.get(name, PROVIDER_THROTTLE_MAP["default"]).get(cls, None) or \
|
||||
PROVIDER_THROTTLE_MAP["default"].get(cls, None)
|
||||
|
||||
|
||||
if throttle_data:
|
||||
throttle_delta, throttle_description = throttle_data
|
||||
else:
|
||||
throttle_delta, throttle_description = datetime.timedelta(minutes=10), "10 minutes"
|
||||
|
||||
|
||||
throttle_until = datetime.datetime.now() + throttle_delta
|
||||
|
||||
|
||||
if cls_name not in VALID_COUNT_EXCEPTIONS or throttled_count(name):
|
||||
if cls_name == 'ValueError' and exception.args[0].startswith('unsupported pickle protocol'):
|
||||
for fn in subliminal_cache_region.backend.all_filenames:
|
||||
|
|
@ -210,24 +231,18 @@ def throttled_count(name):
|
|||
throttle_count[name]['count'] = value
|
||||
else:
|
||||
throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))}
|
||||
|
||||
|
||||
else:
|
||||
throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))}
|
||||
|
||||
if throttle_count[name]['count'] < 5:
|
||||
if throttle_count[name]['time'] > datetime.datetime.now():
|
||||
logging.info("Provider %s throttle count %s of 5, waiting 5sec and trying again", name,
|
||||
throttle_count[name]['count'])
|
||||
time.sleep(5)
|
||||
return False
|
||||
else:
|
||||
throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))}
|
||||
logging.info("Provider %s throttle count %s of 5, waiting 5sec and trying again", name,
|
||||
throttle_count[name]['count'])
|
||||
time.sleep(5)
|
||||
return False
|
||||
else:
|
||||
|
||||
if throttle_count[name]['count'] >= 5:
|
||||
return True
|
||||
if throttle_count[name]['time'] <= datetime.datetime.now():
|
||||
throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))}
|
||||
logging.info("Provider %s throttle count %s of 5, waiting 5sec and trying again", name,
|
||||
throttle_count[name]['count'])
|
||||
time.sleep(5)
|
||||
return False
|
||||
|
||||
|
||||
def update_throttled_provider():
|
||||
|
|
@ -242,9 +257,7 @@ def update_throttled_provider():
|
|||
|
||||
if reason:
|
||||
now = datetime.datetime.now()
|
||||
if now < until:
|
||||
pass
|
||||
else:
|
||||
if now >= until:
|
||||
logging.info("Using %s again after %s, (disabled because: %s)", provider, throttle_desc, reason)
|
||||
del tp[provider]
|
||||
set_throttled_providers(str(tp))
|
||||
|
|
|
|||
Loading…
Reference in New Issue