Merge branch 'development'

.gitignore

@@ -8,6 +8,7 @@ cachefile.dbm
 bazarr.pid
 /venv
 /data
+/.vscode
 
 # Allow
 !*.dll

README.md

@@ -70,6 +70,7 @@ If you need something that is not already part of Bazarr, feel free to create a
 * TVSubtitles
 * Wizdom
 * XSubs
+* Yavka.net
 * Zimuku
 
 ## Screenshot

bazarr.py

@@ -2,29 +2,26 @@
 
 import os
 import platform
-import signal
 import subprocess
 import sys
 import time
+import atexit
 
 from bazarr.get_args import args
-from libs.six import PY3
 
 
 def check_python_version():
     python_version = platform.python_version_tuple()
-    minimum_py2_tuple = (2, 7, 13)
     minimum_py3_tuple = (3, 7, 0)
-    minimum_py2_str = ".".join(str(i) for i in minimum_py2_tuple)
     minimum_py3_str = ".".join(str(i) for i in minimum_py3_tuple)
 
-    if (int(python_version[0]) == minimum_py3_tuple[0] and int(python_version[1]) < minimum_py3_tuple[1]) or \
+    if int(python_version[0]) < minimum_py3_tuple[0]:
-            (int(python_version[0]) != minimum_py3_tuple[0] and int(python_version[0]) != minimum_py2_tuple[0]):
         print("Python " + minimum_py3_str + " or greater required. "
               "Current version is " + platform.python_version() + ". Please upgrade Python.")
         sys.exit(1)
-    elif int(python_version[0]) == minimum_py2_tuple[0] and int(python_version[1]) < minimum_py2_tuple[1]:
+    elif (int(python_version[0]) == minimum_py3_tuple[0] and int(python_version[1]) < minimum_py3_tuple[1]) or \
-        print("Python " + minimum_py2_str + " or greater required. "
+            (int(python_version[0]) != minimum_py3_tuple[0]):
+        print("Python " + minimum_py3_str + " or greater required. "
               "Current version is " + platform.python_version() + ". Please upgrade Python.")
         sys.exit(1)
 
@@ -34,156 +31,58 @@ check_python_version()
 dir_name = os.path.dirname(__file__)
 
 
-class ProcessRegistry:
+def start_bazarr():
-
-    def register(self, process):
-        pass
-
-    def unregister(self, process):
-        pass
-
-
-class DaemonStatus(ProcessRegistry):
-
-    def __init__(self):
-        self.__should_stop = False
-        self.__processes = set()
-
-    def register(self, process):
-        self.__processes.add(process)
-
-    def unregister(self, process):
-        self.__processes.remove(process)
-
-    @staticmethod
-    def __wait_for_processes(processes, timeout):
-        """
-        Waits all the provided processes for the specified amount of time in seconds.
-        """
-        reference_ts = time.time()
-        elapsed = 0
-        remaining_processes = list(processes)
-        while elapsed < timeout and len(remaining_processes) > 0:
-            remaining_time = timeout - elapsed
-            for ep in list(remaining_processes):
-                if ep.poll() is not None:
-                    remaining_processes.remove(ep)
-                else:
-                    if remaining_time > 0:
-                        if PY3:
-                            try:
-                                ep.wait(remaining_time)
-                                remaining_processes.remove(ep)
-                            except subprocess.TimeoutExpired:
-                                pass
-                        else:
-                            # In python 2 there is no such thing as some mechanism to wait with a timeout
-                            time.sleep(1)
-                        elapsed = time.time() - reference_ts
-                        remaining_time = timeout - elapsed
-        return remaining_processes
-
-    @staticmethod
-    def __send_signal(processes, signal_no, live_processes=None):
-        """
-        Sends to every single of the specified processes the given signal and (if live_processes is not None) append to
-        it processes which are still alive.
-        """
-        for ep in processes:
-            if ep.poll() is None:
-                if live_processes is not None:
-                    live_processes.append(ep)
-                try:
-                    ep.send_signal(signal_no)
-                except Exception as e:
-                    print('Failed sending signal %s to process %s because of an unexpected error: %s' % (
-                        signal_no, ep.pid, e))
-        return live_processes
-
-    def stop(self):
-        """
-        Flags this instance as should stop and terminates as smoothly as possible children processes.
-        """
-        self.__should_stop = True
-        live_processes = DaemonStatus.__send_signal(self.__processes, signal.SIGINT, list())
-        live_processes = DaemonStatus.__wait_for_processes(live_processes, 120)
-        DaemonStatus.__send_signal(live_processes, signal.SIGTERM)
-
-    def should_stop(self):
-        return self.__should_stop
-
-
-def start_bazarr(process_registry=ProcessRegistry()):
     script = [sys.executable, "-u", os.path.normcase(os.path.join(dir_name, 'bazarr', 'main.py'))] + sys.argv[1:]
+    ep = subprocess.Popen(script, stdout=None, stderr=None, stdin=subprocess.DEVNULL)
+    atexit.register(lambda: ep.kill())
 
-    print("Bazarr starting...")
+
-    if PY3:
+def check_status():
-        ep = subprocess.Popen(script, stdout=None, stderr=None, stdin=subprocess.DEVNULL)
+    if os.path.exists(stopfile):
-    else:
+        try:
-        ep = subprocess.Popen(script, stdout=None, stderr=None, stdin=None)
+            os.remove(stopfile)
-    process_registry.register(ep)
+        except Exception:
-    try:
+            print('Unable to delete stop file.')
-        ep.wait()
+        finally:
-        process_registry.unregister(ep)
+            print('Bazarr exited.')
-    except KeyboardInterrupt:
+            sys.exit(0)
-        pass
+
+    if os.path.exists(restartfile):
+        try:
+            os.remove(restartfile)
+        except Exception:
+            print('Unable to delete restart file.')
+        else:
+            print("Bazarr is restarting...")
+            start_bazarr()
 
 
 if __name__ == '__main__':
-    restartfile = os.path.normcase(os.path.join(args.config_dir, 'bazarr.restart'))
+    restartfile = os.path.join(args.config_dir, 'bazarr.restart')
-    stopfile = os.path.normcase(os.path.join(args.config_dir, 'bazarr.stop'))
+    stopfile = os.path.join(args.config_dir, 'bazarr.stop')
 
+    # Cleanup leftover files
     try:
         os.remove(restartfile)
-    except Exception:
+    except FileNotFoundError:
         pass
 
     try:
         os.remove(stopfile)
-    except Exception:
+    except FileNotFoundError:
         pass
 
-
+    # Initial start of main bazarr process
-    def daemon(bazarr_runner=lambda: start_bazarr()):
+    print("Bazarr starting...")
-        if os.path.exists(stopfile):
+    start_bazarr()
-            try:
-                os.remove(stopfile)
-            except Exception:
-                print('Unable to delete stop file.')
-            else:
-                print('Bazarr exited.')
-                sys.exit(0)
-
-        if os.path.exists(restartfile):
-            try:
-                os.remove(restartfile)
-            except Exception:
-                print('Unable to delete restart file.')
-            else:
-                bazarr_runner()
-
-
-    bazarr_runner = lambda: start_bazarr()
-
-    should_stop = lambda: False
-
-    if PY3:
-        daemonStatus = DaemonStatus()
-
-        def shutdown():
-            # indicates that everything should stop
-            daemonStatus.stop()
-            # emulate a Ctrl C command on itself (bypasses the signal thing but, then, emulates the "Ctrl+C break")
-            os.kill(os.getpid(), signal.SIGINT)
-
-        signal.signal(signal.SIGTERM, lambda signal_no, frame: shutdown())
-
-        should_stop = lambda: daemonStatus.should_stop()
-        bazarr_runner = lambda: start_bazarr(daemonStatus)
-
-    bazarr_runner()
 
     # Keep the script running forever until stop is requested through term or keyboard interrupt
-    while not should_stop():
+    while True:
-        daemon(bazarr_runner)
+        check_status()
-        time.sleep(1)
+        try:
+            if sys.platform.startswith('win'):
+                time.sleep(5)
+            else:
+                os.wait()
+        except (KeyboardInterrupt, SystemExit):
+            pass

bazarr/check_update.py

@@ -34,6 +34,8 @@ def gitconfig():
         logging.debug('BAZARR Settings git email')
         config_write.set_value("user", "email", "bazarr@fake.email")
 
+    config_write.release()
+
 
 def check_and_apply_update():
     check_releases()

bazarr/config.py

@@ -104,7 +104,8 @@ defaults = {
     },
     'legendasdivx': {
         'username': '',
-        'password': ''
+        'password': '',
+        'skip_wrong_fps': 'False'
     },
     'legendastv': {
         'username': '',
@@ -150,6 +151,7 @@ else:
     settings = simpleconfigparser(defaults=defaults)
 settings.read(os.path.join(args.config_dir, 'config', 'config.ini'))
 
+settings.general.base_url = settings.general.base_url if settings.general.base_url else '/'
 base_url = settings.general.base_url
 
 

bazarr/get_providers.py

@@ -8,12 +8,24 @@ import time
 
 from get_args import args
 from config import settings
-from subliminal_patch.exceptions import TooManyRequests, APIThrottled, ParseResponseError
+from subliminal_patch.exceptions import TooManyRequests, APIThrottled, ParseResponseError, IPAddressBlocked
 from subliminal.exceptions import DownloadLimitExceeded, ServiceUnavailable
 from subliminal import region as subliminal_cache_region
 
+def time_until_end_of_day(dt=None):
+    # type: (datetime.datetime) -> datetime.timedelta
+    """
+    Get timedelta until end of day on the datetime passed, or current time.
+    """
+    if dt is None:
+        dt = datetime.datetime.now()
+    tomorrow = dt + datetime.timedelta(days=1)
+    return datetime.datetime.combine(tomorrow, datetime.time.min) - dt
+
+hours_until_end_of_day = time_until_end_of_day().seconds // 3600 + 1
+
 VALID_THROTTLE_EXCEPTIONS = (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled,
-                             ParseResponseError)
+                             ParseResponseError, IPAddressBlocked)
 VALID_COUNT_EXCEPTIONS = ('TooManyRequests', 'ServiceUnavailable', 'APIThrottled')
 
 PROVIDER_THROTTLE_MAP = {
@@ -32,9 +44,16 @@ PROVIDER_THROTTLE_MAP = {
     "addic7ed": {
         DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"),
         TooManyRequests: (datetime.timedelta(minutes=5), "5 minutes"),
+        IPAddressBlocked: (datetime.timedelta(hours=1), "1 hours"),
+
     },
     "titulky": {
         DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours")
+    },
+    "legendasdivx": {
+        TooManyRequests: (datetime.timedelta(hours=3), "3 hours"),
+        DownloadLimitExceeded: (datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))),
+        IPAddressBlocked: (datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))),
     }
 }
 
@@ -116,6 +135,7 @@ def get_providers_auth():
                      },
         'legendasdivx': {'username': settings.legendasdivx.username,
                        'password': settings.legendasdivx.password,
+                       'skip_wrong_fps': settings.legendasdivx.getboolean('skip_wrong_fps'),
                        },
         'legendastv': {'username': settings.legendastv.username,
                        'password': settings.legendastv.password,

bazarr/get_subtitle.py

@@ -207,7 +207,7 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro
                                                      path_decoder=force_unicode
                                                      )
                 except Exception as e:
-                    logging.exception('BAZARR Error saving Subtitles file to disk for this file:' + path)
+                    logging.exception('BAZARR Error saving Subtitles file to disk for this file:' + path + ': ' + repr(e))
                     pass
                 else:
                     saved_any = True
@@ -473,11 +473,14 @@ def manual_download_subtitle(path, language, audio_language, hi, forced, subtitl
     logging.debug('BAZARR Ended manually downloading Subtitles for file: ' + path)
 
 
-def manual_upload_subtitle(path, language, forced, title, scene_name, media_type, subtitle):
+def manual_upload_subtitle(path, language, forced, title, scene_name, media_type, subtitle, audio_language):
     logging.debug('BAZARR Manually uploading subtitles for this file: ' + path)
 
     single = settings.general.getboolean('single_language')
 
+    use_postprocessing = settings.general.getboolean('use_postprocessing')
+    postprocessing_cmd = settings.general.postprocessing_cmd
+
     chmod = int(settings.general.chmod, 8) if not sys.platform.startswith(
         'win') and settings.general.getboolean('chmod_enabled') else None
 
@@ -540,6 +543,20 @@ def manual_upload_subtitle(path, language, forced, title, scene_name, media_type
         os.chmod(subtitle_path, chmod)
 
     message = language_from_alpha3(language) + (" forced" if forced else "") + " Subtitles manually uploaded."
+    
+    uploaded_language_code3 = language 
+    uploaded_language = language_from_alpha3(uploaded_language_code3)
+    uploaded_language_code2 = alpha2_from_alpha3(uploaded_language_code3)
+    audio_language_code2 = alpha2_from_language(audio_language)
+    audio_language_code3 = alpha3_from_language(audio_language)
+
+
+    if use_postprocessing is True:
+        command = pp_replace(postprocessing_cmd, path, subtitle_path, uploaded_language,
+                             uploaded_language_code2, uploaded_language_code3, audio_language, 
+                             audio_language_code2, audio_language_code3, forced)
+        postprocessing(command, path)
+
 
     if media_type == 'series':
         reversed_path = path_replace_reverse(path)
@@ -973,7 +990,10 @@ def refine_from_ffprobe(path, video):
                 video.video_codec = data['video'][0]['codec']
         if 'frame_rate' in data['video'][0]:
             if not video.fps:
-                video.fps = data['video'][0]['frame_rate']
+                if isinstance(data['video'][0]['frame_rate'], float):
+                    video.fps = data['video'][0]['frame_rate']
+                else:
+                    video.fps = data['video'][0]['frame_rate'].magnitude
 
     if 'audio' not in data:
         logging.debug('BAZARR FFprobe was unable to find audio tracks in the file!')

bazarr/list_subtitles.py

@@ -40,7 +40,7 @@ def store_subtitles(original_path, reversed_path):
                 subtitle_languages = embedded_subs_reader.list_languages(reversed_path)
                 for subtitle_language, subtitle_forced, subtitle_codec in subtitle_languages:
                     try:
-                        if settings.general.getboolean("ignore_pgs_subs") and subtitle_codec == "hdmv_pgs_subtitle":
+                        if settings.general.getboolean("ignore_pgs_subs") and subtitle_codec == "PGS":
                             logging.debug("BAZARR skipping pgs sub for language: " + str(alpha2_from_alpha3(subtitle_language)))
                             continue
 
@@ -116,7 +116,7 @@ def store_subtitles_movie(original_path, reversed_path):
                 subtitle_languages = embedded_subs_reader.list_languages(reversed_path)
                 for subtitle_language, subtitle_forced, subtitle_codec in subtitle_languages:
                     try:
-                        if settings.general.getboolean("ignore_pgs_subs") and subtitle_codec == "hdmv_pgs_subtitle":
+                        if settings.general.getboolean("ignore_pgs_subs") and subtitle_codec == "PGS":
                             logging.debug("BAZARR skipping pgs sub for language: " + str(alpha2_from_alpha3(subtitle_language)))
                             continue
 
@@ -383,7 +383,7 @@ def guess_external_subtitles(dest_folder, subtitles):
                     logging.debug('BAZARR detected encoding %r', guess)
                     if guess["confidence"] < 0.6:
                         raise UnicodeError
-                    if guess["confidence"] < 0.8 or guess["encoding"] == "ascii":
+                    if guess["encoding"] == "ascii":
                         guess["encoding"] = "utf-8"
                     text = text.decode(guess["encoding"])
                     detected_language = guess_language(text)

bazarr/main.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 
-bazarr_version = '0.8.4.3'
+bazarr_version = '0.8.4.4'
 
 import os
 os.environ["SZ_USER_AGENT"] = "Bazarr/1"
@@ -65,7 +65,7 @@ from notifier import send_notifications, send_notifications_movie
 from check_update import check_and_apply_update
 from subliminal_patch.extensions import provider_registry as provider_manager
 from subliminal_patch.core import SUBTITLE_EXTENSIONS
-
+from subliminal.cache import region
 
 scheduler = Scheduler()
 
@@ -222,7 +222,7 @@ def doShutdown():
         else:
             stop_file.write(six.text_type(''))
             stop_file.close()
-            sys.exit(0)
+            os._exit(0)
 
 
 @route(base_url + 'restart')
@@ -243,7 +243,7 @@ def restart():
             logging.info('Bazarr is being restarted...')
             restart_file.write(six.text_type(''))
             restart_file.close()
-            sys.exit(0)
+            os._exit(0)
 
 
 @route(base_url + 'wizard')
@@ -401,12 +401,19 @@ def save_wizard():
     else:
         settings_opensubtitles_skip_wrong_fps = 'True'
     
+    settings_legendasdivx_skip_wrong_fps = request.forms.get('settings_legendasdivx_skip_wrong_fps')
+    if settings_legendasdivx_skip_wrong_fps is None:
+        settings_legendasdivx_skip_wrong_fps = 'False'
+    else:
+        settings_legendasdivx_skip_wrong_fps = 'True'
+
     settings.addic7ed.username = request.forms.get('settings_addic7ed_username')
     settings.addic7ed.password = request.forms.get('settings_addic7ed_password')
     settings.addic7ed.random_agents = text_type(settings_addic7ed_random_agents)
     settings.assrt.token = request.forms.get('settings_assrt_token')
     settings.legendasdivx.username = request.forms.get('settings_legendasdivx_username')
     settings.legendasdivx.password = request.forms.get('settings_legendasdivx_password')
+    settings.legendasdivx.skip_wrong_fps = text_type(settings_legendasdivx_skip_wrong_fps)
     settings.legendastv.username = request.forms.get('settings_legendastv_username')
     settings.legendastv.password = request.forms.get('settings_legendastv_password')
     settings.opensubtitles.username = request.forms.get('settings_opensubtitles_username')
@@ -1536,13 +1543,26 @@ def save_settings():
         settings_opensubtitles_skip_wrong_fps = 'False'
     else:
         settings_opensubtitles_skip_wrong_fps = 'True'
-    
+
+    if (settings.opensubtitles.username != request.forms.get('settings_opensubtitles_username') or
+        settings.opensubtitles.password != request.forms.get('settings_opensubtitles_password') or
+        settings.opensubtitles.vip != text_type(settings_opensubtitles_vip)):
+        region.delete("os_token")
+        region.delete("os_server_url")
+
+    settings_legendasdivx_skip_wrong_fps = request.forms.get('settings_legendasdivx_skip_wrong_fps')
+    if settings_legendasdivx_skip_wrong_fps is None:
+        settings_legendasdivx_skip_wrong_fps = 'False'
+    else:
+        settings_legendasdivx_skip_wrong_fps = 'True'
+
     settings.addic7ed.username = request.forms.get('settings_addic7ed_username')
     settings.addic7ed.password = request.forms.get('settings_addic7ed_password')
     settings.addic7ed.random_agents = text_type(settings_addic7ed_random_agents)
     settings.assrt.token = request.forms.get('settings_assrt_token')
     settings.legendasdivx.username = request.forms.get('settings_legendasdivx_username')
     settings.legendasdivx.password = request.forms.get('settings_legendasdivx_password')
+    settings.legendasdivx.skip_wrong_fps = text_type(settings_legendasdivx_skip_wrong_fps)
     settings.legendastv.username = request.forms.get('settings_legendastv_username')
     settings.legendastv.password = request.forms.get('settings_legendastv_password')
     settings.opensubtitles.username = request.forms.get('settings_opensubtitles_username')
@@ -1848,14 +1868,17 @@ def perform_manual_upload_subtitle():
     authorize()
     ref = request.environ['HTTP_REFERER']
 
-    episodePath = request.forms.episodePath
+    episodePath = request.forms.get('episodePath')
-    sceneName = request.forms.sceneName
+    sceneName = request.forms.get('sceneName')
     language = request.forms.get('language')
     forced = True if request.forms.get('forced') == '1' else False
     upload = request.files.get('upload')
     sonarrSeriesId = request.forms.get('sonarrSeriesId')
     sonarrEpisodeId = request.forms.get('sonarrEpisodeId')
-    title = request.forms.title
+    title = request.forms.get('title')
+    
+    data = database.execute("SELECT audio_language FROM table_shows WHERE sonarrSeriesId=?", (sonarrSeriesId,), only_one=True)
+    audio_language = data['audio_language']
 
     _, ext = os.path.splitext(upload.filename)
 
@@ -1869,7 +1892,8 @@ def perform_manual_upload_subtitle():
                                         title=title,
                                         scene_name=sceneName,
                                         media_type='series',
-                                        subtitle=upload)
+                                        subtitle=upload,
+                                        audio_language=audio_language)
 
         if result is not None:
             message = result[0]
@@ -1988,13 +2012,16 @@ def perform_manual_upload_subtitle_movie():
     authorize()
     ref = request.environ['HTTP_REFERER']
 
-    moviePath = request.forms.moviePath
+    moviePath = request.forms.get('moviePath')
-    sceneName = request.forms.sceneName
+    sceneName = request.forms.get('sceneName')
     language = request.forms.get('language')
     forced = True if request.forms.get('forced') == '1' else False
     upload = request.files.get('upload')
     radarrId = request.forms.get('radarrId')
-    title = request.forms.title
+    title = request.forms.get('title')
+    
+    data = database.execute("SELECT audio_language FROM table_movies WHERE radarrId=?", (radarrId,), only_one=True)
+    audio_language = data['audio_language']
 
     _, ext = os.path.splitext(upload.filename)
 
@@ -2008,7 +2035,8 @@ def perform_manual_upload_subtitle_movie():
                                         title=title,
                                         scene_name=sceneName,
                                         media_type='movie',
-                                        subtitle=upload)
+                                        subtitle=upload,
+                                        audio_language=audio_language)
 
         if result is not None:
             message = result[0]
@@ -2093,10 +2121,11 @@ def api_history():
 @custom_auth_basic(check_credentials)
 def test_url(protocol, url):
     authorize()
-    url = six.moves.urllib.parse.unquote(url)
+    url = protocol + "://" + six.moves.urllib.parse.unquote(url)
     try:
-        result = requests.get(protocol + "://" + url, allow_redirects=False, verify=False).json()['version']
+        result = requests.get(url, allow_redirects=False, verify=False).json()['version']
-    except:
+    except Exception as e:
+        logging.exception('BAZARR cannot successfully contact this URL: ' + url)
         return dict(status=False)
     else:
         return dict(status=True, version=result)

bazarr/scheduler.py

@@ -18,8 +18,6 @@ from apscheduler.triggers.cron import CronTrigger
 from apscheduler.triggers.date import DateTrigger
 from apscheduler.events import EVENT_JOB_SUBMITTED, EVENT_JOB_EXECUTED, EVENT_JOB_ERROR
 from datetime import datetime
-import pytz
-from tzlocal import get_localzone
 from calendar import day_name
 import pretty
 from six import PY2
@@ -30,10 +28,7 @@ class Scheduler:
     def __init__(self):
         self.__running_tasks = []
 
-        if str(get_localzone()) == "local":
+        self.aps_scheduler = BackgroundScheduler()
-            self.aps_scheduler = BackgroundScheduler(timezone=pytz.timezone('UTC'))
-        else:
-            self.aps_scheduler = BackgroundScheduler()
 
         # task listener
         def task_listener_add(event):

libs/subliminal/providers/tvsubtitles.py

@@ -209,7 +209,7 @@ class TVsubtitlesProvider(Provider):
             if subtitles:
                 return subtitles
         else:
-            logger.error('No show id found for %r (%r)', video.series, {'year': video.year})
+            logger.debug('No show id found for %r (%r)', video.series, {'year': video.year})
 
         return []
 

libs/subliminal_patch/exceptions.py

@@ -7,11 +7,13 @@ class TooManyRequests(ProviderError):
     """Exception raised by providers when too many requests are made."""
     pass
 
-
 class APIThrottled(ProviderError):
     pass
 
-
 class ParseResponseError(ProviderError):
     """Exception raised by providers when they are not able to parse the response."""
     pass
+
+class IPAddressBlocked(ProviderError):
+	"""Exception raised when providers block requests from IP Address."""
+	pass

libs/subliminal_patch/providers/addic7ed.py

@@ -9,13 +9,14 @@ from random import randint
 
 from dogpile.cache.api import NO_VALUE
 from requests import Session
+from requests.exceptions import ConnectionError
 from subliminal.cache import region
 from subliminal.exceptions import DownloadLimitExceeded, AuthenticationError, ConfigurationError
 from subliminal.providers.addic7ed import Addic7edProvider as _Addic7edProvider, \
     Addic7edSubtitle as _Addic7edSubtitle, ParserBeautifulSoup
 from subliminal.subtitle import fix_line_ending
 from subliminal_patch.utils import sanitize
-from subliminal_patch.exceptions import TooManyRequests
+from subliminal_patch.exceptions import TooManyRequests, IPAddressBlocked
 from subliminal_patch.pitcher import pitchers, load_verification, store_verification
 from subzero.language import Language
 
@@ -91,15 +92,19 @@ class Addic7edProvider(_Addic7edProvider):
         # login
         if self.username and self.password:
             def check_verification(cache_region):
-                rr = self.session.get(self.server_url + 'panel.php', allow_redirects=False, timeout=10,
+                try:
-                                      headers={"Referer": self.server_url})
+                    rr = self.session.get(self.server_url + 'panel.php', allow_redirects=False, timeout=10,
-                if rr.status_code == 302:
+                                          headers={"Referer": self.server_url})
-                    logger.info('Addic7ed: Login expired')
+                    if rr.status_code == 302:
-                    cache_region.delete("addic7ed_data")
+                        logger.info('Addic7ed: Login expired')
-                else:
+                        cache_region.delete("addic7ed_data")
-                    logger.info('Addic7ed: Re-using old login')
+                    else:
-                    self.logged_in = True
+                        logger.info('Addic7ed: Re-using old login')
-                    return True
+                        self.logged_in = True
+                        return True
+                except ConnectionError as e:
+                    logger.debug("Addic7ed: There was a problem reaching the server: %s." % e)
+                    raise IPAddressBlocked("Addic7ed: Your IP is temporarily blocked.")
 
             if load_verification("addic7ed", self.session, callback=check_verification):
                 return

libs/subliminal_patch/providers/argenteam.py

@@ -55,7 +55,7 @@ class ArgenteamSubtitle(Subtitle):
             return self._release_info
 
         combine = []
-        for attr in ("format", "version", "video_codec"):
+        for attr in ("format", "version"):
             value = getattr(self, attr)
             if value:
                 combine.append(value)
@@ -76,9 +76,11 @@ class ArgenteamSubtitle(Subtitle):
             if video.series and (sanitize(self.title) in (
                      sanitize(name) for name in [video.series] + video.alternative_series)):
                 matches.add('series')
+
             # season
             if video.season and self.season == video.season:
                 matches.add('season')
+
             # episode
             if video.episode and self.episode == video.episode:
                 matches.add('episode')
@@ -87,6 +89,9 @@ class ArgenteamSubtitle(Subtitle):
             if video.tvdb_id and str(self.tvdb_id) == str(video.tvdb_id):
                 matches.add('tvdb_id')
 
+            # year (year is not available for series, but we assume it matches)
+            matches.add('year')
+
         elif isinstance(video, Movie) and self.movie_kind == 'movie':
             # title
             if video.title and (sanitize(self.title) in (
@@ -230,29 +235,29 @@ class ArgenteamProvider(Provider, ProviderSubtitleArchiveMixin):
         has_multiple_ids = len(argenteam_ids) > 1
         for aid in argenteam_ids:
             response = self.session.get(url, params={'id': aid}, timeout=10)
-
             response.raise_for_status()
             content = response.json()
 
-            imdb_id = year = None
+            if content is not None:  # eg https://argenteam.net/api/v1/episode?id=11534
-            returned_title = title
+                imdb_id = year = None
-            if not is_episode and "info" in content:
+                returned_title = title
-                imdb_id = content["info"].get("imdb")
+                if not is_episode and "info" in content:
-                year = content["info"].get("year")
+                    imdb_id = content["info"].get("imdb")
-                returned_title = content["info"].get("title", title)
+                    year = content["info"].get("year")
+                    returned_title = content["info"].get("title", title)
 
-            for r in content['releases']:
+                for r in content['releases']:
-                for s in r['subtitles']:
+                    for s in r['subtitles']:
-                    movie_kind = "episode" if is_episode else "movie"
+                        movie_kind = "episode" if is_episode else "movie"
-                    page_link = self.BASE_URL + movie_kind + "/" + str(aid)
+                        page_link = self.BASE_URL + movie_kind + "/" + str(aid)
-                    # use https and new domain
+                        # use https and new domain
-                    download_link = s['uri'].replace('http://www.argenteam.net/', self.BASE_URL)
+                        download_link = s['uri'].replace('http://www.argenteam.net/', self.BASE_URL)
-                    sub = ArgenteamSubtitle(language, page_link, download_link, movie_kind, returned_title,
+                        sub = ArgenteamSubtitle(language, page_link, download_link, movie_kind, returned_title,
-                                            season, episode, year, r.get('team'), r.get('tags'),
+                                                season, episode, year, r.get('team'), r.get('tags'),
-                                            r.get('source'), r.get('codec'), content.get("tvdb"), imdb_id,
+                                                r.get('source'), r.get('codec'), content.get("tvdb"), imdb_id,
-                                            asked_for_release_group=video.release_group,
+                                                asked_for_release_group=video.release_group,
-                                            asked_for_episode=episode)
+                                                asked_for_episode=episode)
-                    subtitles.append(sub)
+                        subtitles.append(sub)
 
             if has_multiple_ids:
                 time.sleep(self.multi_result_throttle)

libs/subliminal_patch/providers/legendasdivx.py

@@ -3,19 +3,27 @@ from __future__ import absolute_import
 import logging
 import io
 import os
-import rarfile
+import re
 import zipfile
+from time import sleep
+from urllib.parse import quote
+from requests.exceptions import HTTPError
+import rarfile
 
-from requests import Session
 from guessit import guessit
-from subliminal_patch.exceptions import ParseResponseError
+from subliminal.cache import region
-from subliminal_patch.providers import Provider
+from subliminal.exceptions import ConfigurationError, AuthenticationError, ServiceUnavailable, DownloadLimitExceeded
 from subliminal.providers import ParserBeautifulSoup
-from subliminal_patch.subtitle import Subtitle
+from subliminal.subtitle import SUBTITLE_EXTENSIONS, fix_line_ending, guess_matches
+from subliminal.utils import sanitize, sanitize_release_group
 from subliminal.video import Episode, Movie
-from subliminal.subtitle import SUBTITLE_EXTENSIONS, fix_line_ending,guess_matches
+from subliminal_patch.exceptions import TooManyRequests, IPAddressBlocked
+from subliminal_patch.http import RetryingCFSession
+from subliminal_patch.providers import Provider
+from subliminal_patch.score import get_scores, framerate_equal
+from subliminal_patch.subtitle import Subtitle
 from subzero.language import Language
-from subliminal_patch.score import get_scores
+from dogpile.cache.api import NO_VALUE
 
 logger = logging.getLogger(__name__)
 
@@ -23,15 +31,18 @@ class LegendasdivxSubtitle(Subtitle):
     """Legendasdivx Subtitle."""
     provider_name = 'legendasdivx'
 
-    def __init__(self, language, video, data):
+    def __init__(self, language, video, data, skip_wrong_fps=True):
         super(LegendasdivxSubtitle, self).__init__(language)
         self.language = language
         self.page_link = data['link']
-        self.hits=data['hits']
+        self.hits = data['hits']
-        self.exact_match=data['exact_match']
+        self.exact_match = data['exact_match']
-        self.description=data['description'].lower()
+        self.description = data['description']
         self.video = video
-        self.videoname =data['videoname']
+        self.sub_frame_rate = data['frame_rate']
+        self.uploader = data['uploader']
+        self.wrong_fps = False
+        self.skip_wrong_fps = skip_wrong_fps
 
     @property
     def id(self):
@@ -44,40 +55,67 @@ class LegendasdivxSubtitle(Subtitle):
     def get_matches(self, video):
         matches = set()
 
-        if self.videoname.lower() in self.description:
+        # if skip_wrong_fps = True no point to continue if they don't match
-            matches.update(['title'])
+        subtitle_fps = None
-            matches.update(['season'])
+        try:
-            matches.update(['episode'])
+            subtitle_fps = float(self.sub_frame_rate)
+        except ValueError:
+            pass
 
-        # episode
+        # check fps match and skip based on configuration
-        if video.title and video.title.lower() in self.description:
+        if video.fps and subtitle_fps and not framerate_equal(video.fps, subtitle_fps):
+            self.wrong_fps = True
+
+            if self.skip_wrong_fps:
+                logger.debug("Legendasdivx :: Skipping subtitle due to FPS mismatch (expected: %s, got: %s)", video.fps, self.sub_frame_rate)
+                # not a single match :)
+                return set()
+            logger.debug("Legendasdivx :: Frame rate mismatch (expected: %s, got: %s, but continuing...)", video.fps, self.sub_frame_rate)
+
+        description = sanitize(self.description)
+
+        video_filename = video.name
+        video_filename = os.path.basename(video_filename)
+        video_filename, _ = os.path.splitext(video_filename)
+        video_filename = sanitize_release_group(video_filename)
+
+        if sanitize(video_filename) in description:
             matches.update(['title'])
-        if video.year and '{:04d}'.format(video.year) in self.description:
+            # relying people won' use just S01E01 for the file name
+            if isinstance(video, Episode):
+                matches.update(['series'])
+                matches.update(['season'])
+                matches.update(['episode'])
+
+        # can match both movies and series
+        if video.year and '{:04d}'.format(video.year) in description:
             matches.update(['year'])
 
+        # match movie title (include alternative movie names)
+        if isinstance(video, Movie):
+            if video.title:
+                for movie_name in [video.title] + video.alternative_titles:
+                    if sanitize(movie_name) in description:
+                        matches.update(['title'])
+
         if isinstance(video, Episode):
-            # already matched in search query
+            if video.title and sanitize(video.title) in description:
-            if video.season and 's{:02d}'.format(video.season) in self.description:
+                matches.update(['title'])
-                matches.update(['season'])
+            if video.series:
-            if video.episode and 'e{:02d}'.format(video.episode) in self.description:
+                for series_name in [video.series] + video.alternative_series:
-                matches.update(['episode'])
+                    if sanitize(series_name) in description:
-            if video.episode and video.season and video.series:
-                if '{}.s{:02d}e{:02d}'.format(video.series.lower(),video.season,video.episode) in self.description:
                         matches.update(['series'])
-                        matches.update(['season'])
+            if video.season and 's{:02d}'.format(video.season) in description:
-                        matches.update(['episode'])
+                matches.update(['season'])
-                if '{} s{:02d}e{:02d}'.format(video.series.lower(),video.season,video.episode) in self.description:
+            if video.episode and 'e{:02d}'.format(video.episode) in description:
-                    matches.update(['series'])
+                matches.update(['episode'])
-                    matches.update(['season'])
-                    matches.update(['episode'])
 
         # release_group
-        if video.release_group  and video.release_group.lower() in self.description:
+        if video.release_group and sanitize_release_group(video.release_group) in sanitize_release_group(description):
             matches.update(['release_group'])
 
         # resolution
-
+        if video.resolution and video.resolution.lower() in description:
-        if video.resolution and video.resolution.lower() in self.description:
             matches.update(['resolution'])
 
         # format
@@ -87,9 +125,9 @@ class LegendasdivxSubtitle(Subtitle):
             if formats[0] == "web-dl":
                 formats.append("webdl")
                 formats.append("webrip")
-                formats.append("web ")
+                formats.append("web")
             for frmt in formats:
-                if frmt.lower() in self.description:
+                if frmt in description:
                     matches.update(['format'])
                     break
 
@@ -97,21 +135,16 @@ class LegendasdivxSubtitle(Subtitle):
         if video.video_codec:
             video_codecs = [video.video_codec.lower()]
             if video_codecs[0] == "h264":
-                formats.append("x264")
+                video_codecs.append("x264")
             elif video_codecs[0] == "h265":
-                formats.append("x265")
+                video_codecs.append("x265")
-            for vc in formats:
+            for vc in video_codecs:
-                if vc.lower() in self.description:
+                if vc in description:
                     matches.update(['video_codec'])
                     break
 
-        # running guessit on a huge description may break guessit
-        # matches |= guess_matches(video, guessit(self.description))
         return matches
 
-
-
-
 class LegendasdivxProvider(Provider):
     """Legendasdivx Provider."""
     languages = {Language('por', 'BR')} | {Language('por')}
@@ -121,147 +154,223 @@ class LegendasdivxProvider(Provider):
         'User-Agent': os.environ.get("SZ_USER_AGENT", "Sub-Zero/2"),
         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
         'Origin': 'https://www.legendasdivx.pt',
-        'Referer': 'https://www.legendasdivx.pt',
+        'Referer': 'https://www.legendasdivx.pt'
-        'Pragma': 'no-cache',
-        'Cache-Control': 'no-cache'
     }
     loginpage = site + '/forum/ucp.php?mode=login'
     searchurl = site + '/modules.php?name=Downloads&file=jz&d_op=search&op=_jz00&query={query}'
-    language_list = list(languages)
+    download_link = site + '/modules.php{link}'
 
-    def __init__(self, username, password):
+    def __init__(self, username, password, skip_wrong_fps=True):
+        # make sure login credentials are configured.
+        if any((username, password)) and not all((username, password)):
+            raise ConfigurationError('Legendasdivx.pt :: Username and password must be specified')
         self.username = username
         self.password = password
+        self.skip_wrong_fps = skip_wrong_fps
 
     def initialize(self):
-        self.session = Session()
+        logger.debug("Legendasdivx.pt :: Creating session for requests")
-        self.login()
+        self.session = RetryingCFSession()
+        # re-use PHP Session if present
+        prev_cookies = region.get("legendasdivx_cookies2")
+        if prev_cookies != NO_VALUE:
+            logger.debug("Legendasdivx.pt :: Re-using previous legendasdivx cookies: %s", prev_cookies)
+            self.session.cookies.update(prev_cookies)
+        # login if session has expired
+        else:
+            logger.debug("Legendasdivx.pt :: Session cookies not found!")
+            self.session.headers.update(self.headers)
+            self.login()
 
     def terminate(self):
-        self.logout()
+        # session close
         self.session.close()
 
     def login(self):
-        logger.info('Logging in')
+        logger.debug('Legendasdivx.pt :: Logging in')
-        self.headers['Referer'] = self.site + '/index.php'
-        self.session.headers.update(self.headers.items())
-        res = self.session.get(self.loginpage)
-        bsoup = ParserBeautifulSoup(res.content, ['lxml'])
-
-        _allinputs = bsoup.findAll('input')
-        fields = {}
-        for field in _allinputs:
-            fields[field.get('name')] = field.get('value')
-
-        fields['username'] = self.username
-        fields['password'] = self.password
-        fields['autologin'] = 'on'
-        fields['viewonline'] = 'on'
-
-        self.headers['Referer'] = self.loginpage
-        self.session.headers.update(self.headers.items())
-        res = self.session.post(self.loginpage, fields)
         try:
-            logger.debug('Got session id %s' %
+            res = self.session.get(self.loginpage)
-                         self.session.cookies.get_dict()['PHPSESSID'])
+            res.raise_for_status()
-        except KeyError as e:
+            bsoup = ParserBeautifulSoup(res.content, ['lxml'])
-            logger.error(repr(e))
+
-            logger.error("Didn't get session id, check your credentials")
+            _allinputs = bsoup.findAll('input')
-            return False
+            data = {}
+            # necessary to set 'sid' for POST request
+            for field in _allinputs:
+                data[field.get('name')] = field.get('value')
+
+            data['username'] = self.username
+            data['password'] = self.password
+
+            res = self.session.post(self.loginpage, data)
+            res.raise_for_status()
+            # make sure we're logged in
+            logger.debug('Legendasdivx.pt :: Logged in successfully: PHPSESSID: %s', self.session.cookies.get_dict()['PHPSESSID'])
+            cj = self.session.cookies.copy()
+            store_cks = ("PHPSESSID", "phpbb3_2z8zs_sid", "phpbb3_2z8zs_k", "phpbb3_2z8zs_u", "lang")
+            for cn in iter(self.session.cookies.keys()):
+                if cn not in store_cks:
+                    del cj[cn]
+            # store session cookies on cache
+            logger.debug("Legendasdivx.pt :: Storing legendasdivx session cookies: %r", cj)
+            region.set("legendasdivx_cookies2", cj)
+
+        except KeyError:
+            logger.error("Legendasdivx.pt :: Couldn't get session ID, check your credentials")
+            raise AuthenticationError("Legendasdivx.pt :: Couldn't get session ID, check your credentials")
+        except HTTPError as e:
+            if "bloqueado" in res.text.lower():
+                logger.error("LegendasDivx.pt :: Your IP is blocked on this server.")
+                raise IPAddressBlocked("LegendasDivx.pt :: Your IP is blocked on this server.")
+            logger.error("Legendasdivx.pt :: HTTP Error %s", e)
+            raise TooManyRequests("Legendasdivx.pt :: HTTP Error %s", e)
         except Exception as e:
-            logger.error(repr(e))
+            logger.error("LegendasDivx.pt :: Uncaught error: %r", e)
-            logger.error('uncached error #legendasdivx #AA')
+            raise ServiceUnavailable("LegendasDivx.pt :: Uncaught error: %r", e)
-            return False
 
-        return True
+    def _process_page(self, video, bsoup):
 
-    def logout(self):
-        # need to figure this out
-        return True
-
-    def _process_page(self, video, bsoup, querytext, videoname):
         subtitles = []
-        _allsubs = bsoup.findAll("div", {"class": "sub_box"})
-        lang = Language.fromopensubtitles("pob")
-        for _subbox in _allsubs:
-            hits=0
-            for th in _subbox.findAll("th", {"class": "color2"}):
-                if th.string == 'Hits:':
-                    hits = int(th.parent.find("td").string)
-                if th.string == 'Idioma:':
-                    lang = th.parent.find("td").find ("img").get ('src')
-                    if 'brazil' in lang:
-                        lang = Language.fromopensubtitles('pob')
-                    else:
-                        lang = Language.fromopensubtitles('por')
 
-            description = _subbox.find("td", {"class": "td_desc brd_up"})
+        _allsubs = bsoup.findAll("div", {"class": "sub_box"})
-            download = _subbox.find("a", {"class": "sub_download"})
+
+        for _subbox in _allsubs:
+
+            hits = 0
+            for th in _subbox.findAll("th"):
+                if th.text == 'Hits:':
+                    hits = int(th.find_next("td").text)
+                if th.text == 'Idioma:':
+                    lang = th.find_next("td").find("img").get('src')
+                    if 'brazil' in lang.lower():
+                        lang = Language.fromopensubtitles('pob')
+                    elif 'portugal' in lang.lower():
+                        lang = Language.fromopensubtitles('por')
+                    else:
+                        continue
+                if th.text == "Frame Rate:":
+                    frame_rate = th.find_next("td").text.strip()
+
+            # get description for matches
+            description = _subbox.find("td", {"class": "td_desc brd_up"}).get_text()
+
+            # get subtitle link from footer
+            sub_footer = _subbox.find("div", {"class": "sub_footer"})
+            download = sub_footer.find("a", {"class": "sub_download"}) if sub_footer else None
+
+            # sometimes 'a' tag is not found and returns None. Most likely HTML format error!
             try:
-                # sometimes BSoup just doesn't get the link
+                download_link = self.download_link.format(link=download.get('href'))
-                logger.debug(download.get('href'))
+                logger.debug("Legendasdivx.pt :: Found subtitle link on: %s ", download_link)
-            except Exception as e:
+            except:
-                logger.warning('skipping subbox on %s' % self.searchurl.format(query=querytext))
+                logger.debug("Legendasdivx.pt :: Couldn't find download link. Trying next...")
                 continue
 
+            # get subtitle uploader
+            sub_header = _subbox.find("div", {"class" :"sub_header"})
+            uploader = sub_header.find("a").text if sub_header else 'anonymous'
+
             exact_match = False
-            if video.name.lower() in description.get_text().lower():
+            if video.name.lower() in description.lower():
                 exact_match = True
-            data = {'link': self.site + '/modules.php' + download.get('href'),
+
+            data = {'link': download_link,
                     'exact_match': exact_match,
                     'hits': hits,
-                    'videoname': videoname,
+                    'uploader': uploader,
-                    'description': description.get_text() }
+                    'frame_rate': frame_rate,
+                    'description': description
+                    }
             subtitles.append(
-                LegendasdivxSubtitle(lang, video, data)
+                LegendasdivxSubtitle(lang, video, data, skip_wrong_fps=self.skip_wrong_fps)
             )
         return subtitles
 
-    def query(self, video, language):
+    def query(self, video, languages):
-        try:
-            logger.debug('Got session id %s' %
-                         self.session.cookies.get_dict()['PHPSESSID'])
-        except Exception as e:
-            self.login()
 
-        language_ids = '0'
-        if isinstance(language, (tuple, list, set)):
-            if len(language) == 1:
-                language_ids = ','.join(sorted(l.opensubtitles for l in language))
-                if language_ids == 'por':
-                    language_ids = '&form_cat=28'
-                else:
-                    language_ids = '&form_cat=29'
-
-        videoname = video.name
-        videoname = os.path.basename(videoname)
-        videoname, _ = os.path.splitext(videoname)
-        # querytext = videoname.lower()
         _searchurl = self.searchurl
-        if video.imdb_id is None:
-            if isinstance(video, Episode):
-                querytext = "{} S{:02d}E{:02d}".format(video.series, video.season, video.episode)
-            elif isinstance(video, Movie):
-                querytext = video.title
-        else:
-            querytext = video.imdb_id
 
+        if isinstance(video, Movie):
+            querytext = video.imdb_id if video.imdb_id else video.title
 
-        # querytext = querytext.replace(
+        if isinstance(video, Episode):
-        #     ".", "+").replace("[", "").replace("]", "")
+            querytext = '"{} S{:02d}E{:02d}"'.format(video.series, video.season, video.episode)
-        if language_ids != '0':
+            querytext = quote(quote(querytext))
-            querytext = querytext + language_ids
+
-        self.headers['Referer'] = self.site + '/index.php'
+        # language query filter
-        self.session.headers.update(self.headers.items())
+        if isinstance(languages, (tuple, list, set)):
-        res = self.session.get(_searchurl.format(query=querytext))
+            language_ids = ','.join(sorted(l.opensubtitles for l in languages))
-        # form_cat=28 = br
+            if 'por' in language_ids: # prioritize portuguese subtitles
-        # form_cat=29 = pt
+                lang_filter = '&form_cat=28'
-        if "A legenda não foi encontrada" in res.text:
+            elif 'pob' in language_ids:
-            logger.warning('%s not found', querytext)
+                lang_filter = '&form_cat=29'
-            return []
+            else:
+                lang_filter = ''
+
+        querytext = querytext + lang_filter if lang_filter else querytext
+
+        try:
+            # sleep for a 1 second before another request
+            sleep(1)
+            self.headers['Referer'] = self.site + '/index.php'
+            self.session.headers.update(self.headers)
+            res = self.session.get(_searchurl.format(query=querytext), allow_redirects=False)
+            res.raise_for_status()
+            if (res.status_code == 200 and "A legenda não foi encontrada" in res.text):
+                logger.warning('Legendasdivx.pt :: query %s return no results!', querytext)
+                # for series, if no results found, try again just with series and season (subtitle packs)
+                if isinstance(video, Episode):
+                    logger.debug("Legendasdivx.pt :: trying again with just series and season on query.")
+                    querytext = re.sub("(e|E)(\d{2})", "", querytext)
+                    res = self.session.get(_searchurl.format(query=querytext), allow_redirects=False)
+                    res.raise_for_status()
+                    if (res.status_code == 200 and "A legenda não foi encontrada" in res.text):
+                        logger.warning('Legendasdivx.pt :: query %s return no results (for series and season only).', querytext)
+                        return []
+            if res.status_code == 302: # got redirected to login page.
+                # seems that our session cookies are no longer valid... clean them from cache
+                region.delete("legendasdivx_cookies2")
+                logger.debug("Legendasdivx.pt :: Logging in again. Cookies have expired!")
+                # login and try again
+                self.login()
+                res = self.session.get(_searchurl.format(query=querytext))
+                res.raise_for_status()
+        except HTTPError as e:
+            if "bloqueado" in res.text.lower():
+                logger.error("LegendasDivx.pt :: Your IP is blocked on this server.")
+                raise IPAddressBlocked("LegendasDivx.pt :: Your IP is blocked on this server.")
+            logger.error("Legendasdivx.pt :: HTTP Error %s", e)
+            raise TooManyRequests("Legendasdivx.pt :: HTTP Error %s", e)
+        except Exception as e:
+            logger.error("LegendasDivx.pt :: Uncaught error: %r", e)
+            raise ServiceUnavailable("LegendasDivx.pt :: Uncaught error: %r", e)
 
         bsoup = ParserBeautifulSoup(res.content, ['html.parser'])
-        subtitles = self._process_page(video, bsoup, querytext, videoname)
+
+        # search for more than 10 results (legendasdivx uses pagination)
+        # don't throttle - maximum results = 6 * 10
+        MAX_PAGES = 6
+
+        # get number of pages bases on results found
+        page_header = bsoup.find("div", {"class": "pager_bar"})
+        results_found = re.search(r'\((.*?) encontradas\)', page_header.text).group(1) if page_header else 0
+        logger.debug("Legendasdivx.pt :: Found %s subtitles", str(results_found))
+        num_pages = (int(results_found) // 10) + 1
+        num_pages = min(MAX_PAGES, num_pages)
+
+        # process first page
+        subtitles = self._process_page(video, bsoup)
+
+        # more pages?
+        if num_pages > 1:
+            for num_page in range(2, num_pages+1):
+                sleep(1) # another 1 sec before requesting...
+                _search_next = self.searchurl.format(query=querytext) + "&page={0}".format(str(num_page))
+                logger.debug("Legendasdivx.pt :: Moving on to next page: %s", _search_next)
+                res = self.session.get(_search_next)
+                next_page = ParserBeautifulSoup(res.content, ['html.parser'])
+                subs = self._process_page(video, next_page)
+                subtitles.extend(subs)
 
         return subtitles
 
@@ -269,34 +378,47 @@ class LegendasdivxProvider(Provider):
         return self.query(video, languages)
 
     def download_subtitle(self, subtitle):
-        res = self.session.get(subtitle.page_link)
-        if res:
-            if res.text == '500':
-                raise ValueError('Error 500 on server')
 
-            archive = self._get_archive(res.content)
+        try:
-            # extract the subtitle
+            res = self.session.get(subtitle.page_link)
+            res.raise_for_status()
+        except HTTPError as e:
+            if "bloqueado" in res.text.lower():
+                logger.error("LegendasDivx.pt :: Your IP is blocked on this server.")
+                raise IPAddressBlocked("LegendasDivx.pt :: Your IP is blocked on this server.")
+            logger.error("Legendasdivx.pt :: HTTP Error %s", e)
+            raise TooManyRequests("Legendasdivx.pt :: HTTP Error %s", e)
+        except Exception as e:
+            logger.error("LegendasDivx.pt :: Uncaught error: %r", e)
+            raise ServiceUnavailable("LegendasDivx.pt :: Uncaught error: %r", e)
+
+        # make sure we haven't maxed out our daily limit
+        if (res.status_code == 200 and 'limite de downloads diário atingido' in res.text.lower()):
+            logger.error("LegendasDivx.pt :: Daily download limit reached!")
+            raise DownloadLimitExceeded("Legendasdivx.pt :: Daily download limit reached!")
+
+        archive = self._get_archive(res.content)
+        # extract the subtitle
+        if archive:
             subtitle_content = self._get_subtitle_from_archive(archive, subtitle)
-            subtitle.content = fix_line_ending(subtitle_content)
+            if subtitle_content:
-            subtitle.normalize()
+                subtitle.content = fix_line_ending(subtitle_content)
-
+                subtitle.normalize()
-            return subtitle
+                return subtitle
-        raise ValueError('Problems conecting to the server')
+        return
 
     def _get_archive(self, content):
         # open the archive
-        # stole^H^H^H^H^H inspired from subvix provider
         archive_stream = io.BytesIO(content)
         if rarfile.is_rarfile(archive_stream):
-            logger.debug('Identified rar archive')
+            logger.debug('Legendasdivx.pt :: Identified rar archive')
             archive = rarfile.RarFile(archive_stream)
         elif zipfile.is_zipfile(archive_stream):
-            logger.debug('Identified zip archive')
+            logger.debug('Legendasdivx.pt :: Identified zip archive')
             archive = zipfile.ZipFile(archive_stream)
         else:
-            # raise ParseResponseError('Unsupported compressed format')
+            logger.error('Legendasdivx.pt :: Unsupported compressed format')
-            raise Exception('Unsupported compressed format')
+            return None
-
         return archive
 
     def _get_subtitle_from_archive(self, archive, subtitle):
@@ -305,7 +427,7 @@ class LegendasdivxProvider(Provider):
         _tmp.remove('.txt')
         _subtitle_extensions = tuple(_tmp)
         _max_score = 0
-        _scores = get_scores (subtitle.video)
+        _scores = get_scores(subtitle.video)
 
         for name in archive.namelist():
             # discard hidden files
@@ -316,26 +438,27 @@ class LegendasdivxProvider(Provider):
             if not name.lower().endswith(_subtitle_extensions):
                 continue
 
-            _guess = guessit (name)
+            _guess = guessit(name)
             if isinstance(subtitle.video, Episode):
-                logger.debug ("guessing %s" % name)
+                logger.debug("Legendasdivx.pt :: guessing %s", name)
-                logger.debug("subtitle S{}E{} video S{}E{}".format(_guess['season'],_guess['episode'],subtitle.video.season,subtitle.video.episode))
+                logger.debug("Legendasdivx.pt :: subtitle S%sE%s video S%sE%s", _guess['season'], _guess['episode'], subtitle.video.season, subtitle.video.episode)
 
                 if subtitle.video.episode != _guess['episode'] or subtitle.video.season != _guess['season']:
-                    logger.debug('subtitle does not match video, skipping')
+                    logger.debug('Legendasdivx.pt :: subtitle does not match video, skipping')
                     continue
 
             matches = set()
-            matches |= guess_matches (subtitle.video, _guess)
+            matches |= guess_matches(subtitle.video, _guess)
-            logger.debug('srt matches: %s' % matches)
+            logger.debug('Legendasdivx.pt :: sub matches: %s', matches)
-            _score = sum ((_scores.get (match, 0) for match in matches))
+            _score = sum((_scores.get(match, 0) for match in matches))
             if _score > _max_score:
                 _max_name = name
                 _max_score = _score
-                logger.debug("new max: {} {}".format(name, _score))
+                logger.debug("Legendasdivx.pt :: new max: %s %s", name, _score)
 
         if _max_score > 0:
-            logger.debug("returning from archive: {} scored {}".format(_max_name, _max_score))
+            logger.debug("Legendasdivx.pt :: returning from archive: %s scored %s", _max_name, _max_score)
             return archive.read(_max_name)
 
-        raise ParseResponseError('Can not find the subtitle in the compressed file')
+        logger.error("Legendasdivx.pt :: No subtitle found on compressed file. Max score was 0")
+        return None

libs/subliminal_patch/providers/opensubtitles.py

@@ -44,6 +44,12 @@ class OpenSubtitlesSubtitle(_OpenSubtitlesSubtitle):
         self.wrong_fps = False
         self.skip_wrong_fps = skip_wrong_fps
 
+    def get_fps(self):
+        try:
+            return float(self.fps)
+        except:
+            return None
+
     def get_matches(self, video, hearing_impaired=False):
         matches = super(OpenSubtitlesSubtitle, self).get_matches(video)
 
@@ -138,11 +144,9 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider):
         return ServerProxy(url, SubZeroRequestsTransport(use_https=self.use_ssl, timeout=timeout or self.timeout,
                                                          user_agent=os.environ.get("SZ_USER_AGENT", "Sub-Zero/2")))
 
-    def log_in(self, server_url=None):
+    def log_in_url(self, server_url):
-        if server_url:
+        self.token = None
-            self.terminate()
+        self.server = self.get_server_proxy(server_url)
-
-            self.server = self.get_server_proxy(server_url)
 
         response = self.retry(
             lambda: checked(
@@ -155,6 +159,25 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider):
         logger.debug('Logged in with token %r', self.token[:10]+"X"*(len(self.token)-10))
 
         region.set("os_token", bytearray(self.token, encoding='utf-8'))
+        region.set("os_server_url", bytearray(server_url, encoding='utf-8'))
+
+    def log_in(self):
+        logger.info('Logging in')
+
+        try:
+            self.log_in_url(self.vip_url if self.is_vip else self.default_url)
+
+        except Unauthorized:
+            if self.is_vip:
+                logger.info("VIP server login failed, falling back")
+                try:
+                    self.log_in_url(self.default_url)
+                except Unauthorized:
+                    pass
+
+        if not self.token:
+            logger.error("Login failed, please check your credentials")
+            raise Unauthorized
 
     def use_token_or_login(self, func):
         if not self.token:
@@ -167,45 +190,18 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider):
             return func()
 
     def initialize(self):
-        if self.is_vip:
+        token_cache = region.get("os_token")
-            self.server = self.get_server_proxy(self.vip_url)
+        url_cache = region.get("os_server_url")
-            logger.info("Using VIP server")
+
+        if token_cache is not NO_VALUE and url_cache is not NO_VALUE:
+            self.token = token_cache.decode("utf-8")
+            self.server = self.get_server_proxy(url_cache.decode("utf-8"))
+            logger.debug("Using previous login token: %r", self.token[:10] + "X" * (len(self.token) - 10))
         else:
-            self.server = self.get_server_proxy(self.default_url)
+            self.server = None
+            self.token = None
 
-        logger.info('Logging in')
-
-        token = str(region.get("os_token"))
-        if token is not NO_VALUE:
-            try:
-                logger.debug('Trying previous token: %r', token[:10]+"X"*(len(token)-10))
-                checked(lambda: self.server.NoOperation(token))
-                self.token = token
-                logger.debug("Using previous login token: %r", token[:10]+"X"*(len(token)-10))
-                return
-            except (NoSession, Unauthorized):
-                logger.debug('Token not valid.')
-                pass
-
-        try:
-            self.log_in()
-
-        except Unauthorized:
-            if self.is_vip:
-                logger.info("VIP server login failed, falling back")
-                self.log_in(self.default_url)
-                if self.token:
-                    return
-
-            logger.error("Login failed, please check your credentials")
-                
     def terminate(self):
-        if self.token:
-            try:
-                checked(lambda: self.server.LogOut(self.token))
-            except:
-                logger.error("Logout failed: %s", traceback.format_exc())
-
         self.server = None
         self.token = None
 

libs/subliminal_patch/providers/subssabbz.py

@@ -13,7 +13,6 @@ from guessit import guessit
 from subliminal_patch.providers import Provider
 from subliminal_patch.subtitle import Subtitle
 from subliminal_patch.utils import sanitize, fix_inconsistent_naming
-from subliminal.exceptions import ProviderError
 from subliminal.utils import sanitize_release_group
 from subliminal.subtitle import guess_matches
 from subliminal.video import Episode, Movie
@@ -35,25 +34,31 @@ def fix_tv_naming(title):
                                            "Marvel's Luke Cage": "Luke Cage",
                                            "Marvel's Iron Fist": "Iron Fist",
                                            "Marvel's Jessica Jones": "Jessica Jones",
-                                           "DC's Legends of Tomorrow": "Legends of Tomorrow"
+                                           "DC's Legends of Tomorrow": "Legends of Tomorrow",
+                                           "Doctor Who (2005)": "Doctor Who",
                                            }, True)
 
 class SubsSabBzSubtitle(Subtitle):
     """SubsSabBz Subtitle."""
     provider_name = 'subssabbz'
 
-    def __init__(self, langauge, filename, type, video, link):
+    def __init__(self, langauge, filename, type, video, link, fps, num_cds):
         super(SubsSabBzSubtitle, self).__init__(langauge)
         self.langauge = langauge
         self.filename = filename
         self.page_link = link
         self.type = type
         self.video = video
+        self.fps = fps
+        self.num_cds = num_cds
         self.release_info = os.path.splitext(filename)[0]
 
     @property
     def id(self):
-        return self.filename
+        return self.page_link + self.filename
+
+    def get_fps(self):
+        return self.fps
 
     def make_picklable(self):
         self.content = None
@@ -75,13 +80,21 @@ class SubsSabBzSubtitle(Subtitle):
         if video_filename == subtitle_filename:
              matches.add('hash')
 
-        matches |= guess_matches(video, guessit(self.filename, {'type': self.type}))
+        if video.year and self.year == video.year:
+            matches.add('year')
+
+        if isinstance(video, Movie):
+            if video.imdb_id and self.imdb_id == video.imdb_id:
+                matches.add('imdb_id')
+
+        matches |= guess_matches(video, guessit(self.title, {'type': self.type, 'allowed_countries': [None]}))
+        matches |= guess_matches(video, guessit(self.filename, {'type': self.type, 'allowed_countries': [None]}))
         return matches
 
 
 class SubsSabBzProvider(Provider):
     """SubsSabBz Provider."""
-    languages = {Language('por', 'BR')} | {Language(l) for l in [
+    languages = {Language(l) for l in [
         'bul', 'eng'
     ]}
 
@@ -135,19 +148,51 @@ class SubsSabBzProvider(Provider):
         soup = BeautifulSoup(response.content, 'lxml')
         rows = soup.findAll('tr', {'class': 'subs-row'})
 
-        # Search on first 20 rows only
+        # Search on first 25 rows only
-        for row in rows[:20]:
+        for row in rows[:25]:
             a_element_wrapper = row.find('td', { 'class': 'c2field' })
             if a_element_wrapper:
                 element = a_element_wrapper.find('a')
                 if element:
                     link = element.get('href')
-                    element = row.find('a', href = re.compile(r'.*showuser=.*'))
+                    notes = element.get('onmouseover')
-                    uploader = element.get_text() if element else None
+                    title = element.get_text()
+
+                    try:
+                        year = int(str(element.next_sibling).strip(' ()'))
+                    except:
+                        year = None
+
+                    td = row.findAll('td')
+
+                    try:
+                        num_cds = int(td[6].get_text())
+                    except:
+                        num_cds = None
+
+                    try:
+                        fps = float(td[7].get_text())
+                    except:
+                        fps = None
+
+                    try:
+                        uploader = td[8].get_text()
+                    except:
+                        uploader = None
+
+                    try:
+                        imdb_id = re.findall(r'imdb.com/title/(tt\d+)/?$', td[9].find('a').get('href'))[0]
+                    except:
+                        imdb_id = None
+
                     logger.info('Found subtitle link %r', link)
-                    sub = self.download_archive_and_add_subtitle_files(link, language, video)
+                    sub = self.download_archive_and_add_subtitle_files(link, language, video, fps, num_cds)
-                    for s in sub: 
+                    for s in sub:
+                        s.title = title
+                        s.notes = notes
+                        s.year = year
                         s.uploader = uploader
+                        s.imdb_id = imdb_id
                     subtitles = subtitles + sub
         return subtitles
 
@@ -159,23 +204,24 @@ class SubsSabBzProvider(Provider):
             pass
         else:
             seeking_subtitle_file = subtitle.filename
-            arch = self.download_archive_and_add_subtitle_files(subtitle.page_link, subtitle.language, subtitle.video)
+            arch = self.download_archive_and_add_subtitle_files(subtitle.page_link, subtitle.language, subtitle.video,
+                                                                subtitle.fps, subtitle.num_cds)
             for s in arch:
                 if s.filename == seeking_subtitle_file:
                     subtitle.content = s.content
 
-    def process_archive_subtitle_files(self, archiveStream, language, video, link):
+    def process_archive_subtitle_files(self, archiveStream, language, video, link, fps, num_cds):
         subtitles = []
         type = 'episode' if isinstance(video, Episode) else 'movie'
-        for file_name in archiveStream.namelist():
+        for file_name in sorted(archiveStream.namelist()):
             if file_name.lower().endswith(('.srt', '.sub')):
                 logger.info('Found subtitle file %r', file_name)
-                subtitle = SubsSabBzSubtitle(language, file_name, type, video, link)
+                subtitle = SubsSabBzSubtitle(language, file_name, type, video, link, fps, num_cds)
-                subtitle.content = archiveStream.read(file_name)
+                subtitle.content = fix_line_ending(archiveStream.read(file_name))
                 subtitles.append(subtitle)
         return subtitles
 
-    def download_archive_and_add_subtitle_files(self, link, language, video ):
+    def download_archive_and_add_subtitle_files(self, link, language, video, fps, num_cds):
         logger.info('Downloading subtitle %r', link)
         request = self.session.get(link, headers={
             'Referer': 'http://subs.sab.bz/index.php?'
@@ -184,8 +230,9 @@ class SubsSabBzProvider(Provider):
 
         archive_stream = io.BytesIO(request.content)
         if is_rarfile(archive_stream):
-            return self.process_archive_subtitle_files( RarFile(archive_stream), language, video, link )
+            return self.process_archive_subtitle_files(RarFile(archive_stream), language, video, link, fps, num_cds)
         elif is_zipfile(archive_stream):
-            return self.process_archive_subtitle_files( ZipFile(archive_stream), language, video, link )
+            return self.process_archive_subtitle_files(ZipFile(archive_stream), language, video, link, fps, num_cds)
         else:
-            raise ValueError('Not a valid archive')
+            logger.error('Ignore unsupported archive %r', request.headers)
+            return []

libs/subliminal_patch/providers/subsunacs.py

@@ -13,7 +13,6 @@ from guessit import guessit
 from subliminal_patch.providers import Provider
 from subliminal_patch.subtitle import Subtitle
 from subliminal_patch.utils import sanitize, fix_inconsistent_naming
-from subliminal.exceptions import ProviderError
 from subliminal.utils import sanitize_release_group
 from subliminal.subtitle import guess_matches
 from subliminal.video import Episode, Movie
@@ -34,25 +33,31 @@ def fix_tv_naming(title):
     return fix_inconsistent_naming(title, {"Marvel's Daredevil": "Daredevil",
                                            "Marvel's Luke Cage": "Luke Cage",
                                            "Marvel's Iron Fist": "Iron Fist",
-                                           "DC's Legends of Tomorrow": "Legends of Tomorrow"
+                                           "DC's Legends of Tomorrow": "Legends of Tomorrow",
+                                           "Doctor Who (2005)": "Doctor Who",
                                            }, True)
 
 class SubsUnacsSubtitle(Subtitle):
     """SubsUnacs Subtitle."""
     provider_name = 'subsunacs'
 
-    def __init__(self, langauge, filename, type, video, link):
+    def __init__(self, langauge, filename, type, video, link, fps, num_cds):
         super(SubsUnacsSubtitle, self).__init__(langauge)
         self.langauge = langauge
         self.filename = filename
         self.page_link = link
         self.type = type
         self.video = video
+        self.fps = fps
+        self.num_cds = num_cds
         self.release_info = os.path.splitext(filename)[0]
 
     @property
     def id(self):
-        return self.filename
+        return self.page_link + self.filename
+
+    def get_fps(self):
+        return self.fps
 
     def make_picklable(self):
         self.content = None
@@ -74,13 +79,17 @@ class SubsUnacsSubtitle(Subtitle):
         if video_filename == subtitle_filename:
              matches.add('hash')
 
-        matches |= guess_matches(video, guessit(self.filename, {'type': self.type}))
+        if video.year and self.year == video.year:
+            matches.add('year')
+
+        matches |= guess_matches(video, guessit(self.title, {'type': self.type, 'allowed_countries': [None]}))
+        matches |= guess_matches(video, guessit(self.filename, {'type': self.type, 'allowed_countries': [None]}))
         return matches
 
 
 class SubsUnacsProvider(Provider):
     """SubsUnacs Provider."""
-    languages = {Language('por', 'BR')} | {Language(l) for l in [
+    languages = {Language(l) for l in [
         'bul', 'eng'
     ]}
 
@@ -145,11 +154,43 @@ class SubsUnacsProvider(Provider):
                 element = a_element_wrapper.find('a', {'class': 'tooltip'})
                 if element:
                     link = element.get('href')
-                    element = row.find('a', href = re.compile(r'.*/search\.php\?t=1\&(memid|u)=.*'))
+                    notes = element.get('title')
-                    uploader = element.get_text() if element else None
+                    title = element.get_text()
+
+                    try:
+                        year = int(element.find_next_sibling('span', {'class' : 'smGray'}).text.strip('\xa0()'))
+                    except:
+                        year = None
+
+                    td = row.findAll('td')
+
+                    try:
+                        num_cds = int(td[1].get_text())
+                    except:
+                        num_cds = None
+
+                    try:
+                        fps = float(td[2].get_text())
+                    except:
+                        fps = None
+
+                    try:
+                        rating = float(td[3].find('img').get('title'))
+                    except:
+                        rating = None
+
+                    try:
+                        uploader = td[5].get_text()
+                    except:
+                        uploader = None
+
                     logger.info('Found subtitle link %r', link)
-                    sub = self.download_archive_and_add_subtitle_files('https://subsunacs.net' + link, language, video)
+                    sub = self.download_archive_and_add_subtitle_files('https://subsunacs.net' + link, language, video, fps, num_cds)
-                    for s in sub: 
+                    for s in sub:
+                        s.title = title
+                        s.notes = notes
+                        s.year = year
+                        s.rating = rating
                         s.uploader = uploader
                     subtitles = subtitles + sub
         return subtitles
@@ -162,28 +203,29 @@ class SubsUnacsProvider(Provider):
             pass
         else:
             seeking_subtitle_file = subtitle.filename
-            arch = self.download_archive_and_add_subtitle_files(subtitle.page_link, subtitle.language, subtitle.video)
+            arch = self.download_archive_and_add_subtitle_files(subtitle.page_link, subtitle.language, subtitle.video,
+                                                                subtitle.fps, subtitle.num_cds)
             for s in arch:
                 if s.filename == seeking_subtitle_file:
                     subtitle.content = s.content
 
-    def process_archive_subtitle_files(self, archiveStream, language, video, link):
+    def process_archive_subtitle_files(self, archiveStream, language, video, link, fps, num_cds):
         subtitles = []
         type = 'episode' if isinstance(video, Episode) else 'movie'
-        for file_name in archiveStream.namelist():
+        for file_name in sorted(archiveStream.namelist()):
             if file_name.lower().endswith(('.srt', '.sub', '.txt')):
                 file_is_txt = True if file_name.lower().endswith('.txt') else False
                 if file_is_txt and re.search(r'subsunacs\.net|танете част|прочети|^read ?me|procheti', file_name, re.I): 
                     logger.info('Ignore readme txt file %r', file_name)
                     continue
                 logger.info('Found subtitle file %r', file_name)
-                subtitle = SubsUnacsSubtitle(language, file_name, type, video, link)
+                subtitle = SubsUnacsSubtitle(language, file_name, type, video, link, fps, num_cds)
-                subtitle.content = archiveStream.read(file_name)
+                subtitle.content = fix_line_ending(archiveStream.read(file_name))
                 if file_is_txt == False or subtitle.is_valid():
                     subtitles.append(subtitle)
         return subtitles
 
-    def download_archive_and_add_subtitle_files(self, link, language, video ):
+    def download_archive_and_add_subtitle_files(self, link, language, video, fps, num_cds):
         logger.info('Downloading subtitle %r', link)
         request = self.session.get(link, headers={
             'Referer': 'https://subsunacs.net/search.php'
@@ -192,8 +234,9 @@ class SubsUnacsProvider(Provider):
 
         archive_stream = io.BytesIO(request.content)
         if is_rarfile(archive_stream):
-            return self.process_archive_subtitle_files( RarFile(archive_stream), language, video, link )
+            return self.process_archive_subtitle_files(RarFile(archive_stream), language, video, link, fps, num_cds)
         elif is_zipfile(archive_stream):
-            return self.process_archive_subtitle_files( ZipFile(archive_stream), language, video, link )
+            return self.process_archive_subtitle_files(ZipFile(archive_stream), language, video, link, fps, num_cds)
         else:
-            raise ValueError('Not a valid archive')
+            logger.error('Ignore unsupported archive %r', request.headers)
+            return []

libs/subliminal_patch/providers/xsubs.py

@@ -19,6 +19,8 @@ from subliminal.video import Episode
 logger = logging.getLogger(__name__)
 article_re = re.compile(r'^([A-Za-z]{1,3}) (.*)$')
 episode_re = re.compile(r'^(\d+)(-(\d+))*$')
+episode_name_re = re.compile(r'^(.*?)( [\[(].{2,4}[\])])*$')
+series_sanitize_re = re.compile(r'^(.*?)( \[\D+\])*$')
 
 
 class XSubsSubtitle(Subtitle):
@@ -143,7 +145,11 @@ class XSubsProvider(Provider):
         for show_category in soup.findAll('seriesl'):
             if show_category.attrs['category'] == u'Σειρές':
                 for show in show_category.findAll('series'):
-                    show_ids[sanitize(show.text)] = int(show['srsid'])
+                    series = show.text
+                    series_match = series_sanitize_re.match(series)
+                    if series_match:
+                        series = series_match.group(1)
+                    show_ids[sanitize(series)] = int(show['srsid'])
                 break
         logger.debug('Found %d show ids', len(show_ids))
 
@@ -195,6 +201,9 @@ class XSubsProvider(Provider):
         soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
 
         series = soup.find('name').text
+        series_match = episode_name_re.match(series)
+        if series_match:
+            series = series_match.group(1)
 
         # loop over season rows
         seasons = soup.findAll('series_group')

libs/subliminal_patch/providers/yavkanet.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
 import logging
-import re
 import io
 import os
 from random import randint
@@ -13,7 +12,6 @@ from guessit import guessit
 from subliminal_patch.providers import Provider
 from subliminal_patch.subtitle import Subtitle
 from subliminal_patch.utils import sanitize
-from subliminal.exceptions import ProviderError
 from subliminal.utils import sanitize_release_group
 from subliminal.subtitle import guess_matches
 from subliminal.video import Episode, Movie
@@ -27,18 +25,22 @@ class YavkaNetSubtitle(Subtitle):
     """YavkaNet Subtitle."""
     provider_name = 'yavkanet'
 
-    def __init__(self, langauge, filename, type, video, link):
+    def __init__(self, langauge, filename, type, video, link, fps):
         super(YavkaNetSubtitle, self).__init__(langauge)
         self.langauge = langauge
         self.filename = filename
         self.page_link = link
         self.type = type
         self.video = video
+        self.fps = fps
         self.release_info = os.path.splitext(filename)[0]
 
     @property
     def id(self):
-        return self.filename
+        return self.page_link + self.filename
+
+    def get_fps(self):
+        return self.fps
 
     def make_picklable(self):
         self.content = None
@@ -60,7 +62,11 @@ class YavkaNetSubtitle(Subtitle):
         if video_filename == subtitle_filename:
              matches.add('hash')
 
-        matches |= guess_matches(video, guessit(self.filename, {'type': self.type}))
+        if video.year and self.year == video.year:
+            matches.add('year')
+
+        matches |= guess_matches(video, guessit(self.title, {'type': self.type, 'allowed_countries': [None]}))
+        matches |= guess_matches(video, guessit(self.filename, {'type': self.type, 'allowed_countries': [None]}))
         return matches
 
 
@@ -122,18 +128,34 @@ class YavkaNetProvider(Provider):
             return subtitles
 
         soup = BeautifulSoup(response.content, 'lxml')
-        rows = soup.findAll('tr', {'class': 'info'})
+        rows = soup.findAll('tr')
         
-        # Search on first 20 rows only
+        # Search on first 25 rows only
-        for row in rows[:20]:
+        for row in rows[:25]:
             element = row.find('a', {'class': 'selector'})
             if element:
                 link = element.get('href')
+                notes = element.get('content')
+                title = element.get_text()
+
+                try:
+                    year = int(element.find_next_sibling('span').text.strip('()'))
+                except:
+                    year = None
+
+                try:
+                    fps = float(row.find('span', {'title': 'Кадри в секунда'}).text.strip())
+                except:
+                    fps = None
+
                 element = row.find('a', {'class': 'click'})
                 uploader = element.get_text() if element else None
                 logger.info('Found subtitle link %r', link)
-                sub = self.download_archive_and_add_subtitle_files('http://yavka.net/' + link, language, video)
+                sub = self.download_archive_and_add_subtitle_files('http://yavka.net/' + link, language, video, fps)
-                for s in sub: 
+                for s in sub:
+                    s.title = title
+                    s.notes = notes
+                    s.year = year
                     s.uploader = uploader
                 subtitles = subtitles + sub
         return subtitles
@@ -146,23 +168,24 @@ class YavkaNetProvider(Provider):
             pass
         else:
             seeking_subtitle_file = subtitle.filename
-            arch = self.download_archive_and_add_subtitle_files(subtitle.page_link, subtitle.language, subtitle.video)
+            arch = self.download_archive_and_add_subtitle_files(subtitle.page_link, subtitle.language, subtitle.video,
+                                                                subtitle.fps)
             for s in arch:
                 if s.filename == seeking_subtitle_file:
                     subtitle.content = s.content
 
-    def process_archive_subtitle_files(self, archiveStream, language, video, link):
+    def process_archive_subtitle_files(self, archiveStream, language, video, link, fps):
         subtitles = []
         type = 'episode' if isinstance(video, Episode) else 'movie'
         for file_name in archiveStream.namelist():
             if file_name.lower().endswith(('.srt', '.sub')):
                 logger.info('Found subtitle file %r', file_name)
-                subtitle = YavkaNetSubtitle(language, file_name, type, video, link)
+                subtitle = YavkaNetSubtitle(language, file_name, type, video, link, fps)
-                subtitle.content = archiveStream.read(file_name)
+                subtitle.content = fix_line_ending(archiveStream.read(file_name))
                 subtitles.append(subtitle)
         return subtitles
 
-    def download_archive_and_add_subtitle_files(self, link, language, video ):
+    def download_archive_and_add_subtitle_files(self, link, language, video, fps):
         logger.info('Downloading subtitle %r', link)
         request = self.session.get(link, headers={
             'Referer': 'http://yavka.net/subtitles.php'
@@ -171,9 +194,9 @@ class YavkaNetProvider(Provider):
 
         archive_stream = io.BytesIO(request.content)
         if is_rarfile(archive_stream):
-            return self.process_archive_subtitle_files( RarFile(archive_stream), language, video, link )
+            return self.process_archive_subtitle_files(RarFile(archive_stream), language, video, link, fps)
         elif is_zipfile(archive_stream):
-            return self.process_archive_subtitle_files( ZipFile(archive_stream), language, video, link )
+            return self.process_archive_subtitle_files(ZipFile(archive_stream), language, video, link, fps)
         else:
-            raise ValueError('Not a valid archive')
+            logger.error('Ignore unsupported archive %r', request.headers)
-        
+            return []        

libs/subliminal_patch/subtitle.py

@@ -89,6 +89,13 @@ class Subtitle(Subtitle_):
     def numeric_id(self):
         raise NotImplemented
 
+    def get_fps(self):
+        """
+        :return: frames per second or None if not supported
+        :rtype: float
+        """
+        return None
+
     def make_picklable(self):
         """
         some subtitle instances might have unpicklable objects stored; clean them up here 
@@ -264,10 +271,14 @@ class Subtitle(Subtitle_):
                 else:
                     logger.info("Got format: %s", subs.format)
             except pysubs2.UnknownFPSError:
-                # if parsing failed, suggest our media file's fps
+                # if parsing failed, use frame rate from provider
-                logger.info("No FPS info in subtitle. Using our own media FPS for the MicroDVD subtitle: %s",
+                sub_fps = self.get_fps()
-                            self.plex_media_fps)
+                if not isinstance(sub_fps, float) or sub_fps < 10.0:
-                subs = pysubs2.SSAFile.from_string(text, fps=self.plex_media_fps)
+                    # or use our media file's fps as a fallback
+                    sub_fps = self.plex_media_fps
+                    logger.info("No FPS info in subtitle. Using our own media FPS for the MicroDVD subtitle: %s",
+                                self.plex_media_fps)
+                subs = pysubs2.SSAFile.from_string(text, fps=sub_fps)
 
             unicontent = self.pysubs2_to_unicode(subs)
             self.content = unicontent.encode(self._guessed_encoding)

libs/tzlocal/unix.py

@@ -84,11 +84,10 @@ def _get_localzone(_root='/'):
                     if not etctz:
                         continue
                     tz = pytz.timezone(etctz.replace(' ', '_'))
-                    # Disabling this offset valdation due to issue with some timezone: https://github.com/regebro/tzlocal/issues/80
+                    if _root == '/':
-                    # if _root == '/':
                         # We are using a file in etc to name the timezone.
                         # Verify that the timezone specified there is actually used:
-                        # utils.assert_tz_offset(tz)
+                        utils.assert_tz_offset(tz)
                     return tz
 
         except IOError:

libs/tzlocal/utils.py

@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
+import time
 import datetime
+import calendar
 
 
 def get_system_offset():
@@ -11,8 +13,14 @@ def get_system_offset():
 
     To keep compatibility with Windows, we're always importing time module here.
     """
-    import time
+
-    if time.daylight and time.localtime().tm_isdst > 0:
+    localtime = calendar.timegm(time.localtime())
+    gmtime = calendar.timegm(time.gmtime())
+    offset = gmtime - localtime
+    # We could get the localtime and gmtime on either side of a second switch
+    # so we check that the difference is less than one minute, because nobody
+    # has that small DST differences.
+    if abs(offset - time.altzone) < 60:
         return -time.altzone
     else:
         return -time.timezone

libs/tzlocal/windows_tz.py

@@ -87,6 +87,7 @@ win_tz = {'AUS Central Standard Time': 'Australia/Darwin',
  'Pacific Standard Time (Mexico)': 'America/Tijuana',
  'Pakistan Standard Time': 'Asia/Karachi',
  'Paraguay Standard Time': 'America/Asuncion',
+ 'Qyzylorda Standard Time': 'Asia/Qyzylorda',
  'Romance Standard Time': 'Europe/Paris',
  'Russia Time Zone 10': 'Asia/Srednekolymsk',
  'Russia Time Zone 11': 'Asia/Kamchatka',
@@ -127,6 +128,7 @@ win_tz = {'AUS Central Standard Time': 'Australia/Darwin',
  'Ulaanbaatar Standard Time': 'Asia/Ulaanbaatar',
  'Venezuela Standard Time': 'America/Caracas',
  'Vladivostok Standard Time': 'Asia/Vladivostok',
+ 'Volgograd Standard Time': 'Europe/Volgograd',
  'W. Australia Standard Time': 'Australia/Perth',
  'W. Central Africa Standard Time': 'Africa/Lagos',
  'W. Europe Standard Time': 'Europe/Berlin',
@@ -287,7 +289,7 @@ tz_win = {'Africa/Abidjan': 'Greenwich Standard Time',
  'America/Mendoza': 'Argentina Standard Time',
  'America/Menominee': 'Central Standard Time',
  'America/Merida': 'Central Standard Time (Mexico)',
- 'America/Metlakatla': 'Pacific Standard Time',
+ 'America/Metlakatla': 'Alaskan Standard Time',
  'America/Mexico_City': 'Central Standard Time (Mexico)',
  'America/Miquelon': 'Saint Pierre Standard Time',
  'America/Moncton': 'Atlantic Standard Time',
@@ -347,13 +349,13 @@ tz_win = {'Africa/Abidjan': 'Greenwich Standard Time',
  'America/Winnipeg': 'Central Standard Time',
  'America/Yakutat': 'Alaskan Standard Time',
  'America/Yellowknife': 'Mountain Standard Time',
- 'Antarctica/Casey': 'W. Australia Standard Time',
+ 'Antarctica/Casey': 'Singapore Standard Time',
  'Antarctica/Davis': 'SE Asia Standard Time',
  'Antarctica/DumontDUrville': 'West Pacific Standard Time',
  'Antarctica/Macquarie': 'Central Pacific Standard Time',
  'Antarctica/Mawson': 'West Asia Standard Time',
  'Antarctica/McMurdo': 'New Zealand Standard Time',
- 'Antarctica/Palmer': 'Magallanes Standard Time',
+ 'Antarctica/Palmer': 'SA Eastern Standard Time',
  'Antarctica/Rothera': 'SA Eastern Standard Time',
  'Antarctica/South_Pole': 'New Zealand Standard Time',
  'Antarctica/Syowa': 'E. Africa Standard Time',
@@ -424,7 +426,7 @@ tz_win = {'Africa/Abidjan': 'Greenwich Standard Time',
  'Asia/Pyongyang': 'North Korea Standard Time',
  'Asia/Qatar': 'Arab Standard Time',
  'Asia/Qostanay': 'Central Asia Standard Time',
- 'Asia/Qyzylorda': 'West Asia Standard Time',
+ 'Asia/Qyzylorda': 'Qyzylorda Standard Time',
  'Asia/Rangoon': 'Myanmar Standard Time',
  'Asia/Riyadh': 'Arab Standard Time',
  'Asia/Saigon': 'SE Asia Standard Time',
@@ -592,7 +594,7 @@ tz_win = {'Africa/Abidjan': 'Greenwich Standard Time',
  'Europe/Vatican': 'W. Europe Standard Time',
  'Europe/Vienna': 'W. Europe Standard Time',
  'Europe/Vilnius': 'FLE Standard Time',
- 'Europe/Volgograd': 'Russian Standard Time',
+ 'Europe/Volgograd': 'Volgograd Standard Time',
  'Europe/Warsaw': 'Central European Standard Time',
  'Europe/Zagreb': 'Central European Standard Time',
  'Europe/Zaporozhye': 'FLE Standard Time',

libs/version.txt

@@ -1,4 +1,4 @@
-apprise=0.8.2
+apprise=0.8.5
 apscheduler=3.5.1
 babelfish=0.5.5
 backports.functools-lru-cache=1.5
@@ -13,7 +13,6 @@ gitpython=2.1.9
 guessit=2.1.4
 guess_language-spirit=0.5.3
 knowit=0.3.0-dev
-peewee=3.9.6
 py-pretty=1
 pycountry=18.2.23
 pyga=2.6.1
@@ -25,6 +24,6 @@ six=1.11.0
 SimpleConfigParser=0.1.0 <-- modified version: do not update!!!
 stevedore=1.28.0
 subliminal=2.1.0dev
-tzlocal=1.5.1
+tzlocal=2.1b1
 urllib3=1.23
 Js2Py=0.63 <-- modified: manually merged from upstream: https://github.com/PiotrDabkowski/Js2Py/pull/192/files

views/providers.tpl

@@ -265,6 +265,24 @@
                                     </div>
                                 </div>
                             </div>
+                            <div class="middle aligned row">
+                                <div class="right aligned six wide column">
+                                    <label>Skip wrong FPS</label>
+                                </div>
+                                <div class="one wide column">
+                                    <div id="settings_legendasdivx_skip_wrong_fps" class="ui toggle checkbox" data-ldfps={{settings.legendasdivx.getboolean('skip_wrong_fps')}}>
+                                        <input type="checkbox" name="settings_legendasdivx_skip_wrong_fps">
+                                        <label></label>
+                                    </div>
+                                </div>
+                                <div class="collapsed column">
+                                <div class="collapsed center aligned column">
+                                    <div class="ui basic icon" data-tooltip="Skip Subtitles with a mismatched FPS value; might lead to more results when disabled but also to more false-positives." data-inverted="">
+                                        <i class="help circle large icon"></i>
+                                    </div>
+                                </div>
+                                </div>
+                            </div>
                         </div>
 
                         <div class="middle aligned row">
@@ -500,8 +518,9 @@
                             </div>
                         </div>
                         <div id="regielive_option" class="ui grid container">
-
+                        
                         </div>
+
                         <div class="middle aligned row">
                             <div class="right aligned four wide column">
                                 <label>Subdivx</label>
@@ -733,7 +752,7 @@
                             </div>
                         </div>
                         <div id="titlovi_option" class="ui grid container">
-                        	<div class="middle aligned row">
+                            <div class="middle aligned row">
                                 <div class="right aligned six wide column">
                                     <label>Username</label>
                                 </div>
@@ -755,6 +774,28 @@
                             </div>
                         </div>
 
+                        <div class="middle aligned row">
+                            <div class="right aligned four wide column">
+                                <label>Titrari.ro</label>
+                            </div>
+                            <div class="one wide column">
+                                <div id="titrari" class="ui toggle checkbox provider">
+                                    <input type="checkbox">
+                                    <label></label>
+                                </div>
+                            </div>
+                            <div class="collapsed column">
+                                <div class="collapsed center aligned column">
+                                    <div class="ui basic icon" data-tooltip="Romanian Subtitles Provider." data-inverted="">
+                                        <i class="help circle large icon"></i>
+                                     </div>
+                                 </div>
+                             </div>
+                        </div>
+                        <div id="titrari_option" class="ui grid container">
+
+                        </div>
+
                         <div class="middle aligned row">
                             <div class="right aligned four wide column">
                                 <label>TVSubtitles</label>
@@ -811,7 +852,7 @@
                             </div>
                         </div>
                         <div id="xsubs_option" class="ui grid container">
-                        	<div class="middle aligned row">
+                            <div class="middle aligned row">
                                 <div class="right aligned six wide column">
                                     <label>Username</label>
                                 </div>
@@ -922,6 +963,12 @@
                         $("#settings_opensubtitles_skip_wrong_fps").checkbox('uncheck');
                     }
 
+                    if ($('#settings_legendasdivx_skip_wrong_fps').data("ldfps") === "True") {
+                        $("#settings_legendasdivx_skip_wrong_fps").checkbox('check');
+                    } else {
+                        $("#settings_legendasdivx_skip_wrong_fps").checkbox('uncheck');
+                    }
+
                     $('#settings_providers').dropdown('clear');
                     $('#settings_providers').dropdown('set selected',{{!enabled_providers}});
                     $('#settings_providers').dropdown();
@@ -949,4 +996,4 @@
                             $('#'+$(this).parent().attr('id')+'_option').hide();
                         }
                     });
-                </script>
+                </script>

views/system.tpl

@@ -403,15 +403,12 @@
     });
 
 	$('#shutdown').on('click', function(){
-		$.ajax({
+        document.open();
-			url: "{{base_url}}shutdown",
+        document.write('Bazarr has shutdown.');
-			async: false
+        document.close();
+	    $.ajax({
+			url: "{{base_url}}shutdown"
 		})
-		.always(function(){
-			document.open();
-			document.write('Bazarr has shutdown.');
-			document.close();
-		});
 	});
 
     $('#logout').on('click', function(){
@@ -422,11 +419,9 @@
 		$('#loader_text').text("Bazarr is restarting, please wait...");
 		$.ajax({
             url: "{{base_url}}restart",
-            async: true,
+            async: true
-            error: (function () {
-                setTimeout(function () { setInterval(ping, 2000); }, 8000);
-            })
         });
+		setTimeout(function () { setInterval(ping, 2000); }, 8000);
 	});
 
     % from config import settings