Add chmod, provider throttle, multithread options

bazarr/config.py

@@ -32,7 +32,10 @@ defaults = {
         'minimum_score_movie': '70',
         'use_embedded_subs': 'True',
         'adaptive_searching': 'False',
-        'enabled_providers': ''
+        'enabled_providers': '',
+        'throtteled_providers': '',
+        'multithreading': 'True',
+        'chmod': '0640'
     },
     'auth': {
         'type': 'None',
@@ -69,16 +72,16 @@ defaults = {
         'username': '',
         'password': '',
         'use_tag_search': 'False',
-        'vip': 'False',
+        'vip': 'True',
-        'ssl': 'False',
+        'ssl': 'True',
         'timeout': '15',
         'skip_wrong_fps': 'False'
     },
     'addic7ed': {
         'username': '',
         'password': '',
-        'random_agents': 'False'
+        'random_agents': 'True'
-},
+    },
     'legendastv': {
         'username': '',
         'password': ''

bazarr/get_providers.py

@@ -1,17 +1,79 @@
 # coding=utf-8
+import os
 import datetime
+import logging
+import subliminal_patch
 
+from get_args import args
 from config import settings
 from subliminal_patch.exceptions import TooManyRequests, APIThrottled
 from subliminal.exceptions import DownloadLimitExceeded, ServiceUnavailable
 
+VALID_THROTTLE_EXCEPTIONS = (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled)
 
+PROVIDER_THROTTLE_MAP = {
+    "default": {
+        TooManyRequests: (datetime.timedelta(hours=1), "1 hour"),
+        DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"),
+        ServiceUnavailable: (datetime.timedelta(minutes=20), "20 minutes"),
+        APIThrottled: (datetime.timedelta(minutes=10), "10 minutes"),
+    },
+    "opensubtitles": {
+        TooManyRequests: (datetime.timedelta(hours=3), "3 hours"),
+        DownloadLimitExceeded: (datetime.timedelta(hours=6), "6 hours"),
+        APIThrottled: (datetime.timedelta(seconds=15), "15 seconds"),
+    },
+    "addic7ed": {
+        DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"),
+        TooManyRequests: (datetime.timedelta(minutes=5), "5 minutes"),
+    }
+}
+
+PROVIDERS_FORCED_OFF = ["addic7ed", "tvsubtitles", "legendastv", "napiprojekt", "shooter", "hosszupuska",
+                        "supersubtitles", "titlovi", "argenteam", "assrt", "subscene"]
+
+if not settings.general.throtteled_providers:
+    tp = {}
+else:
+    tp = eval(str(settings.general.throtteled_providers))
+
+
+def provider_pool():
+    if settings.general.getboolean('multithreading'):
+        return subliminal_patch.core.SZAsyncProviderPool
+    return subliminal_patch.core.SZProviderPool
+
+    
 def get_providers():
+    changed = False
     providers_list = []
     if settings.general.enabled_providers:
         for provider in settings.general.enabled_providers.lower().split(','):
+            reason, until, throttle_desc = tp.get(provider, (None, None, None))
             providers_list.append(provider)
-    else:
+    
+            if reason:
+                now = datetime.datetime.now()
+                if now < until:
+                    logging.info("Not using %s until %s, because of: %s", provider,
+                                 until.strftime("%y/%m/%d %H:%M"), reason)
+                    providers_list.remove(provider)
+                else:
+                    logging.info("Using %s again after %s, (disabled because: %s)", provider, throttle_desc, reason)
+                    del tp[provider]
+                    settings.general.throtteled_providers = str(tp)
+                    changed = True
+
+        if changed:
+            with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle:
+                settings.write(handle)
+
+        # if forced only is enabled: # fixme: Prepared for forced only implementation to remove providers with don't support forced only subtitles
+        #     for provider in providers_list:
+        #         if provider in PROVIDERS_FORCED_OFF:
+        #             providers_list.remove(provider)
+    
+    if not providers_list:
         providers_list = None
     
     return providers_list
@@ -47,3 +109,30 @@ def get_providers_auth():
     }
     
     return providers_auth
+
+
+def provider_throttle(name, exception):
+    cls = getattr(exception, "__class__")
+    cls_name = getattr(cls, "__name__")
+    if cls not in VALID_THROTTLE_EXCEPTIONS:
+        for valid_cls in VALID_THROTTLE_EXCEPTIONS:
+            if isinstance(cls, valid_cls):
+                cls = valid_cls
+    
+    throttle_data = PROVIDER_THROTTLE_MAP.get(name, PROVIDER_THROTTLE_MAP["default"]).get(cls, None) or \
+                    PROVIDER_THROTTLE_MAP["default"].get(cls, None)
+    
+    if not throttle_data:
+        return
+    
+    throttle_delta, throttle_description = throttle_data
+    throttle_until = datetime.datetime.now() + throttle_delta
+    
+    tp[name] = (cls_name, throttle_until, throttle_description)
+    
+    settings.general.throtteled_providers = str(tp)
+    with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle:
+        settings.write(handle)
+    
+    logging.info("Throttling %s for %s, until %s, because of: %s. Exception info: %r", name, throttle_description,
+                 throttle_until.strftime("%y/%m/%d %H:%M"), cls_name, exception.message)

bazarr/get_subtitle.py

@@ -28,9 +28,11 @@ from helper import path_replace, path_replace_movie, path_replace_reverse, \
 from list_subtitles import store_subtitles, list_missing_subtitles, store_subtitles_movie, list_missing_subtitles_movies
 from utils import history_log, history_log_movie
 from notifier import send_notifications, send_notifications_movie
-from get_providers import get_providers, get_providers_auth
+from get_providers import get_providers, get_providers_auth, provider_throttle, provider_pool
 from get_args import args
 from queueconfig import q4ws
+from subliminal_patch.exceptions import TooManyRequests, APIThrottled
+from subliminal.exceptions import DownloadLimitExceeded, ServiceUnavailable
 
 # configure the cache
 
@@ -57,13 +59,17 @@ def get_video(path, title, sceneName, use_scenename, providers=None, media_type=
         # use the sceneName but keep the folder structure for better guessing
         path = os.path.join(os.path.dirname(path), sceneName + os.path.splitext(path)[1])
         dont_use_actual_file = True
-    
     try:
-        video = parse_video(path, hints=hints, providers=providers, dry_run=dont_use_actual_file)
+        if providers:
-        video.used_scene_name = dont_use_actual_file
+            video = parse_video(path, hints=hints, providers=providers, dry_run=dont_use_actual_file)
-        video.original_name = original_name
+            video.used_scene_name = dont_use_actual_file
-        video.original_path = original_path
+            video.original_name = original_name
-        return video
+            video.original_path = original_path
+            return video
+        else:
+            logging.info("BAZARR All providers are throttled")
+            return None
+    
     
     except:
         logging.exception("BAZARR Error trying to get video information for this file: " + path)
@@ -144,102 +150,31 @@ def download_subtitle(path, language, hi, providers, providers_auth, sceneName,
     AsyncProviderPool:
     implement:
         blacklist=None,
-        throttle_callback=None,
         pre_download_hook=None,
         post_download_hook=None,
         language_hook=None
     """
     
-    """
-    throttle_callback:
-    
-    VALID_THROTTLE_EXCEPTIONS = (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled)
-    
-    PROVIDER_THROTTLE_MAP = {
-        "default": {
-            TooManyRequests: (datetime.timedelta(hours=1), "1 hour"),
-            DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"),
-            ServiceUnavailable: (datetime.timedelta(minutes=20), "20 minutes"),
-            APIThrottled: (datetime.timedelta(minutes=10), "10 minutes"),
-        },
-        "opensubtitles": {
-            TooManyRequests: (datetime.timedelta(hours=3), "3 hours"),
-            DownloadLimitExceeded: (datetime.timedelta(hours=6), "6 hours"),
-            APIThrottled: (datetime.timedelta(seconds=15), "15 seconds"),
-        },
-        "addic7ed": {
-            DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"),
-            TooManyRequests: (datetime.timedelta(minutes=5), "5 minutes"),
-        }
-    }
-    
-    throttle_callback gist:
-        def provider_throttle(self, name, exception):
-            cls = getattr(exception, "__class__")
-            cls_name = getattr(cls, "__name__")
-            if cls not in VALID_THROTTLE_EXCEPTIONS:
-                for valid_cls in VALID_THROTTLE_EXCEPTIONS:
-                    if isinstance(cls, valid_cls):
-                        cls = valid_cls
-    
-            throttle_data = PROVIDER_THROTTLE_MAP.get(name, PROVIDER_THROTTLE_MAP["default"]).get(cls, None) or \
-                PROVIDER_THROTTLE_MAP["default"].get(cls, None)
-    
-            if not throttle_data:
-                return
-    
-            throttle_delta, throttle_description = throttle_data
-            throttle_until = datetime.datetime.now() + throttle_delta
-            
-            # save throttle_until together with provider name somewhere, then implement dynamic provider_list based on
-            # that
-            
-    provider_configs=
-                {'addic7ed': {'username': Prefs['provider.addic7ed.username'],
-                              'password': Prefs['provider.addic7ed.password'],
-                              'use_random_agents': cast_bool(Prefs['provider.addic7ed.use_random_agents1']),
-                              },
-                 'opensubtitles': {'username': Prefs['provider.opensubtitles.username'],
-                                   'password': Prefs['provider.opensubtitles.password'],
-                                   'use_tag_search': self.exact_filenames,
-                                   'only_foreign': self.forced_only,
-                                   'also_foreign': self.forced_also,
-                                   'is_vip': cast_bool(Prefs['provider.opensubtitles.is_vip']),
-                                   'use_ssl': os_use_https,
-                                   'timeout': self.advanced.providers.opensubtitles.timeout or 15,
-                                   'skip_wrong_fps': os_skip_wrong_fps,
-                                   },
-                 'podnapisi': {
-                     'only_foreign': self.forced_only,
-                     'also_foreign': self.forced_also,
-                 },
-                 'subscene': {
-                     'only_foreign': self.forced_only,
-                 },
-                 'legendastv': {'username': Prefs['provider.legendastv.username'],
-                                'password': Prefs['provider.legendastv.password'],
-                                },
-                 'assrt': {'token': Prefs['provider.assrt.token'], }
-                 }
-
-    """
-    
     video = get_video(path, title, sceneName, use_scenename, providers=providers, media_type=media_type)
     if video:
         min_score, max_score, scores = get_scores(video, media_type, min_score_movie_perc=int(minimum_score_movie),
                                                   min_score_series_perc=int(minimum_score))
-        
+
-        downloaded_subtitles = download_best_subtitles({video}, language_set, int(min_score), hi,
+        if providers:
-                                                       providers=providers,
+            downloaded_subtitles = download_best_subtitles({video}, language_set, int(min_score), hi,
-                                                       provider_configs=providers_auth,
+                                                           providers=providers,
-                                                       pool_class=SZAsyncProviderPool,
+                                                           provider_configs=providers_auth,
-                                                       compute_score=compute_score,
+                                                           pool_class=provider_pool(),
-                                                       throttle_time=None,  # fixme
+                                                           compute_score=compute_score,
-                                                       blacklist=None,  # fixme
+                                                           throttle_time=None,  # fixme
-                                                       throttle_callback=None,  # fixme
+                                                           blacklist=None,  # fixme
-                                                       pre_download_hook=None,  # fixme
+                                                           throttle_callback=provider_throttle,
-                                                       post_download_hook=None,  # fixme
+                                                           pre_download_hook=None,  # fixme
-                                                       language_hook=None)  # fixme
+                                                           post_download_hook=None,  # fixme
+                                                           language_hook=None)  # fixme
+        else:
+            downloaded_subtitles = None
+            logging.info("BAZARR All providers are throttled")
         
         saved_any = False
         if downloaded_subtitles:
@@ -251,7 +186,7 @@ def download_subtitle(path, language, hi, providers, providers_auth, sceneName,
                     saved_subtitles = save_subtitles(video.original_path, subtitles, single=single,
                                                      tags=None,  # fixme
                                                      directory=None,  # fixme
-                                                     chmod=None,  # fixme
+                                                     chmod=int(settings.general.chmod),  # fixme
                                                      # formats=("srt", "vtt")
                                                      path_decoder=force_unicode
                                                      )
@@ -344,12 +279,16 @@ def manual_search(path, language, hi, providers, providers_auth, sceneName, titl
                                                   min_score_series_perc=int(minimum_score))
         
         try:
-            subtitles = list_subtitles([video], language_set,
+            if providers:
-                                       providers=providers,
+                subtitles = list_subtitles([video], language_set,
-                                       provider_configs=providers_auth,
+                                           providers=providers,
-                                       pool_class=SZAsyncProviderPool,  # fixme: make async optional
+                                           provider_configs=providers_auth,
-                                       throttle_callback=None,  # fixme
+                                           pool_class=provider_pool(),
-                                       language_hook=None)  # fixme
+                                           throttle_callback=provider_throttle,
+                                           language_hook=None)  # fixme
+            else:
+                subtitles = []
+                logging.info("BAZARR All providers are throttled")
         except Exception as e:
             logging.exception("BAZARR Error trying to get subtitle list from provider for this file: " + path)
         else:
@@ -402,12 +341,16 @@ def manual_download_subtitle(path, language, hi, subtitle, provider, providers_a
     if video:
         min_score, max_score, scores = get_scores(video, media_type)
         try:
-            download_subtitles([subtitle], providers={provider}, provider_configs=providers_auth,
+            if provider:
-                                                  pool_class=SZAsyncProviderPool,
+                download_subtitles([subtitle], providers={provider}, provider_configs=providers_auth,
-                                                  throttle_callback=None)  # fixme
+                                   pool_class=provider_pool(),
-            logging.debug('BAZARR Subtitles file downloaded for this file:' + path)
+                                   throttle_callback=provider_throttle)
+                logging.debug('BAZARR Subtitles file downloaded for this file:' + path)
+            else:
+                logging.info("BAZARR All providers are throttled")
+                return None
         except Exception as e:
-            logging.exception('BAZARR Error downloading subtitles for this file ' + path)
+            logging.exception('BAZARR Error downloading subtitles for this file ' + path + e)
             return None
         else:
             if not subtitle.is_valid():
@@ -468,7 +411,8 @@ def manual_download_subtitle(path, language, hi, subtitle, provider, providers_a
                         return message
                 else:
                     logging.error(
-                        "BAZARR Tried to manually download a subtitles for file: " + path + " but we weren't able to do (probably throttled by " + str(subtitle.provider_name) + ". Please retry later or select a subtitles from another provider.")
+                        "BAZARR Tried to manually download a subtitles for file: " + path + " but we weren't able to do (probably throttled by " + str(
+                            subtitle.provider_name) + ". Please retry later or select a subtitles from another provider.")
                     return None
     logging.debug('BAZARR Ended manually downloading subtitles for file: ' + path)
 
@@ -559,7 +503,7 @@ def wanted_download_subtitles(path):
             
             for i in range(len(attempt)):
                 if attempt[i][0] == language:
-                    if search_active(attempt[i][1]) is True:
+                    if search_active(attempt[i][1]):
                         q4ws.append(
                             'Searching ' + str(language_from_alpha2(language)) + ' subtitles for this file: ' + path)
                         message = download_subtitle(path_replace(episode[0]), str(alpha3_from_alpha2(language)),
@@ -648,14 +592,22 @@ def wanted_search_missing_subtitles():
     movies = c.fetchall()
     
     c.close()
-    
+    providers = get_providers()
     if settings.general.getboolean('use_sonarr'):
-        for episode in episodes:
+        if providers:
-            wanted_download_subtitles(episode[0])
+            for episode in episodes:
+                wanted_download_subtitles(episode[0])
+        else:
+            q4ws.append('BAZARR All providers are throttled')
+            logging.info("BAZARR All providers are throttled")
     
     if settings.general.getboolean('use_radarr'):
-        for movie in movies:
+        if providers:
-            wanted_download_subtitles_movie(movie[0])
+            for movie in movies:
+                wanted_download_subtitles_movie(movie[0])
+        else:
+            q4ws.append('BAZARR All providers are throttled')
+            logging.info("BAZARR All providers are throttled")
     
     logging.info('BAZARR Finished searching for missing subtitles. Check histories for more information.')
 

bazarr/main.py

@@ -1103,6 +1103,7 @@ def save_settings():
         settings_general_debug = 'False'
     else:
         settings_general_debug = 'True'
+    settings_general_chmod = request.forms.get('settings_general_chmod')
     settings_general_sourcepath = request.forms.getall('settings_general_sourcepath')
     settings_general_destpath = request.forms.getall('settings_general_destpath')
     settings_general_pathmapping = []
@@ -1138,6 +1139,11 @@ def save_settings():
         settings_general_adaptive_searching = 'False'
     else:
         settings_general_adaptive_searching = 'True'
+    settings_general_multithreading = request.forms.get('settings_general_multithreading')
+    if settings_general_multithreading is None:
+        settings_general_multithreading = 'False'
+    else:
+        settings_general_multithreading = 'True'
     settings_general_minimum_score = request.forms.get('settings_general_minimum_score')
     settings_general_minimum_score_movies = request.forms.get('settings_general_minimum_score_movies')
     settings_general_use_postprocessing = request.forms.get('settings_general_use_postprocessing')
@@ -1170,6 +1176,7 @@ def save_settings():
     settings.general.base_url = text_type(settings_general_baseurl)
     settings.general.path_mappings = text_type(settings_general_pathmapping)
     settings.general.debug = text_type(settings_general_debug)
+    settings.general.chmod = text_type(settings_general_chmod)
     settings.general.branch = text_type(settings_general_branch)
     settings.general.auto_update = text_type(settings_general_automatic)
     settings.general.single_language = text_type(settings_general_single_language)
@@ -1184,6 +1191,7 @@ def save_settings():
     settings.general.minimum_score_movie = text_type(settings_general_minimum_score_movies)
     settings.general.use_embedded_subs = text_type(settings_general_embedded)
     settings.general.adaptive_searching = text_type(settings_general_adaptive_searching)
+    settings.general.multithreading = text_type(settings_general_multithreading)
     
     if after != before:
         configured()

views/settings.tpl

@@ -153,6 +153,26 @@
                             </div>
                         </div>
 
+                        <div class="middle aligned row">
+                            <div class="right aligned four wide column">
+                                <label>Set subtitle file permissions to</label>
+                            </div>
+                            <div class="five wide column">
+                                <div class='field'>
+                                    <div id="settings_chmod" class="ui fluid input">
+                                        <input name="settings_general_chmod" type="text"
+                                               value={{ settings.general.chmod }}>
+                                        <label></label>
+                                    </div>
+                                </div>
+                            </div>
+                            <div class="collapsed center aligned column">
+                                <div class="ui basic icon" data-tooltip="Integer, e.g.: 0775" data-inverted="">
+                                    <i class="help circle large icon"></i>
+                                </div>
+                            </div>
+                        </div>
+
                         <div class="middle aligned row">
                             <div class="right aligned four wide column">
                                 <label>Page size</label>
@@ -1076,6 +1096,26 @@
                                 </div>
                             </div>
                         </div>
+
+                        <div class="middle aligned row">
+                            <div class="right aligned four wide column">
+                                <label>Search enabled providers simultaneously</label>
+                            </div>
+                            <div class="one wide column">
+                                <div id="settings_multithreading" class="ui toggle checkbox"
+                                     data-multithreading={{ settings.general.getboolean('multithreading') }}>
+                                    <input name="settings_general_multithreading" type="checkbox">
+                                    <label></label>
+                                </div>
+                            </div>
+                            <div class="collapsed column">
+                                <div class="collapsed center aligned column">
+                                    <div class="ui basic icon" data-tooltip="Multithreading" data-inverted="">
+                                        <i class="help circle large icon"></i>
+                                    </div>
+                                </div>
+                            </div>
+                        </div>
                     </div>
                 </div>
                 <div class="ui dividing header">Subtitles providers</div>
@@ -1760,6 +1800,12 @@
                 $("#settings_adaptive_searching").checkbox('uncheck');
             }
 
+    if ($('#settings_multithreading').data("multithreading") === "True") {
+        $("#settings_multithreading").checkbox('check');
+    } else {
+        $("#settings_multithreading").checkbox('uncheck');
+    }
+
     if ($('#settings_addic7ed_random_agents').data("randomagents") === "True") {
                 $("#settings_addic7ed_random_agents").checkbox('check');
             } else {
@@ -2057,6 +2103,14 @@
                         }
                     ]
                 },
+                settings_general_chmod: {
+                    rules: [
+                        {
+                            type: 'regExp[^(0[0-7]{3})$]',
+                            prompt: 'Please use only 4-digit integers with leading 0 (e.g.: 775)'
+                        }
+                    ]
+                },
                 settings_auth_password : {
                     depends: 'settings_auth_username',
                     rules : [