diff --git a/bazarr/get_episodes.py b/bazarr/get_episodes.py index ed865f0fb..60d616e1c 100644 --- a/bazarr/get_episodes.py +++ b/bazarr/get_episodes.py @@ -124,6 +124,12 @@ def sync_episodes(): 'audio_codec': audioCodec, 'episode_file_id': episode['episodeFile']['id']}) + # Remove old episodes from DB + removed_episodes = list(set(current_episodes_db_list) - set(current_episodes_sonarr)) + + for removed_episode in removed_episodes: + database.execute("DELETE FROM table_episodes WHERE sonarrEpisodeId=?", (removed_episode,)) + # Update existing episodes in DB episode_in_db_list = [] episodes_in_db = database.execute("SELECT sonarrSeriesId, sonarrEpisodeId, title, path, season, episode, " @@ -146,17 +152,14 @@ def sync_episodes(): # Insert new episodes in DB for added_episode in episodes_to_add: query = dict_converter.convert(added_episode) - database.execute( + result = database.execute( '''INSERT OR IGNORE INTO table_episodes(''' + query.keys_insert + ''') VALUES(''' + query.question_marks + ''')''', query.values) - altered_episodes.append([added_episode['sonarrEpisodeId'], - added_episode['path']]) - - # Remove old episodes from DB - removed_episodes = list(set(current_episodes_db_list) - set(current_episodes_sonarr)) - - for removed_episode in removed_episodes: - database.execute("DELETE FROM table_episodes WHERE sonarrEpisodeId=?", (removed_episode,)) + if result: + altered_episodes.append([added_episode['sonarrEpisodeId'], added_episode['path']]) + else: + logging.debug('BAZARR unable to insert this episode into the database:', + path_replace(added_episode['path'])) # Store subtitles for added or modified episodes for i, altered_episode in enumerate(altered_episodes, 1): diff --git a/bazarr/get_movies.py b/bazarr/get_movies.py index eb73b68b9..2fc6b1f5a 100644 --- a/bazarr/get_movies.py +++ b/bazarr/get_movies.py @@ -206,8 +206,14 @@ def update_movies(): logging.error( 'BAZARR Radarr returned a movie without a file path: ' + movie["path"] + separator + movie['movieFile']['relativePath']) - - # Update or insert movies in DB + + # Remove old movies from DB + removed_movies = list(set(current_movies_db_list) - set(current_movies_radarr)) + + for removed_movie in removed_movies: + database.execute("DELETE FROM table_movies WHERE tmdbId=?", (removed_movie,)) + + # Update movies in DB movies_in_db_list = [] movies_in_db = database.execute("SELECT radarrId, title, path, tmdbId, overview, poster, fanart, " "audio_language, sceneName, monitored, sortTitle, year, " @@ -231,19 +237,17 @@ def update_movies(): # Insert new movies in DB for added_movie in movies_to_add: query = dict_converter.convert(added_movie) - database.execute( + result = database.execute( '''INSERT OR IGNORE INTO table_movies(''' + query.keys_insert + ''') VALUES(''' + query.question_marks + ''')''', query.values) - altered_movies.append([added_movie['tmdbId'], - added_movie['path'], - added_movie['radarrId'], - added_movie['monitored']]) - - # Remove old movies from DB - removed_movies = list(set(current_movies_db_list) - set(current_movies_radarr)) - - for removed_movie in removed_movies: - database.execute("DELETE FROM table_movies WHERE tmdbId=?", (removed_movie,)) + if result: + altered_movies.append([added_movie['tmdbId'], + added_movie['path'], + added_movie['radarrId'], + added_movie['monitored']]) + else: + logging.debug('BAZARR unable to insert this movie into the database:', + path_replace_movie(added_movie['path'])) # Store subtitles for added or modified movies for i, altered_movie in enumerate(altered_movies, 1): diff --git a/bazarr/get_series.py b/bazarr/get_series.py index d9756ffea..223f65d44 100644 --- a/bazarr/get_series.py +++ b/bazarr/get_series.py @@ -15,6 +15,7 @@ from list_subtitles import list_missing_subtitles from database import database, dict_converter from utils import get_sonarr_version import six +from helper import path_replace def update_series(): @@ -123,7 +124,13 @@ def update_series(): 'sortTitle': show['sortTitle'], 'year': show['year'], 'alternateTitles': alternateTitles}) - + + # Remove old series from DB + removed_series = list(set(current_shows_db_list) - set(current_shows_sonarr)) + + for series in removed_series: + database.execute("DELETE FROM table_shows WHERE tvdbId=?",(series,)) + # Update existing series in DB series_in_db_list = [] series_in_db = database.execute("SELECT title, path, tvdbId, sonarrSeriesId, overview, poster, fanart, " @@ -142,16 +149,14 @@ def update_series(): # Insert new series in DB for added_series in series_to_add: query = dict_converter.convert(added_series) - database.execute( + result = database.execute( '''INSERT OR IGNORE INTO table_shows(''' + query.keys_insert + ''') VALUES(''' + query.question_marks + ''')''', query.values) - list_missing_subtitles(no=added_series['sonarrSeriesId']) - - # Remove old series from DB - removed_series = list(set(current_shows_db_list) - set(current_shows_sonarr)) - - for series in removed_series: - database.execute("DELETE FROM table_shows WHERE tvdbId=?",(series,)) + if result: + list_missing_subtitles(no=added_series['sonarrSeriesId']) + else: + logging.debug('BAZARR unable to insert this series into the database:', + path_replace(added_series['path'])) logging.debug('BAZARR All series synced from Sonarr into database.') diff --git a/bazarr/get_subtitle.py b/bazarr/get_subtitle.py index 33cfdd983..4c7cadb91 100644 --- a/bazarr/get_subtitle.py +++ b/bazarr/get_subtitle.py @@ -181,6 +181,8 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce pre_download_hook=None, # fixme post_download_hook=None, # fixme language_hook=None) # fixme + for provider in providers: + track_event(category=provider, action='search', label=language[0]) else: downloaded_subtitles = None logging.info("BAZARR All providers are throttled") @@ -337,6 +339,8 @@ def manual_search(path, language, hi, forced, providers, providers_auth, sceneNa provider_configs=providers_auth, throttle_callback=provider_throttle, language_hook=None) # fixme + for provider in providers: + track_event(category=provider, action='search', label=language[0]) else: subtitles = [] logging.info("BAZARR All providers are throttled") @@ -561,9 +565,15 @@ def series_download_subtitles(no): episodes_details = database.execute("SELECT path, missing_subtitles, sonarrEpisodeId, scene_name " "FROM table_episodes WHERE sonarrSeriesId=? and missing_subtitles!='[]'" + episodes_details_clause, (no,)) + if not episodes_details: + logging.debug("BAZARR no episode for that sonarrSeriesId can be found in database:", str(no)) + return series_details = database.execute("SELECT hearing_impaired, title, forced FROM table_shows WHERE sonarrSeriesId=?", (no,), only_one=True) + if not series_details: + logging.debug("BAZARR no series with that sonarrSeriesId can be found in database:", str(no)) + return providers_list = get_providers() providers_auth = get_providers_auth() @@ -617,6 +627,10 @@ def episode_download_subtitles(no): "table_shows.forced FROM table_episodes LEFT JOIN table_shows on " "table_episodes.sonarrSeriesId = table_shows.sonarrSeriesId " "WHERE sonarrEpisodeId=?" + episodes_details_clause, (no,)) + + if not episodes_details: + logging.debug("BAZARR no episode with that sonarrEpisodeId can be found in database:", str(no)) + return providers_list = get_providers() providers_auth = get_providers_auth() @@ -661,6 +675,10 @@ def movies_download_subtitles(no): movie = database.execute("SELECT path, missing_subtitles, radarrId, sceneName, hearing_impaired, title, forced " "FROM table_movies WHERE radarrId=?" + movie_details_clause, (no,), only_one=True) + + if not movie: + logging.debug("BAZARR no movie with that radarrId can be found in database:", str(no)) + return providers_list = get_providers() providers_auth = get_providers_auth() @@ -999,14 +1017,15 @@ def upgrade_subtitles(): "table_history.score, table_shows.hearing_impaired, " "table_episodes.scene_name, table_episodes.title," "table_episodes.sonarrSeriesId, table_episodes.sonarrEpisodeId," - "MAX(table_history.timestamp), table_shows.languages, table_shows.forced " + "MAX(table_history.timestamp) as timestamp, " + "table_shows.languages, table_shows.forced " "FROM table_history INNER JOIN table_shows on " "table_shows.sonarrSeriesId = table_history.sonarrSeriesId INNER JOIN " "table_episodes on table_episodes.sonarrEpisodeId = " "table_history.sonarrEpisodeId WHERE action IN " "(" + ','.join(map(str, query_actions)) + ") AND timestamp > ? AND " "score is not null" + series_monitored_only_query_string + - "GROUP BY table_history.video_path, table_history.language", + " GROUP BY table_history.video_path, table_history.language", (minimum_timestamp,)) upgradable_episodes_not_perfect = [] @@ -1031,7 +1050,7 @@ def upgrade_subtitles(): upgradable_movies = database.execute("SELECT table_history_movie.video_path, table_history_movie.language, " "table_history_movie.score, table_movies.hearing_impaired, " "table_movies.sceneName, table_movies.title, table_movies.radarrId, " - "MAX(table_history_movie.timestamp), table_movies.languages, " + "MAX(table_history_movie.timestamp) as timestamp, table_movies.languages, " "table_movies.forced FROM table_history_movie INNER JOIN " "table_movies on table_movies.radarrId = table_history_movie.radarrId " "WHERE action IN (" + ','.join(map(str, query_actions)) + diff --git a/bazarr/list_subtitles.py b/bazarr/list_subtitles.py index 1d96eeb73..fc12d259b 100644 --- a/bazarr/list_subtitles.py +++ b/bazarr/list_subtitles.py @@ -78,7 +78,8 @@ def store_subtitles(original_path, reversed_path): logging.debug("BAZARR external subtitles detected: " + "pb:forced") actual_subtitles.append( [str("pb:forced"), path_replace_reverse(subtitle_path)]) - + elif not language: + continue elif str(language) != 'und': logging.debug("BAZARR external subtitles detected: " + str(language)) actual_subtitles.append( @@ -110,7 +111,7 @@ def store_subtitles(original_path, reversed_path): episode = database.execute("SELECT sonarrEpisodeId FROM table_episodes WHERE path=?", (original_path,), only_one=True) - if len(episode): + if episode: logging.debug("BAZARR storing those languages to DB: " + str(actual_subtitles)) list_missing_subtitles(epno=episode['sonarrEpisodeId']) else: @@ -170,6 +171,8 @@ def store_subtitles_movie(original_path, reversed_path): elif str(os.path.splitext(subtitle)[0]).lower().endswith(tuple(brazilian_portuguese_forced)): logging.debug("BAZARR external subtitles detected: " + "pb:forced") actual_subtitles.append([str("pb:forced"), path_replace_reverse_movie(subtitle_path)]) + elif not language: + continue elif str(language) != 'und': logging.debug("BAZARR external subtitles detected: " + str(language)) actual_subtitles.append([str(language), path_replace_reverse_movie(subtitle_path)]) @@ -197,11 +200,11 @@ def store_subtitles_movie(original_path, reversed_path): database.execute("UPDATE table_movies SET subtitles=? WHERE path=?", (str(actual_subtitles), original_path)) - movie = database.execute("SELECT radarrId FROM table_movies WHERE path=?", (original_path,)) + movie = database.execute("SELECT radarrId FROM table_movies WHERE path=?", (original_path,), only_one=True) - if len(movie): + if movie: logging.debug("BAZARR storing those languages to DB: " + str(actual_subtitles)) - list_missing_subtitles_movies(no=movie[0]['radarrId']) + list_missing_subtitles_movies(no=movie['radarrId']) else: logging.debug("BAZARR haven't been able to update existing subtitles to DB : " + str(actual_subtitles)) else: diff --git a/libs/sqlite3worker.py b/libs/sqlite3worker.py index ab7c1b8ce..fc589cc0b 100644 --- a/libs/sqlite3worker.py +++ b/libs/sqlite3worker.py @@ -93,7 +93,7 @@ class Sqlite3Worker(threading.Thread): for token, query, values, only_one in iter(self.sql_queue.get, None): LOGGER.debug("sql_queue: %s", self.sql_queue.qsize()) if token != self.exit_token: - LOGGER.debug("run: %s", query) + LOGGER.debug("run: %s, %s", query, values) self.run_query(token, query, values, only_one) execute_count += 1 # Let the executes build up a little before committing to disk @@ -188,9 +188,9 @@ class Sqlite3Worker(threading.Thread): If it's a select query it will return the results of the query. """ if self.exit_set: - LOGGER.debug("Exit set, not running: %s", query) + LOGGER.debug("Exit set, not running: %s, %s", query, values) return "Exit Called" - LOGGER.debug("execute: %s", query) + LOGGER.debug("execute: %s, %s", query, values) values = values or [] # A token to track this query with. token = str(uuid.uuid4()) diff --git a/libs/subliminal_patch/providers/subtitulamostv.py b/libs/subliminal_patch/providers/subtitulamostv.py new file mode 100644 index 000000000..4ad74a67b --- /dev/null +++ b/libs/subliminal_patch/providers/subtitulamostv.py @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- +import json +import logging +import os +import re +import io + +from babelfish import language_converters +from guessit import guessit +from requests import Session +from subzero.language import Language + +from subliminal import Movie, Episode, ProviderError, __short_version__ +from subliminal.exceptions import AuthenticationError, ConfigurationError, DownloadLimitExceeded, ProviderError +from subliminal_patch.subtitle import Subtitle, guess_matches +from subliminal.subtitle import fix_line_ending, SUBTITLE_EXTENSIONS +from subliminal_patch.providers import Provider + +logger = logging.getLogger(__name__) + +server_url = 'https://subtitulamos.tv/' + + +class SubtitulamosTVSubtitle(Subtitle): + provider_name = 'subtitulamostv' + hash_verifiable = False + + def __init__(self, language, page_link, download_link, description, title, matches, release_info): + super(SubtitulamosTVSubtitle, self).__init__(language, hearing_impaired=False, + page_link=page_link) + self.download_link = download_link + self.description = description.lower() + self.title = title + self.release_info = release_info + self.found_matches = matches + + @property + def id(self): + return self.download_link + + def get_matches(self, video): + matches = self.found_matches + + # release_group + if video.release_group and video.release_group.lower() in self.description: + matches.add('release_group') + + # resolution + if video.resolution and video.resolution.lower() in self.description: + matches.add('resolution') + + # format + if video.format: + formats = [video.format.lower()] + if formats[0] == "web-dl": + formats.append("webdl") + formats.append("webrip") + formats.append("web ") + for frmt in formats: + if frmt.lower() in self.description: + matches.add('format') + break + + # video_codec + if video.video_codec: + video_codecs = [video.video_codec.lower()] + if video_codecs[0] == "h264": + formats.append("x264") + elif video_codecs[0] == "h265": + formats.append("x265") + for vc in formats: + if vc.lower() in self.description: + matches.add('video_codec') + break + + return matches + + +class SubtitulamosTVProvider(Provider): + """Subtitulamostv Provider""" + languages = {Language.fromietf(l) for l in ['en','es']} + video_types = (Episode,) + + def initialize(self): + self.session = Session() + self.session.headers = { + 'User-Agent': os.environ.get("SZ_USER_AGENT", "Sub-Zero/2")} + + def terminate(self): + self.session.close() + + def query(self, languages, video): + # query the server + result = None + year = (" (%d)" % video.year) if video.year else "" + q = "%s%s %dx%02d" % (video.series, year, video.season, video.episode) + logger.debug('Searching subtitles "%s"', q) + + res = self.session.get( + server_url + 'search/query', params={'q':q}, timeout=10) + res.raise_for_status() + result = res.json() + + subtitles = [] + for s in [s for s in result if len(s['episodes'])]: + for e in s['episodes']: + res = self.session.get( + server_url + 'episodes/%d' % e['id'], timeout=10) + res.raise_for_status() + html = res.text + for lang_m in re.finditer(r"
(.*?)<\/div>.*?(?=
|
)", html, re.S): + lang = lang_m.group(1) + language = "es" + if "English" in lang: + language = "en" + logger.debug('Found subtitles in "%s" language.', language) + + for subt_m in re.finditer(r"
(.*?)
.*?(?:.*?
.*?(.*?)

)?", lang_m.group(0), re.S): + matches = set() + if video.alternative_series is None: + if video.series.lower() == s['name'].lower(): + matches.add('series') + elif s['name'].lower() in [video.series.lower()]+map(lambda name: name.lower(), video.alternative_series): + matches.add('series') + if video.season == e['season']: + matches.add('season') + if video.episode == e['number']: + matches.add('episode') + if video.title == e['name']: + matches.add('title') + #if video.year is None or ("(%d)" % video.year) in s['name']: + matches.add('year') + subtitles.append( + SubtitulamosTVSubtitle( + Language.fromietf(language), + server_url + 'episodes/%d' % e['id'], + server_url + subt_m.group(2), + subt_m.group(1)+(subt_m.group(3) if not subt_m.group(3) is None else ""), + e['name'], + matches, + '%s %dx%d,%s,%s' % (s['name'], e['season'], e['number'], subt_m.group(1), lang_m.group(1)), + ) + ) + + return subtitles + + def list_subtitles(self, video, languages): + return self.query(languages, video) + + def download_subtitle(self, subtitle): + logger.info('Downloading subtitle %r', subtitle) + r = self.session.get(subtitle.download_link, timeout=10) + r.raise_for_status() + + r.encoding = "ISO-8859-1" + subtitle_content = r.text + + if subtitle_content: + subtitle.content = fix_line_ending(subtitle_content) + else: + logger.debug('Could not download subtitle from %s', subtitle.download_link) diff --git a/views/historymovies.tpl b/views/historymovies.tpl index 27981f3c3..a10411988 100644 --- a/views/historymovies.tpl +++ b/views/historymovies.tpl @@ -89,7 +89,7 @@
- % upgradable_criteria = (row['timestamp'], row['video_path'], row['score']) + % upgradable_criteria = dict([('timestamp', row['timestamp']), ('video_path', row['video_path']), ('score', row['score'])]) % if upgradable_criteria in upgradable_movies: % if row['languages'] != "None": % desired_languages = ast.literal_eval(str(row['languages'])) diff --git a/views/historyseries.tpl b/views/historyseries.tpl index 4bbf05360..24a97f6d4 100644 --- a/views/historyseries.tpl +++ b/views/historyseries.tpl @@ -104,7 +104,7 @@
- % upgradable_criteria = (row['timestamp'], row['path'], row['score']) + % upgradable_criteria = dict([('timestamp', row['timestamp']), ('video_path', row['path']), ('score', row['score'])]) % if upgradable_criteria in upgradable_episodes: % if row['languages'] != "None": % desired_languages = ast.literal_eval(str(row['languages'])) diff --git a/views/providers.tpl b/views/providers.tpl index 89d929217..e8c694411 100644 --- a/views/providers.tpl +++ b/views/providers.tpl @@ -628,6 +628,28 @@
+
+
+ +
+
+
+ + +
+
+ +
+
+ +
+