From 241049495c0d1f4a8785e8cdfb657608aead4e84 Mon Sep 17 00:00:00 2001 From: morpheus65535 Date: Thu, 29 Apr 2021 21:57:51 -0400 Subject: [PATCH] Revert "Added caching of video files analyzing to prevent useless disk IO" This reverts commit 8e5dbcd8 --- bazarr/database.py | 2 - bazarr/embedded_subs_reader.py | 26 ++++------ bazarr/get_episodes.py | 65 ++++++++++++------------- bazarr/get_movies.py | 86 ++++++++++++++++++---------------- bazarr/list_subtitles.py | 4 +- bazarr/utils.py | 48 +------------------ 6 files changed, 87 insertions(+), 144 deletions(-) diff --git a/bazarr/database.py b/bazarr/database.py index f99b5ca2e..c98e37eab 100644 --- a/bazarr/database.py +++ b/bazarr/database.py @@ -108,7 +108,6 @@ def db_upgrade(): ['table_episodes', 'audio_codec', 'text'], ['table_episodes', 'episode_file_id', 'integer'], ['table_episodes', 'audio_language', 'text'], - ['table_episodes', 'file_ffprobe', 'text'], ['table_movies', 'sortTitle', 'text'], ['table_movies', 'year', 'text'], ['table_movies', 'alternativeTitles', 'text'], @@ -121,7 +120,6 @@ def db_upgrade(): ['table_movies', 'movie_file_id', 'integer'], ['table_movies', 'tags', 'text', '[]'], ['table_movies', 'profileId', 'integer'], - ['table_movies', 'file_ffprobe', 'text'], ['table_history', 'video_path', 'text'], ['table_history', 'language', 'text'], ['table_history', 'provider', 'text'], diff --git a/bazarr/embedded_subs_reader.py b/bazarr/embedded_subs_reader.py index 2193b2bed..fda402d4a 100644 --- a/bazarr/embedded_subs_reader.py +++ b/bazarr/embedded_subs_reader.py @@ -10,28 +10,21 @@ from knowit import api class EmbeddedSubsReader: def __init__(self): self.ffprobe = None - self.cache = None - self.data = None - - def list_languages(self, file, original_path): - from utils import get_binary, cache_get_ffprobe, cache_save_ffprobe - self.cache = cache_get_ffprobe(original_path) - if self.cache['ffprobe'] is not None: - logging.debug('returning cached results for: %s', original_path) - return self.cache['ffprobe'] - + + def list_languages(self, file): + from utils import get_binary self.ffprobe = get_binary("ffprobe") subtitles_list = [] if self.ffprobe: api.initialize({'provider': 'ffmpeg', 'ffmpeg': self.ffprobe}) - self.data = api.know(file) + data = api.know(file) traditional_chinese = ["cht", "tc", "traditional", "zht", "hant", "big5", u"繁", u"雙語"] brazilian_portuguese = ["pt-br", "pob", "pb", "brazilian", "brasil", "brazil"] - if 'subtitle' in self.data: - for detected_language in self.data['subtitle']: + if 'subtitle' in data: + for detected_language in data['subtitle']: if 'language' in detected_language: language = detected_language['language'].alpha3 if language == 'zho' and 'name' in detected_language: @@ -51,11 +44,11 @@ class EmbeddedSubsReader: if os.path.splitext(file)[1] == '.mkv': with open(file, 'rb') as f: try: - self.data = enzyme.MKV(f) + mkv = enzyme.MKV(f) except MalformedMKVError: logging.error('BAZARR cannot analyze this MKV with our built-in MKV parser, you should install ffmpeg: ' + file) else: - for subtitle_track in self.data.subtitle_tracks: + for subtitle_track in mkv.subtitle_tracks: hearing_impaired = False if subtitle_track.name: if 'sdh' in subtitle_track.name.lower(): @@ -63,9 +56,6 @@ class EmbeddedSubsReader: subtitles_list.append([subtitle_track.language, subtitle_track.forced, hearing_impaired, subtitle_track.codec_id]) - if subtitles_list is not None: - cache_save_ffprobe(original_path, self.cache['id'], self.cache['type'], subtitles_list) - return subtitles_list diff --git a/bazarr/get_episodes.py b/bazarr/get_episodes.py index c7efc2d47..0840715a1 100644 --- a/bazarr/get_episodes.py +++ b/bazarr/get_episodes.py @@ -24,7 +24,7 @@ def sync_episodes(): apikey_sonarr = settings.sonarr.apikey # Get current episodes id in DB - current_episodes_db = database.execute("SELECT sonarrEpisodeId, path, sonarrSeriesId, file_ffprobe FROM table_episodes") + current_episodes_db = database.execute("SELECT sonarrEpisodeId, path, sonarrSeriesId FROM table_episodes") current_episodes_db_list = [x['sonarrEpisodeId'] for x in current_episodes_db] @@ -32,14 +32,7 @@ def sync_episodes(): episodes_to_update = [] episodes_to_add = [] altered_episodes = [] - path_episodes = {} - - for item in current_episodes_db: - path_episodes[str(item['sonarrEpisodeId'])] = { - 'path': item['path'], - 'file_ffprobe': item['file_ffprobe'] - } - + # Get sonarrId for each series from database seriesIdList = database.execute("SELECT sonarrSeriesId, title FROM table_shows") @@ -109,33 +102,37 @@ def sync_episodes(): # Add episodes in sonarr to current episode list current_episodes_sonarr.append(episode['id']) - - info = { - 'sonarrSeriesId': episode['seriesId'], - 'sonarrEpisodeId': episode['id'], - 'title': episode['title'], - 'path': episode['episodeFile']['path'], - 'season': episode['seasonNumber'], - 'episode': episode['episodeNumber'], - 'scene_name': sceneName, - 'monitored': str(bool(episode['monitored'])), - 'format': format, - 'resolution': resolution, - 'video_codec': videoCodec, - 'audio_codec': audioCodec, - 'episode_file_id': episode['episodeFile']['id'], - 'audio_language': str(audio_language), - 'file_ffprobe': None, - } - + if episode['id'] in current_episodes_db_list: - if info['path'] == path_episodes[str(info['sonarrEpisodeId'])]['path'] and path_episodes[str(info['sonarrEpisodeId'])]['file_ffprobe'] is not None: - logging.debug('path is the same. Preserving ffprobe old results for: %s.', str(info['sonarrEpisodeId'])) - info['file_ffprobe'] = path_episodes[str(episode['id'])]['file_ffprobe'] - - episodes_to_update.append(info) + episodes_to_update.append({'sonarrSeriesId': episode['seriesId'], + 'sonarrEpisodeId': episode['id'], + 'title': episode['title'], + 'path': episode['episodeFile']['path'], + 'season': episode['seasonNumber'], + 'episode': episode['episodeNumber'], + 'scene_name': sceneName, + 'monitored': str(bool(episode['monitored'])), + 'format': format, + 'resolution': resolution, + 'video_codec': videoCodec, + 'audio_codec': audioCodec, + 'episode_file_id': episode['episodeFile']['id'], + 'audio_language': str(audio_language)}) else: - episodes_to_add.append(info) + episodes_to_add.append({'sonarrSeriesId': episode['seriesId'], + 'sonarrEpisodeId': episode['id'], + 'title': episode['title'], + 'path': episode['episodeFile']['path'], + 'season': episode['seasonNumber'], + 'episode': episode['episodeNumber'], + 'scene_name': sceneName, + 'monitored': str(bool(episode['monitored'])), + 'format': format, + 'resolution': resolution, + 'video_codec': videoCodec, + 'audio_codec': audioCodec, + 'episode_file_id': episode['episodeFile']['id'], + 'audio_language': str(audio_language)}) # Remove old episodes from DB removed_episodes = list(set(current_episodes_db_list) - set(current_episodes_sonarr)) diff --git a/bazarr/get_movies.py b/bazarr/get_movies.py index e9b329118..9ab7fd7d0 100644 --- a/bazarr/get_movies.py +++ b/bazarr/get_movies.py @@ -63,21 +63,14 @@ def update_movies(): return else: # Get current movies in DB - current_movies_db = database.execute("SELECT tmdbId, path, radarrId, file_ffprobe FROM table_movies") - + current_movies_db = database.execute("SELECT tmdbId, path, radarrId FROM table_movies") + current_movies_db_list = [x['tmdbId'] for x in current_movies_db] current_movies_radarr = [] movies_to_update = [] movies_to_add = [] altered_movies = [] - path_movies = {} - - for item in current_movies_db: - path_movies[str(item['tmdbId'])] = { - 'path': item['path'], - 'file_ffprobe': item['file_ffprobe'] - } moviesIdListLength = len(r.json()) for i, movie in enumerate(r.json(), 1): @@ -174,40 +167,51 @@ def update_movies(): # Add movies in radarr to current movies list current_movies_radarr.append(str(movie['tmdbId'])) - - info = { - 'radarrId': int(movie['id']), - 'title': movie['title'], - 'path': movie['path'] + separator + movie['movieFile']['relativePath'], - 'tmdbId': str(movie['tmdbId']), - 'poster': poster, - 'fanart': fanart, - 'audio_language': str(audio_language), - 'sceneName': sceneName, - 'monitored': str(bool(movie['monitored'])), - 'year': str(movie['year']), - 'sortTitle': movie['sortTitle'], - 'alternativeTitles': alternativeTitles, - 'format': format, - 'resolution': resolution, - 'video_codec': videoCodec, - 'audio_codec': audioCodec, - 'overview': overview, - 'imdbId': imdbId, - 'movie_file_id': int(movie['movieFile']['id']), - 'tags': str(tags), - 'file_ffprobe': None, - } - + if str(movie['tmdbId']) in current_movies_db_list: - if info['path'] == path_movies[str(info['tmdbId'])]['path'] and path_movies[str(info['tmdbId'])]['file_ffprobe'] is not None: - logging.debug('path is the same. Preserving ffprobe old results for: %s.', str(info['tmdbId'])) - info['file_ffprobe'] = path_movies[str(info['tmdbId'])]['file_ffprobe'] - - movies_to_update.append(info) + movies_to_update.append({'radarrId': int(movie["id"]), + 'title': movie["title"], + 'path': movie["path"] + separator + movie['movieFile']['relativePath'], + 'tmdbId': str(movie["tmdbId"]), + 'poster': poster, + 'fanart': fanart, + 'audio_language': str(audio_language), + 'sceneName': sceneName, + 'monitored': str(bool(movie['monitored'])), + 'year': str(movie['year']), + 'sortTitle': movie['sortTitle'], + 'alternativeTitles': alternativeTitles, + 'format': format, + 'resolution': resolution, + 'video_codec': videoCodec, + 'audio_codec': audioCodec, + 'overview': overview, + 'imdbId': imdbId, + 'movie_file_id': int(movie['movieFile']['id']), + 'tags': str(tags)}) else: - info['profileId'] = movie_default_profile - movies_to_add.append(info) + movies_to_add.append({'radarrId': int(movie["id"]), + 'title': movie["title"], + 'path': movie["path"] + separator + movie['movieFile']['relativePath'], + 'tmdbId': str(movie["tmdbId"]), + 'subtitles': '[]', + 'overview': overview, + 'poster': poster, + 'fanart': fanart, + 'audio_language': str(audio_language), + 'sceneName': sceneName, + 'monitored': str(bool(movie['monitored'])), + 'sortTitle': movie['sortTitle'], + 'year': str(movie['year']), + 'alternativeTitles': alternativeTitles, + 'format': format, + 'resolution': resolution, + 'video_codec': videoCodec, + 'audio_codec': audioCodec, + 'imdbId': imdbId, + 'movie_file_id': int(movie['movieFile']['id']), + 'tags': str(tags), + 'profileId': movie_default_profile}) else: logging.error( 'BAZARR Radarr returned a movie without a file path: ' + movie["path"] + separator + diff --git a/bazarr/list_subtitles.py b/bazarr/list_subtitles.py index 9e9f8d32d..ff0793173 100644 --- a/bazarr/list_subtitles.py +++ b/bazarr/list_subtitles.py @@ -31,7 +31,7 @@ def store_subtitles(original_path, reversed_path): if settings.general.getboolean('use_embedded_subs'): logging.debug("BAZARR is trying to index embedded subtitles.") try: - subtitle_languages = embedded_subs_reader.list_languages(reversed_path, original_path) + subtitle_languages = embedded_subs_reader.list_languages(reversed_path) for subtitle_language, subtitle_forced, subtitle_hi, subtitle_codec in subtitle_languages: try: if (settings.general.getboolean("ignore_pgs_subs") and subtitle_codec.lower() == "pgs") or \ @@ -145,7 +145,7 @@ def store_subtitles_movie(original_path, reversed_path): if settings.general.getboolean('use_embedded_subs'): logging.debug("BAZARR is trying to index embedded subtitles.") try: - subtitle_languages = embedded_subs_reader.list_languages(reversed_path, original_path) + subtitle_languages = embedded_subs_reader.list_languages(reversed_path) for subtitle_language, subtitle_forced, subtitle_hi, subtitle_codec in subtitle_languages: try: if (settings.general.getboolean("ignore_pgs_subs") and subtitle_codec.lower() == "pgs") or \ diff --git a/bazarr/utils.py b/bazarr/utils.py index 8518cf5c2..ede3cc2d1 100644 --- a/bazarr/utils.py +++ b/bazarr/utils.py @@ -399,53 +399,7 @@ def translate_subtitles_file(video_path, source_srt_file, to_lang, forced, hi): return dest_srt_file - def check_credentials(user, pw): username = settings.auth.username password = settings.auth.password - return hashlib.md5(pw.encode('utf-8')).hexdigest() == password and user == username - - -def cache_get_ffprobe(file): - record = {} - - item = database.execute("SELECT sonarrEpisodeId, file_ffprobe FROM table_episodes WHERE path = ?", - (file,), only_one=True) - - if item is not None: - record['id'] = item['sonarrEpisodeId'] - record['type'] = 'episode' - - if item['file_ffprobe']: - record['ffprobe'] = json.JSONDecoder().decode(item['file_ffprobe']) - else: - record['ffprobe'] = None - - return record - - item = database.execute("SELECT tmdbId, file_ffprobe FROM table_movies WHERE path = ?", - (file,), only_one=True) - if item is not None: - record['id'] = item['tmdbId'] - record['type'] = 'movie' - - if item['file_ffprobe']: - record['ffprobe'] = json.JSONDecoder().decode(item['file_ffprobe']) - else: - record['ffprobe'] = None - - return record - - return {'type': None, 'id': None, 'ffprobe': None} - - -def cache_save_ffprobe(file, record_id, record_type, ffprobe): - logging.debug('Saving ffprobe records [%s, %s, %s] data: %s', record_type, record_id, file, ffprobe) - - if record_type == 'movie': - database.execute("UPDATE table_movies SET file_ffprobe = ? WHERE path = ? AND tmdbId = ?", - (json.JSONEncoder().encode(ffprobe), file, record_id)) - - if record_type == 'episode': - database.execute("UPDATE table_episodes SET file_ffprobe = ? WHERE path = ? AND sonarrEpisodeId = ?", - (json.JSONEncoder().encode(ffprobe), file, record_id)) + return hashlib.md5(pw.encode('utf-8')).hexdigest() == password and user == username \ No newline at end of file