From 7295094ed404e0f75341aa0d6581dbb7d6a031c0 Mon Sep 17 00:00:00 2001 From: morpheus65535 Date: Sat, 11 Sep 2021 08:35:22 -0400 Subject: [PATCH] Caching TMDB queries to database for 24 hours. --- bazarr/database.py | 11 +++ bazarr/indexer/movies/local/movies_indexer.py | 24 +++--- .../indexer/series/local/episodes_indexer.py | 8 +- bazarr/indexer/series/local/series_indexer.py | 24 +++--- bazarr/indexer/tmdb_caching_proxy.py | 76 ++++++++++--------- 5 files changed, 75 insertions(+), 68 deletions(-) diff --git a/bazarr/database.py b/bazarr/database.py index 2cf4bd0c1..3887b770d 100644 --- a/bazarr/database.py +++ b/bazarr/database.py @@ -246,6 +246,17 @@ class TableCustomScoreProfileConditions(BaseModel): table_name = 'table_custom_score_profile_conditions' +class TableTmdbCache(BaseModel): + id = AutoField() + timestamp = IntegerField(null=False) + function = BlobField(null=False) + arguments = BlobField(null=True) + result = BlobField(null=False) + + class Meta: + table_name = 'table_tmdb_cache' + + def init_db(): # Create tables if they don't exists. database.create_tables([System, diff --git a/bazarr/indexer/movies/local/movies_indexer.py b/bazarr/indexer/movies/local/movies_indexer.py index 854b56cbc..755b50c71 100644 --- a/bazarr/indexer/movies/local/movies_indexer.py +++ b/bazarr/indexer/movies/local/movies_indexer.py @@ -6,8 +6,13 @@ import logging from indexer.tmdb_caching_proxy import tmdb from database import TableMoviesRootfolder, TableMovies from indexer.video_prop_reader import VIDEO_EXTENSION, video_prop_reader +from indexer.tmdb_caching_proxy import tmdb_func_cache from list_subtitles import store_subtitles_movie -import subliminal + +WordDelimiterRegex = re.compile(r"(\s|\.|,|_|-|=|\|)+") +PunctuationRegex = re.compile(r"[^\w\s]") +CommonWordRegex = re.compile(r"\b(a|an|the|and|or|of)\b\s?") +DuplicateSpacesRegex = re.compile(r"\s{2,}") def list_movies_directories(root_dir): @@ -48,13 +53,11 @@ def get_movies_match(directory): directory_original = re.sub(r"\(\b(19|20)\d{2}\b\)", '', directory_temp).rstrip() directory = re.sub(r"\s\b(19|20)\d{2}\b", '', directory_original).rstrip() - search = tmdb.Search() try: - movies_temp = search.movie(query=directory) + movies_temp = tmdb_func_cache(tmdb.Search().movie, query=directory) except Exception as e: logging.exception('BAZARR is facing issues indexing movies: {0}'.format(repr(e))) else: - subliminal.region.backend.sync() matching_movies = [] if movies_temp['total_results']: for item in movies_temp['results']: @@ -94,14 +97,12 @@ def get_movies_metadata(tmdbid, root_dir_id, dir_name): .get() if tmdbid: try: - tmdbMovies = tmdb.Movies(id=tmdbid) - movies_info = tmdbMovies.info() - alternative_titles = tmdbMovies.alternative_titles() - external_ids = tmdbMovies.external_ids() + movies_info = tmdb_func_cache(tmdb.Movies(tmdbid).info) + alternative_titles = tmdb_func_cache(tmdb.Movies(tmdbid).alternative_titles) + external_ids = tmdb_func_cache(tmdb.Movies(tmdbid).external_ids) except Exception as e: logging.exception('BAZARR is facing issues indexing movies: {0}'.format(repr(e))) else: - subliminal.region.backend.sync() images_url = 'https://image.tmdb.org/t/p/w500{0}' movie_dir = os.path.join(root_dir_path['path'], dir_name) movie_file = get_movie_file_from_list(movie_dir) @@ -124,11 +125,6 @@ def get_movies_metadata(tmdbid, root_dir_id, dir_name): def normalize_title(title): - WordDelimiterRegex = re.compile(r"(\s|\.|,|_|-|=|\|)+") - PunctuationRegex = re.compile(r"[^\w\s]") - CommonWordRegex = re.compile(r"\b(a|an|the|and|or|of)\b\s?") - DuplicateSpacesRegex = re.compile(r"\s{2,}") - title = title.lower() title = re.sub(WordDelimiterRegex, " ", title) diff --git a/bazarr/indexer/series/local/episodes_indexer.py b/bazarr/indexer/series/local/episodes_indexer.py index d5adad944..2e42fa923 100644 --- a/bazarr/indexer/series/local/episodes_indexer.py +++ b/bazarr/indexer/series/local/episodes_indexer.py @@ -8,8 +8,8 @@ from guessit import guessit from requests.exceptions import HTTPError from database import TableShowsRootfolder, TableShows, TableEpisodes from indexer.video_prop_reader import VIDEO_EXTENSION, video_prop_reader +from indexer.tmdb_caching_proxy import tmdb_func_cache from list_subtitles import store_subtitles -import subliminal def get_series_episodes(series_directory): @@ -32,8 +32,8 @@ def get_episode_metadata(file, tmdbid, series_id): else: episode_number = guessed['episode'][0] try: - tmdbEpisode = tmdb.TV_Episodes(tv_id=tmdbid, season_number=guessed['season'], episode_number=episode_number) - episode_info = tmdbEpisode.info() + episode_info = tmdb_func_cache(tmdb.TV_Episodes(tv_id=tmdbid, season_number=guessed['season'], + episode_number=episode_number).info) except HTTPError: logging.debug(f"BAZARR can't find this episode on TMDB: {file}") episode_info['name'] = 'TBA' @@ -41,8 +41,6 @@ def get_episode_metadata(file, tmdbid, series_id): logging.exception(f'BAZARR is facing issues indexing this episodes: {file}') return False else: - subliminal.region.backend.sync() - episode_metadata = { 'seriesId': series_id, 'title': episode_info['name'], diff --git a/bazarr/indexer/series/local/series_indexer.py b/bazarr/indexer/series/local/series_indexer.py index eb5e8200c..a84e03e98 100644 --- a/bazarr/indexer/series/local/series_indexer.py +++ b/bazarr/indexer/series/local/series_indexer.py @@ -5,7 +5,12 @@ import re import logging from indexer.tmdb_caching_proxy import tmdb from database import TableShowsRootfolder, TableShows -import subliminal +from indexer.tmdb_caching_proxy import tmdb_func_cache + +WordDelimiterRegex = re.compile(r"(\s|\.|,|_|-|=|\|)+") +PunctuationRegex = re.compile(r"[^\w\s]") +CommonWordRegex = re.compile(r"\b(a|an|the|and|or|of)\b\s?") +DuplicateSpacesRegex = re.compile(r"\s{2,}") def list_series_directories(root_dir): @@ -46,13 +51,11 @@ def get_series_match(directory): directory_original = re.sub(r"\(\b(19|20)\d{2}\b\)", '', directory_temp).rstrip() directory = re.sub(r"\s\b(19|20)\d{2}\b", '', directory_original).rstrip() - search = tmdb.Search() try: - series_temp = search.tv(query=directory) + series_temp = tmdb_func_cache(tmdb.Search().tv, query=directory) except Exception as e: logging.exception('BAZARR is facing issues indexing series: {0}'.format(repr(e))) else: - subliminal.region.backend.sync() matching_series = [] if series_temp['total_results']: for item in series_temp['results']: @@ -77,14 +80,12 @@ def get_series_metadata(tmdbid, root_dir_id, dir_name): .get() if tmdbid: try: - tmdbSeries = tmdb.TV(id=tmdbid) - series_info = tmdbSeries.info() - alternative_titles = tmdbSeries.alternative_titles() - external_ids = tmdbSeries.external_ids() + series_info = tmdb_func_cache(tmdb.TV(tmdbid).info) + alternative_titles = tmdb_func_cache(tmdb.TV(tmdbid).alternative_titles) + external_ids = tmdb_func_cache(tmdb.TV(tmdbid).external_ids) except Exception as e: logging.exception('BAZARR is facing issues indexing series: {0}'.format(repr(e))) else: - subliminal.region.backend.sync() images_url = 'https://image.tmdb.org/t/p/w500{0}' series_metadata = { @@ -104,11 +105,6 @@ def get_series_metadata(tmdbid, root_dir_id, dir_name): def normalize_title(title): - WordDelimiterRegex = re.compile(r"(\s|\.|,|_|-|=|\|)+") - PunctuationRegex = re.compile(r"[^\w\s]") - CommonWordRegex = re.compile(r"\b(a|an|the|and|or|of)\b\s?") - DuplicateSpacesRegex = re.compile(r"\s{2,}") - title = title.lower() title = re.sub(WordDelimiterRegex, " ", title) diff --git a/bazarr/indexer/tmdb_caching_proxy.py b/bazarr/indexer/tmdb_caching_proxy.py index 5c6176cfc..d12cb0282 100644 --- a/bazarr/indexer/tmdb_caching_proxy.py +++ b/bazarr/indexer/tmdb_caching_proxy.py @@ -1,45 +1,51 @@ # -*- coding: utf-8 -*- -import json import datetime -import requests +import time +import pickle import tmdbsimple as tmdb -import tmdbsimple.base from subliminal.cache import region +from database import TableTmdbCache CACHE_EXPIRATION_TIME = datetime.timedelta(days=1).total_seconds() - -# Monkey patch to cache everything from TMDB -@region.cache_on_arguments(expiration_time=CACHE_EXPIRATION_TIME) -def _cached_request(self, method, path, params=None, payload=None): - url = self._get_complete_url(path) - params = self._get_params(params) - - if self.session is None: - response = requests.request( - method, - url, - params=params, - data=json.dumps(payload) if payload else payload, - headers=self.headers, - ) - - else: - response = self.session.request( - method, - url, - params=params, - data=json.dumps(payload) if payload else payload, - headers=self.headers, - ) - - response.raise_for_status() - response.encoding = "utf-8" - return response.json() - - -tmdbsimple.base.TMDB._request = _cached_request -tmdbsimple.base.TMDB.session = None tmdb.API_KEY = "e5577e69d409c601acb98d5bfcee31c7" + + +def tmdb_func_cache(func, *args, **kwargs): + try: + pickled_func = pickle.dumps(func, pickle.HIGHEST_PROTOCOL) + pickled_kwargs = pickle.dumps(kwargs, pickle.HIGHEST_PROTOCOL) + except: + return func(**kwargs) + else: + try: + cached_result = TableTmdbCache.select(TableTmdbCache.result) \ + .where((TableTmdbCache.function == pickled_func) & + (TableTmdbCache.arguments == pickled_kwargs) & + (TableTmdbCache.timestamp > (time.time() - CACHE_EXPIRATION_TIME))) \ + .dicts() \ + .get() + except TableTmdbCache.DoesNotExist: + cached_result = None + if cached_result: + try: + pickled_result = pickle.loads(cached_result['result']) + except: + return renew_cache(func, pickled_func, pickled_kwargs, **kwargs) + else: + return pickled_result + else: + return renew_cache(func, pickled_func, pickled_kwargs, **kwargs) + + +def renew_cache(func, pickled_func, pickled_kwargs, **kwargs): + result = func(**kwargs) + TableTmdbCache.insert({ + TableTmdbCache.timestamp: time.time(), + TableTmdbCache.function: pickled_func, + TableTmdbCache.arguments: pickled_kwargs, + TableTmdbCache.result: pickle.dumps(result, pickle.HIGHEST_PROTOCOL) + }).execute() + return result