mirror/bazarr
1
0
Fork 0
mirror of https://github.com/morpheus65535/bazarr synced 2025-02-28 16:45:40 +00:00
Code Issues Releases Wiki Activity

Caching TMDB queries to database for 24 hours.

Browse source
This commit is contained in:
morpheus65535 2021-09-11 08:35:22 -04:00
parent f1389d559f
commit 7295094ed4
5 changed files with 75 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
bazarr/database.py
View file

@ -246,6 +246,17 @@ class TableCustomScoreProfileConditions(BaseModel):
table_name = 'table_custom_score_profile_conditions'
class TableTmdbCache(BaseModel):
id = AutoField()
timestamp = IntegerField(null=False)
function = BlobField(null=False)
arguments = BlobField(null=True)
result = BlobField(null=False)
class Meta:
table_name = 'table_tmdb_cache'
def init_db():
# Create tables if they don't exists.
database.create_tables([System,

24
bazarr/indexer/movies/local/movies_indexer.py
View file

@ -6,8 +6,13 @@ import logging
from indexer.tmdb_caching_proxy import tmdb
from database import TableMoviesRootfolder, TableMovies
from indexer.video_prop_reader import VIDEO_EXTENSION, video_prop_reader
from indexer.tmdb_caching_proxy import tmdb_func_cache
from list_subtitles import store_subtitles_movie
import subliminal
WordDelimiterRegex = re.compile(r"(\s|\.|,|_|-|=|\|)+")
PunctuationRegex = re.compile(r"[^\w\s]")
CommonWordRegex = re.compile(r"\b(a|an|the|and|or|of)\b\s?")
DuplicateSpacesRegex = re.compile(r"\s{2,}")
def list_movies_directories(root_dir):
@ -48,13 +53,11 @@ def get_movies_match(directory):
directory_original = re.sub(r"\(\b(19|20)\d{2}\b\)", '', directory_temp).rstrip()
directory = re.sub(r"\s\b(19|20)\d{2}\b", '', directory_original).rstrip()
search = tmdb.Search()
try:
movies_temp = search.movie(query=directory)
movies_temp = tmdb_func_cache(tmdb.Search().movie, query=directory)
except Exception as e:
logging.exception('BAZARR is facing issues indexing movies: {0}'.format(repr(e)))
else:
subliminal.region.backend.sync()
matching_movies = []
if movies_temp['total_results']:
for item in movies_temp['results']:
@ -94,14 +97,12 @@ def get_movies_metadata(tmdbid, root_dir_id, dir_name):
.get()
if tmdbid:
try:
tmdbMovies = tmdb.Movies(id=tmdbid)
movies_info = tmdbMovies.info()
alternative_titles = tmdbMovies.alternative_titles()
external_ids = tmdbMovies.external_ids()
movies_info = tmdb_func_cache(tmdb.Movies(tmdbid).info)
alternative_titles = tmdb_func_cache(tmdb.Movies(tmdbid).alternative_titles)
external_ids = tmdb_func_cache(tmdb.Movies(tmdbid).external_ids)
except Exception as e:
logging.exception('BAZARR is facing issues indexing movies: {0}'.format(repr(e)))
else:
subliminal.region.backend.sync()
images_url = 'https://image.tmdb.org/t/p/w500{0}'
movie_dir = os.path.join(root_dir_path['path'], dir_name)
movie_file = get_movie_file_from_list(movie_dir)
@ -124,11 +125,6 @@ def get_movies_metadata(tmdbid, root_dir_id, dir_name):
def normalize_title(title):
WordDelimiterRegex = re.compile(r"(\s|\.|,|_|-|=|\|)+")
PunctuationRegex = re.compile(r"[^\w\s]")
CommonWordRegex = re.compile(r"\b(a|an|the|and|or|of)\b\s?")
DuplicateSpacesRegex = re.compile(r"\s{2,}")
title = title.lower()
title = re.sub(WordDelimiterRegex, " ", title)

8
bazarr/indexer/series/local/episodes_indexer.py
View file

@ -8,8 +8,8 @@ from guessit import guessit
from requests.exceptions import HTTPError
from database import TableShowsRootfolder, TableShows, TableEpisodes
from indexer.video_prop_reader import VIDEO_EXTENSION, video_prop_reader
from indexer.tmdb_caching_proxy import tmdb_func_cache
from list_subtitles import store_subtitles
import subliminal
def get_series_episodes(series_directory):
@ -32,8 +32,8 @@ def get_episode_metadata(file, tmdbid, series_id):
else:
episode_number = guessed['episode'][0]
try:
tmdbEpisode = tmdb.TV_Episodes(tv_id=tmdbid, season_number=guessed['season'], episode_number=episode_number)
episode_info = tmdbEpisode.info()
episode_info = tmdb_func_cache(tmdb.TV_Episodes(tv_id=tmdbid, season_number=guessed['season'],
episode_number=episode_number).info)
except HTTPError:
logging.debug(f"BAZARR can't find this episode on TMDB: {file}")
episode_info['name'] = 'TBA'
@ -41,8 +41,6 @@ def get_episode_metadata(file, tmdbid, series_id):
logging.exception(f'BAZARR is facing issues indexing this episodes: {file}')
return False
else:
subliminal.region.backend.sync()
episode_metadata = {
'seriesId': series_id,
'title': episode_info['name'],

24
bazarr/indexer/series/local/series_indexer.py
View file

@ -5,7 +5,12 @@ import re
import logging
from indexer.tmdb_caching_proxy import tmdb
from database import TableShowsRootfolder, TableShows
import subliminal
from indexer.tmdb_caching_proxy import tmdb_func_cache
WordDelimiterRegex = re.compile(r"(\s|\.|,|_|-|=|\|)+")
PunctuationRegex = re.compile(r"[^\w\s]")
CommonWordRegex = re.compile(r"\b(a|an|the|and|or|of)\b\s?")
DuplicateSpacesRegex = re.compile(r"\s{2,}")
def list_series_directories(root_dir):
@ -46,13 +51,11 @@ def get_series_match(directory):
directory_original = re.sub(r"\(\b(19|20)\d{2}\b\)", '', directory_temp).rstrip()
directory = re.sub(r"\s\b(19|20)\d{2}\b", '', directory_original).rstrip()
search = tmdb.Search()
try:
series_temp = search.tv(query=directory)
series_temp = tmdb_func_cache(tmdb.Search().tv, query=directory)
except Exception as e:
logging.exception('BAZARR is facing issues indexing series: {0}'.format(repr(e)))
else:
subliminal.region.backend.sync()
matching_series = []
if series_temp['total_results']:
for item in series_temp['results']:
@ -77,14 +80,12 @@ def get_series_metadata(tmdbid, root_dir_id, dir_name):
.get()
if tmdbid:
try:
tmdbSeries = tmdb.TV(id=tmdbid)
series_info = tmdbSeries.info()
alternative_titles = tmdbSeries.alternative_titles()
external_ids = tmdbSeries.external_ids()
series_info = tmdb_func_cache(tmdb.TV(tmdbid).info)
alternative_titles = tmdb_func_cache(tmdb.TV(tmdbid).alternative_titles)
external_ids = tmdb_func_cache(tmdb.TV(tmdbid).external_ids)
except Exception as e:
logging.exception('BAZARR is facing issues indexing series: {0}'.format(repr(e)))
else:
subliminal.region.backend.sync()
images_url = 'https://image.tmdb.org/t/p/w500{0}'
series_metadata = {
@ -104,11 +105,6 @@ def get_series_metadata(tmdbid, root_dir_id, dir_name):
def normalize_title(title):
WordDelimiterRegex = re.compile(r"(\s|\.|,|_|-|=|\|)+")
PunctuationRegex = re.compile(r"[^\w\s]")
CommonWordRegex = re.compile(r"\b(a|an|the|and|or|of)\b\s?")
DuplicateSpacesRegex = re.compile(r"\s{2,}")
title = title.lower()
title = re.sub(WordDelimiterRegex, " ", title)

76
bazarr/indexer/tmdb_caching_proxy.py
View file

@ -1,45 +1,51 @@
# -*- coding: utf-8 -*-
import json
import datetime
import requests
import time
import pickle
import tmdbsimple as tmdb
import tmdbsimple.base
from subliminal.cache import region
from database import TableTmdbCache
CACHE_EXPIRATION_TIME = datetime.timedelta(days=1).total_seconds()
# Monkey patch to cache everything from TMDB
@region.cache_on_arguments(expiration_time=CACHE_EXPIRATION_TIME)
def _cached_request(self, method, path, params=None, payload=None):
url = self._get_complete_url(path)
params = self._get_params(params)
if self.session is None:
response = requests.request(
method,
url,
params=params,
data=json.dumps(payload) if payload else payload,
headers=self.headers,
)
else:
response = self.session.request(
method,
url,
params=params,
data=json.dumps(payload) if payload else payload,
headers=self.headers,
)
response.raise_for_status()
response.encoding = "utf-8"
return response.json()
tmdbsimple.base.TMDB._request = _cached_request
tmdbsimple.base.TMDB.session = None
tmdb.API_KEY = "e5577e69d409c601acb98d5bfcee31c7"
def tmdb_func_cache(func, *args, **kwargs):
try:
pickled_func = pickle.dumps(func, pickle.HIGHEST_PROTOCOL)
pickled_kwargs = pickle.dumps(kwargs, pickle.HIGHEST_PROTOCOL)
except:
return func(**kwargs)
else:
try:
cached_result = TableTmdbCache.select(TableTmdbCache.result) \
.where((TableTmdbCache.function == pickled_func) &
(TableTmdbCache.arguments == pickled_kwargs) &
(TableTmdbCache.timestamp > (time.time() - CACHE_EXPIRATION_TIME))) \
.dicts() \
.get()
except TableTmdbCache.DoesNotExist:
cached_result = None
if cached_result:
try:
pickled_result = pickle.loads(cached_result['result'])
except:
return renew_cache(func, pickled_func, pickled_kwargs, **kwargs)
else:
return pickled_result
else:
return renew_cache(func, pickled_func, pickled_kwargs, **kwargs)
def renew_cache(func, pickled_func, pickled_kwargs, **kwargs):
result = func(**kwargs)
TableTmdbCache.insert({
TableTmdbCache.timestamp: time.time(),
TableTmdbCache.function: pickled_func,
TableTmdbCache.arguments: pickled_kwargs,
TableTmdbCache.result: pickle.dumps(result, pickle.HIGHEST_PROTOCOL)
}).execute()
return result