mirror of https://github.com/morpheus65535/bazarr
Merge branch 'development' into python3
# Conflicts: # bazarr/get_series.py # bazarr/list_subtitles.py
This commit is contained in:
Louis Vézina
commit
7f80b1de83
|
|
@ -124,6 +124,12 @@ def sync_episodes():
|
|||
'audio_codec': audioCodec,
|
||||
'episode_file_id': episode['episodeFile']['id']})
|
||||
|
||||
# Remove old episodes from DB
|
||||
removed_episodes = list(set(current_episodes_db_list) - set(current_episodes_sonarr))
|
||||
|
||||
for removed_episode in removed_episodes:
|
||||
database.execute("DELETE FROM table_episodes WHERE sonarrEpisodeId=?", (removed_episode,))
|
||||
|
||||
# Update existing episodes in DB
|
||||
episode_in_db_list = []
|
||||
episodes_in_db = database.execute("SELECT sonarrSeriesId, sonarrEpisodeId, title, path, season, episode, "
|
||||
|
|
@ -146,17 +152,14 @@ def sync_episodes():
|
|||
# Insert new episodes in DB
|
||||
for added_episode in episodes_to_add:
|
||||
query = dict_converter.convert(added_episode)
|
||||
database.execute(
|
||||
result = database.execute(
|
||||
'''INSERT OR IGNORE INTO table_episodes(''' + query.keys_insert + ''') VALUES(''' + query.question_marks +
|
||||
''')''', query.values)
|
||||
altered_episodes.append([added_episode['sonarrEpisodeId'],
|
||||
added_episode['path']])
|
||||
|
||||
# Remove old episodes from DB
|
||||
removed_episodes = list(set(current_episodes_db_list) - set(current_episodes_sonarr))
|
||||
|
||||
for removed_episode in removed_episodes:
|
||||
database.execute("DELETE FROM table_episodes WHERE sonarrEpisodeId=?", (removed_episode,))
|
||||
if result:
|
||||
altered_episodes.append([added_episode['sonarrEpisodeId'], added_episode['path']])
|
||||
else:
|
||||
logging.debug('BAZARR unable to insert this episode into the database:',
|
||||
path_replace(added_episode['path']))
|
||||
|
||||
# Store subtitles for added or modified episodes
|
||||
for i, altered_episode in enumerate(altered_episodes, 1):
|
||||
|
|
|
|||
|
|
@ -206,8 +206,14 @@ def update_movies():
|
|||
logging.error(
|
||||
'BAZARR Radarr returned a movie without a file path: ' + movie["path"] + separator +
|
||||
movie['movieFile']['relativePath'])
|
||||
|
||||
# Update or insert movies in DB
|
||||
|
||||
# Remove old movies from DB
|
||||
removed_movies = list(set(current_movies_db_list) - set(current_movies_radarr))
|
||||
|
||||
for removed_movie in removed_movies:
|
||||
database.execute("DELETE FROM table_movies WHERE tmdbId=?", (removed_movie,))
|
||||
|
||||
# Update movies in DB
|
||||
movies_in_db_list = []
|
||||
movies_in_db = database.execute("SELECT radarrId, title, path, tmdbId, overview, poster, fanart, "
|
||||
"audio_language, sceneName, monitored, sortTitle, year, "
|
||||
|
|
@ -231,19 +237,17 @@ def update_movies():
|
|||
# Insert new movies in DB
|
||||
for added_movie in movies_to_add:
|
||||
query = dict_converter.convert(added_movie)
|
||||
database.execute(
|
||||
result = database.execute(
|
||||
'''INSERT OR IGNORE INTO table_movies(''' + query.keys_insert + ''') VALUES(''' +
|
||||
query.question_marks + ''')''', query.values)
|
||||
altered_movies.append([added_movie['tmdbId'],
|
||||
added_movie['path'],
|
||||
added_movie['radarrId'],
|
||||
added_movie['monitored']])
|
||||
|
||||
# Remove old movies from DB
|
||||
removed_movies = list(set(current_movies_db_list) - set(current_movies_radarr))
|
||||
|
||||
for removed_movie in removed_movies:
|
||||
database.execute("DELETE FROM table_movies WHERE tmdbId=?", (removed_movie,))
|
||||
if result:
|
||||
altered_movies.append([added_movie['tmdbId'],
|
||||
added_movie['path'],
|
||||
added_movie['radarrId'],
|
||||
added_movie['monitored']])
|
||||
else:
|
||||
logging.debug('BAZARR unable to insert this movie into the database:',
|
||||
path_replace_movie(added_movie['path']))
|
||||
|
||||
# Store subtitles for added or modified movies
|
||||
for i, altered_movie in enumerate(altered_movies, 1):
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from list_subtitles import list_missing_subtitles
|
|||
from database import database, dict_converter
|
||||
from utils import get_sonarr_version
|
||||
import six
|
||||
from helper import path_replace
|
||||
|
||||
|
||||
def update_series():
|
||||
|
|
@ -123,7 +124,13 @@ def update_series():
|
|||
'sortTitle': show['sortTitle'],
|
||||
'year': show['year'],
|
||||
'alternateTitles': alternateTitles})
|
||||
|
||||
|
||||
# Remove old series from DB
|
||||
removed_series = list(set(current_shows_db_list) - set(current_shows_sonarr))
|
||||
|
||||
for series in removed_series:
|
||||
database.execute("DELETE FROM table_shows WHERE tvdbId=?",(series,))
|
||||
|
||||
# Update existing series in DB
|
||||
series_in_db_list = []
|
||||
series_in_db = database.execute("SELECT title, path, tvdbId, sonarrSeriesId, overview, poster, fanart, "
|
||||
|
|
@ -142,16 +149,14 @@ def update_series():
|
|||
# Insert new series in DB
|
||||
for added_series in series_to_add:
|
||||
query = dict_converter.convert(added_series)
|
||||
database.execute(
|
||||
result = database.execute(
|
||||
'''INSERT OR IGNORE INTO table_shows(''' + query.keys_insert + ''') VALUES(''' +
|
||||
query.question_marks + ''')''', query.values)
|
||||
list_missing_subtitles(no=added_series['sonarrSeriesId'])
|
||||
|
||||
# Remove old series from DB
|
||||
removed_series = list(set(current_shows_db_list) - set(current_shows_sonarr))
|
||||
|
||||
for series in removed_series:
|
||||
database.execute("DELETE FROM table_shows WHERE tvdbId=?",(series,))
|
||||
if result:
|
||||
list_missing_subtitles(no=added_series['sonarrSeriesId'])
|
||||
else:
|
||||
logging.debug('BAZARR unable to insert this series into the database:',
|
||||
path_replace(added_series['path']))
|
||||
|
||||
logging.debug('BAZARR All series synced from Sonarr into database.')
|
||||
|
||||
|
|
|
|||
|
|
@ -181,6 +181,8 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce
|
|||
pre_download_hook=None, # fixme
|
||||
post_download_hook=None, # fixme
|
||||
language_hook=None) # fixme
|
||||
for provider in providers:
|
||||
track_event(category=provider, action='search', label=language[0])
|
||||
else:
|
||||
downloaded_subtitles = None
|
||||
logging.info("BAZARR All providers are throttled")
|
||||
|
|
@ -337,6 +339,8 @@ def manual_search(path, language, hi, forced, providers, providers_auth, sceneNa
|
|||
provider_configs=providers_auth,
|
||||
throttle_callback=provider_throttle,
|
||||
language_hook=None) # fixme
|
||||
for provider in providers:
|
||||
track_event(category=provider, action='search', label=language[0])
|
||||
else:
|
||||
subtitles = []
|
||||
logging.info("BAZARR All providers are throttled")
|
||||
|
|
@ -561,9 +565,15 @@ def series_download_subtitles(no):
|
|||
episodes_details = database.execute("SELECT path, missing_subtitles, sonarrEpisodeId, scene_name "
|
||||
"FROM table_episodes WHERE sonarrSeriesId=? and missing_subtitles!='[]'" +
|
||||
episodes_details_clause, (no,))
|
||||
if not episodes_details:
|
||||
logging.debug("BAZARR no episode for that sonarrSeriesId can be found in database:", str(no))
|
||||
return
|
||||
|
||||
series_details = database.execute("SELECT hearing_impaired, title, forced FROM table_shows WHERE sonarrSeriesId=?",
|
||||
(no,), only_one=True)
|
||||
if not series_details:
|
||||
logging.debug("BAZARR no series with that sonarrSeriesId can be found in database:", str(no))
|
||||
return
|
||||
|
||||
providers_list = get_providers()
|
||||
providers_auth = get_providers_auth()
|
||||
|
|
@ -617,6 +627,10 @@ def episode_download_subtitles(no):
|
|||
"table_shows.forced FROM table_episodes LEFT JOIN table_shows on "
|
||||
"table_episodes.sonarrSeriesId = table_shows.sonarrSeriesId "
|
||||
"WHERE sonarrEpisodeId=?" + episodes_details_clause, (no,))
|
||||
|
||||
if not episodes_details:
|
||||
logging.debug("BAZARR no episode with that sonarrEpisodeId can be found in database:", str(no))
|
||||
return
|
||||
|
||||
providers_list = get_providers()
|
||||
providers_auth = get_providers_auth()
|
||||
|
|
@ -661,6 +675,10 @@ def movies_download_subtitles(no):
|
|||
|
||||
movie = database.execute("SELECT path, missing_subtitles, radarrId, sceneName, hearing_impaired, title, forced "
|
||||
"FROM table_movies WHERE radarrId=?" + movie_details_clause, (no,), only_one=True)
|
||||
|
||||
if not movie:
|
||||
logging.debug("BAZARR no movie with that radarrId can be found in database:", str(no))
|
||||
return
|
||||
|
||||
providers_list = get_providers()
|
||||
providers_auth = get_providers_auth()
|
||||
|
|
@ -999,14 +1017,15 @@ def upgrade_subtitles():
|
|||
"table_history.score, table_shows.hearing_impaired, "
|
||||
"table_episodes.scene_name, table_episodes.title,"
|
||||
"table_episodes.sonarrSeriesId, table_episodes.sonarrEpisodeId,"
|
||||
"MAX(table_history.timestamp), table_shows.languages, table_shows.forced "
|
||||
"MAX(table_history.timestamp) as timestamp, "
|
||||
"table_shows.languages, table_shows.forced "
|
||||
"FROM table_history INNER JOIN table_shows on "
|
||||
"table_shows.sonarrSeriesId = table_history.sonarrSeriesId INNER JOIN "
|
||||
"table_episodes on table_episodes.sonarrEpisodeId = "
|
||||
"table_history.sonarrEpisodeId WHERE action IN "
|
||||
"(" + ','.join(map(str, query_actions)) + ") AND timestamp > ? AND "
|
||||
"score is not null" + series_monitored_only_query_string +
|
||||
"GROUP BY table_history.video_path, table_history.language",
|
||||
" GROUP BY table_history.video_path, table_history.language",
|
||||
(minimum_timestamp,))
|
||||
|
||||
upgradable_episodes_not_perfect = []
|
||||
|
|
@ -1031,7 +1050,7 @@ def upgrade_subtitles():
|
|||
upgradable_movies = database.execute("SELECT table_history_movie.video_path, table_history_movie.language, "
|
||||
"table_history_movie.score, table_movies.hearing_impaired, "
|
||||
"table_movies.sceneName, table_movies.title, table_movies.radarrId, "
|
||||
"MAX(table_history_movie.timestamp), table_movies.languages, "
|
||||
"MAX(table_history_movie.timestamp) as timestamp, table_movies.languages, "
|
||||
"table_movies.forced FROM table_history_movie INNER JOIN "
|
||||
"table_movies on table_movies.radarrId = table_history_movie.radarrId "
|
||||
"WHERE action IN (" + ','.join(map(str, query_actions)) +
|
||||
|
|
|
|||
|
|
@ -78,7 +78,8 @@ def store_subtitles(original_path, reversed_path):
|
|||
logging.debug("BAZARR external subtitles detected: " + "pb:forced")
|
||||
actual_subtitles.append(
|
||||
[str("pb:forced"), path_replace_reverse(subtitle_path)])
|
||||
|
||||
elif not language:
|
||||
continue
|
||||
elif str(language) != 'und':
|
||||
logging.debug("BAZARR external subtitles detected: " + str(language))
|
||||
actual_subtitles.append(
|
||||
|
|
@ -110,7 +111,7 @@ def store_subtitles(original_path, reversed_path):
|
|||
episode = database.execute("SELECT sonarrEpisodeId FROM table_episodes WHERE path=?",
|
||||
(original_path,), only_one=True)
|
||||
|
||||
if len(episode):
|
||||
if episode:
|
||||
logging.debug("BAZARR storing those languages to DB: " + str(actual_subtitles))
|
||||
list_missing_subtitles(epno=episode['sonarrEpisodeId'])
|
||||
else:
|
||||
|
|
@ -170,6 +171,8 @@ def store_subtitles_movie(original_path, reversed_path):
|
|||
elif str(os.path.splitext(subtitle)[0]).lower().endswith(tuple(brazilian_portuguese_forced)):
|
||||
logging.debug("BAZARR external subtitles detected: " + "pb:forced")
|
||||
actual_subtitles.append([str("pb:forced"), path_replace_reverse_movie(subtitle_path)])
|
||||
elif not language:
|
||||
continue
|
||||
elif str(language) != 'und':
|
||||
logging.debug("BAZARR external subtitles detected: " + str(language))
|
||||
actual_subtitles.append([str(language), path_replace_reverse_movie(subtitle_path)])
|
||||
|
|
@ -197,11 +200,11 @@ def store_subtitles_movie(original_path, reversed_path):
|
|||
|
||||
database.execute("UPDATE table_movies SET subtitles=? WHERE path=?",
|
||||
(str(actual_subtitles), original_path))
|
||||
movie = database.execute("SELECT radarrId FROM table_movies WHERE path=?", (original_path,))
|
||||
movie = database.execute("SELECT radarrId FROM table_movies WHERE path=?", (original_path,), only_one=True)
|
||||
|
||||
if len(movie):
|
||||
if movie:
|
||||
logging.debug("BAZARR storing those languages to DB: " + str(actual_subtitles))
|
||||
list_missing_subtitles_movies(no=movie[0]['radarrId'])
|
||||
list_missing_subtitles_movies(no=movie['radarrId'])
|
||||
else:
|
||||
logging.debug("BAZARR haven't been able to update existing subtitles to DB : " + str(actual_subtitles))
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@ class Sqlite3Worker(threading.Thread):
|
|||
for token, query, values, only_one in iter(self.sql_queue.get, None):
|
||||
LOGGER.debug("sql_queue: %s", self.sql_queue.qsize())
|
||||
if token != self.exit_token:
|
||||
LOGGER.debug("run: %s", query)
|
||||
LOGGER.debug("run: %s, %s", query, values)
|
||||
self.run_query(token, query, values, only_one)
|
||||
execute_count += 1
|
||||
# Let the executes build up a little before committing to disk
|
||||
|
|
@ -188,9 +188,9 @@ class Sqlite3Worker(threading.Thread):
|
|||
If it's a select query it will return the results of the query.
|
||||
"""
|
||||
if self.exit_set:
|
||||
LOGGER.debug("Exit set, not running: %s", query)
|
||||
LOGGER.debug("Exit set, not running: %s, %s", query, values)
|
||||
return "Exit Called"
|
||||
LOGGER.debug("execute: %s", query)
|
||||
LOGGER.debug("execute: %s, %s", query, values)
|
||||
values = values or []
|
||||
# A token to track this query with.
|
||||
token = str(uuid.uuid4())
|
||||
|
|
|
|||
|
|
@ -0,0 +1,161 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import io
|
||||
|
||||
from babelfish import language_converters
|
||||
from guessit import guessit
|
||||
from requests import Session
|
||||
from subzero.language import Language
|
||||
|
||||
from subliminal import Movie, Episode, ProviderError, __short_version__
|
||||
from subliminal.exceptions import AuthenticationError, ConfigurationError, DownloadLimitExceeded, ProviderError
|
||||
from subliminal_patch.subtitle import Subtitle, guess_matches
|
||||
from subliminal.subtitle import fix_line_ending, SUBTITLE_EXTENSIONS
|
||||
from subliminal_patch.providers import Provider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
server_url = 'https://subtitulamos.tv/'
|
||||
|
||||
|
||||
class SubtitulamosTVSubtitle(Subtitle):
|
||||
provider_name = 'subtitulamostv'
|
||||
hash_verifiable = False
|
||||
|
||||
def __init__(self, language, page_link, download_link, description, title, matches, release_info):
|
||||
super(SubtitulamosTVSubtitle, self).__init__(language, hearing_impaired=False,
|
||||
page_link=page_link)
|
||||
self.download_link = download_link
|
||||
self.description = description.lower()
|
||||
self.title = title
|
||||
self.release_info = release_info
|
||||
self.found_matches = matches
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
return self.download_link
|
||||
|
||||
def get_matches(self, video):
|
||||
matches = self.found_matches
|
||||
|
||||
# release_group
|
||||
if video.release_group and video.release_group.lower() in self.description:
|
||||
matches.add('release_group')
|
||||
|
||||
# resolution
|
||||
if video.resolution and video.resolution.lower() in self.description:
|
||||
matches.add('resolution')
|
||||
|
||||
# format
|
||||
if video.format:
|
||||
formats = [video.format.lower()]
|
||||
if formats[0] == "web-dl":
|
||||
formats.append("webdl")
|
||||
formats.append("webrip")
|
||||
formats.append("web ")
|
||||
for frmt in formats:
|
||||
if frmt.lower() in self.description:
|
||||
matches.add('format')
|
||||
break
|
||||
|
||||
# video_codec
|
||||
if video.video_codec:
|
||||
video_codecs = [video.video_codec.lower()]
|
||||
if video_codecs[0] == "h264":
|
||||
formats.append("x264")
|
||||
elif video_codecs[0] == "h265":
|
||||
formats.append("x265")
|
||||
for vc in formats:
|
||||
if vc.lower() in self.description:
|
||||
matches.add('video_codec')
|
||||
break
|
||||
|
||||
return matches
|
||||
|
||||
|
||||
class SubtitulamosTVProvider(Provider):
|
||||
"""Subtitulamostv Provider"""
|
||||
languages = {Language.fromietf(l) for l in ['en','es']}
|
||||
video_types = (Episode,)
|
||||
|
||||
def initialize(self):
|
||||
self.session = Session()
|
||||
self.session.headers = {
|
||||
'User-Agent': os.environ.get("SZ_USER_AGENT", "Sub-Zero/2")}
|
||||
|
||||
def terminate(self):
|
||||
self.session.close()
|
||||
|
||||
def query(self, languages, video):
|
||||
# query the server
|
||||
result = None
|
||||
year = (" (%d)" % video.year) if video.year else ""
|
||||
q = "%s%s %dx%02d" % (video.series, year, video.season, video.episode)
|
||||
logger.debug('Searching subtitles "%s"', q)
|
||||
|
||||
res = self.session.get(
|
||||
server_url + 'search/query', params={'q':q}, timeout=10)
|
||||
res.raise_for_status()
|
||||
result = res.json()
|
||||
|
||||
subtitles = []
|
||||
for s in [s for s in result if len(s['episodes'])]:
|
||||
for e in s['episodes']:
|
||||
res = self.session.get(
|
||||
server_url + 'episodes/%d' % e['id'], timeout=10)
|
||||
res.raise_for_status()
|
||||
html = res.text
|
||||
for lang_m in re.finditer(r"<div class=\"subtitle_language\">(.*?)<\/div>.*?(?=<div class=\"subtitle_language\">|<div id=\"subtitle-actions\">)", html, re.S):
|
||||
lang = lang_m.group(1)
|
||||
language = "es"
|
||||
if "English" in lang:
|
||||
language = "en"
|
||||
logger.debug('Found subtitles in "%s" language.', language)
|
||||
|
||||
for subt_m in re.finditer(r"<div class=\"version_name\">(.*?)</div>.*?<a href=\"/(subtitles/\d+/download)\" rel=\"nofollow\">(?:.*?<div class=\"version_comments ?\">.*?</i>(.*?)</p>)?", lang_m.group(0), re.S):
|
||||
matches = set()
|
||||
if video.alternative_series is None:
|
||||
if video.series.lower() == s['name'].lower():
|
||||
matches.add('series')
|
||||
elif s['name'].lower() in [video.series.lower()]+map(lambda name: name.lower(), video.alternative_series):
|
||||
matches.add('series')
|
||||
if video.season == e['season']:
|
||||
matches.add('season')
|
||||
if video.episode == e['number']:
|
||||
matches.add('episode')
|
||||
if video.title == e['name']:
|
||||
matches.add('title')
|
||||
#if video.year is None or ("(%d)" % video.year) in s['name']:
|
||||
matches.add('year')
|
||||
subtitles.append(
|
||||
SubtitulamosTVSubtitle(
|
||||
Language.fromietf(language),
|
||||
server_url + 'episodes/%d' % e['id'],
|
||||
server_url + subt_m.group(2),
|
||||
subt_m.group(1)+(subt_m.group(3) if not subt_m.group(3) is None else ""),
|
||||
e['name'],
|
||||
matches,
|
||||
'%s %dx%d,%s,%s' % (s['name'], e['season'], e['number'], subt_m.group(1), lang_m.group(1)),
|
||||
)
|
||||
)
|
||||
|
||||
return subtitles
|
||||
|
||||
def list_subtitles(self, video, languages):
|
||||
return self.query(languages, video)
|
||||
|
||||
def download_subtitle(self, subtitle):
|
||||
logger.info('Downloading subtitle %r', subtitle)
|
||||
r = self.session.get(subtitle.download_link, timeout=10)
|
||||
r.raise_for_status()
|
||||
|
||||
r.encoding = "ISO-8859-1"
|
||||
subtitle_content = r.text
|
||||
|
||||
if subtitle_content:
|
||||
subtitle.content = fix_line_ending(subtitle_content)
|
||||
else:
|
||||
logger.debug('Could not download subtitle from %s', subtitle.download_link)
|
||||
|
|
@ -89,7 +89,7 @@
|
|||
</div>
|
||||
</td>
|
||||
<td>
|
||||
% upgradable_criteria = (row['timestamp'], row['video_path'], row['score'])
|
||||
% upgradable_criteria = dict([('timestamp', row['timestamp']), ('video_path', row['video_path']), ('score', row['score'])])
|
||||
% if upgradable_criteria in upgradable_movies:
|
||||
% if row['languages'] != "None":
|
||||
% desired_languages = ast.literal_eval(str(row['languages']))
|
||||
|
|
|
|||
|
|
@ -104,7 +104,7 @@
|
|||
</div>
|
||||
</td>
|
||||
<td>
|
||||
% upgradable_criteria = (row['timestamp'], row['path'], row['score'])
|
||||
% upgradable_criteria = dict([('timestamp', row['timestamp']), ('video_path', row['path']), ('score', row['score'])])
|
||||
% if upgradable_criteria in upgradable_episodes:
|
||||
% if row['languages'] != "None":
|
||||
% desired_languages = ast.literal_eval(str(row['languages']))
|
||||
|
|
|
|||
|
|
@ -628,6 +628,28 @@
|
|||
|
||||
</div>
|
||||
|
||||
<div class="middle aligned row">
|
||||
<div class="right aligned four wide column">
|
||||
<label>Subtitulamos.tv</label>
|
||||
</div>
|
||||
<div class="one wide column">
|
||||
<div id="subtitulamostv" class="ui toggle checkbox provider">
|
||||
<input type="checkbox">
|
||||
<label></label>
|
||||
</div>
|
||||
</div>
|
||||
<div class="collapsed column">
|
||||
<div class="collapsed center aligned column">
|
||||
<div class="ui basic icon" data-tooltip="Spanish subtitles provider." data-inverted="">
|
||||
<i class="help circle large icon"></i>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div id="subtitulamostv_option" class="ui grid container">
|
||||
|
||||
</div>
|
||||
|
||||
<div class="middle aligned row">
|
||||
<div class="right aligned four wide column">
|
||||
<label>Supersubtitles</label>
|
||||
|
|
|
|||
Loading…
Reference in New Issue