From b2871be90c4b00c571db902a8441dc42b3c7e529 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Mon, 28 Sep 2020 08:22:00 -0400 Subject: [PATCH 1/3] Implemented imdbId for series. --- bazarr/database.py | 1 + bazarr/get_series.py | 10 +++++++--- bazarr/get_subtitle.py | 4 +++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/bazarr/database.py b/bazarr/database.py index 005d96345..9f5b3a2ef 100644 --- a/bazarr/database.py +++ b/bazarr/database.py @@ -94,6 +94,7 @@ def db_upgrade(): ['table_shows', 'forced', 'text', 'False'], ['table_shows', 'tags', 'text', '[]'], ['table_shows', 'seriesType', 'text', ''], + ['table_shows', 'imdbId', 'text', ''], ['table_episodes', 'format', 'text'], ['table_episodes', 'resolution', 'text'], ['table_episodes', 'video_codec', 'text'], diff --git a/bazarr/get_series.py b/bazarr/get_series.py index fb263ff8f..64c7987e2 100644 --- a/bazarr/get_series.py +++ b/bazarr/get_series.py @@ -83,6 +83,8 @@ def update_series(): tags = [d['label'] for d in tagsDict if d['id'] in show['tags']] + imdbId = show['imdbId'] if 'imdbId' in show else None + # Add shows in Sonarr to current shows list current_shows_sonarr.append(show['id']) @@ -99,7 +101,8 @@ def update_series(): 'year': str(show['year']), 'alternateTitles': alternate_titles, 'tags': str(tags), - 'seriesType': show['seriesType']}) + 'seriesType': show['seriesType'], + 'imdbId': imdbId}) else: series_to_add.append({'title': show["title"], 'path': show["path"], @@ -116,7 +119,8 @@ def update_series(): 'alternateTitles': alternate_titles, 'forced': serie_default_forced, 'tags': str(tags), - 'seriesType': show['seriesType']}) + 'seriesType': show['seriesType'], + 'imdbId': imdbId}) # Remove old series from DB removed_series = list(set(current_shows_db_list) - set(current_shows_sonarr)) @@ -128,7 +132,7 @@ def update_series(): # Update existing series in DB series_in_db_list = [] series_in_db = database.execute("SELECT title, path, tvdbId, sonarrSeriesId, overview, poster, fanart, " - "audio_language, sortTitle, year, alternateTitles, tags, seriesType FROM table_shows") + "audio_language, sortTitle, year, alternateTitles, tags, seriesType, imdbId FROM table_shows") for item in series_in_db: series_in_db_list.append(item) diff --git a/bazarr/get_subtitle.py b/bazarr/get_subtitle.py index 93f52515a..f11f2a736 100644 --- a/bazarr/get_subtitle.py +++ b/bazarr/get_subtitle.py @@ -1027,7 +1027,7 @@ def refine_from_db(path, video): "SELECT table_shows.title as seriesTitle, table_episodes.season, table_episodes.episode, " "table_episodes.title as episodeTitle, table_shows.year, table_shows.tvdbId, " "table_shows.alternateTitles, table_episodes.format, table_episodes.resolution, " - "table_episodes.video_codec, table_episodes.audio_codec, table_episodes.path " + "table_episodes.video_codec, table_episodes.audio_codec, table_episodes.path, table_shows.imdbId " "FROM table_episodes INNER JOIN table_shows on " "table_shows.sonarrSeriesId = table_episodes.sonarrSeriesId " "WHERE table_episodes.path = ?", (path_mappings.path_replace_reverse(path),), only_one=True) @@ -1042,6 +1042,8 @@ def refine_from_db(path, video): # if int(data['year']) > 0: video.year = int(data['year']) video.series_tvdb_id = int(data['tvdbId']) video.alternative_series = ast.literal_eval(data['alternateTitles']) + if data['imdbId'] and not video.series_imdb_id: + video.series_imdb_id = data['imdbId'] if not video.source: video.source = str(data['format']) if not video.resolution: From 81d42dd6dcc41f5664746cf11b92f73e4aca4df8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Tue, 29 Sep 2020 10:35:00 -0400 Subject: [PATCH 2/3] WIP --- bazarr/get_subtitle.py | 3 +- libs/subliminal_patch/providers/subztv.py | 153 ++++++++++++++++++++++ views/settingsproviders.html | 12 ++ 3 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 libs/subliminal_patch/providers/subztv.py diff --git a/bazarr/get_subtitle.py b/bazarr/get_subtitle.py index f11f2a736..61616ecd2 100644 --- a/bazarr/get_subtitle.py +++ b/bazarr/get_subtitle.py @@ -1062,7 +1062,8 @@ def refine_from_db(path, video): # Commented out because Radarr provided so much bad year # if data['year']: # if int(data['year']) > 0: video.year = int(data['year']) - if data['imdbId']: video.imdb_id = data['imdbId'] + if data['imdbId'] and not video.imdb_id: + video.imdb_id = data['imdbId'] video.alternative_titles = ast.literal_eval(data['alternativeTitles']) if not video.source: if data['format']: video.source = data['format'] diff --git a/libs/subliminal_patch/providers/subztv.py b/libs/subliminal_patch/providers/subztv.py new file mode 100644 index 000000000..e5c7efdbf --- /dev/null +++ b/libs/subliminal_patch/providers/subztv.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- +import io +import logging +import os +import re + +from subzero.language import Language +from guessit import guessit +from requests import Session + +from subliminal.providers import ParserBeautifulSoup, Provider +from subliminal.subtitle import SUBTITLE_EXTENSIONS, Subtitle, fix_line_ending, guess_matches +from subliminal.video import Episode, Movie +from subliminal.exceptions import ServiceUnavailable + +from requests.exceptions import RequestException + +logger = logging.getLogger(__name__) + + +class SubztvSubtitle(Subtitle): + """Subztv Subtitle.""" + provider_name = 'subztv' + + def __init__(self, language, page_link, version, uploader, referer): + super(SubztvSubtitle, self).__init__(language, page_link=page_link) + self.version = version + self.release_info = version + self.page_link = page_link + self.uploader = uploader + self.referer = referer + self.hearing_impaired = None + + @property + def id(self): + return self.page_link + + def get_matches(self, video): + matches = set() + + # episode + if isinstance(video, Episode): + # other properties + matches |= guess_matches(video, guessit(self.version, {'type': 'episode'}), partial=True) + # movie + elif isinstance(video, Movie): + # other properties + matches |= guess_matches(video, guessit(self.version, {'type': 'movie'}), partial=True) + + return matches + + +class SubztvProvider(Provider): + """Subztv Provider.""" + languages = {Language('ell')} + server_url = 'https://subztv.online/' + subtitle_class = SubztvSubtitle + + def __init__(self): + self.session = None + + def initialize(self): + self.session = Session() + self.session.headers['User-Agent'] = os.environ.get("SZ_USER_AGENT", "Sub-Zero/2") + + def terminate(self): + self.session.close() + + def query(self, video, imdb_id, season=None, episode=None): + logger.debug('Searching subtitles for %r', imdb_id) + subtitles = [] + search_link = self.server_url + 'en/view/' + imdb_id + + r = self.session.get(search_link, timeout=30) + r.raise_for_status() + + if isinstance(video, Episode): + soup_page = ParserBeautifulSoup(r.content.decode('utf-8', 'ignore'), ['html.parser']) + for item in soup_page.select('div.col-lg-offset-2.col-md-8.text-center.top30.bottom10 > a'): + season_episode = re.search(r'Season (\d+) Episode (\d+)', item.text) + season_number = int(season_episode.group(1)) + episode_number = int(season_episode.group(2)) + if season_number == season and episode_number == episode: + episode_page = item.attrs['href'] + r = self.session.get(episode_page, timeout=30) + soup_subs = ParserBeautifulSoup(r.content.decode('utf-8', 'ignore'), ['html.parser']) + secCode = soup_subs.find('input', {'id': 'secCode'}).get('value') + for subtitles in soup_subs.select('#elSub > tbody > tr'): + subtitle_id = re.search(r'downloadMe\(\'(.*)\'\)', subtitles.contents[2].contents[2].contents[0].attrs['onclick']).group(1) + page_link = self.server_url + 'dll/' + subtitle_id + '/0/' + secCode + language = Language.fromalpha2(subtitles.parent.find('img')['alt']) + version = subtitles.contents[2].contents[4].text.strip() + uploader = subtitles.contents[2].contents[5].contents[0].contents[1].text.strip() + referer = episode_page + + subtitle = self.subtitle_class(language, page_link, version, uploader, referer) + + logger.debug('Found subtitle %r', subtitle) + subtitles.append(subtitle) + else: + pass + elif isinstance(video, Movie): + pass + + return subtitles + + def list_subtitles(self, video, languages): + imdbId = None + subtitles = [] + + if isinstance(video, Episode): + imdbId = video.series_imdb_id + elif isinstance(video, Movie): + imdbId = video.imdb_id + + if not imdbId: + logger.debug('No imdb number available to search with provider') + return subtitles + + # query for subtitles with the imdbId + response = None + status_code = None + try: + if isinstance(video, Episode): + response = self.query(video, imdbId, season=video.season, episode=video.episode) + elif isinstance(video, Movie): + response = self.query(video, imdbId) + for s in response: + if s.language in languages: + subtitles += s + except RequestException as e: + status_code = e.response.status_code + else: + status_code = int(response['status'][:3]) + + if status_code == 503: + raise ServiceUnavailable(str(status_code)) + + return subtitles + + def download_subtitle(self, subtitle): + if isinstance(subtitle, SubztvSubtitle): + # download the subtitle + logger.info('Downloading subtitle %r', subtitle) + r = self.session.get(subtitle.download_link, headers={'Referer': subtitle.referer}, + timeout=30) + r.raise_for_status() + + if not r.content: + logger.debug('Unable to download subtitle. No data returned from provider') + return + + subtitle.content = fix_line_ending(r.content) diff --git a/views/settingsproviders.html b/views/settingsproviders.html index e45658bab..b85a660ee 100644 --- a/views/settingsproviders.html +++ b/views/settingsproviders.html @@ -521,6 +521,18 @@ +
+
+ Subztv.online +
+
+ +
+
+
Supersubtitles From f74d70cfe15442e6af30483d264200c61f343242 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Tue, 13 Oct 2020 13:35:41 -0400 Subject: [PATCH 3/3] WIP --- libs/subliminal_patch/providers/greeksubs.py | 218 +++++++++++++++++++ libs/subliminal_patch/providers/subztv.py | 153 ------------- views/settingsproviders.html | 24 +- 3 files changed, 230 insertions(+), 165 deletions(-) create mode 100644 libs/subliminal_patch/providers/greeksubs.py delete mode 100644 libs/subliminal_patch/providers/subztv.py diff --git a/libs/subliminal_patch/providers/greeksubs.py b/libs/subliminal_patch/providers/greeksubs.py new file mode 100644 index 000000000..009df1ce6 --- /dev/null +++ b/libs/subliminal_patch/providers/greeksubs.py @@ -0,0 +1,218 @@ +# -*- coding: utf-8 -*- +import logging +import re +from random import randint + +from subzero.language import Language +from guessit import guessit +from subliminal_patch.http import RetryingCFSession +from .utils import FIRST_THOUSAND_OR_SO_USER_AGENTS as AGENT_LIST + +from subliminal.providers import ParserBeautifulSoup, Provider +from subliminal.subtitle import SUBTITLE_EXTENSIONS, Subtitle, fix_line_ending, guess_matches +from subliminal.video import Episode, Movie + +logger = logging.getLogger(__name__) + + +class GreekSubsSubtitle(Subtitle): + """GreekSubs Subtitle.""" + provider_name = 'greeksubs' + hearing_impaired_verifiable = False + + def __init__(self, language, page_link, version, uploader, referer): + super(GreekSubsSubtitle, self).__init__(language, page_link=page_link) + self.version = version.replace('-', '.') + self.release_info = version + self.page_link = page_link + self.download_link = page_link + self.uploader = uploader + self.referer = referer + + @property + def id(self): + return self.page_link + + def get_matches(self, video): + matches = set() + + # episode + if isinstance(video, Episode): + # other properties + matches |= guess_matches(video, guessit(self.version, {'type': 'episode'}), partial=True) + # movie + elif isinstance(video, Movie): + # other properties + matches |= guess_matches(video, guessit(self.version, {'type': 'movie'}), partial=True) + + return matches + + +class GreekSubsProvider(Provider): + """GreekSubs Provider.""" + languages = {Language('ell')} + server_url = 'https://greeksubs.net/' + subtitle_class = GreekSubsSubtitle + + def __init__(self): + self.session = None + + def initialize(self): + self.session = RetryingCFSession() + self.session.headers['User-Agent'] = AGENT_LIST[randint(0, len(AGENT_LIST) - 1)] + + def terminate(self): + self.session.close() + + def query(self, video, languages, imdb_id, season=None, episode=None): + logger.debug('Searching subtitles for %r', imdb_id) + subtitles = [] + search_link = self.server_url + 'en/view/' + imdb_id + + r = self.session.get(search_link, timeout=30) + r.raise_for_status() + + soup_page = ParserBeautifulSoup(r.content.decode('utf-8', 'ignore'), ['html.parser']) + + if isinstance(video, Episode): + try: + episodes = soup_page.select('div.col-lg-offset-2.col-md-8.text-center.top30.bottom10 > a') + for item in episodes: + season_episode = re.search(r'Season (\d+) Episode (\d+)', item.text) + season_number = int(season_episode.group(1)) + episode_number = int(season_episode.group(2)) + if season_number == season and episode_number == episode: + episode_page = item.attrs['href'] + r = self.session.get(episode_page, timeout=30) + soup_subs = ParserBeautifulSoup(r.content.decode('utf-8', 'ignore'), ['html.parser']) + try: + secCode = soup_subs.find('input', {'id': 'secCode'}).get('value') + except Exception as e: + logging.debug(e) + else: + for subtitles_item in soup_subs.select('#elSub > tbody > tr'): + try: + subtitle_id = re.search(r'downloadMe\(\'(.*)\'\)', subtitles_item.contents[2].contents[2].contents[0].attrs['onclick']).group(1) + page_link = self.server_url + 'dll/' + subtitle_id + '/0/' + secCode + language = Language.fromalpha2(subtitles_item.parent.find('img')['alt']) + version = subtitles_item.contents[2].contents[4].text.strip() + uploader = subtitles_item.contents[2].contents[5].contents[0].contents[1].text.strip() + referer = episode_page + + r = self.session.get(page_link, + headers={'Referer': referer}, + timeout=30, allow_redirects=False) + r.raise_for_status() + soup_dll = ParserBeautifulSoup(r.content.decode('utf-8', 'ignore'), ['html.parser']) + try: + langcode = soup_dll.find(attrs={"name": 'langcode'}).get('value') + uid = soup_dll.find(attrs={"name": 'uid'}).get('value') + output = soup_dll.find(attrs={"name": 'output'}).get('value') + dll = soup_dll.find(attrs={"name": 'dll'}).get('value') + except Exception as e: + logging.debug(e) + else: + download_req = self.session.post(page_link, data={'langcode': langcode, + 'uid': uid, + 'output': output, + 'dll': dll}, + headers={'Referer': page_link}, timeout=10) + except Exception as e: + logging.debug(e) + else: + if language in languages: + subtitle = self.subtitle_class(language, page_link, version, uploader, referer) + if not download_req.content: + logger.error('Unable to download subtitle. No data returned from provider') + continue + + subtitle.content = download_req.content + + logger.debug('Found subtitle %r', subtitle) + subtitles.append(subtitle) + else: + pass + except Exception as e: + logging.debug(e) + elif isinstance(video, Movie): + try: + soup_subs = ParserBeautifulSoup(r.content.decode('utf-8', 'ignore'), ['html.parser']) + try: + secCode = soup_subs.find('input', {'id': 'secCode'}).get('value') + except Exception as e: + logging.debug(e) + else: + for subtitles_item in soup_subs.select('#elSub > tbody > tr'): + try: + subtitle_id = re.search(r'downloadMe\(\'(.*)\'\)', + subtitles_item.contents[2].contents[2].contents[0].attrs[ + 'onclick']).group(1) + page_link = self.server_url + 'dll/' + subtitle_id + '/0/' + secCode + language = Language.fromalpha2(subtitles_item.parent.find('img')['alt']) + version = subtitles_item.contents[2].contents[4].text.strip() + uploader = subtitles_item.contents[2].contents[5].contents[0].contents[ + 1].text.strip() + referer = page_link + + r = self.session.get(page_link, + headers={'Referer': referer}, + timeout=30, allow_redirects=False) + r.raise_for_status() + soup_dll = ParserBeautifulSoup(r.content.decode('utf-8', 'ignore'), ['html.parser']) + try: + langcode = soup_dll.find(attrs={"name": 'langcode'}).get('value') + uid = soup_dll.find(attrs={"name": 'uid'}).get('value') + output = soup_dll.find(attrs={"name": 'output'}).get('value') + dll = soup_dll.find(attrs={"name": 'dll'}).get('value') + except Exception as e: + logging.debug(e) + else: + download_req = self.session.post(page_link, data={'langcode': langcode, + 'uid': uid, + 'output': output, + 'dll': dll}, + headers={'Referer': page_link}, timeout=10) + except Exception as e: + logging.debug(e) + else: + if language in languages: + subtitle = self.subtitle_class(language, page_link, version, uploader, referer) + if not download_req.content: + logger.error('Unable to download subtitle. No data returned from provider') + continue + + subtitle.content = download_req.content + + logger.debug('Found subtitle %r', subtitle) + subtitles.append(subtitle) + except Exception as e: + logging.debug(e) + + return subtitles + + def list_subtitles(self, video, languages): + imdbId = None + subtitles = [] + + if isinstance(video, Episode): + imdbId = video.series_imdb_id + elif isinstance(video, Movie): + imdbId = video.imdb_id + + if not imdbId: + logger.debug('No imdb number available to search with provider') + return subtitles + + # query for subtitles with the imdbId + subtitles = [] + + if isinstance(video, Episode): + subtitles = self.query(video, languages, imdbId, season=video.season, episode=video.episode) + elif isinstance(video, Movie): + subtitles = self.query(video, languages, imdbId) + + return subtitles + + def download_subtitle(self, subtitle): + if isinstance(subtitle, GreekSubsSubtitle): + subtitle.content = fix_line_ending(subtitle.content) diff --git a/libs/subliminal_patch/providers/subztv.py b/libs/subliminal_patch/providers/subztv.py deleted file mode 100644 index e5c7efdbf..000000000 --- a/libs/subliminal_patch/providers/subztv.py +++ /dev/null @@ -1,153 +0,0 @@ -# -*- coding: utf-8 -*- -import io -import logging -import os -import re - -from subzero.language import Language -from guessit import guessit -from requests import Session - -from subliminal.providers import ParserBeautifulSoup, Provider -from subliminal.subtitle import SUBTITLE_EXTENSIONS, Subtitle, fix_line_ending, guess_matches -from subliminal.video import Episode, Movie -from subliminal.exceptions import ServiceUnavailable - -from requests.exceptions import RequestException - -logger = logging.getLogger(__name__) - - -class SubztvSubtitle(Subtitle): - """Subztv Subtitle.""" - provider_name = 'subztv' - - def __init__(self, language, page_link, version, uploader, referer): - super(SubztvSubtitle, self).__init__(language, page_link=page_link) - self.version = version - self.release_info = version - self.page_link = page_link - self.uploader = uploader - self.referer = referer - self.hearing_impaired = None - - @property - def id(self): - return self.page_link - - def get_matches(self, video): - matches = set() - - # episode - if isinstance(video, Episode): - # other properties - matches |= guess_matches(video, guessit(self.version, {'type': 'episode'}), partial=True) - # movie - elif isinstance(video, Movie): - # other properties - matches |= guess_matches(video, guessit(self.version, {'type': 'movie'}), partial=True) - - return matches - - -class SubztvProvider(Provider): - """Subztv Provider.""" - languages = {Language('ell')} - server_url = 'https://subztv.online/' - subtitle_class = SubztvSubtitle - - def __init__(self): - self.session = None - - def initialize(self): - self.session = Session() - self.session.headers['User-Agent'] = os.environ.get("SZ_USER_AGENT", "Sub-Zero/2") - - def terminate(self): - self.session.close() - - def query(self, video, imdb_id, season=None, episode=None): - logger.debug('Searching subtitles for %r', imdb_id) - subtitles = [] - search_link = self.server_url + 'en/view/' + imdb_id - - r = self.session.get(search_link, timeout=30) - r.raise_for_status() - - if isinstance(video, Episode): - soup_page = ParserBeautifulSoup(r.content.decode('utf-8', 'ignore'), ['html.parser']) - for item in soup_page.select('div.col-lg-offset-2.col-md-8.text-center.top30.bottom10 > a'): - season_episode = re.search(r'Season (\d+) Episode (\d+)', item.text) - season_number = int(season_episode.group(1)) - episode_number = int(season_episode.group(2)) - if season_number == season and episode_number == episode: - episode_page = item.attrs['href'] - r = self.session.get(episode_page, timeout=30) - soup_subs = ParserBeautifulSoup(r.content.decode('utf-8', 'ignore'), ['html.parser']) - secCode = soup_subs.find('input', {'id': 'secCode'}).get('value') - for subtitles in soup_subs.select('#elSub > tbody > tr'): - subtitle_id = re.search(r'downloadMe\(\'(.*)\'\)', subtitles.contents[2].contents[2].contents[0].attrs['onclick']).group(1) - page_link = self.server_url + 'dll/' + subtitle_id + '/0/' + secCode - language = Language.fromalpha2(subtitles.parent.find('img')['alt']) - version = subtitles.contents[2].contents[4].text.strip() - uploader = subtitles.contents[2].contents[5].contents[0].contents[1].text.strip() - referer = episode_page - - subtitle = self.subtitle_class(language, page_link, version, uploader, referer) - - logger.debug('Found subtitle %r', subtitle) - subtitles.append(subtitle) - else: - pass - elif isinstance(video, Movie): - pass - - return subtitles - - def list_subtitles(self, video, languages): - imdbId = None - subtitles = [] - - if isinstance(video, Episode): - imdbId = video.series_imdb_id - elif isinstance(video, Movie): - imdbId = video.imdb_id - - if not imdbId: - logger.debug('No imdb number available to search with provider') - return subtitles - - # query for subtitles with the imdbId - response = None - status_code = None - try: - if isinstance(video, Episode): - response = self.query(video, imdbId, season=video.season, episode=video.episode) - elif isinstance(video, Movie): - response = self.query(video, imdbId) - for s in response: - if s.language in languages: - subtitles += s - except RequestException as e: - status_code = e.response.status_code - else: - status_code = int(response['status'][:3]) - - if status_code == 503: - raise ServiceUnavailable(str(status_code)) - - return subtitles - - def download_subtitle(self, subtitle): - if isinstance(subtitle, SubztvSubtitle): - # download the subtitle - logger.info('Downloading subtitle %r', subtitle) - r = self.session.get(subtitle.download_link, headers={'Referer': subtitle.referer}, - timeout=30) - r.raise_for_status() - - if not r.content: - logger.debug('Unable to download subtitle. No data returned from provider') - return - - subtitle.content = fix_line_ending(r.content) diff --git a/views/settingsproviders.html b/views/settingsproviders.html index c7ed07b32..1feaa12cc 100644 --- a/views/settingsproviders.html +++ b/views/settingsproviders.html @@ -132,6 +132,18 @@
+
+
+ GreekSubs (previously Subztv.online) +
+
+ +
+
+
GreekSubtitles @@ -533,18 +545,6 @@
-
-
- Subztv.online -
-
- -
-
-
Supersubtitles