mirror
/
bazarr
mirror of https://github.com/morpheus65535/bazarr
1
0
Fork 0
Code Issues Releases Wiki Activity

Implemented words/regex ban list for subtitles

This commit is contained in:
morpheus65535 2021-12-11 07:44:53 -05:00 committed by GitHub
parent a87a1fad8f
commit 63b326aa2f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 141 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
bazarr/api/episodes/episodes_subtitles.py
View File

@ -6,7 +6,7 @@ from flask import request
from flask_restful import Resource
from subliminal_patch.core import SUBTITLE_EXTENSIONS
from database import TableEpisodes, get_audio_profile_languages
from database import TableEpisodes, get_audio_profile_languages, get_profile_id
from ..utils import authenticate
from helper import path_mappings
from get_providers import get_providers, get_providers_auth
@ -55,7 +55,8 @@ class EpisodesSubtitles(Resource):
try:
result = download_subtitle(episodePath, language, audio_language, hi, forced, providers_list,
providers_auth, sceneName, title, 'series')
providers_auth, sceneName, title, 'series',
profile_id=get_profile_id(episode_id=sonarrEpisodeId))
if result is not None:
message = result[0]
path = result[1]

5
bazarr/api/movies/movies_subtitles.py
View File

@ -6,7 +6,7 @@ from flask import request
from flask_restful import Resource
from subliminal_patch.core import SUBTITLE_EXTENSIONS
from database import TableMovies, get_audio_profile_languages
from database import TableMovies, get_audio_profile_languages, get_profile_id
from ..utils import authenticate
from helper import path_mappings
from get_providers import get_providers, get_providers_auth
@ -57,7 +57,8 @@ class MoviesSubtitles(Resource):
try:
result = download_subtitle(moviePath, language, audio_language, hi, forced, providers_list,
providers_auth, sceneName, title, 'movie')
providers_auth, sceneName, title, 'movie',
profile_id=get_profile_id(movie_id=radarrId))
if result is not None:
message = result[0]
path = result[1]

5
bazarr/api/providers/providers_episodes.py
View File

@ -3,7 +3,7 @@
from flask import request, jsonify
from flask_restful import Resource
from database import TableEpisodes, TableShows, get_audio_profile_languages
from database import TableEpisodes, TableShows, get_audio_profile_languages, get_profile_id
from helper import path_mappings
from get_providers import get_providers, get_providers_auth
from get_subtitle import manual_search, manual_download_subtitle
@ -76,7 +76,8 @@ class ProviderEpisodes(Resource):
try:
result = manual_download_subtitle(episodePath, language, audio_language, hi, forced, subtitle,
selected_provider, providers_auth, sceneName, title, 'series')
selected_provider, providers_auth, sceneName, title, 'series',
profile_id=get_profile_id(episode_id=sonarrEpisodeId))
if result is not None:
message = result[0]
path = result[1]

5
bazarr/api/providers/providers_movies.py
View File

@ -3,7 +3,7 @@
from flask import request, jsonify
from flask_restful import Resource
from database import TableMovies, get_audio_profile_languages
from database import TableMovies, get_audio_profile_languages, get_profile_id
from helper import path_mappings
from get_providers import get_providers, get_providers_auth
from get_subtitle import manual_search, manual_download_subtitle
@ -77,7 +77,8 @@ class ProviderMovies(Resource):
try:
result = manual_download_subtitle(moviePath, language, audio_language, hi, forced, subtitle,
selected_provider, providers_auth, sceneName, title, 'movie')
selected_provider, providers_auth, sceneName, title, 'movie',
profile_id=get_profile_id(movie_id=radarrId))
if result is not None:
message = result[0]
path = result[1]

8
bazarr/api/system/settings.py
View File

@ -56,7 +56,9 @@ class SystemSettings(Resource):
TableLanguagesProfiles.update({
TableLanguagesProfiles.name: item['name'],
TableLanguagesProfiles.cutoff: item['cutoff'] if item['cutoff'] != 'null' else None,
TableLanguagesProfiles.items: json.dumps(item['items'])
TableLanguagesProfiles.items: json.dumps(item['items']),
TableLanguagesProfiles.mustContain: item['mustContain'],
TableLanguagesProfiles.mustNotContain: item['mustNotContain'],
})\
.where(TableLanguagesProfiles.profileId == item['profileId'])\
.execute()
@ -67,7 +69,9 @@ class SystemSettings(Resource):
TableLanguagesProfiles.profileId: item['profileId'],
TableLanguagesProfiles.name: item['name'],
TableLanguagesProfiles.cutoff: item['cutoff'] if item['cutoff'] != 'null' else None,
TableLanguagesProfiles.items: json.dumps(item['items'])
TableLanguagesProfiles.items: json.dumps(item['items']),
TableLanguagesProfiles.mustContain: item['must_contain'],
TableLanguagesProfiles.mustNotContain: item['must_not_contain'],
}).execute()
for profileId in existing:
# Unassign this profileId from series and movies

36
bazarr/database.py
View File

@ -136,6 +136,8 @@ class TableLanguagesProfiles(BaseModel):
items = TextField()
name = TextField()
profileId = AutoField()
mustContain = TextField(null=True)
mustNotContain = TextField(null=True)
class Meta:
table_name = 'table_languages_profiles'
@ -329,7 +331,9 @@ def migrate_db():
migrator.add_column('table_history_movie', 'provider', TextField(null=True)),
migrator.add_column('table_history_movie', 'score', TextField(null=True)),
migrator.add_column('table_history_movie', 'subs_id', TextField(null=True)),
migrator.add_column('table_history_movie', 'subtitles_path', TextField(null=True))
migrator.add_column('table_history_movie', 'subtitles_path', TextField(null=True)),
migrator.add_column('table_languages_profiles', 'mustContain', TextField(null=True)),
migrator.add_column('table_languages_profiles', 'mustNotContain', TextField(null=True)),
)
@ -394,10 +398,16 @@ def update_profile_id_list():
profile_id_list = TableLanguagesProfiles.select(TableLanguagesProfiles.profileId,
TableLanguagesProfiles.name,
TableLanguagesProfiles.cutoff,
TableLanguagesProfiles.items).dicts()
TableLanguagesProfiles.items,
TableLanguagesProfiles.mustContain,
TableLanguagesProfiles.mustNotContain).dicts()
profile_id_list = list(profile_id_list)
for profile in profile_id_list:
profile['items'] = json.loads(profile['items'])
profile['mustContain'] = ast.literal_eval(profile['mustContain']) if profile['mustContain'] else \
profile['mustContain']
profile['mustNotContain'] = ast.literal_eval(profile['mustNotContain']) if profile['mustNotContain'] else \
profile['mustNotContain']
def get_profiles_list(profile_id=None):
@ -422,7 +432,7 @@ def get_desired_languages(profile_id):
if profile_id and profile_id != 'null':
for profile in profile_id_list:
profileId, name, cutoff, items = profile.values()
profileId, name, cutoff, items, mustContain, mustNotContain = profile.values()
if profileId == int(profile_id):
languages = [x['language'] for x in items]
break
@ -438,7 +448,7 @@ def get_profile_id_name(profile_id):
if profile_id and profile_id != 'null':
for profile in profile_id_list:
profileId, name, cutoff, items = profile.values()
profileId, name, cutoff, items, mustContain, mustNotContain = profile.values()
if profileId == int(profile_id):
name_from_id = name
break
@ -455,7 +465,7 @@ def get_profile_cutoff(profile_id):
if profile_id and profile_id != 'null':
cutoff_language = []
for profile in profile_id_list:
profileId, name, cutoff, items = profile.values()
profileId, name, cutoff, items, mustContain, mustNotContain = profile.values()
if cutoff:
if profileId == int(profile_id):
for item in items:
@ -498,6 +508,22 @@ def get_audio_profile_languages(series_id=None, episode_id=None, movie_id=None):
return audio_languages
def get_profile_id(series_id=None, episode_id=None, movie_id=None):
if series_id:
profileId = TableShows.get(TableShows.sonarrSeriesId == series_id).profileId
elif episode_id:
profileId = TableShows.select(TableShows.profileId)\
.join(TableEpisodes, on=(TableShows.sonarrSeriesId == TableEpisodes.sonarrSeriesId))\
.where(TableEpisodes.sonarrEpisodeId == episode_id)\
.get().profileId
elif movie_id:
profileId = TableMovies.get(TableMovies.radarrId == movie_id).profileId
else:
return None
return profileId
def convert_list_to_clause(arr: list):
if isinstance(arr, list):
return f"({','.join(str(x) for x in arr)})"

18
bazarr/get_subtitle.py
View File

@ -84,7 +84,7 @@ def get_video(path, title, sceneName, providers=None, media_type="movie"):
def download_subtitle(path, language, audio_language, hi, forced, providers, providers_auth, sceneName, title,
media_type, forced_minimum_score=None, is_upgrade=False):
media_type, forced_minimum_score=None, is_upgrade=False, profile_id=None):
# fixme: supply all missing languages, not only one, to hit providers only once who support multiple languages in
# one query
@ -158,6 +158,7 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro
compute_score=compute_score,
throttle_time=None, # fixme
blacklist=get_blacklist(media_type=media_type),
ban_list=get_ban_list(profile_id),
throttle_callback=provider_throttle,
score_obj=handler,
pre_download_hook=None, # fixme
@ -361,6 +362,7 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title,
providers=providers,
provider_configs=providers_auth,
blacklist=get_blacklist(media_type=media_type),
ban_list=get_ban_list(profileId),
throttle_callback=provider_throttle,
language_hook=None) # fixme
@ -375,6 +377,7 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title,
providers=['subscene'],
provider_configs=providers_auth,
blacklist=get_blacklist(media_type=media_type),
ban_list=get_ban_list(profileId),
throttle_callback=provider_throttle,
language_hook=None) # fixme
providers_auth['subscene']['only_foreign'] = False
@ -466,7 +469,7 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title,
def manual_download_subtitle(path, language, audio_language, hi, forced, subtitle, provider, providers_auth, sceneName,
title, media_type):
title, media_type, profile_id):
logging.debug('BAZARR Manually downloading Subtitles for this file: ' + path)
if settings.general.getboolean('utf8_encode'):
@ -498,6 +501,7 @@ def manual_download_subtitle(path, language, audio_language, hi, forced, subtitl
provider_configs=providers_auth,
pool_class=provider_pool(),
blacklist=get_blacklist(media_type=media_type),
ban_list=get_ban_list(profile_id),
throttle_callback=provider_throttle)
logging.debug('BAZARR Subtitles file downloaded for this file:' + path)
else:
@ -1706,6 +1710,7 @@ def _get_lang_obj(alpha3):
return sub.subzero_language()
def _get_scores(media_type, min_movie=None, min_ep=None):
series = "series" == media_type
handler = series_score if series else movie_score
@ -1713,3 +1718,12 @@ def _get_scores(media_type, min_movie=None, min_ep=None):
min_ep = min_ep or (240 * 100 / handler.max_score)
min_score_ = int(min_ep if series else min_movie)
return handler.get_scores(min_score_)
def get_ban_list(profile_id):
if profile_id:
profile = get_profiles_list(profile_id)
if profile:
return {'must_contain': profile['mustContain'] or [],
'must_not_contain': profile['mustNotContain'] or []}
return None

2
frontend/src/@types/api.d.ts vendored
View File

@ -33,6 +33,8 @@ declare namespace Language {
profileId: number;
cutoff: number | null;
items: ProfileItem[];
mustContain: string[];
mustNotContain: string[];
}
}

25
frontend/src/Settings/Languages/modal.tsx
View File

@ -13,6 +13,7 @@ import {
ActionButton,
BaseModal,
BaseModalProps,
Chips,
LanguageSelector,
Selector,
SimpleTable,
@ -31,6 +32,8 @@ function createDefaultProfile(): Language.Profile {
name: "",
items: [],
cutoff: null,
mustContain: [],
mustNotContain: [],
};
}
@ -260,6 +263,28 @@ const LanguagesProfileModal: FunctionComponent<Props & BaseModalProps> = (
></Selector>
<Message>Ignore others if existing</Message>
</Input>
<Input name="Release info must contain">
<Chips
value={current.mustContain}
onChange={(mc) => updateProfile("mustContain", mc)}
></Chips>
<Message>
Subtitles release info must include one of those words or they will be
excluded from search results (regex supported).
</Message>
</Input>
<Input name="Release info must not contain">
<Chips
value={current.mustNotContain}
onChange={(mnc: string[]) => {
updateProfile("mustNotContain", mnc);
}}
></Chips>
<Message>
Subtitles release info including one of those words (case insensitive)
will be excluded from search results (regex supported).
</Message>
</Input>
</BaseModal>
);
};

34
frontend/src/Settings/Languages/table.tsx
View File

@ -94,6 +94,40 @@ const Table: FunctionComponent = () => {
});
},
},
{
Header: "Must contain",
accessor: "mustContain",
Cell: (row) => {
const items = row.value;
if (!items) {
return false;
}
return items.map((v) => {
return (
<Badge className={"mx-1"} variant={"secondary"}>
{v}
</Badge>
);
});
},
},
{
Header: "Must not contain",
accessor: "mustNotContain",
Cell: (row) => {
const items = row.value;
if (!items) {
return false;
}
return items.map((v) => {
return (
<Badge className={"mx-1"} variant={"secondary"}>
{v}
</Badge>
);
});
},
},
{
accessor: "profileId",
Cell: ({ row, update }) => {

17
libs/subliminal_patch/core.py
View File

@ -66,7 +66,7 @@ def remove_crap_from_fn(fn):
class SZProviderPool(ProviderPool):
def __init__(self, providers=None, provider_configs=None, blacklist=None, throttle_callback=None,
def __init__(self, providers=None, provider_configs=None, blacklist=None, ban_list=None, throttle_callback=None,
pre_download_hook=None, post_download_hook=None, language_hook=None):
#: Name of providers to use
self.providers = providers
@ -82,6 +82,9 @@ class SZProviderPool(ProviderPool):
self.blacklist = blacklist or []
#: Should be a dict of 2 lists of strings
self.ban_list = ban_list or {'must_contain': [], 'must_not_contain': []}
self.throttle_callback = throttle_callback
self.pre_download_hook = pre_download_hook
@ -184,6 +187,16 @@ class SZProviderPool(ProviderPool):
if (str(provider), str(s.id)) in self.blacklist:
logger.info("Skipping blacklisted subtitle: %s", s)
continue
if hasattr(s, 'release_info'):
if s.release_info is not None:
if any([x for x in self.ban_list["must_not_contain"]
if re.search(x, s.release_info, flags=re.IGNORECASE) is not None]):
logger.info("Skipping subtitle because release name contains prohibited string: %s", s)
continue
if any([x for x in self.ban_list["must_contain"]
if re.search(x, s.release_info, flags=re.IGNORECASE) is None]):
logger.info("Skipping subtitle because release name does not contains required string: %s", s)
continue
if s.id in seen:
continue
s.plex_media_fps = float(video.fps) if video.fps else None
@ -506,7 +519,7 @@ class SZAsyncProviderPool(SZProviderPool):
return provider, provider_subtitles
def list_subtitles(self, video, languages, blacklist=None):
def list_subtitles(self, video, languages, blacklist=None, ban_list=None):
if is_windows_special_path:
return super(SZAsyncProviderPool, self).list_subtitles(video, languages)