2018-12-15 00:36:28 +00:00
import ast
2018-10-04 18:16:49 +00:00
import gc
import logging
2018-12-15 00:36:28 +00:00
import os
2018-10-04 18:16:49 +00:00
import sqlite3
2018-12-15 00:36:28 +00:00
from itertools import islice
import enzyme
2018-10-04 18:16:49 +00:00
import langdetect
from bs4 import UnicodeDammit
2018-12-15 00:36:28 +00:00
from subliminal import core
2018-10-04 18:16:49 +00:00
2018-12-15 00:36:28 +00:00
from get_argv import config_dir
2018-10-04 18:16:49 +00:00
from get_languages import alpha2_from_alpha3
2018-12-15 00:36:28 +00:00
from config import settings
from helper import path_replace , path_replace_movie , path_replace_reverse , \
path_replace_reverse_movie
2018-10-04 18:16:49 +00:00
gc . enable ( )
2018-12-15 00:36:28 +00:00
2018-10-04 18:16:49 +00:00
def store_subtitles ( file ) :
2018-12-05 01:30:03 +00:00
logging . debug ( ' BAZARR started subtitles indexing for this file: ' + file )
2018-10-04 18:16:49 +00:00
actual_subtitles = [ ]
if os . path . exists ( file ) :
if os . path . splitext ( file ) [ 1 ] == ' .mkv ' :
2018-12-05 01:30:03 +00:00
logging . debug ( " BAZARR is trying to index embedded subtitles. " )
2018-10-04 18:16:49 +00:00
try :
with open ( file , ' rb ' ) as f :
mkv = enzyme . MKV ( f )
for subtitle_track in mkv . subtitle_tracks :
try :
2018-12-15 00:36:28 +00:00
if alpha2_from_alpha3 ( subtitle_track . language ) is not None :
2018-12-05 01:30:03 +00:00
lang = str ( alpha2_from_alpha3 ( subtitle_track . language ) )
logging . debug ( " BAZARR embedded subtitles detected: " + lang )
actual_subtitles . append ( [ lang , None ] )
2018-10-04 18:16:49 +00:00
except :
2018-12-05 01:30:03 +00:00
logging . debug ( " BAZARR unable to index this unrecognized language: " + subtitle_track . language )
2018-10-04 18:16:49 +00:00
pass
2018-12-05 01:30:03 +00:00
except Exception as e :
logging . exception ( " BAZARR error when trying to analyze this mkv file: " + file )
2018-10-04 18:16:49 +00:00
pass
2018-12-05 01:30:03 +00:00
else :
logging . debug ( " BAZARR This file isn ' t an .mkv file. " )
2018-10-04 18:16:49 +00:00
brazilian_portuguese = [ " .pt-br " , " .pob " , " pb " ]
try :
subtitles = core . search_external_subtitles ( file )
2018-12-05 01:30:03 +00:00
except Exception as e :
logging . exception ( " BAZARR unable to index external subtitles. " )
2018-10-04 18:16:49 +00:00
pass
else :
for subtitle , language in subtitles . iteritems ( ) :
2018-12-15 00:36:28 +00:00
if str ( os . path . splitext ( subtitle ) [ 0 ] ) . lower ( ) . endswith ( tuple ( brazilian_portuguese ) ) :
2018-12-05 01:30:03 +00:00
logging . debug ( " BAZARR external subtitles detected: " + " pb " )
2018-10-04 18:16:49 +00:00
actual_subtitles . append ( [ str ( " pb " ) , path_replace_reverse ( os . path . join ( os . path . dirname ( file ) , subtitle ) ) ] )
elif str ( language ) != ' und ' :
2018-12-05 01:30:03 +00:00
logging . debug ( " BAZARR external subtitles detected: " + str ( language ) )
2018-10-04 18:16:49 +00:00
actual_subtitles . append ( [ str ( language ) , path_replace_reverse ( os . path . join ( os . path . dirname ( file ) , subtitle ) ) ] )
else :
2018-12-05 01:30:03 +00:00
if os . path . splitext ( subtitle ) [ 1 ] != " .sub " :
logging . debug ( " BAZARR falling back to file content analysis to detect language. " )
with open ( path_replace ( os . path . join ( os . path . dirname ( file ) , subtitle ) ) , ' r ' ) as f :
text = list ( islice ( f , 100 ) )
text = ' ' . join ( text )
encoding = UnicodeDammit ( text )
try :
text = text . decode ( encoding . original_encoding )
detected_language = langdetect . detect ( text )
except Exception as e :
logging . exception ( ' BAZARR Error trying to detect language for this subtitles file: ' + path_replace ( os . path . join ( os . path . dirname ( file ) , subtitle ) ) + ' You should try to delete this subtitles file manually and ask Bazarr to download it again. ' )
else :
if len ( detected_language ) > 0 :
logging . debug ( " BAZARR external subtitles detected and analysis guessed this language: " + str ( detected_language ) )
actual_subtitles . append ( [ str ( detected_language ) , path_replace_reverse ( os . path . join ( os . path . dirname ( file ) , subtitle ) ) ] )
2018-12-15 00:42:09 +00:00
conn_db = sqlite3 . connect ( os . path . join ( config_dir , ' db ' , ' bazarr.db ' ) , timeout = 30 )
2018-12-05 01:30:03 +00:00
c_db = conn_db . cursor ( )
logging . debug ( " BAZARR storing those languages to DB: " + str ( actual_subtitles ) )
c_db . execute ( " UPDATE table_episodes SET subtitles = ? WHERE path = ? " , ( str ( actual_subtitles ) , path_replace_reverse ( file ) ) )
conn_db . commit ( )
c_db . close ( )
else :
logging . debug ( " BAZARR this file doesn ' t seems to exist or isn ' t accessible. " )
logging . debug ( ' BAZARR ended subtitles indexing for this file: ' + file )
2018-10-04 18:16:49 +00:00
return actual_subtitles
def store_subtitles_movie ( file ) :
2018-12-05 01:30:03 +00:00
logging . debug ( ' BAZARR started subtitles indexing for this file: ' + file )
2018-10-04 18:16:49 +00:00
actual_subtitles = [ ]
if os . path . exists ( file ) :
if os . path . splitext ( file ) [ 1 ] == ' .mkv ' :
2018-12-05 01:30:03 +00:00
logging . debug ( " BAZARR is trying to index embedded subtitles. " )
2018-10-04 18:16:49 +00:00
try :
with open ( file , ' rb ' ) as f :
mkv = enzyme . MKV ( f )
for subtitle_track in mkv . subtitle_tracks :
try :
2018-12-15 00:36:28 +00:00
if alpha2_from_alpha3 ( subtitle_track . language ) is not None :
2018-12-05 01:30:03 +00:00
lang = str ( alpha2_from_alpha3 ( subtitle_track . language ) )
logging . debug ( " BAZARR embedded subtitles detected: " + lang )
actual_subtitles . append ( [ lang , None ] )
2018-10-04 18:16:49 +00:00
except :
2018-12-05 01:30:03 +00:00
logging . debug ( " BAZARR unable to index this unrecognized language: " + subtitle_track . language )
2018-10-04 18:16:49 +00:00
pass
2018-12-05 01:30:03 +00:00
except Exception as e :
logging . exception ( " BAZARR error when trying to analyze this mkv file: " + file )
2018-10-04 18:16:49 +00:00
pass
2018-12-05 01:30:03 +00:00
else :
logging . debug ( " BAZARR This file isn ' t an .mkv file. " )
2018-10-04 18:16:49 +00:00
2018-12-05 01:30:03 +00:00
brazilian_portuguese = [ " .pt-br " , " .pob " , " pb " ]
try :
subtitles = core . search_external_subtitles ( file )
except Exception as e :
logging . exception ( " BAZARR unable to index external subtitles. " )
pass
else :
for subtitle , language in subtitles . iteritems ( ) :
if str ( os . path . splitext ( subtitle ) [ 0 ] ) . lower ( ) . endswith ( tuple ( brazilian_portuguese ) ) is True :
logging . debug ( " BAZARR external subtitles detected: " + " pb " )
actual_subtitles . append ( [ str ( " pb " ) , path_replace_reverse_movie ( os . path . join ( os . path . dirname ( file ) , subtitle ) ) ] )
elif str ( language ) != ' und ' :
logging . debug ( " BAZARR external subtitles detected: " + str ( language ) )
actual_subtitles . append ( [ str ( language ) , path_replace_reverse_movie ( os . path . join ( os . path . dirname ( file ) , subtitle ) ) ] )
else :
if os . path . splitext ( subtitle ) [ 1 ] != " .sub " :
logging . debug ( " BAZARR falling back to file content analysis to detect language. " )
with open ( path_replace_movie ( os . path . join ( os . path . dirname ( file ) , subtitle ) ) , ' r ' ) as f :
text = list ( islice ( f , 100 ) )
text = ' ' . join ( text )
encoding = UnicodeDammit ( text )
try :
text = text . decode ( encoding . original_encoding )
detected_language = langdetect . detect ( text )
except Exception as e :
logging . exception ( ' BAZARR Error trying to detect language for this subtitles file: ' + path_replace ( os . path . join ( os . path . dirname ( file ) , subtitle ) ) + ' You should try to delete this subtitles file manually and ask Bazarr to download it again. ' )
else :
if len ( detected_language ) > 0 :
logging . debug ( " BAZARR external subtitles detected and analysis guessed this language: " + str ( detected_language ) )
actual_subtitles . append ( [ str ( detected_language ) , path_replace_reverse_movie ( os . path . join ( os . path . dirname ( file ) , subtitle ) ) ] )
2018-10-04 18:16:49 +00:00
2018-12-15 00:36:28 +00:00
conn_db = sqlite3 . connect ( os . path . join ( config_dir , ' db ' , ' bazarr.db ' ) , timeout = 30 )
2018-10-04 18:16:49 +00:00
c_db = conn_db . cursor ( )
2018-12-05 01:30:03 +00:00
logging . debug ( " BAZARR storing those languages to DB: " + str ( actual_subtitles ) )
2018-10-04 18:16:49 +00:00
c_db . execute ( " UPDATE table_movies SET subtitles = ? WHERE path = ? " , ( str ( actual_subtitles ) , path_replace_reverse_movie ( file ) ) )
conn_db . commit ( )
c_db . close ( )
2018-12-05 01:30:03 +00:00
else :
logging . debug ( " BAZARR this file doesn ' t seems to exist or isn ' t accessible. " )
logging . debug ( ' BAZARR ended subtitles indexing for this file: ' + file )
2018-10-04 18:16:49 +00:00
return actual_subtitles
def list_missing_subtitles ( * no ) :
query_string = ' '
try :
query_string = " WHERE table_shows.sonarrSeriesId = " + str ( no [ 0 ] )
except :
pass
2018-12-15 00:36:28 +00:00
conn_db = sqlite3 . connect ( os . path . join ( config_dir , ' db ' , ' bazarr.db ' ) , timeout = 30 )
2018-10-04 18:16:49 +00:00
c_db = conn_db . cursor ( )
episodes_subtitles = c_db . execute ( " SELECT table_episodes.sonarrEpisodeId, table_episodes.subtitles, table_shows.languages FROM table_episodes INNER JOIN table_shows on table_episodes.sonarrSeriesId = table_shows.sonarrSeriesId " + query_string ) . fetchall ( )
c_db . close ( )
missing_subtitles_global = [ ]
2018-12-15 00:36:28 +00:00
use_embedded_subs = settings . general . use_embedded_subs
2018-10-04 18:16:49 +00:00
for episode_subtitles in episodes_subtitles :
actual_subtitles_temp = [ ]
actual_subtitles = [ ]
desired_subtitles = [ ]
missing_subtitles = [ ]
2018-12-15 00:36:28 +00:00
if episode_subtitles [ 1 ] is not None :
if use_embedded_subs :
2018-10-04 18:16:49 +00:00
actual_subtitles = ast . literal_eval ( episode_subtitles [ 1 ] )
else :
actual_subtitles_temp = ast . literal_eval ( episode_subtitles [ 1 ] )
for subtitle in actual_subtitles_temp :
2018-12-15 00:36:28 +00:00
if subtitle [ 1 ] is not None :
2018-10-04 18:16:49 +00:00
actual_subtitles . append ( subtitle )
2018-12-15 00:36:28 +00:00
if episode_subtitles [ 2 ] is not None :
2018-10-04 18:16:49 +00:00
desired_subtitles = ast . literal_eval ( episode_subtitles [ 2 ] )
actual_subtitles_list = [ ]
2018-12-15 00:36:28 +00:00
if desired_subtitles is None :
2018-10-04 18:16:49 +00:00
missing_subtitles_global . append ( tuple ( [ ' [] ' , episode_subtitles [ 0 ] ] ) )
else :
for item in actual_subtitles :
if item [ 0 ] == " pt-BR " :
actual_subtitles_list . append ( " pb " )
else :
actual_subtitles_list . append ( item [ 0 ] )
missing_subtitles = list ( set ( desired_subtitles ) - set ( actual_subtitles_list ) )
missing_subtitles_global . append ( tuple ( [ str ( missing_subtitles ) , episode_subtitles [ 0 ] ] ) )
2018-12-15 00:36:28 +00:00
conn_db = sqlite3 . connect ( os . path . join ( config_dir , ' db ' , ' bazarr.db ' ) , timeout = 30 )
2018-10-04 18:16:49 +00:00
c_db = conn_db . cursor ( )
c_db . executemany ( " UPDATE table_episodes SET missing_subtitles = ? WHERE sonarrEpisodeId = ? " , ( missing_subtitles_global ) )
conn_db . commit ( )
c_db . close ( )
def list_missing_subtitles_movies ( * no ) :
query_string = ' '
try :
query_string = " WHERE table_movies.radarrId = " + str ( no [ 0 ] )
except :
pass
2018-12-15 00:36:28 +00:00
conn_db = sqlite3 . connect ( os . path . join ( config_dir , ' db ' , ' bazarr.db ' ) , timeout = 30 )
2018-10-04 18:16:49 +00:00
c_db = conn_db . cursor ( )
movies_subtitles = c_db . execute ( " SELECT radarrId, subtitles, languages FROM table_movies " + query_string ) . fetchall ( )
c_db . close ( )
missing_subtitles_global = [ ]
2018-12-15 00:36:28 +00:00
use_embedded_subs = settings . general . use_embedded_subs
2018-10-04 18:16:49 +00:00
for movie_subtitles in movies_subtitles :
actual_subtitles_temp = [ ]
actual_subtitles = [ ]
desired_subtitles = [ ]
missing_subtitles = [ ]
2018-12-15 00:36:28 +00:00
if movie_subtitles [ 1 ] is not None :
if use_embedded_subs :
2018-10-04 18:16:49 +00:00
actual_subtitles = ast . literal_eval ( movie_subtitles [ 1 ] )
else :
actual_subtitles_temp = ast . literal_eval ( movie_subtitles [ 1 ] )
for subtitle in actual_subtitles_temp :
2018-12-15 00:36:28 +00:00
if subtitle [ 1 ] is not None :
2018-10-04 18:16:49 +00:00
actual_subtitles . append ( subtitle )
2018-12-15 00:36:28 +00:00
if movie_subtitles [ 2 ] is not None :
2018-10-04 18:16:49 +00:00
desired_subtitles = ast . literal_eval ( movie_subtitles [ 2 ] )
actual_subtitles_list = [ ]
2018-12-15 00:36:28 +00:00
if desired_subtitles is None :
2018-10-04 18:16:49 +00:00
missing_subtitles_global . append ( tuple ( [ ' [] ' , movie_subtitles [ 0 ] ] ) )
else :
for item in actual_subtitles :
if item [ 0 ] == " pt-BR " :
actual_subtitles_list . append ( " pb " )
else :
actual_subtitles_list . append ( item [ 0 ] )
missing_subtitles = list ( set ( desired_subtitles ) - set ( actual_subtitles_list ) )
missing_subtitles_global . append ( tuple ( [ str ( missing_subtitles ) , movie_subtitles [ 0 ] ] ) )
2018-12-15 00:36:28 +00:00
conn_db = sqlite3 . connect ( os . path . join ( config_dir , ' db ' , ' bazarr.db ' ) , timeout = 30 )
2018-10-04 18:16:49 +00:00
c_db = conn_db . cursor ( )
c_db . executemany ( " UPDATE table_movies SET missing_subtitles = ? WHERE radarrId = ? " , ( missing_subtitles_global ) )
conn_db . commit ( )
c_db . close ( )
2018-12-15 00:36:28 +00:00
2018-10-04 18:16:49 +00:00
def series_full_scan_subtitles ( ) :
2018-12-15 00:36:28 +00:00
conn_db = sqlite3 . connect ( os . path . join ( config_dir , ' db ' , ' bazarr.db ' ) , timeout = 30 )
2018-10-04 18:16:49 +00:00
c_db = conn_db . cursor ( )
episodes = c_db . execute ( " SELECT path FROM table_episodes " ) . fetchall ( )
c_db . close ( )
for episode in episodes :
store_subtitles ( path_replace ( episode [ 0 ] ) )
gc . collect ( )
def movies_full_scan_subtitles ( ) :
2018-12-15 00:36:28 +00:00
conn_db = sqlite3 . connect ( os . path . join ( config_dir , ' db ' , ' bazarr.db ' ) , timeout = 30 )
2018-10-04 18:16:49 +00:00
c_db = conn_db . cursor ( )
movies = c_db . execute ( " SELECT path FROM table_movies " ) . fetchall ( )
c_db . close ( )
for movie in movies :
store_subtitles_movie ( path_replace_movie ( movie [ 0 ] ) )
gc . collect ( )
def series_scan_subtitles ( no ) :
2018-12-15 00:36:28 +00:00
conn_db = sqlite3 . connect ( os . path . join ( config_dir , ' db ' , ' bazarr.db ' ) , timeout = 30 )
2018-10-04 18:16:49 +00:00
c_db = conn_db . cursor ( )
episodes = c_db . execute ( " SELECT path FROM table_episodes WHERE sonarrSeriesId = ? " , ( no , ) ) . fetchall ( )
c_db . close ( )
for episode in episodes :
store_subtitles ( path_replace ( episode [ 0 ] ) )
list_missing_subtitles ( no )
def movies_scan_subtitles ( no ) :
2018-12-15 00:36:28 +00:00
conn_db = sqlite3 . connect ( os . path . join ( config_dir , ' db ' , ' bazarr.db ' ) , timeout = 30 )
2018-10-04 18:16:49 +00:00
c_db = conn_db . cursor ( )
movies = c_db . execute ( " SELECT path FROM table_movies WHERE radarrId = ? " , ( no , ) ) . fetchall ( )
c_db . close ( )
for movie in movies :
store_subtitles_movie ( path_replace_movie ( movie [ 0 ] ) )
list_missing_subtitles_movies ( no )