From a9518a3ed9ff59c17cd41802ba1730239ae67343 Mon Sep 17 00:00:00 2001 From: evilhero Date: Tue, 12 Apr 2016 14:53:18 -0400 Subject: [PATCH] FIX: Problems with manual post-processing not scanning in alternate search names properly, FIX: Update Dynamic Names field within comics table, FIX: Annuals + manual post-processing --- mylar/PostProcessor.py | 22 ++++++++++++++-------- mylar/__init__.py | 11 ++++++++--- mylar/filechecker.py | 36 +++++++++++++++++++++++++++--------- mylar/helpers.py | 4 +++- 4 files changed, 52 insertions(+), 21 deletions(-) diff --git a/mylar/PostProcessor.py b/mylar/PostProcessor.py index 5cab7e6e..615640c4 100755 --- a/mylar/PostProcessor.py +++ b/mylar/PostProcessor.py @@ -259,7 +259,8 @@ class PostProcessor(object): as_d = filechecker.FileChecker(AlternateSearch=aldb['AlternateSearch'].decode('utf-8')) as_dinfo = as_d.altcheck() alt_list.append({'AS_Alt': as_dinfo['AS_Alt'], - 'AS_Tuple': as_dinfo['AS_Tuple']}) + 'AS_Tuple': as_dinfo['AS_Tuple'], + 'AS_DyComicName': aldb['DynamicComicName']}) manual_list = [] @@ -269,15 +270,20 @@ class PostProcessor(object): mod_seriesname = as_dinfo['mod_seriesname'] loopchk = [] for x in alt_list: - if mod_seriesname in x['AS_Alt']: - for ab in x['AS_Alt']: - loopchk.append('%' + re.sub('\|', '%', ab) + '%') + cname = x['AS_DyComicName'] + for ab in x['AS_Alt']: + if re.sub('[\|\s]', '', mod_seriesname.lower()).strip() in re.sub('[\|\s]', '', ab.lower()).strip(): + if not any(re.sub('[\|\s]', '', cname.lower()) == x for x in loopchk): + loopchk.append(re.sub('[\|\s]', '', cname.lower())) + + if 'annual' in mod_seriesname.lower(): + mod_seriesname = re.sub('annual', '', mod_seriesname, flags=re.I).strip() #make sure we add back in the original parsed filename here. - loopchk.append('%' + re.sub('\|', '%', mod_seriesname) + '%') - - tmpsql = "SELECT * FROM comics WHERE DynamicComicName LIKE ({seq}) COLLATE NOCASE".format(seq=','.join(['?'] *(len(loopchk)))) - comicseries = myDB.select(tmpsql, loopchk) + if not any(re.sub('[\|\s]', '', mod_seriesname).lower() == x for x in loopchk): + loopchk.append(re.sub('[\|\s]', '', mod_seriesname.lower())) + tmpsql = "SELECT * FROM comics WHERE DynamicComicName IN ({seq}) COLLATE NOCASE".format(seq=','.join('?' * len(loopchk))) + comicseries = myDB.select(tmpsql, tuple(loopchk)) if comicseries is None: logger.error(module + ' No Series in Watchlist - checking against Story Arcs (just in case). If I do not find anything, maybe you should be running Import?') diff --git a/mylar/__init__.py b/mylar/__init__.py index 2ee5d59d..973b5da1 100755 --- a/mylar/__init__.py +++ b/mylar/__init__.py @@ -96,6 +96,7 @@ CONFIG_VERSION = None DB_FILE = None DBCHOICE = None +DYNAMIC_UPDATE = 0 #these are used depending on dbchoice. DBUSER = None @@ -431,7 +432,7 @@ def check_setting_str(config, cfg_name, item_name, def_val, log=True): def initialize(): with INIT_LOCK: - global __INITIALIZED__, DBCHOICE, DBUSER, DBPASS, DBNAME, COMICVINE_API, DEFAULT_CVAPI, CVAPI_RATE, CV_HEADERS, BLACKLISTED_PUBLISHERS, FULL_PATH, PROG_DIR, VERBOSE, DAEMON, UPCOMING_SNATCHED, COMICSORT, DATA_DIR, CONFIG_FILE, CFG, CONFIG_VERSION, LOG_DIR, CACHE_DIR, MAX_LOGSIZE, OLDCONFIG_VERSION, OS_DETECT, \ + global __INITIALIZED__, DBCHOICE, DBUSER, DBPASS, DBNAME, DYNAMIC_UPDATE, COMICVINE_API, DEFAULT_CVAPI, CVAPI_RATE, CV_HEADERS, BLACKLISTED_PUBLISHERS, FULL_PATH, PROG_DIR, VERBOSE, DAEMON, UPCOMING_SNATCHED, COMICSORT, DATA_DIR, CONFIG_FILE, CFG, CONFIG_VERSION, LOG_DIR, CACHE_DIR, MAX_LOGSIZE, OLDCONFIG_VERSION, OS_DETECT, \ queue, LOCAL_IP, EXT_IP, HTTP_PORT, HTTP_HOST, HTTP_USERNAME, HTTP_PASSWORD, HTTP_ROOT, ENABLE_HTTPS, HTTPS_CERT, HTTPS_KEY, HTTPS_FORCE_ON, HOST_RETURN, API_ENABLED, API_KEY, DOWNLOAD_APIKEY, LAUNCH_BROWSER, GIT_PATH, SAFESTART, NOWEEKLY, AUTO_UPDATE, \ IMPORT_STATUS, IMPORT_FILES, IMPORT_TOTALFILES, IMPORT_CID_COUNT, IMPORT_PARSED_COUNT, IMPORT_FAILURE_COUNT, CHECKENABLED, \ CURRENT_VERSION, LATEST_VERSION, CHECK_GITHUB, CHECK_GITHUB_ON_STARTUP, CHECK_GITHUB_INTERVAL, GIT_USER, GIT_BRANCH, USER_AGENT, DESTINATION_DIR, MULTIPLE_DEST_DIRS, CREATE_FOLDERS, DELETE_REMOVE_DIR, \ @@ -482,6 +483,7 @@ def initialize(): DBUSER = check_setting_str(CFG, 'General', 'dbuser', '') DBPASS = check_setting_str(CFG, 'General', 'dbpass', '') DBNAME = check_setting_str(CFG, 'General', 'dbname', '') + DYNAMIC_UPDATE = check_setting_int(CFG, 'General', 'dynamic_update', 0) COMICVINE_API = check_setting_str(CFG, 'General', 'comicvine_api', '') if not COMICVINE_API: @@ -1233,7 +1235,7 @@ def config_write(): new_config['General']['dbuser'] = DBUSER new_config['General']['dbpass'] = DBPASS new_config['General']['dbname'] = DBNAME - + new_config['General']['dynamic_update'] = DYNAMIC_UPDATE if COMICVINE_API is None or COMICVINE_API == '': new_config['General']['comicvine_api'] = COMICVINE_API else: @@ -1676,7 +1678,10 @@ def dbcheck(): try: c.execute('SELECT DynamicComicName from comics') - dynamic_upgrade = False + if DYNAMIC_UPDATE < 1: + dynamic_upgrade = True + else: + dynamic_upgrade = False except sqlite3.OperationalError: c.execute('ALTER TABLE comics ADD COLUMN DynamicComicName TEXT') dynamic_upgrade = True diff --git a/mylar/filechecker.py b/mylar/filechecker.py index 996278df..29e47a64 100755 --- a/mylar/filechecker.py +++ b/mylar/filechecker.py @@ -357,13 +357,18 @@ class FileChecker(object): 'mod_position': self.char_file_position(modfilename, sf, lastmod_position), 'validcountchk': validcountchk}) - if '\xe2' in sf: + if '\xe2' in sf: #(maybe \u221e) logger.fdebug('[SPECIAL-CHARACTER ISSUE] Possible issue # : ' + sf) possible_issuenumbers.append({'number': sf, 'position': split_file.index(sf), 'mod_position': self.char_file_position(modfilename, sf, lastmod_position), 'validcountchk': validcountchk}) + #if '\xbc' in sf: + # '0.25' + #if '\xbe' in sf:: + # '0.75' + count = None found = False @@ -686,7 +691,10 @@ class FileChecker(object): logger.fdebug('series title possibly: ' + series_name) #check for annual in title(s) here. - + if mylar.ANNUALS_ON: + if 'annual' in series_name.lower(): + issue_number = 'Annual ' + str(issue_number) + series_name = re.sub('annual', '', series_name, flags=re.I).strip() #if path_list is not None: # clocation = os.path.join(path, path_list, filename) #else: @@ -752,10 +760,16 @@ class FileChecker(object): nspace_seriesname = re.sub(' ', '', mod_seriesname) nspace_watchcomic = re.sub(' ', '', mod_watchcomic) - logger.fdebug('Possible Alternate Names to match against (if necessary): ' + str(self.AS_Alt)) + if self.AS_Alt != '127372873872871091383 abdkhjhskjhkjdhakajhf': + logger.fdebug('Possible Alternate Names to match against (if necessary): ' + str(self.AS_Alt)) justthedigits = series_info['issue_number'] + if mylar.ANNUALS_ON: + if 'annual' in series_name.lower(): + justthedigits = 'Annual ' + series_info['issue_number'] + nspace_seriesname = re.sub('annual', '', nspace_seriesname.lower()).strip() + if re.sub('\|','', nspace_seriesname.lower()).strip() == re.sub('\|', '', nspace_watchcomic.lower()).strip() or any(re.sub('[\|\s]','', x.lower()).strip() == re.sub('[\|\s]','', nspace_seriesname.lower()).strip() for x in self.AS_Alt): logger.fdebug('[MATCH: ' + series_info['series_name'] + '] ' + filename) enable_annual = False @@ -784,12 +798,12 @@ class FileChecker(object): if 'biannual' in nspace_seriesname.lower(): if mylar.FOLDER_SCAN_LOG_VERBOSE: logger.fdebug('[FILECHECKER] BiAnnual detected - wouldn\'t Deadpool be proud?') - nspace_seriesname = re.sub('biannual', 'biannual', nspace_seriesname).strip() + nspace_seriesname = re.sub('biannual', '', nspace_seriesname).strip() enable_annual = True else: if mylar.FOLDER_SCAN_LOG_VERBOSE: logger.fdebug('[FILECHECKER] Annual detected - proceeding cautiously.') - nspace_seriesname = re.sub('annual', 'annual', nspace_seriesname).strip() + nspace_seriesname = re.sub('annual', '', nspace_seriesname).strip() enable_annual = False if mylar.FOLDER_SCAN_LOG_VERBOSE: @@ -942,10 +956,13 @@ class FileChecker(object): AS_Tuple = [] if self.AlternateSearch is not None and self.AlternateSearch != 'None': chkthealt = self.AlternateSearch.split('##') - if chkthealt == 0: - altsearchcomic = self.watchcomic - for calt in chkthealt: - #logger.info('calt: ' + calt) + #logger.info('[' + str(len(chkthealt)) + '] chkthealt: ' + str(chkthealt)) + i = 0 + while (i <= len(chkthealt)): + try: + calt = chkthealt[i] + except IndexError: + break AS_tupled = False AS_Alternate = re.sub('##', '', calt) if '!!' in AS_Alternate: @@ -971,6 +988,7 @@ class FileChecker(object): AS_Tuple.append({"ComicID": AS_ComicID, "AS_Alternate": altsearchcomic}) AS_Alt.append(altsearchcomic) + i+=1 else: #create random characters so it will never match. altsearchcomic = "127372873872871091383 abdkhjhskjhkjdhakajhf" diff --git a/mylar/helpers.py b/mylar/helpers.py index 7dea0413..0c5b60e6 100755 --- a/mylar/helpers.py +++ b/mylar/helpers.py @@ -1141,7 +1141,7 @@ def upgrade_dynamic(): for cl in clist: cl_d = mylar.filechecker.FileChecker(watchcomic=cl['ComicName']) cl_dyninfo = cl_d.dynamic_replace(cl['ComicName']) - dynamic_list.append({'DynamicComicName': cl_dyninfo['mod_seriesname'], + dynamic_list.append({'DynamicComicName': re.sub('[\|\s]','', cl_dyninfo['mod_seriesname'].lower()).strip(), 'ComicID': cl['ComicID']}) if len(dynamic_list) > 0: @@ -1151,6 +1151,8 @@ def upgrade_dynamic(): myDB.upsert("Comics", newVal, CtrlVal) logger.info('Finshed updating ' + str(len(dynamic_list)) + ' entries within the db.') + mylar.DYNAMIC_UPDATE = 1 + mylar.config_write() return def checkFolder():