From 5503a363ba3e96851bdfd382ec9a75d86fc66b42 Mon Sep 17 00:00:00 2001 From: evilhero Date: Mon, 10 Dec 2018 12:05:42 -0500 Subject: [PATCH] FIX: Fix for filechecker returning traceback on some invalid date patterns, FIX: When a title contained the word special, but was not a part of a series (ie. annual integration) would fail to recognize it for post-processing --- mylar/PostProcessor.py | 132 +++++++++++++++++++++++------------------ mylar/filechecker.py | 49 +++++++++++---- 2 files changed, 109 insertions(+), 72 deletions(-) diff --git a/mylar/PostProcessor.py b/mylar/PostProcessor.py index a08d9b98..11db48c0 100755 --- a/mylar/PostProcessor.py +++ b/mylar/PostProcessor.py @@ -421,10 +421,11 @@ class PostProcessor(object): self.matched = False as_d = filechecker.FileChecker() as_dinfo = as_d.dynamic_replace(helpers.conversion(fl['series_name'])) + orig_seriesname = as_dinfo['mod_seriesname'] mod_seriesname = as_dinfo['mod_seriesname'] loopchk = [] if fl['alt_series'] is not None: - logger.info('%s Alternate series naming detected: %s' % (module, fl['alt_series'])) + logger.fdebug('%s Alternate series naming detected: %s' % (module, fl['alt_series'])) as_sinfo = as_d.dynamic_replace(helpers.conversion(fl['alt_series'])) mod_altseriesname = as_sinfo['mod_seriesname'] if all([mylar.CONFIG.ANNUALS_ON, 'annual' in mod_altseriesname.lower()]) or all([mylar.CONFIG.ANNUALS_ON, 'special' in mod_altseriesname.lower()]): @@ -456,31 +457,56 @@ class PostProcessor(object): tmpsql = "SELECT * FROM comics WHERE DynamicComicName IN ({seq}) COLLATE NOCASE".format(seq=','.join('?' * len(loopchk))) comicseries = myDB.select(tmpsql, tuple(loopchk)) - if comicseries is None: - logger.error(module + ' No Series in Watchlist - checking against Story Arcs (just in case). If I do not find anything, maybe you should be running Import?') - break - else: - watchvals = [] - for wv in comicseries: - #do some extra checks in here to ignore these types: - #check for Paused status / - #check for Ended status and 100% completion of issues. - if wv['Status'] == 'Paused' or (wv['Have'] == wv['Total'] and not any(['Present' in wv['ComicPublished'], helpers.now()[:4] in wv['ComicPublished']])): - logger.warn(wv['ComicName'] + ' [' + wv['ComicYear'] + '] is either Paused or in an Ended status with 100% completion. Ignoring for match.') - continue - wv_comicname = wv['ComicName'] - wv_comicpublisher = wv['ComicPublisher'] - wv_alternatesearch = wv['AlternateSearch'] - wv_comicid = wv['ComicID'] + if not comicseries: + if (['special' in mod_seriesname.lower(), mylar.CONFIG.ANNUALS_ON, orig_seriesname != mod_seriesname]): + loopchk.append(re.sub('[\|\s]', '', orig_seriesname.lower())) + tmpsql = "SELECT * FROM comics WHERE DynamicComicName IN ({seq}) COLLATE NOCASE".format(seq=','.join('?' * len(loopchk))) + comicseries = myDB.select(tmpsql, tuple(loopchk)) + if not comicseries: + logger.error(module + ' No Series in Watchlist - checking against Story Arcs (just in case). If I do not find anything, maybe you should be running Import?') + break - wv_seriesyear = wv['ComicYear'] - wv_comicversion = wv['ComicVersion'] - wv_publisher = wv['ComicPublisher'] - wv_total = wv['Total'] - if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE: - logger.fdebug('Queuing to Check: ' + wv['ComicName'] + ' [' + str(wv['ComicYear']) + '] -- ' + str(wv['ComicID'])) + watchvals = [] + for wv in comicseries: + #do some extra checks in here to ignore these types: + #check for Paused status / + #check for Ended status and 100% completion of issues. + if wv['Status'] == 'Paused' or (wv['Have'] == wv['Total'] and not any(['Present' in wv['ComicPublished'], helpers.now()[:4] in wv['ComicPublished']])): + logger.warn(wv['ComicName'] + ' [' + wv['ComicYear'] + '] is either Paused or in an Ended status with 100% completion. Ignoring for match.') + continue + wv_comicname = wv['ComicName'] + wv_comicpublisher = wv['ComicPublisher'] + wv_alternatesearch = wv['AlternateSearch'] + wv_comicid = wv['ComicID'] + if all([wv['Type'] != 'Print', wv['Type'] != 'Digital']) or wv['Corrected_Type'] == 'TPB': + wv_type = 'TPB' + else: + wv_type = None + wv_seriesyear = wv['ComicYear'] + wv_comicversion = wv['ComicVersion'] + wv_publisher = wv['ComicPublisher'] + wv_total = wv['Total'] + if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE: + logger.fdebug('Queuing to Check: ' + wv['ComicName'] + ' [' + str(wv['ComicYear']) + '] -- ' + str(wv['ComicID'])) - #force it to use the Publication Date of the latest issue instead of the Latest Date (which could be anything) + #force it to use the Publication Date of the latest issue instead of the Latest Date (which could be anything) + latestdate = myDB.select('SELECT IssueDate from issues WHERE ComicID=? order by ReleaseDate DESC', [wv['ComicID']]) + if latestdate: + tmplatestdate = latestdate[0][0] + if tmplatestdate[:4] != wv['LatestDate'][:4]: + if tmplatestdate[:4] > wv['LatestDate'][:4]: + latestdate = tmplatestdate + else: + latestdate = wv['LatestDate'] + else: + latestdate = tmplatestdate + else: + latestdate = wv['LatestDate'] + + if latestdate == '0000-00-00' or latestdate == 'None' or latestdate is None: + logger.fdebug('Forcing a refresh of series: ' + wv_comicname + ' as it appears to have incomplete issue dates.') + updater.dbUpdate([wv_comicid]) + logger.fdebug('Refresh complete for ' + wv_comicname + '. Rechecking issue dates for completion.') latestdate = myDB.select('SELECT IssueDate from issues WHERE ComicID=? order by ReleaseDate DESC', [wv['ComicID']]) if latestdate: tmplatestdate = latestdate[0][0] @@ -494,41 +520,25 @@ class PostProcessor(object): else: latestdate = wv['LatestDate'] + logger.fdebug('Latest Date (after forced refresh) set to :' + str(latestdate)) + if latestdate == '0000-00-00' or latestdate == 'None' or latestdate is None: - logger.fdebug('Forcing a refresh of series: ' + wv_comicname + ' as it appears to have incomplete issue dates.') - updater.dbUpdate([wv_comicid]) - logger.fdebug('Refresh complete for ' + wv_comicname + '. Rechecking issue dates for completion.') - latestdate = myDB.select('SELECT IssueDate from issues WHERE ComicID=? order by ReleaseDate DESC', [wv['ComicID']]) - if latestdate: - tmplatestdate = latestdate[0][0] - if tmplatestdate[:4] != wv['LatestDate'][:4]: - if tmplatestdate[:4] > wv['LatestDate'][:4]: - latestdate = tmplatestdate - else: - latestdate = wv['LatestDate'] - else: - latestdate = tmplatestdate - else: - latestdate = wv['LatestDate'] + logger.fdebug('Unable to properly attain the Latest Date for series: ' + wv_comicname + '. Cannot check against this series for post-processing.') + continue - logger.fdebug('Latest Date (after forced refresh) set to :' + str(latestdate)) - - if latestdate == '0000-00-00' or latestdate == 'None' or latestdate is None: - logger.fdebug('Unable to properly attain the Latest Date for series: ' + wv_comicname + '. Cannot check against this series for post-processing.') - continue - - watchvals.append({"ComicName": wv_comicname, - "ComicPublisher": wv_comicpublisher, - "AlternateSearch": wv_alternatesearch, - "ComicID": wv_comicid, - "WatchValues": {"SeriesYear": wv_seriesyear, - "LatestDate": latestdate, - "ComicVersion": wv_comicversion, - "Publisher": wv_publisher, - "Total": wv_total, - "ComicID": wv_comicid, - "IsArc": False} - }) + watchvals.append({"ComicName": wv_comicname, + "ComicPublisher": wv_comicpublisher, + "AlternateSearch": wv_alternatesearch, + "ComicID": wv_comicid, + "WatchValues": {"SeriesYear": wv_seriesyear, + "LatestDate": latestdate, + "ComicVersion": wv_comicversion, + "Type": wv_type, + "Publisher": wv_publisher, + "Total": wv_total, + "ComicID": wv_comicid, + "IsArc": False} + }) ccnt=0 nm=0 @@ -539,9 +549,13 @@ class PostProcessor(object): nm+=1 continue else: - temploc= watchmatch['justthedigits'].replace('_', ' ') + if cs['WatchValues']['Type'] == 'TPB' and cs['WatchValues']['Total'] > 1: + just_the_digits = re.sub('[^0-9]', '', watchmatch['seriesvolume']).strip() + else: + just_the_digits = watchmatch['justthedigits'] + temploc= just_the_digits.replace('_', ' ') temploc = re.sub('[\#\']', '', temploc) - logger.info('temploc: %s' % temploc) + logger.fdebug('temploc: %s' % temploc) datematch = "False" if any(['annual' in temploc.lower(), 'special' in temploc.lower()]) and mylar.CONFIG.ANNUALS_ON is True: diff --git a/mylar/filechecker.py b/mylar/filechecker.py index 465c37dc..83c3ef4e 100755 --- a/mylar/filechecker.py +++ b/mylar/filechecker.py @@ -1445,7 +1445,7 @@ class FileChecker(object): return {'AS_Alt': AS_Alt, 'AS_Tuple': AS_Tuple} - def checkthedate(self, txt, fulldate=False): + def checkthedate(self, txt, fulldate=False, cnt=0): # txt='''\ # Jan 19, 1990 # January 19, 1990 @@ -1457,8 +1457,9 @@ class FileChecker(object): # January1990''' fmts = ('%Y','%b %d, %Y','%B %d, %Y','%B %d %Y','%m/%d/%Y','%m/%d/%y','(%m/%d/%Y)','%b %Y','%B%Y','%b %d,%Y','%m-%Y','%B %Y','%Y-%m-%d','%Y-%m','%Y%m') - + mnths = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec') parsed=[] + if fulldate is False: for e in txt.splitlines(): for fmt in fmts: @@ -1470,14 +1471,36 @@ class FileChecker(object): pass else: for e in txt.split(): - logger.info('word: %s' % e) - for fmt in fmts: - try: - t = dt.datetime.strptime(e, fmt) - parsed.append((e, fmt, t)) - break - except ValueError as err: - pass + if cnt == 0: + for x in mnths: + mnth = re.sub('\.', '', e.lower()) + if x.lower() in mnth and len(mnth) <= 4: + add_date = x + ' ' + cnt+=1 + break + + elif cnt == 1: + issnumb = re.sub(',', '', e).strip() + if issnumb.isdigit() and int(issnumb) < 31: + add_date += issnumb + ', ' + cnt+=1 + elif cnt == 2: + possyear = helpers.cleanhtml(re.sub('\.', '', e).strip()) + if possyear.isdigit() and int(possyear) > 1970 and int(possyear) < 2020: + add_date += possyear + cnt +=1 + if cnt == 3: + return self.checkthedate(add_date, fulldate=False, cnt=-1) + + + if cnt <= 0: + for fmt in fmts: + try: + t = dt.datetime.strptime(e, fmt) + parsed.append((e, fmt, t)) + break + except ValueError as err: + pass # check that all the cases are handled success={t[0] for t in parsed} @@ -1485,13 +1508,13 @@ class FileChecker(object): if e not in success: pass #print e - dateyear = None + dateline = None #logger.info('parsed: %s' % parsed) for t in parsed: - # logger.fdebug('"{:20}" => "{:20}" => {}'.format(*t) - if fulldate is False: + #logger.fdebug('"{:20}" => "{:20}" => {}'.format(*t)) + if fulldate is False and cnt != -1: dateline = t[2].year else: dateline = t[2].strftime('%Y-%m-%d')