mirror of https://github.com/evilhero/mylar
FIX: Fix for filechecker returning traceback on some invalid date patterns, FIX: When a title contained the word special, but was not a part of a series (ie. annual integration) would fail to recognize it for post-processing
This commit is contained in:
evilhero
parent
25267995cc
commit
5503a363ba
|
|
@ -421,10 +421,11 @@ class PostProcessor(object):
|
|||
self.matched = False
|
||||
as_d = filechecker.FileChecker()
|
||||
as_dinfo = as_d.dynamic_replace(helpers.conversion(fl['series_name']))
|
||||
orig_seriesname = as_dinfo['mod_seriesname']
|
||||
mod_seriesname = as_dinfo['mod_seriesname']
|
||||
loopchk = []
|
||||
if fl['alt_series'] is not None:
|
||||
logger.info('%s Alternate series naming detected: %s' % (module, fl['alt_series']))
|
||||
logger.fdebug('%s Alternate series naming detected: %s' % (module, fl['alt_series']))
|
||||
as_sinfo = as_d.dynamic_replace(helpers.conversion(fl['alt_series']))
|
||||
mod_altseriesname = as_sinfo['mod_seriesname']
|
||||
if all([mylar.CONFIG.ANNUALS_ON, 'annual' in mod_altseriesname.lower()]) or all([mylar.CONFIG.ANNUALS_ON, 'special' in mod_altseriesname.lower()]):
|
||||
|
|
@ -456,31 +457,56 @@ class PostProcessor(object):
|
|||
tmpsql = "SELECT * FROM comics WHERE DynamicComicName IN ({seq}) COLLATE NOCASE".format(seq=','.join('?' * len(loopchk)))
|
||||
comicseries = myDB.select(tmpsql, tuple(loopchk))
|
||||
|
||||
if comicseries is None:
|
||||
logger.error(module + ' No Series in Watchlist - checking against Story Arcs (just in case). If I do not find anything, maybe you should be running Import?')
|
||||
break
|
||||
else:
|
||||
watchvals = []
|
||||
for wv in comicseries:
|
||||
#do some extra checks in here to ignore these types:
|
||||
#check for Paused status /
|
||||
#check for Ended status and 100% completion of issues.
|
||||
if wv['Status'] == 'Paused' or (wv['Have'] == wv['Total'] and not any(['Present' in wv['ComicPublished'], helpers.now()[:4] in wv['ComicPublished']])):
|
||||
logger.warn(wv['ComicName'] + ' [' + wv['ComicYear'] + '] is either Paused or in an Ended status with 100% completion. Ignoring for match.')
|
||||
continue
|
||||
wv_comicname = wv['ComicName']
|
||||
wv_comicpublisher = wv['ComicPublisher']
|
||||
wv_alternatesearch = wv['AlternateSearch']
|
||||
wv_comicid = wv['ComicID']
|
||||
if not comicseries:
|
||||
if (['special' in mod_seriesname.lower(), mylar.CONFIG.ANNUALS_ON, orig_seriesname != mod_seriesname]):
|
||||
loopchk.append(re.sub('[\|\s]', '', orig_seriesname.lower()))
|
||||
tmpsql = "SELECT * FROM comics WHERE DynamicComicName IN ({seq}) COLLATE NOCASE".format(seq=','.join('?' * len(loopchk)))
|
||||
comicseries = myDB.select(tmpsql, tuple(loopchk))
|
||||
if not comicseries:
|
||||
logger.error(module + ' No Series in Watchlist - checking against Story Arcs (just in case). If I do not find anything, maybe you should be running Import?')
|
||||
break
|
||||
|
||||
wv_seriesyear = wv['ComicYear']
|
||||
wv_comicversion = wv['ComicVersion']
|
||||
wv_publisher = wv['ComicPublisher']
|
||||
wv_total = wv['Total']
|
||||
if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
|
||||
logger.fdebug('Queuing to Check: ' + wv['ComicName'] + ' [' + str(wv['ComicYear']) + '] -- ' + str(wv['ComicID']))
|
||||
watchvals = []
|
||||
for wv in comicseries:
|
||||
#do some extra checks in here to ignore these types:
|
||||
#check for Paused status /
|
||||
#check for Ended status and 100% completion of issues.
|
||||
if wv['Status'] == 'Paused' or (wv['Have'] == wv['Total'] and not any(['Present' in wv['ComicPublished'], helpers.now()[:4] in wv['ComicPublished']])):
|
||||
logger.warn(wv['ComicName'] + ' [' + wv['ComicYear'] + '] is either Paused or in an Ended status with 100% completion. Ignoring for match.')
|
||||
continue
|
||||
wv_comicname = wv['ComicName']
|
||||
wv_comicpublisher = wv['ComicPublisher']
|
||||
wv_alternatesearch = wv['AlternateSearch']
|
||||
wv_comicid = wv['ComicID']
|
||||
if all([wv['Type'] != 'Print', wv['Type'] != 'Digital']) or wv['Corrected_Type'] == 'TPB':
|
||||
wv_type = 'TPB'
|
||||
else:
|
||||
wv_type = None
|
||||
wv_seriesyear = wv['ComicYear']
|
||||
wv_comicversion = wv['ComicVersion']
|
||||
wv_publisher = wv['ComicPublisher']
|
||||
wv_total = wv['Total']
|
||||
if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
|
||||
logger.fdebug('Queuing to Check: ' + wv['ComicName'] + ' [' + str(wv['ComicYear']) + '] -- ' + str(wv['ComicID']))
|
||||
|
||||
#force it to use the Publication Date of the latest issue instead of the Latest Date (which could be anything)
|
||||
#force it to use the Publication Date of the latest issue instead of the Latest Date (which could be anything)
|
||||
latestdate = myDB.select('SELECT IssueDate from issues WHERE ComicID=? order by ReleaseDate DESC', [wv['ComicID']])
|
||||
if latestdate:
|
||||
tmplatestdate = latestdate[0][0]
|
||||
if tmplatestdate[:4] != wv['LatestDate'][:4]:
|
||||
if tmplatestdate[:4] > wv['LatestDate'][:4]:
|
||||
latestdate = tmplatestdate
|
||||
else:
|
||||
latestdate = wv['LatestDate']
|
||||
else:
|
||||
latestdate = tmplatestdate
|
||||
else:
|
||||
latestdate = wv['LatestDate']
|
||||
|
||||
if latestdate == '0000-00-00' or latestdate == 'None' or latestdate is None:
|
||||
logger.fdebug('Forcing a refresh of series: ' + wv_comicname + ' as it appears to have incomplete issue dates.')
|
||||
updater.dbUpdate([wv_comicid])
|
||||
logger.fdebug('Refresh complete for ' + wv_comicname + '. Rechecking issue dates for completion.')
|
||||
latestdate = myDB.select('SELECT IssueDate from issues WHERE ComicID=? order by ReleaseDate DESC', [wv['ComicID']])
|
||||
if latestdate:
|
||||
tmplatestdate = latestdate[0][0]
|
||||
|
|
@ -494,41 +520,25 @@ class PostProcessor(object):
|
|||
else:
|
||||
latestdate = wv['LatestDate']
|
||||
|
||||
logger.fdebug('Latest Date (after forced refresh) set to :' + str(latestdate))
|
||||
|
||||
if latestdate == '0000-00-00' or latestdate == 'None' or latestdate is None:
|
||||
logger.fdebug('Forcing a refresh of series: ' + wv_comicname + ' as it appears to have incomplete issue dates.')
|
||||
updater.dbUpdate([wv_comicid])
|
||||
logger.fdebug('Refresh complete for ' + wv_comicname + '. Rechecking issue dates for completion.')
|
||||
latestdate = myDB.select('SELECT IssueDate from issues WHERE ComicID=? order by ReleaseDate DESC', [wv['ComicID']])
|
||||
if latestdate:
|
||||
tmplatestdate = latestdate[0][0]
|
||||
if tmplatestdate[:4] != wv['LatestDate'][:4]:
|
||||
if tmplatestdate[:4] > wv['LatestDate'][:4]:
|
||||
latestdate = tmplatestdate
|
||||
else:
|
||||
latestdate = wv['LatestDate']
|
||||
else:
|
||||
latestdate = tmplatestdate
|
||||
else:
|
||||
latestdate = wv['LatestDate']
|
||||
logger.fdebug('Unable to properly attain the Latest Date for series: ' + wv_comicname + '. Cannot check against this series for post-processing.')
|
||||
continue
|
||||
|
||||
logger.fdebug('Latest Date (after forced refresh) set to :' + str(latestdate))
|
||||
|
||||
if latestdate == '0000-00-00' or latestdate == 'None' or latestdate is None:
|
||||
logger.fdebug('Unable to properly attain the Latest Date for series: ' + wv_comicname + '. Cannot check against this series for post-processing.')
|
||||
continue
|
||||
|
||||
watchvals.append({"ComicName": wv_comicname,
|
||||
"ComicPublisher": wv_comicpublisher,
|
||||
"AlternateSearch": wv_alternatesearch,
|
||||
"ComicID": wv_comicid,
|
||||
"WatchValues": {"SeriesYear": wv_seriesyear,
|
||||
"LatestDate": latestdate,
|
||||
"ComicVersion": wv_comicversion,
|
||||
"Publisher": wv_publisher,
|
||||
"Total": wv_total,
|
||||
"ComicID": wv_comicid,
|
||||
"IsArc": False}
|
||||
})
|
||||
watchvals.append({"ComicName": wv_comicname,
|
||||
"ComicPublisher": wv_comicpublisher,
|
||||
"AlternateSearch": wv_alternatesearch,
|
||||
"ComicID": wv_comicid,
|
||||
"WatchValues": {"SeriesYear": wv_seriesyear,
|
||||
"LatestDate": latestdate,
|
||||
"ComicVersion": wv_comicversion,
|
||||
"Type": wv_type,
|
||||
"Publisher": wv_publisher,
|
||||
"Total": wv_total,
|
||||
"ComicID": wv_comicid,
|
||||
"IsArc": False}
|
||||
})
|
||||
|
||||
ccnt=0
|
||||
nm=0
|
||||
|
|
@ -539,9 +549,13 @@ class PostProcessor(object):
|
|||
nm+=1
|
||||
continue
|
||||
else:
|
||||
temploc= watchmatch['justthedigits'].replace('_', ' ')
|
||||
if cs['WatchValues']['Type'] == 'TPB' and cs['WatchValues']['Total'] > 1:
|
||||
just_the_digits = re.sub('[^0-9]', '', watchmatch['seriesvolume']).strip()
|
||||
else:
|
||||
just_the_digits = watchmatch['justthedigits']
|
||||
temploc= just_the_digits.replace('_', ' ')
|
||||
temploc = re.sub('[\#\']', '', temploc)
|
||||
logger.info('temploc: %s' % temploc)
|
||||
logger.fdebug('temploc: %s' % temploc)
|
||||
datematch = "False"
|
||||
|
||||
if any(['annual' in temploc.lower(), 'special' in temploc.lower()]) and mylar.CONFIG.ANNUALS_ON is True:
|
||||
|
|
|
|||
|
|
@ -1445,7 +1445,7 @@ class FileChecker(object):
|
|||
return {'AS_Alt': AS_Alt,
|
||||
'AS_Tuple': AS_Tuple}
|
||||
|
||||
def checkthedate(self, txt, fulldate=False):
|
||||
def checkthedate(self, txt, fulldate=False, cnt=0):
|
||||
# txt='''\
|
||||
# Jan 19, 1990
|
||||
# January 19, 1990
|
||||
|
|
@ -1457,8 +1457,9 @@ class FileChecker(object):
|
|||
# January1990'''
|
||||
|
||||
fmts = ('%Y','%b %d, %Y','%B %d, %Y','%B %d %Y','%m/%d/%Y','%m/%d/%y','(%m/%d/%Y)','%b %Y','%B%Y','%b %d,%Y','%m-%Y','%B %Y','%Y-%m-%d','%Y-%m','%Y%m')
|
||||
|
||||
mnths = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec')
|
||||
parsed=[]
|
||||
|
||||
if fulldate is False:
|
||||
for e in txt.splitlines():
|
||||
for fmt in fmts:
|
||||
|
|
@ -1470,14 +1471,36 @@ class FileChecker(object):
|
|||
pass
|
||||
else:
|
||||
for e in txt.split():
|
||||
logger.info('word: %s' % e)
|
||||
for fmt in fmts:
|
||||
try:
|
||||
t = dt.datetime.strptime(e, fmt)
|
||||
parsed.append((e, fmt, t))
|
||||
break
|
||||
except ValueError as err:
|
||||
pass
|
||||
if cnt == 0:
|
||||
for x in mnths:
|
||||
mnth = re.sub('\.', '', e.lower())
|
||||
if x.lower() in mnth and len(mnth) <= 4:
|
||||
add_date = x + ' '
|
||||
cnt+=1
|
||||
break
|
||||
|
||||
elif cnt == 1:
|
||||
issnumb = re.sub(',', '', e).strip()
|
||||
if issnumb.isdigit() and int(issnumb) < 31:
|
||||
add_date += issnumb + ', '
|
||||
cnt+=1
|
||||
elif cnt == 2:
|
||||
possyear = helpers.cleanhtml(re.sub('\.', '', e).strip())
|
||||
if possyear.isdigit() and int(possyear) > 1970 and int(possyear) < 2020:
|
||||
add_date += possyear
|
||||
cnt +=1
|
||||
if cnt == 3:
|
||||
return self.checkthedate(add_date, fulldate=False, cnt=-1)
|
||||
|
||||
|
||||
if cnt <= 0:
|
||||
for fmt in fmts:
|
||||
try:
|
||||
t = dt.datetime.strptime(e, fmt)
|
||||
parsed.append((e, fmt, t))
|
||||
break
|
||||
except ValueError as err:
|
||||
pass
|
||||
|
||||
# check that all the cases are handled
|
||||
success={t[0] for t in parsed}
|
||||
|
|
@ -1485,13 +1508,13 @@ class FileChecker(object):
|
|||
if e not in success:
|
||||
pass #print e
|
||||
|
||||
dateyear = None
|
||||
dateline = None
|
||||
|
||||
#logger.info('parsed: %s' % parsed)
|
||||
|
||||
for t in parsed:
|
||||
# logger.fdebug('"{:20}" => "{:20}" => {}'.format(*t)
|
||||
if fulldate is False:
|
||||
#logger.fdebug('"{:20}" => "{:20}" => {}'.format(*t))
|
||||
if fulldate is False and cnt != -1:
|
||||
dateline = t[2].year
|
||||
else:
|
||||
dateline = t[2].strftime('%Y-%m-%d')
|
||||
|
|
|
|||
Loading…
Reference in New Issue