From e86b5ce00f35288f4ddd87d1387559bf1babeaa2 Mon Sep 17 00:00:00 2001 From: evilhero Date: Wed, 18 Sep 2013 00:49:24 -0400 Subject: [PATCH] FIX:(#537)digitsvalid error, FIX:(#539) allowance for infinity symbol as Issue# (file-check fix for it next), IMP: Autocorrect ComicTagger path if .exe/.py given within path, IMP: Manual Run detection improvements (volume inclusions), IMP: Reading order # appending to storyarc filenames as option, IMP: story-arc filename/search revisions --- mylar/PostProcessor.py | 6 ++- mylar/filechecker.py | 109 ++++++++++++++++++++++++++++++++--------- mylar/helpers.py | 13 ++++- mylar/importer.py | 5 +- mylar/webserve.py | 66 +++++++++++++++++++++---- 5 files changed, 162 insertions(+), 37 deletions(-) diff --git a/mylar/PostProcessor.py b/mylar/PostProcessor.py index 9f22949f..adc3dce9 100755 --- a/mylar/PostProcessor.py +++ b/mylar/PostProcessor.py @@ -191,6 +191,8 @@ class PostProcessor(object): watchvals = {} for cs in comicseries: watchvals = {"SeriesYear": cs['ComicYear'], + "LatestDate": cs['LatestDate'], + "ComicVersion": cs['ComicVersion'], "Total": cs['Total']} watchmatch = filechecker.listFiles(self.nzb_folder,cs['ComicName'],cs['AlternateSearch'], manual=watchvals) if watchmatch is None: @@ -377,7 +379,9 @@ class PostProcessor(object): #if from a StoryArc, check to see if we're appending the ReadingOrder to the filename if mylar.READ2FILENAME: issuearcid = re.sub('S', '', issueid) - arcdata = myDB.action("SELECT * FROM readinglist WHERE IssueARCID=?",[issuearcid]).fetchone() + logger.fdebug('issuearcid:' + str(issuearcid)) + arcdata = myDB.action("SELECT * FROM readinglist WHERE IssueArcID=?",[issuearcid]).fetchone() + logger.fdebug('readingorder#: ' + str(arcdata['ReadingOrder'])) if int(arcdata['ReadingOrder']) < 10: readord = "00" + str(arcdata['ReadingOrder']) elif int(arcdata['ReadingOrder']) > 10 and int(arcdata['ReadingOrder']) < 99: readord = "0" + str(arcdata['ReadingOrder']) else: readord = str(arcdata['ReadingOrder']) diff --git a/mylar/filechecker.py b/mylar/filechecker.py index 3368a9a5..6fb3e1eb 100755 --- a/mylar/filechecker.py +++ b/mylar/filechecker.py @@ -27,7 +27,7 @@ def file2comicmatch(watchmatch): #print ("match: " + str(watchmatch)) pass -def listFiles(dir,watchcomic,AlternateSearch=None,manual=None): +def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): # use AlternateSearch to check for filenames that follow that naming pattern # ie. Star Trek TNG Doctor Who Assimilation won't get hits as the @@ -48,7 +48,7 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None): '\;', '.', '-', - '\!', + '!', '\$', '\%', '\+', @@ -248,6 +248,12 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None): else: jtd_len = len(cchk)# + nonocount + if sarc and mylar.READ2FILENAME: + removest = subname.find(' ') # the - gets removed above so we test for the first blank space... + if subname[:removest].isdigit(): + jtd_len += removest + 1 # +1 to account for space in place of - + logger.fdebug('adjusted jtd_len to : ' + str(removest) + ' because of story-arc reading order tags') + logger.fdebug('nonocount [' + str(nonocount) + '] cchk [' + cchk + '] length [' + str(len(cchk)) + ']') #if detectand: @@ -274,6 +280,8 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None): justthedigits = justthedigits_1.split(' ', 1)[0] + digitsvalid = "false" + try: tmpthedigits = justthedigits_1.split(' ', 1)[1] logger.fdebug('If the series has a decimal, this should be a number [' + tmpthedigits + ']') @@ -286,7 +294,6 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None): if justthedigits.lower() == 'annual': logger.fdebug('ANNUAL DETECTED [' + poss_alpha + ']') justthedigits += ' ' + poss_alpha - digitsvalid = "true" else: justthedigits += '.' + poss_alpha logger.fdebug('DECIMAL ISSUE DETECTED [' + justthedigits + ']') @@ -306,10 +313,14 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None): logger.fdebug('JUSTTHEDIGITS [' + justthedigits + ']' ) if justthedigits.isdigit(): digitsvalid = "true" -# else: -# if '.' in justthedigits: -# logger.fdebug('DECIMAL ISSUE DETECTED') -# digitsvalid = "true" + else: + if '.' in justthedigits: + tmpdec = justthedigits.find('.') + b4dec = justthedigits[:tmpdec] + a4dec = justthedigits[tmpdec+1:] + if a4dec.isdigit() and b4dec.isdigit(): + logger.fdebug('DECIMAL ISSUE DETECTED') + digitsvalid = "true" # else: # logger.fdebug('NO DECIMALS DETECTED') # digitsvalid = "false" @@ -352,39 +363,89 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None): logger.fdebug('subname = ' + subname.lower()) comyear = manual['SeriesYear'] issuetotal = manual['Total'] + comicvolume = manual['ComicVersion'] logger.fdebug('SeriesYear: ' + str(comyear)) logger.fdebug('IssueTotal: ' + str(issuetotal)) + logger.fdebug('Comic Volume: ' + str(comicvolume)) + logger.fdebug('volume detected: ' + str(volrem)) #set the issue/year threshold here. # 2013 - (24issues/12) = 2011. - minyear = int(comyear) - (int(issuetotal) / 12) + #minyear = int(comyear) - (int(issuetotal) / 12) + maxyear = manual['LatestDate'][:4] # yyyy-mm-dd + #subnm defined at being of module. len_sm = len(subnm) #print ("there are " + str(lenm) + " words.") cnt = 0 yearmatch = "false" + vers4year = "no" + vers4vol = "no" - while (cnt < len_sm): - if subnm[cnt] is None: break - if subnm[cnt] == ' ': - pass + for ct in subsplit: + if ct.lower().startswith('v') and ct[1:].isdigit(): + logger.fdebug("possible versioning..checking") + #we hit a versioning # - account for it + if ct[1:].isdigit(): + if len(ct[1:]) == 4: #v2013 + logger.fdebug("Version detected as " + str(ct)) + vers4year = "yes" #re.sub("[^0-9]", " ", str(ct)) #remove the v + break + else: + if len(ct) < 4: + logger.fdebug("Version detected as " + str(ct)) + vers4vol = str(ct) + break + logger.fdebug("false version detection..ignoring.") + + if vers4year is not "no" or vers4vol is not "no": + yearmatch = "false" + + if comicvolume: #is not "None" and comicvolume is not None: + D_ComicVersion = re.sub("[^0-9]", "", comicvolume) + if D_ComicVersion == '': + D_ComicVersion = 0 else: - logger.fdebug(str(cnt) + ". Bracket Word: " + str(subnm[cnt])) + D_ComicVersion = 0 - if subnm[cnt][:-2] == '19' or subnm[cnt][:-2] == '20': - logger.fdebug("year detected: " + str(subnm[cnt])) - result_comyear = subnm[cnt] - if int(result_comyear) >= int(minyear): - logger.fdebug(str(result_comyear) + ' is within the series range of ' + str(minyear) + '-' + str(comyear)) - yearmatch = "true" - break + F_ComicVersion = re.sub("[^0-9]", "", volrem) + S_ComicVersion = str(comyear) + logger.fdebug("FCVersion: " + str(F_ComicVersion)) + logger.fdebug("DCVersion: " + str(D_ComicVersion)) + logger.fdebug("SCVersion: " + str(S_ComicVersion)) + + #if annualize == "true" and int(ComicYear) == int(F_ComicVersion): + # logger.fdebug("We matched on versions for annuals " + str(volrem)) + + if int(F_ComicVersion) == int(D_ComicVersion) or int(F_ComicVersion) == int(S_ComicVersion): + logger.fdebug("We matched on versions..." + str(volrem)) + yearmatch = "true" + else: + logger.fdebug("Versions wrong. Ignoring possible match.") + + else: + while (cnt < len_sm): + if subnm[cnt] is None: break + if subnm[cnt] == ' ': + pass else: - logger.fdebug(str(result_comyear) + ' - not right - year not within series range of ' + str(minyear) + '-' + str(comyear)) - yearmatch = "false" - break - cnt+=1 + logger.fdebug(str(cnt) + ". Bracket Word: " + str(subnm[cnt])) + + if subnm[cnt][:-2] == '19' or subnm[cnt][:-2] == '20': + logger.fdebug("year detected: " + str(subnm[cnt])) + result_comyear = subnm[cnt] + if int(result_comyear) <= int(maxyear): + logger.fdebug(str(result_comyear) + ' is within the series range of ' + str(comyear) + '-' + str(maxyear)) + #still possible for incorrect match if multiple reboots of series end/start in same year + yearmatch = "true" + break + else: + logger.fdebug(str(result_comyear) + ' - not right - year not within series range of ' + str(comyear) + '-' + str(maxyear)) + yearmatch = "false" + break + cnt+=1 if yearmatch == "false": continue diff --git a/mylar/helpers.py b/mylar/helpers.py index c828b658..62d0cbf8 100755 --- a/mylar/helpers.py +++ b/mylar/helpers.py @@ -645,6 +645,9 @@ def issuedigits(issnum): elif u'\xbe' in issnum: issnum = .75 int_issnum = int(issnum) * 1000 + elif u'\u221e' in issnum: + #issnum = utf-8 will encode the infinity symbol without any help + int_issnum = 9999999999 * 1000 # set 9999999999 for integer value of issue elif '.' in issnum or ',' in issnum: #logger.fdebug('decimal detected.') if ',' in issnum: issnum = re.sub(',','.', issnum) @@ -660,7 +663,7 @@ def issuedigits(issnum): try: int_issnum = (int(issb4dec) * 1000) + (int(issaftdec) * 10) except ValueError: - logger.error('This has no issue # for me to get - Either a Graphic Novel or one-shot.') + logger.fdebug('This has no issue # for me to get - Either a Graphic Novel or one-shot.') int_issnum = 999999999999999 else: try: @@ -777,3 +780,11 @@ def urlretrieve(urlfile, fpath): f.write(data) print "Read %s bytes"%len(data) +def renamefile_readingorder(readorder): + import logger + logger.fdebug('readingorder#: ' + str(readorder)) + if int(readorder) < 10: readord = "00" + str(readorder) + elif int(readorder) > 10 and int(readorder) < 99: readord = "0" + str(readorder) + else: readord = str(readorder) + + return readord diff --git a/mylar/importer.py b/mylar/importer.py index 2575aa01..955ad48f 100755 --- a/mylar/importer.py +++ b/mylar/importer.py @@ -312,7 +312,7 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None): #comlocation = mylar.DESTINATION_DIR + "/" + comicdir + " (" + comic['ComicYear'] + ")" if mylar.DESTINATION_DIR == "": - logger.error('There is no general directory specified - please specify in Config/Post-Processing.') + logger.error('There is no Comic Location Path specified - please specify one in Config/Web Interface.') return if mylar.REPLACE_SPACES: #mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot @@ -608,6 +608,9 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None): elif u'\xbe' in issnum: issnum = .75 int_issnum = int(issnum) * 1000 + elif u'\u221e' in issnum: + #issnum = utf-8 will encode the infinity symbol without any help + int_issnum = 9999999999 * 1000 # set 9999999999 for integer value of issue elif '.' in issnum or ',' in issnum: if ',' in issnum: issnum = re.sub(',','.', issnum) issst = str(issnum).find('.') diff --git a/mylar/webserve.py b/mylar/webserve.py index 3defba00..1d8e01bb 100755 --- a/mylar/webserve.py +++ b/mylar/webserve.py @@ -1125,23 +1125,25 @@ class WebInterface(object): GCDissue = int(GCDissue) / 1000 if '.' not in str(GCDissue): GCDissue = str(GCDissue) + ".00" logger.fdebug("issue converted to " + str(GCDissue)) - isschk = myDB.action("SELECT * FROM issues WHERE ComicName=? AND Issue_Number=?", [comic['ComicName'], str(GCDissue)]).fetchone() + isschk = myDB.action("SELECT * FROM issues WHERE ComicName=? AND Issue_Number=? AND ComicID=?", [comic['ComicName'], str(GCDissue), comic['ComicID']]).fetchone() else: - isschk = myDB.action("SELECT * FROM issues WHERE ComicName=? AND Issue_Number=?", [comic['ComicName'], arc['IssueNumber']]).fetchone() + isschk = myDB.action("SELECT * FROM issues WHERE ComicName=? AND Issue_Number=? AND ComicID=?", [comic['ComicName'], arc['IssueNumber'], comic['ComicID']]).fetchone() if isschk is None: logger.fdebug("we matched on name, but issue " + str(arc['IssueNumber']) + " doesn't exist for " + comic['ComicName']) else: #this gets ugly - if the name matches and the issue, it could still be wrong series #use series year to break it down further. + logger.fdebug('COMIC-comicyear: ' + str(int(comic['ComicYear']))) + logger.fdebug('ARC-seriesyear: ' + str(int(arc['SeriesYear']))) if int(comic['ComicYear']) != int(arc['SeriesYear']): logger.fdebug("Series years are different - discarding match. " + str(comic['ComicYear']) + " != " + str(arc['SeriesYear'])) else: logger.fdebug("issue #: " + str(arc['IssueNumber']) + " is present!") - print isschk - print ("Comicname: " + arc['ComicName']) - #print ("ComicID: " + str(isschk['ComicID'])) - print ("Issue: " + arc['IssueNumber']) - print ("IssueArcID: " + arc['IssueArcID']) + logger.fdebug('isschk: ' + str(isschk)) + logger.fdebug("Comicname: " + arc['ComicName']) + logger.fdebug("ComicID: " + str(isschk['ComicID'])) + logger.fdebug("Issue: " + str(arc['IssueNumber'])) + logger.fdebug("IssueArcID: " + str(arc['IssueArcID'])) #gather the matches now. arc_match.append({ "match_storyarc": arc['storyarc'], @@ -1150,6 +1152,7 @@ class WebInterface(object): "match_issue": arc['IssueNumber'], "match_issuearcid": arc['IssueArcID'], "match_seriesyear": comic['ComicYear'], + "match_readingorder": arc['ReadingOrder'], "match_filedirectory": comic['ComicLocation']}) matcheroso = "yes" break @@ -1160,6 +1163,36 @@ class WebInterface(object): "IssueNumber": arc['IssueNumber'], "IssueYear": arc['IssueYear']}) + dstloc = os.path.join(mylar.DESTINATION_DIR, 'StoryArcs', arc['storyarc']) + logger.fdebug('destination location set to : ' + dstloc) + filechk = filechecker.listFiles(dstloc, arc['ComicName'], sarc='true') + fn = 0 + fccnt = filechk['comiccount'] + while (fn < fccnt): + haveissue = "no" + issuedupe = "no" + try: + tmpfc = filechk['comiclist'][fn] + except IndexError: + break + temploc = tmpfc['JusttheDigits'].replace('_', ' ') + fcdigit = helpers.issuedigits(arc['IssueNumber']) + int_iss = helpers.issuedigits(temploc) + if int_iss == fcdigit: + logger.fdebug(arc['ComicName'] + ' Issue #' + arc['IssueNumber'] + ' already present in StoryArc directory.') + #update readinglist db to reflect status. + if mylar.READ2FILENAME: + readorder = helpers.renamefile_readingorder(arc['ReadingOrder']) + dfilename = str(readorder) + "-" + tmpfc['ComicFilename'] + else: + dfilename = tmpfc['ComicFilename'] + + newVal = {"Status": "Downloaded", + "Location": dfilename} #tmpfc['ComicFilename']} + ctrlVal = {"IssueArcID": arc['IssueArcID'] } + myDB.upsert("readinglist",newVal,ctrlVal) + fn+=1 + logger.fdebug("we matched on " + str(len(arc_match)) + " issues") for m_arc in arc_match: @@ -1187,13 +1220,23 @@ class WebInterface(object): myDB.upsert("readinglist",newVal,ctrlVal) logger.info("Already have " + issue['ComicName'] + " :# " + str(issue['Issue_Number'])) if issue['Status'] == 'Downloaded': + issloc = os.path.join(m_arc['match_filedirectory'], issue['Location']) + logger.fdebug('source location set to : ' + issloc) + dstloc = os.path.join(mylar.DESTINATION_DIR, 'StoryArcs', m_arc['match_storyarc']) + logger.fdebug('destination location set to : ' + dstloc) + logger.fdebug('attempting to copy into StoryArc directory') #copy into StoryArc directory... - issloc = os.path.join(m_arc['match_filedirectory'], issue['Location']) - logger.fdebug('issloc set to : ' + issloc) if os.path.isfile(issloc): - dstloc = os.path.join(mylar.DESTINATION_DIR, 'StoryArcs', m_arc['match_storyarc']) if not os.path.isfile(dstloc): + if mylar.READ2FILENAME: + readorder = helpers.renamefile_readingorder(m_arc['match_readingorder']) + dfilename = str(readorder) + "-" + issue['Location'] + else: + dfilename = issue['Location'] + + dstloc = os.path.join(mylar.DESTINATION_DIR, 'StoryArcs', m_arc['match_storyarc'], dfilename) + logger.fdebug('copying ' + issloc + ' to ' + dstloc) shutil.copy(issloc, dstloc) else: @@ -2182,6 +2225,9 @@ class WebInterface(object): if mylar.CMTAGGER_PATH is None or mylar.CMTAGGER_PATH == '': logger.info("ComicTagger Path not set - defaulting to Mylar Program Directory : " + mylar.PROG_DIR) mylar.CMTAGGER_PATH = mylar.PROG_DIR + if 'comictagger.exe' in mylar.CMTAGGER_PATH.lower() or 'comictagger.py' in mylar.CMTAGGER_PATH.lower(): + mylar.CMTAGGER_PATH = re.sub(os.path.basename(mylar.CMTAGGER_PATH), '', mylar.CMTAGGER_PATH) + logger.fdebug("Removed application name from ComicTagger path") # Write the config mylar.config_write()