# This file is part of Mylar. # # Mylar is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Mylar is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Mylar. If not, see . import time import datetime from xml.dom.minidom import parseString import urllib2 import shlex import re import os import itertools import mylar from mylar import db, logger, helpers, filechecker def dbUpdate(ComicIDList=None, calledfrom=None): myDB = db.DBConnection() #print "comicidlist:" + str(ComicIDList) if ComicIDList is None: if mylar.UPDATE_ENDED: logger.info('Updating only Continuing Series (option enabled) - this might cause problems with the pull-list matching for rebooted series') comiclist = [] completelist = myDB.select('SELECT LatestDate, ComicPublished, ForceContinuing, NewPublish, LastUpdated, ComicID, ComicName from comics WHERE Status="Active" or Status="Loading" order by LatestDate DESC, LastUpdated ASC') for comlist in completelist: if comlist['LatestDate'] is None: recentstatus = 'Loading' elif comlist['ComicPublished'] is None or comlist['ComicPublished'] == '' or comlist['LatestDate'] is None: recentstatus = 'Unknown' elif comlist['ForceContinuing'] == 1: recentstatus = 'Continuing' elif 'present' in comlist['ComicPublished'].lower() or (helpers.today()[:4] in comlist['LatestDate']): latestdate = comlist['LatestDate'] c_date = datetime.date(int(latestdate[:4]), int(latestdate[5:7]), 1) n_date = datetime.date.today() recentchk = (n_date - c_date).days if comlist['NewPublish']: recentstatus = 'Continuing' else: if recentchk < 55: recentstatus = 'Continuing' else: recentstatus = 'Ended' else: recentstatus = 'Ended' if recentstatus == 'Continuing': comiclist.append({"LatestDate": comlist['LatestDate'], "LastUpdated": comlist['LastUpdated'], "ComicID": comlist['ComicID'], "ComicName": comlist['ComicName']}) else: comiclist = myDB.select('SELECT LatestDate, LastUpdated, ComicID, ComicName from comics WHERE Status="Active" or Status="Loading" order by LatestDate DESC, LastUpdated ASC') else: comiclist = ComicIDList if calledfrom is None: logger.info('Starting update for %i active comics' % len(comiclist)) cnt = 1 for comic in comiclist: if ComicIDList is None: ComicID = comic['ComicID'] ComicName = comic['ComicName'] c_date = comic['LastUpdated'] if c_date is None: logger.error(ComicName + ' failed during a previous add /refresh as it has no Last Update timestamp. Forcing refresh now.') else: c_obj_date = datetime.datetime.strptime(c_date, "%Y-%m-%d %H:%M:%S") n_date = datetime.datetime.now() absdiff = abs(n_date - c_obj_date) hours = (absdiff.days * 24 * 60 * 60 + absdiff.seconds) / 3600.0 if hours < 5: logger.info(ComicName + '[' + str(ComicID) + '] Was refreshed less than 5 hours ago. Skipping Refresh at this time.') cnt +=1 continue logger.info('[' + str(cnt) + '/' + str(len(comiclist)) + '] Refreshing :' + ComicName + ' [' + str(ComicID) + ']') else: ComicID = comic logger.fdebug('Refreshing :' + str(ComicID)) mismatch = "no" if not mylar.CV_ONLY or ComicID[:1] == "G": CV_EXcomicid = myDB.selectone("SELECT * from exceptions WHERE ComicID=?", [ComicID]).fetchone() if CV_EXcomicid is None: pass else: if CV_EXcomicid['variloop'] == '99': mismatch = "yes" if ComicID[:1] == "G": mylar.importer.GCDimport(ComicID) else: importer.addComictoDB(ComicID, mismatch) else: if mylar.CV_ONETIMER == 1: logger.fdebug("CV_OneTimer option enabled...") #in order to update to JUST CV_ONLY, we need to delete the issues for a given series so it's a clea$ logger.fdebug("Gathering the status of all issues for the series.") issues = myDB.select('SELECT * FROM issues WHERE ComicID=?', [ComicID]) if not issues: #if issues are None it's probably a bad refresh/maxed out API that resulted in the issue data #getting wiped out and not refreshed. Setting whack=True will force a complete refresh. logger.fdebug('No issue data available. This is Whack.') whack = True else: #check for series that are numerically out of whack (ie. 5/4) logger.fdebug('Checking how out of whack the series is.') whack = helpers.havetotals(refreshit=ComicID) if calledfrom == 'weekly': if whack == True: logger.info('Series is out of whack. Forcibly refreshing series to ensure everything is in order.') return True else: return False annload = [] #initiate the list here so we don't error out below. if mylar.ANNUALS_ON: #now we load the annuals into memory to pass through to importer when refreshing so that it can #refresh even the manually added annuals. annual_load = myDB.select('SELECT * FROM annuals WHERE ComicID=?', [ComicID]) logger.fdebug('checking annual db') for annthis in annual_load: if not any(d['ReleaseComicID'] == annthis['ReleaseComicID'] for d in annload): #print 'matched on annual' annload.append({ 'ReleaseComicID': annthis['ReleaseComicID'], 'ReleaseComicName': annthis['ReleaseComicName'], 'ComicID': annthis['ComicID'], 'ComicName': annthis['ComicName'] }) #print 'added annual' issues += annual_load #myDB.select('SELECT * FROM annuals WHERE ComicID=?', [ComicID]) #store the issues' status for a given comicid, after deleting and readding, flip the status back to$ logger.fdebug("Deleting all issue data.") myDB.action('DELETE FROM issues WHERE ComicID=?', [ComicID]) myDB.action('DELETE FROM annuals WHERE ComicID=?', [ComicID]) logger.fdebug("Refreshing the series and pulling in new data using only CV.") if whack == False: mylar.importer.addComictoDB(ComicID, mismatch, calledfrom='dbupdate', annload=annload) #reload the annuals here. issues_new = myDB.select('SELECT * FROM issues WHERE ComicID=?', [ComicID]) annuals = [] ann_list = [] if mylar.ANNUALS_ON: annuals_list = myDB.select('SELECT * FROM annuals WHERE ComicID=?', [ComicID]) ann_list += annuals_list issues_new += annuals_list logger.fdebug("Attempting to put the Status' back how they were.") icount = 0 #the problem - the loop below will not match on NEW issues that have been refreshed that weren't present in the #db before (ie. you left Mylar off for abit, and when you started it up it pulled down new issue information) #need to test if issuenew['Status'] is None, but in a seperate loop below. fndissue = [] for issue in issues: for issuenew in issues_new: #logger.fdebug(str(issue['Issue_Number']) + ' - issuenew:' + str(issuenew['IssueID']) + ' : ' + str(issuenew['Status'])) #logger.fdebug(str(issue['Issue_Number']) + ' - issue:' + str(issue['IssueID']) + ' : ' + str(issue['Status'])) try: if issuenew['IssueID'] == issue['IssueID'] and (issuenew['Status'] != issue['Status'] or issue['IssueDate_Edit'] is not None): ctrlVAL = {"IssueID": issue['IssueID']} #if the status is None and the original status is either Downloaded / Archived, keep status & stats if issuenew['Status'] == None and (issue['Status'] == 'Downloaded' or issue['Status'] == 'Archived'): newVAL = {"Location": issue['Location'], "ComicSize": issue['ComicSize'], "Status": issue['Status']} #if the status is now Downloaded/Snatched, keep status & stats (downloaded only) elif issuenew['Status'] == 'Downloaded' or issue['Status'] == 'Snatched': newVAL = {"Location": issue['Location'], "ComicSize": issue['ComicSize']} if issuenew['Status'] == 'Downloaded': newVAL['Status'] = issuenew['Status'] else: newVAL['Status'] = issue['Status'] elif issue['Status'] == 'Archived': newVAL = {"Status": issue['Status'], "Location": issue['Location'], "ComicSize": issue['ComicSize']} else: #change the status to the previous status newVAL = {"Status": issue['Status']} if newVAL['Status'] == None: datechk = re.sub('-', '', newissue['ReleaseDate']).strip() # converts date to 20140718 format if mylar.AUTOWANT_ALL: newVAL = {"Status": "Wanted"} elif int(datechk) >= int(nowtime) and mylar.AUTOWANT_UPCOMING: newVAL = {"Status": "Wanted"} else: newVAL = {"Status": "Skipped"} if issue['IssueDate_Edit']: logger.info('[#' + str(issue['Issue_Number']) + '] detected manually edited Issue Date.') logger.info('new value : ' + str(issue['IssueDate']) + ' ... cv value : ' + str(issuenew['IssueDate'])) newVAL['IssueDate'] = issue['IssueDate'] newVAL['IssueDate_Edit'] = issue['IssueDate_Edit'] if any(d['IssueID'] == str(issue['IssueID']) for d in ann_list): #logger.fdebug("annual detected for " + str(issue['IssueID']) + " #: " + str(issue['Issue_Number'])) myDB.upsert("Annuals", newVAL, ctrlVAL) else: #logger.fdebug('#' + str(issue['Issue_Number']) + ' writing issuedata: ' + str(newVAL)) myDB.upsert("Issues", newVAL, ctrlVAL) fndissue.append({"IssueID": issue['IssueID']}) icount+=1 break except: logger.warn('Something is out of whack somewhere with the series.') #if it's an annual (ie. deadpool-2011 ) on a refresh will throw index errors for some reason. logger.info("In the process of converting the data to CV, I changed the status of " + str(icount) + " issues.") issuesnew = myDB.select('SELECT * FROM issues WHERE ComicID=? AND Status is NULL', [ComicID]) if mylar.AUTOWANT_UPCOMING: newstatus = "Wanted" else: newstatus = "Skipped" newiss = [] for iss in issuesnew: newiss.append({"IssueID": iss['IssueID'], "Status": newstatus, "Annual": False}) if mylar.ANNUALS_ON: annualsnew = myDB.select('SELECT * FROM annuals WHERE ComicID=? AND Status is NULL', [ComicID]) for ann in annualsnew: newiss.append({"IssueID": iss['IssueID'], "Status": newstatus, "Annual": True}) if len(newiss) > 0: for newi in newiss: ctrlVAL = {"IssueID": newi['IssueID']} newVAL = {"Status": newi['Status']} #logger.fdebug('writing issuedata: ' + str(newVAL)) if newi['Annual'] == True: myDB.upsert("Annuals", newVAL, ctrlVAL) else: myDB.upsert("Issues", newVAL, ctrlVAL) logger.info('I have added ' + str(len(newiss)) + ' new issues for this series that were not present before.') forceRescan(ComicID) else: mylar.importer.addComictoDB(ComicID, mismatch, annload=annload) else: mylar.importer.addComictoDB(ComicID, mismatch) cnt +=1 time.sleep(5) #pause for 5 secs so dont hammer CV and get 500 error logger.info('Update complete') def latest_update(ComicID, LatestIssue, LatestDate): # here we add to comics.latest #logger.info(str(ComicID) + ' - updating latest_date to : ' + str(LatestDate)) myDB = db.DBConnection() latestCTRLValueDict = {"ComicID": ComicID} newlatestDict = {"LatestIssue": str(LatestIssue), "LatestDate": str(LatestDate)} myDB.upsert("comics", newlatestDict, latestCTRLValueDict) def upcoming_update(ComicID, ComicName, IssueNumber, IssueDate, forcecheck=None, futurepull=None, altissuenumber=None): # here we add to upcoming table... myDB = db.DBConnection() dspComicName = ComicName #to make sure that the word 'annual' will be displayed on screen if 'annual' in ComicName.lower(): adjComicName = re.sub("\\bannual\\b", "", ComicName.lower()) # for use with comparisons. logger.fdebug('annual detected - adjusting name to : ' + adjComicName) else: adjComicName = ComicName controlValue = {"ComicID": ComicID} newValue = {"ComicName": adjComicName, "IssueNumber": str(IssueNumber), "DisplayComicName": dspComicName, "IssueDate": str(IssueDate)} #let's refresh the series here just to make sure if an issue is available/not. mismatch = "no" CV_EXcomicid = myDB.selectone("SELECT * from exceptions WHERE ComicID=?", [ComicID]).fetchone() if CV_EXcomicid is None: pass else: if CV_EXcomicid['variloop'] == '99': mismatch = "yes" lastupdatechk = myDB.selectone("SELECT * FROM comics WHERE ComicID=?", [ComicID]).fetchone() if lastupdatechk is None: pullupd = "yes" else: c_date = lastupdatechk['LastUpdated'] if c_date is None: logger.error(lastupdatechk['ComicName'] + ' failed during a previous add /refresh. Please either delete and readd the series, or try a refresh of the series.') return c_obj_date = datetime.datetime.strptime(c_date, "%Y-%m-%d %H:%M:%S") n_date = datetime.datetime.now() absdiff = abs(n_date - c_obj_date) hours = (absdiff.days * 24 * 60 * 60 + absdiff.seconds) / 3600.0 # no need to hammer the refresh # let's check it every 5 hours (or more) #pullupd = "yes" if 'annual' in ComicName.lower(): if mylar.ANNUALS_ON: issuechk = myDB.selectone("SELECT * FROM annuals WHERE ComicID=? AND Issue_Number=?", [ComicID, IssueNumber]).fetchone() else: logger.fdebug('Annual detected, but annuals not enabled. Ignoring result.') return else: issuechk = myDB.selectone("SELECT * FROM issues WHERE ComicID=? AND Issue_Number=?", [ComicID, IssueNumber]).fetchone() if issuechk is None and altissuenumber is not None: logger.info('altissuenumber is : ' + str(altissuenumber)) issuechk = myDB.selectone("SELECT * FROM issues WHERE ComicID=? AND Int_IssueNumber=?", [ComicID, helpers.issuedigits(altissuenumber)]).fetchone() if issuechk is None: if futurepull is None: og_status = None logger.fdebug(adjComicName + ' Issue: ' + str(IssueNumber) + ' not present in listings to mark for download...updating comic and adding to Upcoming Wanted Releases.') # we need to either decrease the total issue count, OR indicate that an issue is upcoming. upco_results = myDB.select("SELECT COUNT(*) FROM UPCOMING WHERE ComicID=?", [ComicID]) upco_iss = upco_results[0][0] #logger.info("upco_iss: " + str(upco_iss)) if int(upco_iss) > 0: #logger.info("There is " + str(upco_iss) + " of " + str(ComicName) + " that's not accounted for") newKey = {"ComicID": ComicID} newVal = {"not_updated_db": str(upco_iss)} myDB.upsert("comics", newVal, newKey) elif int(upco_iss) <=0 and lastupdatechk['not_updated_db']: #if not_updated_db has a value, and upco_iss is > 0, let's zero it back out cause it's updated now. newKey = {"ComicID": ComicID} newVal = {"not_updated_db": ""} myDB.upsert("comics", newVal, newKey) if hours > 5 or forcecheck == 'yes': pullupd = "yes" logger.fdebug('Now Refreshing comic ' + ComicName + ' to make sure it is up-to-date') if ComicID[:1] == "G": mylar.importer.GCDimport(ComicID, pullupd) else: mylar.importer.updateissuedata(ComicID, ComicName, calledfrom='weeklycheck')#mylar.importer.addComictoDB(ComicID,mismatch,pullupd) else: logger.fdebug('It has not been longer than 5 hours since we last did this...we will wait so we do not hammer things.') logger.fdebug('linking ComicID to Pull-list to reflect status.') downstats = {"ComicID": ComicID, "IssueID": None, "Status": None} return downstats else: # if futurepull is not None, let's just update the status and ComicID # NOTE: THIS IS CREATING EMPTY ENTRIES IN THE FUTURE TABLE. ??? nKey = {"ComicID": ComicID} nVal = {"Status": "Wanted"} myDB.upsert("future", nVal, nKey) return if issuechk is not None: if issuechk['Issue_Number'] == IssueNumber or issuechk['Issue_Number'] == altissuenumber: og_status = issuechk['Status'] #check for 'out-of-whack' series here. whackness = dbUpdate([ComicID], calledfrom='weekly') if whackness == True: if any([issuechk['Status'] == 'Downloaded', issuechk['Status'] == 'Archived', issuechk['Status'] == 'Snatched']): logger.fdebug('Forcibly maintaining status of : ' + og_status + ' for #' + issuechk['Issue_Number'] + ' to ensure integrity.') logger.fdebug('Comic series has an incorrect total count. Forcily refreshing series to ensure data is current.') dbUpdate([ComicID]) issuechk = myDB.selectone("SELECT * FROM issues WHERE ComicID=? AND Int_IssueNumber=?", [ComicID, helpers.issuedigits(IssueNumber)]).fetchone() if issuechk['Status'] != og_status and (issuechk['Status'] != 'Downloaded' or issuechk['Status'] != 'Archived' or issuechk['Status'] != 'Snatched'): logger.fdebug('Forcibly changing status of ' + issuechk['Status'] + ' back to ' + og_status + ' for #' + issuechk['Issue_Number'] + ' to stop repeated downloads.') else: logger.fdebug('[' + issuechk['Status'] + '] / [' + og_status + '] Status has not changed during refresh or is marked as being Wanted/Skipped correctly.') og_status = issuechk['Status'] else: logger.fdebug('Comic series already up-to-date ... no need to refresh at this time.') logger.fdebug('Available to be marked for download - checking...' + adjComicName + ' Issue: ' + str(issuechk['Issue_Number'])) logger.fdebug('...Existing status: ' + og_status) control = {"IssueID": issuechk['IssueID']} newValue['IssueID'] = issuechk['IssueID'] if og_status == "Snatched": values = {"Status": "Snatched"} newValue['Status'] = "Snatched" elif og_status == "Downloaded": values = {"Status": "Downloaded"} newValue['Status'] = "Downloaded" #if the status is Downloaded and it's on the pullist - let's mark it so everyone can bask in the glory elif og_status == "Wanted": values = {"Status": "Wanted"} newValue['Status'] = "Wanted" elif og_status == "Archived": values = {"Status": "Archived"} newValue['Status'] = "Archived" elif og_status == 'Failed': if mylar.FAILED_DOWNLOAD_HANDLING: if mylar.FAILED_AUTO: values = {"Status": "Wanted"} newValue['Status'] = "Wanted" else: values = {"Status": "Failed"} newValue['Status'] = "Failed" else: values = {"Status": "Skipped"} newValue['Status'] = "Skipped" else: values = {"Status": "Skipped"} newValue['Status'] = "Skipped" #was in wrong place :( else: logger.fdebug('Issues do not match for some reason...weekly new issue: ' + str(IssueNumber)) return if mylar.AUTOWANT_UPCOMING: #for issues not in db - to be added to Upcoming table. if og_status is None: newValue['Status'] = "Wanted" logger.fdebug('...Changing Status to Wanted and throwing it in the Upcoming section since it is not published yet.') #this works for issues existing in DB... elif og_status == "Skipped": newValue['Status'] = "Wanted" values = {"Status": "Wanted"} logger.fdebug('...New status of Wanted') elif og_status == "Wanted": logger.fdebug('...Status already Wanted .. not changing.') else: logger.fdebug('...Already have issue - keeping existing status of : ' + og_status) if issuechk is None: myDB.upsert("upcoming", newValue, controlValue) logger.fdebug('updating Pull-list to reflect status.') downstats = {"Status": newValue['Status'], "ComicID": ComicID, "IssueID": None} return downstats else: logger.fdebug('--attempt to find errant adds to Wanted list') logger.fdebug('UpcomingNewValue: ' + str(newValue)) logger.fdebug('UpcomingcontrolValue: ' + str(controlValue)) if issuechk['IssueDate'] == '0000-00-00' and newValue['IssueDate'] != '0000-00-00': logger.fdebug('Found a 0000-00-00 issue - force updating series to try and get it proper.') dateVal = {"IssueDate": newValue['IssueDate'], "ComicName": issuechk['ComicName'], "Status": newValue['Status'], "IssueNumber": issuechk['Issue_Number']} logger.fdebug('updating date in upcoming table to : ' + str(newValue['IssueDate']) + '[' + newValue['Status'] + ']') logger.fdebug('ComicID:' + str(controlValue)) myDB.upsert("upcoming", dateVal, controlValue) logger.fdebug('Temporarily putting the Issue Date for ' + str(issuechk['Issue_Number']) + ' to ' + str(newValue['IssueDate'])) values = {"IssueDate": newValue['IssueDate']} #if ComicID[:1] == "G": mylar.importer.GCDimport(ComicID,pullupd='yes') #else: mylar.importer.addComictoDB(ComicID,mismatch,pullupd='yes') if 'annual' in ComicName.lower(): myDB.upsert("annuals", values, control) else: myDB.upsert("issues", values, control) if any([og_status == 'Downloaded', og_status == 'Archived', og_status == 'Snatched', og_status == 'Wanted', newValue['Status'] == 'Wanted']): logger.fdebug('updating Pull-list to reflect status change: ' + og_status + '[' + newValue['Status'] + ']') if og_status != 'Skipped': downstats = {"Status": og_status, "ComicID": issuechk['ComicID'], "IssueID": issuechk['IssueID']} else: downstats = {"Status": newValue['Status'], "ComicID": issuechk['ComicID'], "IssueID": issuechk['IssueID']} return downstats def weekly_update(ComicName, IssueNumber, CStatus, CID, futurepull=None, altissuenumber=None): if futurepull: logger.fdebug('future_update of table : ' + str(ComicName) + ' #:' + str(IssueNumber) + ' to a status of ' + str(CStatus)) else: logger.fdebug('weekly_update of table : ' + str(ComicName) + ' #:' + str(IssueNumber) + ' to a status of ' + str(CStatus)) if altissuenumber: logger.fdebug('weekly_update of table : ' + str(ComicName) + ' (Alternate Issue #):' + str(altissuenumber) + ' to a status of ' + str(CStatus)) # here we update status of weekly table... # added Issue to stop false hits on series' that have multiple releases in a week # added CStatus to update status flags on Pullist screen myDB = db.DBConnection() if futurepull is None: issuecheck = myDB.selectone("SELECT * FROM weekly WHERE COMIC=? AND ISSUE=?", [ComicName, IssueNumber]).fetchone() else: issuecheck = myDB.selectone("SELECT * FROM future WHERE COMIC=? AND ISSUE=?", [ComicName, IssueNumber]).fetchone() if issuecheck is not None: controlValue = {"COMIC": str(ComicName), "ISSUE": str(IssueNumber)} try: if CID['IssueID']: cidissueid = CID['IssueID'] else: cidissueid = None except: cidissueid = None if CStatus: newValue = {"STATUS": CStatus} else: if mylar.AUTOWANT_UPCOMING: newValue = {"STATUS": "Wanted"} else: newValue = {"STATUS": "Skipped"} #setting this here regardless, as it will be a match for a watchlist hit at this point anyways - so link it here what's availalbe. newValue['ComicID'] = CID['ComicID'] newValue['IssueID'] = cidissueid if futurepull is None: myDB.upsert("weekly", newValue, controlValue) else: logger.fdebug('checking ' + str(issuecheck['ComicID']) + ' status of : ' + str(CStatus)) if issuecheck['ComicID'] is not None and CStatus != None: newValue = {"STATUS": "Wanted", "ComicID": issuecheck['ComicID']} logger.fdebug('updating value: ' + str(newValue)) logger.fdebug('updating control: ' + str(controlValue)) myDB.upsert("future", newValue, controlValue) def newpullcheck(ComicName, ComicID, issue=None): # When adding a new comic, let's check for new issues on this week's pullist and update. mylar.weeklypull.pullitcheck(comic1off_name=ComicName, comic1off_id=ComicID, issue=issue) return def no_searchresults(ComicID): # when there's a mismatch between CV & GCD - let's change the status to # something other than 'Loaded' myDB = db.DBConnection() controlValue = {"ComicID": ComicID} newValue = {"Status": "Error", "LatestDate": "Error", "LatestIssue": "Error"} myDB.upsert("comics", newValue, controlValue) def nzblog(IssueID, NZBName, ComicName, SARC=None, IssueArcID=None, id=None, prov=None, alt_nzbname=None): myDB = db.DBConnection() newValue = {'NZBName': NZBName} if SARC: IssueID = 'S' + str(IssueArcID) newValue['SARC'] = SARC if IssueID is None or IssueID == 'None': #if IssueID is None, it's a one-off download from the pull-list. #give it a generic ID above the last one so it doesn't throw an error later. logger.fdebug("Story Arc (SARC) detected as: " + str(SARC)) if mylar.HIGHCOUNT == 0: IssueID = '900000' else: IssueID = int(mylar.HIGHCOUNT) + 1 controlValue = {"IssueID": IssueID, "Provider": prov} if id: logger.info('setting the nzbid for this download grabbed by ' + prov + ' in the nzblog to : ' + str(id)) newValue['ID'] = id if alt_nzbname: logger.info('setting the alternate nzbname for this download grabbed by ' + prov + ' in the nzblog to : ' + alt_nzbname) newValue['AltNZBName'] = alt_nzbname #check if it exists already in the log. chkd = myDB.selectone('SELECT * FROM nzblog WHERE IssueID=? and Provider=?', [IssueID, prov]).fetchone() if chkd is None: pass else: if chkd['AltNZBName'] is None or chkd['AltNZBName'] == '': #we need to wipe the entry so we can re-update with the alt-nzbname if required myDB.action('DELETE FROM nzblog WHERE IssueID=? and Provider=?', [IssueID, prov]) logger.fdebug('Deleted stale entry from nzblog for IssueID: ' + str(IssueID) + ' [' + prov + ']') myDB.upsert("nzblog", newValue, controlValue) def foundsearch(ComicID, IssueID, mode=None, down=None, provider=None, SARC=None, IssueArcID=None, module=None): # When doing a Force Search (Wanted tab), the resulting search calls this to update. # this is all redudant code that forceRescan already does. # should be redone at some point so that instead of rescanning entire # series directory, it just scans for the issue it just downloaded and # and change the status to Snatched accordingly. It is not to increment the have count # at this stage as it's not downloaded - just the .nzb has been snatched and sent to SAB. if module is None: module = '' module += '[UPDATER]' myDB = db.DBConnection() modcomicname = False logger.fdebug(module + ' comicid: ' + str(ComicID)) logger.fdebug(module + ' issueid: ' + str(IssueID)) if mode != 'story_arc': comic = myDB.selectone('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone() ComicName = comic['ComicName'] if mode == 'want_ann': issue = myDB.selectone('SELECT * FROM annuals WHERE IssueID=?', [IssueID]).fetchone() if ComicName != issue['ReleaseComicName'] + ' Annual': ComicName = issue['ReleaseComicName'] modcomicname = True else: issue = myDB.selectone('SELECT * FROM issues WHERE IssueID=?', [IssueID]).fetchone() CYear = issue['IssueDate'][:4] else: issue = myDB.selectone('SELECT * FROM readinglist WHERE IssueArcID=?', [IssueArcID]).fetchone() ComicName = issue['ComicName'] CYear = issue['IssueYEAR'] if down is None: # update the status to Snatched (so it won't keep on re-downloading!) logger.info(module + ' Updating status to snatched') logger.fdebug(module + ' Provider is ' + provider) newValue = {"Status": "Snatched"} if mode == 'story_arc': cValue = {"IssueArcID": IssueArcID} snatchedupdate = {"IssueArcID": IssueArcID} myDB.upsert("readinglist", newValue, cValue) # update the snatched DB snatchedupdate = {"IssueID": IssueArcID, "Status": "Snatched", "Provider": provider } else: if mode == 'want_ann': controlValue = {"IssueID": IssueID} myDB.upsert("annuals", newValue, controlValue) else: controlValue = {"IssueID": IssueID} myDB.upsert("issues", newValue, controlValue) # update the snatched DB snatchedupdate = {"IssueID": IssueID, "Status": "Snatched", "Provider": provider } if mode == 'story_arc': IssueNum = issue['IssueNumber'] newsnatchValues = {"ComicName": ComicName, "ComicID": 'None', "Issue_Number": IssueNum, "DateAdded": helpers.now(), "Status": "Snatched" } else: if modcomicname: IssueNum = issue['Issue_Number'] else: if mode == 'want_ann': IssueNum = "Annual " + issue['Issue_Number'] else: IssueNum = issue['Issue_Number'] newsnatchValues = {"ComicName": ComicName, "ComicID": ComicID, "Issue_Number": IssueNum, "DateAdded": helpers.now(), "Status": "Snatched" } myDB.upsert("snatched", newsnatchValues, snatchedupdate) #this will update the weeklypull list immediately after sntaching to reflect the new status. #-is ugly, should be linked directly to other table (IssueID should be populated in weekly pull at this point hopefully). chkit = myDB.selectone("SELECT * FROM weekly WHERE ComicID=? AND IssueID=?", [ComicID, IssueID]).fetchone() if chkit is not None: ctlVal = {"ComicID": ComicID, "IssueID": IssueID} newVal = {"Status": "Snatched"} myDB.upsert("weekly", newVal, ctlVal) logger.info(module + ' Updated the status (Snatched) complete for ' + ComicName + ' Issue: ' + str(IssueNum)) else: if down == 'PP': logger.info(module + ' Setting status to Post-Processed in history.') downstatus = 'Post-Processed' else: logger.info(module + ' Setting status to Downloaded in history.') downstatus = 'Downloaded' if mode == 'want_ann': if modcomicname: IssueNum = issue['Issue_Number'] else: IssueNum = "Annual " + issue['Issue_Number'] elif mode == 'story_arc': IssueNum = issue['IssueNumber'] IssueID = IssueArcID else: IssueNum = issue['Issue_Number'] snatchedupdate = {"IssueID": IssueID, "Status": downstatus, "Provider": provider } newsnatchValues = {"ComicName": ComicName, "ComicID": ComicID, "Issue_Number": IssueNum, "DateAdded": helpers.now(), "Status": downstatus } myDB.upsert("snatched", newsnatchValues, snatchedupdate) if mode == 'story_arc': cValue = {"IssueArcID": IssueArcID} nValue = {"Status": "Downloaded"} myDB.upsert("readinglist", nValue, cValue) else: controlValue = {"IssueID": IssueID} newValue = {"Status": "Downloaded"} if mode == 'want_ann': myDB.upsert("annuals", newValue, controlValue) else: myDB.upsert("issues", newValue, controlValue) #this will update the weeklypull list immediately after post-processing to reflect the new status. chkit = myDB.selectone("SELECT * FROM weekly WHERE ComicID=? AND IssueID=? AND Status='Snatched'", [ComicID, IssueID]).fetchone() if chkit is not None: ctlVal = {"ComicID": ComicID, "IssueID": IssueID} newVal = {"Status": "Downloaded"} myDB.upsert("weekly", newVal, ctlVal) logger.info(module + ' Updating Status (' + downstatus + ') now complete for ' + ComicName + ' issue: ' + IssueNum) return def forceRescan(ComicID, archive=None, module=None): if module is None: module = '' module += '[FILE-RESCAN]' myDB = db.DBConnection() # file check to see if issue exists rescan = myDB.selectone('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone() if rescan['AlternateSearch'] is not None: altnames = rescan['AlternateSearch'] + '##' else: altnames = '' annscan = myDB.select('SELECT * FROM annuals WHERE ComicID=?', [ComicID]) if annscan is None: pass else: for ascan in annscan: #logger.info('ReleaseComicName: ' + ascan['ReleaseComicName']) if ascan['ReleaseComicName'] not in altnames: altnames += ascan['ReleaseComicName'] + '!!' + ascan['ReleaseComicID'] + '##' altnames = altnames[:-2] logger.info(module + ' Now checking files for ' + rescan['ComicName'] + ' (' + str(rescan['ComicYear']) + ') in ' + rescan['ComicLocation']) fca = [] if archive is None: tmpval = filechecker.listFiles(dir=rescan['ComicLocation'], watchcomic=rescan['ComicName'], Publisher=rescan['ComicPublisher'], AlternateSearch=altnames) comiccnt = int(tmpval['comiccount']) logger.fdebug(module + 'comiccnt is:' + str(comiccnt)) fca.append(tmpval) if mylar.MULTIPLE_DEST_DIRS is not None and mylar.MULTIPLE_DEST_DIRS != 'None' and os.path.join(mylar.MULTIPLE_DEST_DIRS, os.path.basename(rescan['ComicLocation'])) != rescan['ComicLocation']: logger.fdebug(module + 'multiple_dest_dirs:' + mylar.MULTIPLE_DEST_DIRS) logger.fdebug(module + 'dir: ' + rescan['ComicLocation']) logger.fdebug(module + 'os.path.basename: ' + os.path.basename(rescan['ComicLocation'])) pathdir = os.path.join(mylar.MULTIPLE_DEST_DIRS, os.path.basename(rescan['ComicLocation'])) logger.info(module + ' Now checking files for ' + rescan['ComicName'] + ' (' + str(rescan['ComicYear']) + ') in :' + pathdir) tmpv = filechecker.listFiles(dir=pathdir, watchcomic=rescan['ComicName'], Publisher=rescan['ComicPublisher'], AlternateSearch=altnames) logger.fdebug(module + 'tmpv filecount: ' + str(tmpv['comiccount'])) comiccnt += int(tmpv['comiccount']) fca.append(tmpv) else: fca.append(filechecker.listFiles(dir=archive, watchcomic=rescan['ComicName'], Publisher=rescan['ComicPublisher'], AlternateSearch=rescan['AlternateSearch'])) fcb = [] fc = {} #if len(fca) > 0: for ca in fca: i = 0 while True: try: cla = ca['comiclist'][i] except (IndexError, KeyError) as e: break fcb.append({"ComicFilename": cla['ComicFilename'], "ComicLocation": cla['ComicLocation'], "ComicSize": cla['ComicSize'], "JusttheDigits": cla['JusttheDigits'], "AnnualComicID": cla['AnnualComicID']}) i+=1 fc['comiclist'] = fcb is_cnt = myDB.select("SELECT COUNT(*) FROM issues WHERE ComicID=?", [ComicID]) iscnt = is_cnt[0][0] #iscnt = rescan['Total'] havefiles = 0 if mylar.ANNUALS_ON: an_cnt = myDB.select("SELECT COUNT(*) FROM annuals WHERE ComicID=?", [ComicID]) anncnt = an_cnt[0][0] else: anncnt = 0 fccnt = comiccnt #int(fc['comiccount']) issnum = 1 fcnew = [] fn = 0 issuedupechk = [] annualdupechk = [] issueexceptdupechk = [] mc_issue = [] mc_issuenumber = [] reissues = myDB.select('SELECT * FROM issues WHERE ComicID=?', [ComicID]) multiple_check = myDB.select('SELECT * FROM issues WHERE ComicID=? GROUP BY Int_IssueNumber HAVING (COUNT(Int_IssueNumber) > 1)', [ComicID]) if len(multiple_check) == 0: logger.fdebug('No issues with identical issue numbering were detected for this series') mc_issuenumber = None else: logger.fdebug('Multiple issues with identical numbering were detected. Attempting to accomodate.') for mc in multiple_check: mc_issuenumber.append({"Int_IssueNumber": mc['Int_IssueNumber']}) if not mc_issuenumber is None: for mciss in mc_issuenumber: mchk = myDB.select('SELECT * FROM issues WHERE ComicID=? AND Int_IssueNumber=?', [ComicID, mciss['Int_IssueNumber']]) for mck in mchk: mc_issue.append({"Int_IssueNumber": mck['Int_IssueNumber'], "IssueYear": mck['IssueDate'][:4], "IssueID": mck['IssueID']}) logger.fdebug('mc_issue:' + str(mc_issue)) issID_to_ignore = [] issID_to_ignore.append(str(ComicID)) issID_to_write = [] ANNComicID = None while (fn < fccnt): haveissue = "no" issuedupe = "no" annualdupe = "no" try: tmpfc = fc['comiclist'][fn] except IndexError: logger.fdebug(module + ' Unable to properly retrieve a file listing for the given series.') logger.fdebug(module + ' Probably because the filenames being scanned are not in a parseable format') if fn == 0: return else: break temploc= tmpfc['JusttheDigits'].replace('_', ' ') temploc = re.sub('[\#\']', '', temploc) logger.fdebug(module + ' temploc: ' + str(temploc)) if 'annual' not in temploc.lower(): #remove the extension here extensions = ('.cbr', '.cbz', '.cb7') if temploc.lower().endswith(extensions): logger.fdebug(module + ' Removed extension for issue: ' + str(temploc)) temploc = temploc[:-4] fcnew_af = re.findall('[^\()]+', temploc) fcnew = shlex.split(fcnew_af[0]) fcn = len(fcnew) n = 0 while True: try: reiss = reissues[n] except IndexError: break int_iss = helpers.issuedigits(reiss['Issue_Number']) issyear = reiss['IssueDate'][:4] old_status = reiss['Status'] issname = reiss['IssueName'] fnd_iss_except = 'None' fcdigit = helpers.issuedigits(temploc) if int(fcdigit) == int_iss: logger.fdebug(module + ' [' + str(reiss['IssueID']) + '] Issue match - fcdigit: ' + str(fcdigit) + ' ... int_iss: ' + str(int_iss)) if '-' in temploc and temploc.find(reiss['Issue_Number']) > temploc.find('-'): logger.fdebug(module + ' I have detected a possible Title in the filename') logger.fdebug(module + ' the issue # has occured after the -, so I assume that it is part of the Title') break #baseline these to default to normal scanning multiplechk = False issuedupe = "no" foundchk = False #check here if muliple identical numbering issues exist for the series if len(mc_issue) > 1: for mi in mc_issue: if mi['Int_IssueNumber'] == int_iss: if mi['IssueID'] == reiss['IssueID']: logger.fdebug(module + ' IssueID matches to multiple issues : ' + str(mi['IssueID']) + '. Checking dupe.') logger.fdebug(module + ' miISSUEYEAR: ' + str(mi['IssueYear']) + ' -- issyear : ' + str(issyear)) if any(mi['IssueID'] == d['issueid'] for d in issuedupechk): logger.fdebug(module + ' IssueID already within dupe. Checking next if available.') multiplechk = True break if (mi['IssueYear'] in tmpfc['ComicFilename']) and (issyear == mi['IssueYear']): logger.fdebug(module + ' Matched to year within filename : ' + str(issyear)) multiplechk = False break else: logger.fdebug(module + ' Did not match to year within filename : ' + str(issyear)) multiplechk = True if multiplechk == True: n+=1 continue #this will detect duplicate filenames within the same directory. for di in issuedupechk: if di['fcdigit'] == fcdigit: #base off of config - base duplication keep on filesize or file-type (or both) logger.fdebug('[DUPECHECK] Duplicate issue detected [' + di['filename'] + '] [' + tmpfc['ComicFilename'] + ']') # mylar.DUPECONSTRAINT = 'filesize' / 'filetype-cbr' / 'filetype-cbz' logger.fdebug('[DUPECHECK] Based on duplication preferences I will retain based on : ' + mylar.DUPECONSTRAINT) removedupe = False if 'cbr' in mylar.DUPECONSTRAINT or 'cbz' in mylar.DUPECONSTRAINT: if 'cbr' in mylar.DUPECONSTRAINT: #this has to be configured in config - either retain cbr or cbz. if tmpfc['ComicFilename'].endswith('.cbz'): #keep di['filename'] logger.fdebug('[DUPECHECK-CBR PRIORITY] [#' + reiss['Issue_Number'] + '] Retaining currently scanned in file : ' + di['filename']) issuedupe = "yes" break else: #keep tmpfc['ComicFilename'] logger.fdebug('[DUPECHECK-CBR PRIORITY] [#' + reiss['Issue_Number'] + '] Retaining newly scanned in file : ' + tmpfc['ComicFilename']) removedupe = True elif 'cbz' in mylar.DUPECONSTRAINT: if tmpfc['ComicFilename'].endswith('.cbr'): #keep di['filename'] logger.fdebug('[DUPECHECK-CBZ PRIORITY] [#' + reiss['Issue_Number'] + '] Retaining currently scanned in filename : ' + di['filename']) issuedupe = "yes" break else: #keep tmpfc['ComicFilename'] logger.fdebug('[DUPECHECK-CBZ PRIORITY] [#' + reiss['Issue_Number'] + '] Retaining newly scanned in filename : ' + tmpfc['ComicFilename']) removedupe = True if mylar.DUPECONSTRAINT == 'filesize': if tmpfc['ComicSize'] <= di['filesize']: logger.fdebug('[DUPECHECK-FILESIZE PRIORITY] [#' + reiss['Issue_Number'] + '] Retaining currently scanned in filename : ' + di['filename']) issuedupe = "yes" break else: logger.fdebug('[DUPECHECK-FILESIZE PRIORITY] [#' + reiss['Issue_Number'] + '] Retaining newly scanned in filename : ' + tmpfc['ComicFilename']) removedupe = True if removedupe: #need to remove the entry from issuedupechk so can add new one. #tuple(y for y in x if y) for x in a issuedupe_temp = [] tmphavefiles = 0 for x in issuedupechk: #logger.fdebug('Comparing x: ' + x['filename'] + ' to di:' + di['filename']) if x['filename'] != di['filename']: #logger.fdebug('Matched.') issuedupe_temp.append(x) tmphavefiles+=1 issuedupechk = issuedupe_temp havefiles = tmphavefiles logger.fdebug(issuedupechk) foundchk = False break if issuedupe == "no": if foundchk == False: logger.fdebug(module + ' Matched...issue: ' + rescan['ComicName'] + '#' + reiss['Issue_Number'] + ' --- ' + str(int_iss)) havefiles+=1 haveissue = "yes" isslocation = str(tmpfc['ComicFilename']) issSize = str(tmpfc['ComicSize']) logger.fdebug(module + ' .......filename: ' + str(isslocation)) logger.fdebug(module + ' .......filesize: ' + str(tmpfc['ComicSize'])) # to avoid duplicate issues which screws up the count...let's store the filename issues then # compare earlier... issuedupechk.append({'fcdigit': fcdigit, 'filename': tmpfc['ComicFilename'], 'filesize': tmpfc['ComicSize'], 'issueyear': issyear, 'issueid': reiss['IssueID']}) break if issuedupe == "yes": logger.fdebug(module + ' I should break out here because of a dupe.') break if haveissue == "yes" or issuedupe == "yes": break n+=1 else: if tmpfc['AnnualComicID']: ANNComicID = tmpfc['AnnualComicID'] logger.fdebug(module + ' Forcing ComicID to ' + str(ANNComicID) + ' in case of duplicate numbering across volumes.') reannuals = myDB.select('SELECT * FROM annuals WHERE ComicID=? AND ReleaseComicID=?', [ComicID, ANNComicID]) else: reannuals = myDB.select('SELECT * FROM annuals WHERE ComicID=?', [ComicID]) ANNComicID = ComicID # annual inclusion here. #logger.fdebug("checking " + str(temploc)) fcnew = shlex.split(str(temploc)) fcn = len(fcnew) n = 0 reann = None while True: try: reann = reannuals[n] except IndexError: break int_iss = helpers.issuedigits(reann['Issue_Number']) logger.fdebug(module + ' int_iss:' + str(int_iss)) issyear = reann['IssueDate'][:4] old_status = reann['Status'] fcdigit = helpers.issuedigits(re.sub('annual', '', temploc.lower()).strip()) logger.fdebug(module + ' fcdigit:' + str(fcdigit)) if int(fcdigit) == int_iss: logger.fdebug(module + ' [' + str(ANNComicID) + '] Annual match - issue : ' + str(int_iss)) #baseline these to default to normal scanning multiplechk = False annualdupe = "no" foundchk = False #check here if muliple identical numbering issues exist for the series if len(mc_issue) > 1: for mi in mc_issue: if mi['Int_IssueNumber'] == int_iss: if mi['IssueID'] == reann['IssueID']: logger.fdebug(module + ' IssueID matches to multiple issues : ' + str(mi['IssueID']) + '. Checking dupe.') logger.fdebug(module + ' miISSUEYEAR: ' + str(mi['IssueYear']) + ' -- issyear : ' + str(issyear)) if any(mi['IssueID'] == d['issueid'] for d in issuedupechk): logger.fdebug(module + ' IssueID already within dupe. Checking next if available.') multiplechk = True break if (mi['IssueYear'] in tmpfc['ComicFilename']) and (issyear == mi['IssueYear']): logger.fdebug(module + ' Matched to year within filename : ' + str(issyear)) multiplechk = False break else: logger.fdebug(module + ' Did not match to year within filename : ' + str(issyear)) multiplechk = True if multiplechk == True: n+=1 continue #this will detect duplicate filenames within the same directory. for di in annualdupechk: if di['fcdigit'] == fcdigit and di['issueid'] == reann['IssueID']: #base off of config - base duplication keep on filesize or file-type (or both) logger.fdebug('[DUPECHECK] Duplicate issue detected [' + di['filename'] + '] [' + tmpfc['ComicFilename'] + ']') # mylar.DUPECONSTRAINT = 'filesize' / 'filetype-cbr' / 'filetype-cbz' logger.fdebug('[DUPECHECK] Based on duplication preferences I will retain based on : ' + mylar.DUPECONSTRAINT) removedupe = False if 'cbr' in mylar.DUPECONSTRAINT or 'cbz' in mylar.DUPECONSTRAINT: if 'cbr' in mylar.DUPECONSTRAINT: #this has to be configured in config - either retain cbr or cbz. if tmpfc['ComicFilename'].endswith('.cbz'): #keep di['filename'] logger.fdebug('[DUPECHECK-CBR PRIORITY] [#' + reann['Issue_Number'] + '] Retaining currently scanned in file : ' + di['filename']) annualdupe = "yes" break else: #keep tmpfc['ComicFilename'] logger.fdebug('[DUPECHECK-CBR PRIORITY] [#' + reann['Issue_Number'] + '] Retaining newly scanned in file : ' + tmpfc['ComicFilename']) removedupe = True elif 'cbz' in mylar.DUPECONSTRAINT: if tmpfc['ComicFilename'].endswith('.cbr'): #keep di['filename'] logger.fdebug('[DUPECHECK-CBZ PRIORITY] [#' + reann['Issue_Number'] + '] Retaining currently scanned in filename : ' + di['filename']) annualdupe = "yes" break else: #keep tmpfc['ComicFilename'] logger.fdebug('[DUPECHECK-CBZ PRIORITY] [#' + reann['Issue_Number'] + '] Retaining newly scanned in filename : ' + tmpfc['ComicFilename']) removedupe = True if mylar.DUPECONSTRAINT == 'filesize': if tmpfc['ComicSize'] <= di['filesize']: logger.fdebug('[DUPECHECK-FILESIZE PRIORITY] [#' + reann['Issue_Number'] + '] Retaining currently scanned in filename : ' + di['filename']) annualdupe = "yes" break else: logger.fdebug('[DUPECHECK-FILESIZE PRIORITY] [#' + reann['Issue_Number'] + '] Retaining newly scanned in filename : ' + tmpfc['ComicFilename']) removedupe = True if removedupe: #need to remove the entry from issuedupechk so can add new one. #tuple(y for y in x if y) for x in a annualdupe_temp = [] tmphavefiles = 0 for x in annualdupechk: logger.fdebug('Comparing x: ' + x['filename'] + ' to di:' + di['filename']) if x['filename'] != di['filename']: logger.fdebug('Matched.') annualdupe_temp.append(x) tmphavefiles+=1 annualdupechk = annualdupe_temp havefiles = tmphavefiles logger.fdebug(annualdupechk) foundchk = False break if annualdupe == "no": if foundchk == False: logger.fdebug(module + ' Matched...annual issue: ' + rescan['ComicName'] + '#' + str(reann['Issue_Number']) + ' --- ' + str(int_iss)) havefiles+=1 haveissue = "yes" isslocation = str(tmpfc['ComicFilename']) issSize = str(tmpfc['ComicSize']) logger.fdebug(module + ' .......filename: ' + str(isslocation)) logger.fdebug(module + ' .......filesize: ' + str(tmpfc['ComicSize'])) # to avoid duplicate issues which screws up the count...let's store the filename issues then # compare earlier... annualdupechk.append({'fcdigit': int(fcdigit), 'anncomicid': ANNComicID, 'filename': tmpfc['ComicFilename'], 'filesize': tmpfc['ComicSize'], 'issueyear': issyear, 'issueid': reann['IssueID']}) break if annualdupe == "yes": logger.fdebug(module + ' I should break out here because of a dupe.') break if haveissue == "yes" or annualdupe == "yes": break n+=1 if issuedupe == "yes" or annualdupe == "yes": pass else: #we have the # of comics, now let's update the db. #even if we couldn't find the physical issue, check the status. #-- if annuals aren't enabled, this will bugger out. writeit = True try: if mylar.ANNUALS_ON: if 'annual' in temploc.lower(): if reann is None: logger.fdebug(module + ' Annual present in location, but series does not have any annuals attached to it - Ignoring') writeit = False else: iss_id = reann['IssueID'] else: iss_id = reiss['IssueID'] else: if 'annual' in temploc.lower(): logger.fdebug(module + ' Annual support not enabled, but annual issue present within directory. Ignoring annual.') writeit = False else: iss_id = reiss['IssueID'] except: logger.warn(module + ' An error occured trying to get the relevant issue data. This is probably due to the series not having proper issue data.') logger.warn(module + ' you should either Refresh the series, and/or submit an issue on github in regards to the series and the error.') return if writeit == True: logger.fdebug(module + ' issueID to write to db:' + str(iss_id)) controlValueDict = {"IssueID": iss_id} #if Archived, increase the 'Have' count. #if archive: # issStatus = "Archived" if haveissue == "yes": issStatus = "Downloaded" newValueDict = {"Location": isslocation, "ComicSize": issSize, "Status": issStatus } issID_to_ignore.append(str(iss_id)) if ANNComicID: # if 'annual' in temploc.lower(): #issID_to_write.append({"tableName": "annuals", # "newValueDict": newValueDict, # "controlValueDict": controlValueDict}) myDB.upsert("annuals", newValueDict, controlValueDict) ANNComicID = None else: #issID_to_write.append({"tableName": "issues", # "valueDict": newValueDict, # "keyDict": controlValueDict}) myDB.upsert("issues", newValueDict, controlValueDict) fn+=1 # if len(issID_to_write) > 0: # for iss in issID_to_write: # logger.info('writing ' + str(iss)) # writethis = myDB.upsert(iss['tableName'], iss['valueDict'], iss['keyDict']) logger.fdebug(module + ' IssueID to ignore: ' + str(issID_to_ignore)) #here we need to change the status of the ones we DIDN'T FIND above since the loop only hits on FOUND issues. update_iss = [] tmpsql = "SELECT * FROM issues WHERE ComicID=? AND IssueID not in ({seq})".format(seq=','.join(['?'] *(len(issID_to_ignore) -1))) chkthis = myDB.select(tmpsql, issID_to_ignore) # chkthis = None if chkthis is None: pass else: for chk in chkthis: old_status = chk['Status'] #logger.fdebug('old_status:' + str(old_status)) if old_status == "Skipped": if mylar.AUTOWANT_ALL: issStatus = "Wanted" else: issStatus = "Skipped" elif old_status == "Archived": issStatus = "Archived" elif old_status == "Downloaded": issStatus = "Archived" elif old_status == "Wanted": issStatus = "Wanted" elif old_status == "Ignored": issStatus = "Ignored" elif old_status == "Snatched": #this is needed for torrents, or else it'll keep on queuing.. issStatus = "Snatched" else: issStatus = "Skipped" #logger.fdebug("new status: " + str(issStatus)) update_iss.append({"IssueID": chk['IssueID'], "Status": issStatus}) if len(update_iss) > 0: i = 0 #do it like this to avoid DB locks... for ui in update_iss: controlValueDict = {"IssueID": ui['IssueID']} newStatusValue = {"Status": ui['Status']} myDB.upsert("issues", newStatusValue, controlValueDict) i+=1 logger.info(module + ' Updated the status of ' + str(i) + ' issues for ' + rescan['ComicName'] + ' (' + str(rescan['ComicYear']) + ') that were not found.') logger.info(module + ' Total files located: ' + str(havefiles)) foundcount = havefiles arcfiles = 0 arcanns = 0 # if filechecker returns 0 files (it doesn't find any), but some issues have a status of 'Archived' # the loop below won't work...let's adjust :) arcissues = myDB.select("SELECT count(*) FROM issues WHERE ComicID=? and Status='Archived'", [ComicID]) if int(arcissues[0][0]) > 0: arcfiles = arcissues[0][0] arcannuals = myDB.select("SELECT count(*) FROM annuals WHERE ComicID=? and Status='Archived'", [ComicID]) if int(arcannuals[0][0]) > 0: arcanns = arcannuals[0][0] if arcfiles > 0 or arcanns > 0: arcfiles = arcfiles + arcanns havefiles = havefiles + arcfiles logger.fdebug(module + ' Adjusting have total to ' + str(havefiles) + ' because of this many archive files:' + str(arcfiles)) ignorecount = 0 if mylar.IGNORE_HAVETOTAL: # if this is enabled, will increase Have total as if in Archived Status ignores = myDB.select("SELECT count(*) FROM issues WHERE ComicID=? AND Status='Ignored'", [ComicID]) if int(ignores[0][0]) > 0: ignorecount = ignores[0][0] havefiles = havefiles + ignorecount logger.fdebug(module + ' Adjusting have total to ' + str(havefiles) + ' because of this many Ignored files:' + str(ignorecount)) snatchedcount = 0 if mylar.SNATCHED_HAVETOTAL: # if this is enabled, will increase Have total as if in Archived Status snatches = myDB.select("SELECT count(*) FROM issues WHERE ComicID=? AND Status='Snatched'", [ComicID]) if int(snatches[0][0]) > 0: snatchedcount = snatches[0][0] havefiles = havefiles + snatchedcount logger.fdebug(module + ' Adjusting have total to ' + str(havefiles) + ' because of this many Snatched files:' + str(snatchedcount)) #now that we are finished... #adjust for issues that have been marked as Downloaded, but aren't found/don't exist. #do it here, because above loop only cycles though found comics using filechecker. downissues = myDB.select("SELECT * FROM issues WHERE ComicID=? and Status='Downloaded'", [ComicID]) downissues += myDB.select("SELECT * FROM annuals WHERE ComicID=? and Status='Downloaded'", [ComicID]) if downissues is None: pass else: archivedissues = 0 #set this to 0 so it tallies correctly. for down in downissues: #print "downlocation:" + str(down['Location']) #remove special characters from #temploc = rescan['ComicLocation'].replace('_', ' ') #temploc = re.sub('[\#\'\/\.]', '', temploc) #print ("comiclocation: " + str(rescan['ComicLocation'])) #print ("downlocation: " + str(down['Location'])) if down['Location'] is None: logger.fdebug(module + ' Location does not exist which means file was not downloaded successfully, or was moved.') controlValue = {"IssueID": down['IssueID']} newValue = {"Status": "Archived"} myDB.upsert("issues", newValue, controlValue) archivedissues+=1 pass else: comicpath = os.path.join(rescan['ComicLocation'], down['Location']) if os.path.exists(comicpath): continue #print "Issue exists - no need to change status." else: if mylar.MULTIPLE_DEST_DIRS is not None and mylar.MULTIPLE_DEST_DIRS != 'None': if os.path.exists(os.path.join(mylar.MULTIPLE_DEST_DIRS, os.path.basename(rescan['ComicLocation']))): #logger.fdebug('Issue(s) currently exist and found within multiple destination directory location') continue #print "Changing status from Downloaded to Archived - cannot locate file" controlValue = {"IssueID": down['IssueID']} newValue = {"Status": "Archived"} myDB.upsert("issues", newValue, controlValue) archivedissues+=1 totalarc = arcfiles + archivedissues havefiles = havefiles + archivedissues #arcfiles already tallied in havefiles in above segment logger.fdebug(module + ' arcfiles : ' + str(arcfiles)) logger.fdebug(module + ' havefiles: ' + str(havefiles)) logger.fdebug(module + ' I have changed the status of ' + str(archivedissues) + ' issues to a status of Archived, as I now cannot locate them in the series directory.') #combined total for dispay total purposes only. combined_total = iscnt + anncnt #(rescan['Total'] + anncnt) #let's update the total count of comics that was found. #store just the total of issues, since annuals gets tracked seperately. controlValueStat = {"ComicID": rescan['ComicID']} newValueStat = {"Have": havefiles, "Total": iscnt} myDB.upsert("comics", newValueStat, controlValueStat) logger.info(module + ' I have physically found ' + str(foundcount) + ' issues, ignored ' + str(ignorecount) + ' issues, snatched ' + str(snatchedcount) + ' issues, and accounted for ' + str(totalarc) + ' in an Archived state [ Total Issue Count: ' + str(havefiles) + ' / ' + str(combined_total) + ' ]') return