mirror of https://github.com/evilhero/mylar
1662 lines
90 KiB
Python
Executable File
1662 lines
90 KiB
Python
Executable File
# This file is part of Mylar.
|
|
#
|
|
# Mylar is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# Mylar is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with Mylar. If not, see <http://www.gnu.org/licenses/>.
|
|
import time
|
|
import datetime
|
|
from xml.dom.minidom import parseString
|
|
import urllib2
|
|
import shlex
|
|
import operator
|
|
import re
|
|
import os
|
|
import itertools
|
|
import sys
|
|
import exceptions
|
|
|
|
import mylar
|
|
from mylar import db, logger, helpers, filechecker
|
|
|
|
def dbUpdate(ComicIDList=None, calledfrom=None, sched=False):
|
|
if mylar.IMPORTLOCK:
|
|
logger.info('Import is currently running - deferring this until the next scheduled run sequence.')
|
|
return
|
|
myDB = db.DBConnection()
|
|
if ComicIDList is None:
|
|
if mylar.CONFIG.UPDATE_ENDED:
|
|
logger.info('Updating only Continuing Series (option enabled) - this might cause problems with the pull-list matching for rebooted series')
|
|
comiclist = []
|
|
completelist = myDB.select('SELECT LatestDate, ComicPublished, ForceContinuing, NewPublish, LastUpdated, ComicID, ComicName, Corrected_SeriesYear, Corrected_Type, ComicYear from comics WHERE Status="Active" or Status="Loading" order by LastUpdated DESC, LatestDate ASC')
|
|
for comlist in completelist:
|
|
if comlist['LatestDate'] is None:
|
|
recentstatus = 'Loading'
|
|
elif comlist['ComicPublished'] is None or comlist['ComicPublished'] == '' or comlist['LatestDate'] is None:
|
|
recentstatus = 'Unknown'
|
|
elif comlist['ForceContinuing'] == 1:
|
|
recentstatus = 'Continuing'
|
|
elif 'present' in comlist['ComicPublished'].lower() or (helpers.today()[:4] in comlist['LatestDate']):
|
|
latestdate = comlist['LatestDate']
|
|
c_date = datetime.date(int(latestdate[:4]), int(latestdate[5:7]), 1)
|
|
n_date = datetime.date.today()
|
|
recentchk = (n_date - c_date).days
|
|
if comlist['NewPublish']:
|
|
recentstatus = 'Continuing'
|
|
else:
|
|
if recentchk < 55:
|
|
recentstatus = 'Continuing'
|
|
else:
|
|
recentstatus = 'Ended'
|
|
else:
|
|
recentstatus = 'Ended'
|
|
|
|
if recentstatus == 'Continuing':
|
|
comiclist.append({"LatestDate": comlist['LatestDate'],
|
|
"LastUpdated": comlist['LastUpdated'],
|
|
"ComicID": comlist['ComicID'],
|
|
"ComicName": comlist['ComicName'],
|
|
"ComicYear": comlist['ComicYear'],
|
|
"Corrected_SeriesYear": comlist['Corrected_SeriesYear'],
|
|
"Corrected_Type": comlist['Corrected_Type']})
|
|
|
|
else:
|
|
comiclist = myDB.select('SELECT LatestDate, LastUpdated, ComicID, ComicName, ComicYear, Corrected_SeriesYear, Corrected_Type from comics WHERE Status="Active" or Status="Loading" order by LastUpdated DESC, latestDate ASC')
|
|
else:
|
|
comiclist = []
|
|
comiclisting = ComicIDList
|
|
for cl in comiclisting:
|
|
comiclist += myDB.select('SELECT ComicID, ComicName, ComicYear, Corrected_SeriesYear, Corrected_Type, LastUpdated from comics WHERE ComicID=? order by LastUpdated DESC, LatestDate ASC', [cl])
|
|
|
|
if all([sched is False, calledfrom is None]):
|
|
logger.info('Starting update for %i active comics' % len(comiclist))
|
|
|
|
cnt = 1
|
|
|
|
if sched is True:
|
|
logger.fdebug('Refresh sequence set to fire every %s minutes for %s day(s)' % (mylar.DBUPDATE_INTERVAL, mylar.CONFIG.REFRESH_CACHE))
|
|
|
|
for comic in sorted(comiclist, key=operator.itemgetter('LastUpdated'), reverse=True):
|
|
dspyear = comic['ComicYear']
|
|
csyear = None
|
|
fixed_type = None
|
|
|
|
if comic['Corrected_Type'] is not None:
|
|
fixed_type = comic['Corrected_Type']
|
|
|
|
if comic['Corrected_SeriesYear'] is not None:
|
|
csyear = comic['Corrected_SeriesYear']
|
|
if int(csyear) != int(comic['ComicYear']):
|
|
comic['ComicYear'] = csyear
|
|
dspyear = csyear
|
|
|
|
if ComicIDList is None:
|
|
ComicID = comic['ComicID']
|
|
ComicName = comic['ComicName']
|
|
c_date = comic['LastUpdated']
|
|
if c_date is None:
|
|
logger.error(ComicName + ' failed during a previous add /refresh as it has no Last Update timestamp. Forcing refresh now.')
|
|
else:
|
|
c_obj_date = datetime.datetime.strptime(c_date, "%Y-%m-%d %H:%M:%S")
|
|
n_date = datetime.datetime.now()
|
|
absdiff = abs(n_date - c_obj_date)
|
|
hours = (absdiff.days * 24 * 60 * 60 + absdiff.seconds) / 3600.0
|
|
cache_hours = mylar.CONFIG.REFRESH_CACHE * 24
|
|
if hours < cache_hours:
|
|
#logger.fdebug('%s [%s] Was refreshed less than %s hours ago. Skipping Refresh at this time.' % (ComicName, ComicID, cache_hours))
|
|
cnt +=1
|
|
continue
|
|
logger.info('[%s/%s] Refreshing :%s (%s) [%s]' % (cnt, len(comiclist), ComicName, dspyear, ComicID))
|
|
else:
|
|
ComicID = comic['ComicID']
|
|
ComicName = comic['ComicName']
|
|
|
|
logger.info('Refreshing/Updating: %s (%s) [%s]' % (ComicName, dspyear, ComicID))
|
|
|
|
mismatch = "no"
|
|
if not mylar.CONFIG.CV_ONLY or ComicID[:1] == "G":
|
|
|
|
CV_EXcomicid = myDB.selectone("SELECT * from exceptions WHERE ComicID=?", [ComicID]).fetchone()
|
|
if CV_EXcomicid is None: pass
|
|
else:
|
|
if CV_EXcomicid['variloop'] == '99':
|
|
mismatch = "yes"
|
|
if ComicID[:1] == "G":
|
|
mylar.importer.GCDimport(ComicID)
|
|
else:
|
|
cchk = importer.addComictoDB(ComicID, mismatch)
|
|
else:
|
|
if mylar.CONFIG.CV_ONETIMER == 1:
|
|
if sched is True:
|
|
helpers.job_management(write=True, job='DB Updater', current_run=helpers.utctimestamp(), status='Running')
|
|
mylar.UPDATER_STATUS = 'Running'
|
|
logger.fdebug("CV_OneTimer option enabled...")
|
|
#in order to update to JUST CV_ONLY, we need to delete the issues for a given series so it's a clea$
|
|
logger.fdebug("Gathering the status of all issues for the series.")
|
|
issues = myDB.select('SELECT * FROM issues WHERE ComicID=?', [ComicID])
|
|
|
|
if not issues:
|
|
#if issues are None it's probably a bad refresh/maxed out API that resulted in the issue data
|
|
#getting wiped out and not refreshed. Setting whack=True will force a complete refresh.
|
|
logger.fdebug('No issue data available. This is Whack.')
|
|
whack = True
|
|
else:
|
|
#check for series that are numerically out of whack (ie. 5/4)
|
|
logger.fdebug('Checking how out of whack the series is.')
|
|
whack = helpers.havetotals(refreshit=ComicID)
|
|
|
|
if calledfrom == 'weekly':
|
|
if whack == True:
|
|
logger.info('Series is out of whack. Forcibly refreshing series to ensure everything is in order.')
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
annload = [] #initiate the list here so we don't error out below.
|
|
|
|
if mylar.CONFIG.ANNUALS_ON:
|
|
#now we load the annuals into memory to pass through to importer when refreshing so that it can
|
|
#refresh even the manually added annuals.
|
|
annual_load = myDB.select('SELECT * FROM annuals WHERE ComicID=?', [ComicID])
|
|
logger.fdebug('checking annual db')
|
|
for annthis in annual_load:
|
|
if not any(d['ReleaseComicID'] == annthis['ReleaseComicID'] for d in annload):
|
|
#print 'matched on annual'
|
|
annload.append({
|
|
'ReleaseComicID': annthis['ReleaseComicID'],
|
|
'ReleaseComicName': annthis['ReleaseComicName'],
|
|
'ComicID': annthis['ComicID'],
|
|
'ComicName': annthis['ComicName']
|
|
})
|
|
#print 'added annual'
|
|
issues += annual_load #myDB.select('SELECT * FROM annuals WHERE ComicID=?', [ComicID])
|
|
#store the issues' status for a given comicid, after deleting and readding, flip the status back to$
|
|
#logger.fdebug("Deleting all issue data.")
|
|
#myDB.action('DELETE FROM issues WHERE ComicID=?', [ComicID])
|
|
#myDB.action('DELETE FROM annuals WHERE ComicID=?', [ComicID])
|
|
logger.fdebug("Refreshing the series and pulling in new data using only CV.")
|
|
|
|
if whack == False:
|
|
chkstatus = mylar.importer.addComictoDB(ComicID, mismatch, calledfrom='dbupdate', annload=annload, csyear=csyear, fixed_type=fixed_type)
|
|
if chkstatus['status'] == 'complete':
|
|
#delete the data here if it's all valid.
|
|
logger.fdebug("Deleting all old issue data to make sure new data is clean...")
|
|
myDB.action('DELETE FROM issues WHERE ComicID=?', [ComicID])
|
|
myDB.action('DELETE FROM annuals WHERE ComicID=?', [ComicID])
|
|
mylar.importer.issue_collection(chkstatus['issuedata'], nostatus='True')
|
|
#need to update annuals at this point too....
|
|
if chkstatus['anndata'] is not None:
|
|
mylar.importer.manualAnnual(annchk=chkstatus['anndata'])
|
|
else:
|
|
logger.warn('There was an error when refreshing this series - Make sure directories are writable/exist, etc')
|
|
return
|
|
|
|
issues_new = myDB.select('SELECT * FROM issues WHERE ComicID=?', [ComicID])
|
|
annuals = []
|
|
ann_list = []
|
|
#reload the annuals here.
|
|
if mylar.CONFIG.ANNUALS_ON:
|
|
annuals_list = myDB.select('SELECT * FROM annuals WHERE ComicID=?', [ComicID])
|
|
ann_list += annuals_list
|
|
issues_new += annuals_list
|
|
|
|
logger.fdebug("Attempting to put the Status' back how they were.")
|
|
icount = 0
|
|
#the problem - the loop below will not match on NEW issues that have been refreshed that weren't present in the
|
|
#db before (ie. you left Mylar off for abit, and when you started it up it pulled down new issue information)
|
|
#need to test if issuenew['Status'] is None, but in a seperate loop below.
|
|
fndissue = []
|
|
for issue in issues:
|
|
for issuenew in issues_new:
|
|
#logger.fdebug(str(issue['Issue_Number']) + ' - issuenew:' + str(issuenew['IssueID']) + ' : ' + str(issuenew['Status']))
|
|
#logger.fdebug(str(issue['Issue_Number']) + ' - issue:' + str(issue['IssueID']) + ' : ' + str(issue['Status']))
|
|
try:
|
|
if issuenew['IssueID'] == issue['IssueID']:
|
|
newVAL = None
|
|
ctrlVAL = {"IssueID": issue['IssueID']}
|
|
if any([issuenew['Status'] != issue['Status'], issue['IssueDate_Edit'] is not None]):
|
|
#if the status is None and the original status is either Downloaded / Archived, keep status & stats
|
|
if issuenew['Status'] == None and (issue['Status'] == 'Downloaded' or issue['Status'] == 'Archived'):
|
|
newVAL = {"Location": issue['Location'],
|
|
"ComicSize": issue['ComicSize'],
|
|
"Status": issue['Status']}
|
|
#if the status is now Downloaded/Snatched, keep status & stats (downloaded only)
|
|
elif issuenew['Status'] == 'Downloaded' or issue['Status'] == 'Snatched':
|
|
newVAL = {"Location": issue['Location'],
|
|
"ComicSize": issue['ComicSize']}
|
|
if issuenew['Status'] == 'Downloaded':
|
|
newVAL['Status'] = issuenew['Status']
|
|
else:
|
|
newVAL['Status'] = issue['Status']
|
|
|
|
elif issue['Status'] == 'Archived':
|
|
newVAL = {"Status": issue['Status'],
|
|
"Location": issue['Location'],
|
|
"ComicSize": issue['ComicSize']}
|
|
else:
|
|
#change the status to the previous status
|
|
newVAL = {"Status": issue['Status']}
|
|
|
|
if all([issuenew['Status'] == None, issue['Status'] == 'Skipped']):
|
|
if issuenew['ReleaseDate'] == '0000-00-00':
|
|
dk = re.sub('-', '', issue['IssueDate']).strip()
|
|
else:
|
|
dk = re.sub('-', '', issuenew['ReleaseDate']).strip() # converts date to 20140718 format
|
|
if dk == '0000-00-00':
|
|
logger.warn('Issue Data is invalid for Issue Number %s. Marking this issue as Skipped' % issue['Issue_Number'])
|
|
newVAL = {"Status": "Skipped"}
|
|
else:
|
|
datechk = datetime.datetime.strptime(dk, "%Y%m%d")
|
|
nowdate = datetime.datetime.now()
|
|
now_week = datetime.datetime.strftime(nowdate, "%Y%U")
|
|
issue_week = datetime.datetime.strftime(datechk, "%Y%U")
|
|
if mylar.CONFIG.AUTOWANT_ALL:
|
|
newVAL = {"Status": "Wanted"}
|
|
elif issue_week >= now_week:
|
|
logger.fdebug('Issue_week: %s -- now_week: %s' % (issue_week, now_week))
|
|
logger.fdebug('Issue date [%s] is in/beyond current week - marking as Wanted.' % dk)
|
|
newVAL = {"Status": "Wanted"}
|
|
else:
|
|
newVAL = {"Status": "Skipped"}
|
|
|
|
if newVAL is not None:
|
|
if issue['IssueDate_Edit']:
|
|
logger.fdebug('[#' + str(issue['Issue_Number']) + '] detected manually edited Issue Date.')
|
|
logger.fdebug('new value : ' + str(issue['IssueDate']) + ' ... cv value : ' + str(issuenew['IssueDate']))
|
|
newVAL['IssueDate'] = issue['IssueDate']
|
|
newVAL['IssueDate_Edit'] = issue['IssueDate_Edit']
|
|
|
|
if any(d['IssueID'] == str(issue['IssueID']) for d in ann_list):
|
|
logger.fdebug("annual detected for " + str(issue['IssueID']) + " #: " + str(issue['Issue_Number']))
|
|
myDB.upsert("Annuals", newVAL, ctrlVAL)
|
|
else:
|
|
#logger.fdebug('#' + str(issue['Issue_Number']) + ' writing issuedata: ' + str(newVAL))
|
|
myDB.upsert("Issues", newVAL, ctrlVAL)
|
|
fndissue.append({"IssueID": issue['IssueID']})
|
|
icount+=1
|
|
break
|
|
except (RuntimeError, TypeError, ValueError, OSError) as e:
|
|
logger.warn('Something is out of whack somewhere with the series: %s' % e)
|
|
#if it's an annual (ie. deadpool-2011 ) on a refresh will throw index errors for some reason.
|
|
except:
|
|
logger.warn('Unexpected Error: %s' % sys.exc_info()[0])
|
|
raise
|
|
|
|
logger.info("In the process of converting the data to CV, I changed the status of " + str(icount) + " issues.")
|
|
|
|
issuesnew = myDB.select('SELECT * FROM issues WHERE ComicID=? AND Status is NULL', [ComicID])
|
|
|
|
if mylar.CONFIG.AUTOWANT_UPCOMING:
|
|
newstatus = "Wanted"
|
|
else:
|
|
newstatus = "Skipped"
|
|
|
|
newiss = []
|
|
|
|
for iss in issuesnew:
|
|
newiss.append({"IssueID": iss['IssueID'],
|
|
"Status": newstatus,
|
|
"Annual": False})
|
|
|
|
if mylar.CONFIG.ANNUALS_ON:
|
|
annualsnew = myDB.select('SELECT * FROM annuals WHERE ComicID=? AND Status is NULL', [ComicID])
|
|
|
|
for ann in annualsnew:
|
|
newiss.append({"IssueID": iss['IssueID'],
|
|
"Status": newstatus,
|
|
"Annual": True})
|
|
|
|
if len(newiss) > 0:
|
|
for newi in newiss:
|
|
ctrlVAL = {"IssueID": newi['IssueID']}
|
|
newVAL = {"Status": newi['Status']}
|
|
#logger.fdebug('writing issuedata: ' + str(newVAL))
|
|
if newi['Annual'] == True:
|
|
myDB.upsert("Annuals", newVAL, ctrlVAL)
|
|
else:
|
|
myDB.upsert("Issues", newVAL, ctrlVAL)
|
|
|
|
logger.info('I have added ' + str(len(newiss)) + ' new issues for this series that were not present before.')
|
|
forceRescan(ComicID)
|
|
|
|
else:
|
|
chkstatus = mylar.importer.addComictoDB(ComicID, mismatch, annload=annload, csyear=csyear)
|
|
#if cchk:
|
|
# #delete the data here if it's all valid.
|
|
# #logger.fdebug("Deleting all old issue data to make sure new data is clean...")
|
|
# myDB.action('DELETE FROM issues WHERE ComicID=?', [ComicID])
|
|
# myDB.action('DELETE FROM annuals WHERE ComicID=?', [ComicID])
|
|
# mylar.importer.issue_collection(cchk, nostatus='True')
|
|
# #need to update annuals at this point too....
|
|
# if annchk:
|
|
# mylar.importer.manualAnnual(annchk=annchk)
|
|
|
|
else:
|
|
chkstatus = mylar.importer.addComictoDB(ComicID, mismatch)
|
|
|
|
cnt += 1
|
|
if all([sched is False, calledfrom != 'refresh']):
|
|
time.sleep(15) #pause for 15 secs so dont hammer CV and get 500 error
|
|
else:
|
|
break
|
|
|
|
helpers.job_management(write=True, job='DB Updater', last_run_completed=helpers.utctimestamp(), status='Waiting')
|
|
mylar.UPDATER_STATUS = 'Waiting'
|
|
logger.info('Update complete')
|
|
|
|
def latest_update(ComicID, LatestIssue, LatestDate):
|
|
# here we add to comics.latest
|
|
logger.fdebug(str(ComicID) + ' - updating latest_date to : ' + str(LatestDate))
|
|
myDB = db.DBConnection()
|
|
latestCTRLValueDict = {"ComicID": ComicID}
|
|
newlatestDict = {"LatestIssue": str(LatestIssue),
|
|
"LatestDate": str(LatestDate)}
|
|
myDB.upsert("comics", newlatestDict, latestCTRLValueDict)
|
|
|
|
def upcoming_update(ComicID, ComicName, IssueNumber, IssueDate, forcecheck=None, futurepull=None, altissuenumber=None, weekinfo=None):
|
|
# here we add to upcoming table...
|
|
myDB = db.DBConnection()
|
|
dspComicName = ComicName #to make sure that the word 'annual' will be displayed on screen
|
|
if 'annual' in ComicName.lower():
|
|
adjComicName = re.sub("\\bannual\\b", "", ComicName.lower()) # for use with comparisons.
|
|
logger.fdebug('annual detected - adjusting name to : ' + adjComicName)
|
|
else:
|
|
adjComicName = ComicName
|
|
controlValue = {"ComicID": ComicID}
|
|
newValue = {"ComicName": adjComicName,
|
|
"IssueNumber": str(IssueNumber),
|
|
"DisplayComicName": dspComicName,
|
|
"IssueDate": str(IssueDate)}
|
|
|
|
#let's refresh the series here just to make sure if an issue is available/not.
|
|
mismatch = "no"
|
|
CV_EXcomicid = myDB.selectone("SELECT * from exceptions WHERE ComicID=?", [ComicID]).fetchone()
|
|
if CV_EXcomicid is None:
|
|
pass
|
|
else:
|
|
if CV_EXcomicid['variloop'] == '99':
|
|
mismatch = "yes"
|
|
if mylar.CONFIG.ALT_PULL != 2 or mylar.PULLBYFILE is True:
|
|
lastupdatechk = myDB.selectone("SELECT * FROM comics WHERE ComicID=?", [ComicID]).fetchone()
|
|
if lastupdatechk is None:
|
|
pullupd = "yes"
|
|
else:
|
|
c_date = lastupdatechk['LastUpdated']
|
|
if c_date is None:
|
|
logger.error(lastupdatechk['ComicName'] + ' failed during a previous add /refresh. Please either delete and readd the series, or try a refresh of the series.')
|
|
return
|
|
c_obj_date = datetime.datetime.strptime(c_date, "%Y-%m-%d %H:%M:%S")
|
|
n_date = datetime.datetime.now()
|
|
absdiff = abs(n_date - c_obj_date)
|
|
hours = (absdiff.days * 24 * 60 * 60 + absdiff.seconds) / 3600.0
|
|
else:
|
|
#if it's at this point and the refresh is None, odds are very good that it's already up-to-date so let it flow thru
|
|
if mylar.CONFIG.PULL_REFRESH is None:
|
|
mylar.CONFIG.PULL_REFRESH = datetime.datetime.today().replace(second=0,microsecond=0)
|
|
#update the PULL_REFRESH
|
|
#mylar.config_write()
|
|
logger.fdebug('pull_refresh: ' + str(mylar.CONFIG.PULL_REFRESH))
|
|
c_obj_date = datetime.datetime.strptime(str(mylar.CONFIG.PULL_REFRESH),"%Y-%m-%d %H:%M:%S")
|
|
#logger.fdebug('c_obj_date: ' + str(c_obj_date))
|
|
n_date = datetime.datetime.now()
|
|
#logger.fdebug('n_date: ' + str(n_date))
|
|
absdiff = abs(n_date - c_obj_date)
|
|
#logger.fdebug('absdiff: ' + str(absdiff))
|
|
hours = (absdiff.days * 24 * 60 * 60 + absdiff.seconds) / 3600.0
|
|
#logger.fdebug('hours: ' + str(hours))
|
|
|
|
if any(['annual' in ComicName.lower(), 'special' in ComicName.lower()]):
|
|
if mylar.CONFIG.ANNUALS_ON:
|
|
logger.info('checking: ' + str(ComicID) + ' -- issue#: ' + str(IssueNumber))
|
|
issuechk = myDB.selectone("SELECT * FROM annuals WHERE ComicID=? AND Issue_Number=?", [ComicID, IssueNumber]).fetchone()
|
|
else:
|
|
logger.fdebug('Non-standard issue detected (annual/special/etc), but Annual Integration is not enabled. Ignoring result.')
|
|
return
|
|
else:
|
|
issuechk = myDB.selectone("SELECT * FROM issues WHERE ComicID=? AND Issue_Number=?", [ComicID, IssueNumber]).fetchone()
|
|
if issuechk is None and altissuenumber is not None:
|
|
logger.info('altissuenumber is : ' + str(altissuenumber))
|
|
issuechk = myDB.selectone("SELECT * FROM issues WHERE ComicID=? AND Int_IssueNumber=?", [ComicID, helpers.issuedigits(altissuenumber)]).fetchone()
|
|
if issuechk is None:
|
|
if futurepull is None:
|
|
og_status = None
|
|
if mylar.CONFIG.ALT_PULL != 2 or mylar.PULLBYFILE is True:
|
|
logger.fdebug(adjComicName + ' Issue: ' + str(IssueNumber) + ' not present in listings to mark for download...updating comic and adding to Upcoming Wanted Releases.')
|
|
# we need to either decrease the total issue count, OR indicate that an issue is upcoming.
|
|
upco_results = myDB.select("SELECT COUNT(*) FROM UPCOMING WHERE ComicID=?", [ComicID])
|
|
upco_iss = upco_results[0][0]
|
|
#logger.info("upco_iss: " + str(upco_iss))
|
|
if int(upco_iss) > 0:
|
|
#logger.info("There is " + str(upco_iss) + " of " + str(ComicName) + " that's not accounted for")
|
|
newKey = {"ComicID": ComicID}
|
|
newVal = {"not_updated_db": str(upco_iss)}
|
|
myDB.upsert("comics", newVal, newKey)
|
|
elif int(upco_iss) <=0 and lastupdatechk['not_updated_db']:
|
|
#if not_updated_db has a value, and upco_iss is > 0, let's zero it back out cause it's updated now.
|
|
newKey = {"ComicID": ComicID}
|
|
newVal = {"not_updated_db": ""}
|
|
myDB.upsert("comics", newVal, newKey)
|
|
|
|
if hours > 5 or forcecheck == 'yes':
|
|
pullupd = "yes"
|
|
logger.fdebug('Now Refreshing comic ' + ComicName + ' to make sure it is up-to-date')
|
|
if ComicID[:1] == "G":
|
|
mylar.importer.GCDimport(ComicID, pullupd)
|
|
else:
|
|
cchk = mylar.importer.updateissuedata(ComicID, ComicName, calledfrom='weeklycheck') #mylar.importer.addComictoDB(ComicID,mismatch,pullupd)
|
|
else:
|
|
logger.fdebug('It has not been longer than 5 hours since we last did this...we will wait so we do not hammer things.')
|
|
else:
|
|
logger.fdebug('[WEEKLY-PULL] Walksoftly has been enabled. ComicID/IssueID control given to the ninja to monitor.')
|
|
logger.fdebug('hours: ' + str(hours) + ' -- forcecheck: ' + str(forcecheck))
|
|
if hours > 2 or forcecheck == 'yes':
|
|
logger.fdebug('weekinfo:' + str(weekinfo))
|
|
mylar.CONFIG.PULL_REFRESH = datetime.datetime.today().replace(second=0,microsecond=0)
|
|
#update the PULL_REFRESH
|
|
#mylar.config_write()
|
|
chkitout = mylar.locg.locg(weeknumber=str(weekinfo['weeknumber']),year=str(weekinfo['year']))
|
|
|
|
logger.fdebug('linking ComicID to Pull-list to reflect status.')
|
|
downstats = {"ComicID": ComicID,
|
|
"IssueID": None,
|
|
"Status": None}
|
|
return downstats
|
|
else:
|
|
# if futurepull is not None, let's just update the status and ComicID
|
|
# NOTE: THIS IS CREATING EMPTY ENTRIES IN THE FUTURE TABLE. ???
|
|
nKey = {"ComicID": ComicID}
|
|
nVal = {"Status": "Wanted"}
|
|
myDB.upsert("future", nVal, nKey)
|
|
return
|
|
if issuechk is not None:
|
|
if issuechk['Issue_Number'] == IssueNumber or issuechk['Issue_Number'] == altissuenumber:
|
|
og_status = issuechk['Status']
|
|
#check for 'out-of-whack' series here.
|
|
whackness = dbUpdate([ComicID], calledfrom='weekly', sched=False)
|
|
if any([whackness == True, og_status is None]):
|
|
if any([issuechk['Status'] == 'Downloaded', issuechk['Status'] == 'Archived', issuechk['Status'] == 'Snatched']):
|
|
logger.fdebug('Forcibly maintaining status of : ' + og_status + ' for #' + issuechk['Issue_Number'] + ' to ensure integrity.')
|
|
logger.fdebug('Comic series has an incorrect total count. Forcily refreshing series to ensure data is current.')
|
|
dbUpdate([ComicID])
|
|
issuechk = myDB.selectone("SELECT * FROM issues WHERE ComicID=? AND Int_IssueNumber=?", [ComicID, helpers.issuedigits(IssueNumber)]).fetchone()
|
|
if issuechk['Status'] != og_status and (issuechk['Status'] != 'Downloaded' or issuechk['Status'] != 'Archived' or issuechk['Status'] != 'Snatched'):
|
|
logger.fdebug('Forcibly changing status of %s back to %s for #%s to stop repeated downloads.' % (issuechk['Status'], og_status, issuechk['Issue_Number']))
|
|
else:
|
|
logger.fdebug('[%s] / [%s] Status has not changed during refresh or is marked as being Wanted/Skipped correctly.' % (issuechk['Status'], og_status))
|
|
og_status = issuechk['Status']
|
|
else:
|
|
logger.fdebug('Comic series already up-to-date ... no need to refresh at this time.')
|
|
|
|
logger.fdebug('Available to be marked for download - checking...' + adjComicName + ' Issue: ' + str(issuechk['Issue_Number']))
|
|
logger.fdebug('...Existing status: ' + og_status)
|
|
control = {"IssueID": issuechk['IssueID']}
|
|
newValue['IssueID'] = issuechk['IssueID']
|
|
if og_status == "Snatched":
|
|
values = {"Status": "Snatched"}
|
|
newValue['Status'] = "Snatched"
|
|
elif og_status == "Downloaded":
|
|
values = {"Status": "Downloaded"}
|
|
newValue['Status'] = "Downloaded"
|
|
#if the status is Downloaded and it's on the pullist - let's mark it so everyone can bask in the glory
|
|
|
|
elif og_status == "Wanted":
|
|
values = {"Status": "Wanted"}
|
|
newValue['Status'] = "Wanted"
|
|
elif og_status == "Archived":
|
|
values = {"Status": "Archived"}
|
|
newValue['Status'] = "Archived"
|
|
elif og_status == 'Failed':
|
|
if mylar.CONFIG.FAILED_DOWNLOAD_HANDLING:
|
|
if mylar.CONFIG.FAILED_AUTO:
|
|
values = {"Status": "Wanted"}
|
|
newValue['Status'] = "Wanted"
|
|
else:
|
|
values = {"Status": "Failed"}
|
|
newValue['Status'] = "Failed"
|
|
else:
|
|
values = {"Status": "Skipped"}
|
|
newValue['Status'] = "Skipped"
|
|
else:
|
|
values = {"Status": "Skipped"}
|
|
newValue['Status'] = "Skipped"
|
|
#was in wrong place :(
|
|
else:
|
|
logger.fdebug('Issues do not match for some reason...weekly new issue: %s' % IssueNumber)
|
|
return
|
|
|
|
if mylar.CONFIG.AUTOWANT_UPCOMING:
|
|
#for issues not in db - to be added to Upcoming table.
|
|
if og_status is None:
|
|
newValue['Status'] = "Wanted"
|
|
logger.fdebug('...Changing Status to Wanted and throwing it in the Upcoming section since it is not published yet.')
|
|
#this works for issues existing in DB...
|
|
elif og_status == "Skipped":
|
|
newValue['Status'] = "Wanted"
|
|
values = {"Status": "Wanted"}
|
|
logger.fdebug('...New status of Wanted')
|
|
elif og_status == "Wanted":
|
|
logger.fdebug('...Status already Wanted .. not changing.')
|
|
else:
|
|
logger.fdebug('...Already have issue - keeping existing status of : ' + og_status)
|
|
|
|
if issuechk is None:
|
|
myDB.upsert("upcoming", newValue, controlValue)
|
|
logger.fdebug('updating Pull-list to reflect status.')
|
|
downstats = {"Status": newValue['Status'],
|
|
"ComicID": ComicID,
|
|
"IssueID": None}
|
|
return downstats
|
|
|
|
else:
|
|
logger.fdebug('--attempt to find errant adds to Wanted list')
|
|
logger.fdebug('UpcomingNewValue: ' + str(newValue))
|
|
logger.fdebug('UpcomingcontrolValue: ' + str(controlValue))
|
|
if issuechk['IssueDate'] == '0000-00-00' and newValue['IssueDate'] != '0000-00-00':
|
|
logger.fdebug('Found a 0000-00-00 issue - force updating series to try and get it proper.')
|
|
dateVal = {"IssueDate": newValue['IssueDate'],
|
|
"ComicName": issuechk['ComicName'],
|
|
"Status": newValue['Status'],
|
|
"IssueNumber": issuechk['Issue_Number']}
|
|
logger.fdebug('updating date in upcoming table to : ' + str(newValue['IssueDate']) + '[' + newValue['Status'] + ']')
|
|
logger.fdebug('ComicID:' + str(controlValue))
|
|
myDB.upsert("upcoming", dateVal, controlValue)
|
|
logger.fdebug('Temporarily putting the Issue Date for ' + str(issuechk['Issue_Number']) + ' to ' + str(newValue['IssueDate']))
|
|
values = {"IssueDate": newValue['IssueDate']}
|
|
#if ComicID[:1] == "G": mylar.importer.GCDimport(ComicID,pullupd='yes')
|
|
#else: mylar.importer.addComictoDB(ComicID,mismatch,pullupd='yes')
|
|
|
|
if any(['annual' in ComicName.lower(), 'special' in ComicName.lower()]):
|
|
myDB.upsert("annuals", values, control)
|
|
else:
|
|
myDB.upsert("issues", values, control)
|
|
|
|
if any([og_status == 'Downloaded', og_status == 'Archived', og_status == 'Snatched', og_status == 'Wanted', newValue['Status'] == 'Wanted']):
|
|
logger.fdebug('updating Pull-list to reflect status change: ' + og_status + '[' + newValue['Status'] + ']')
|
|
if og_status != 'Skipped':
|
|
downstats = {"Status": og_status,
|
|
"ComicID": issuechk['ComicID'],
|
|
"IssueID": issuechk['IssueID']}
|
|
else:
|
|
downstats = {"Status": newValue['Status'],
|
|
"ComicID": issuechk['ComicID'],
|
|
"IssueID": issuechk['IssueID']}
|
|
return downstats
|
|
|
|
|
|
def weekly_update(ComicName, IssueNumber, CStatus, CID, weeknumber, year, altissuenumber=None):
|
|
logger.fdebug('Weekly Update for week ' + str(weeknumber) + '-' + str(year) + ' : ' + str(ComicName) + ' #' + str(IssueNumber) + ' to a status of ' + str(CStatus))
|
|
|
|
if altissuenumber:
|
|
logger.fdebug('weekly_update of table : ' + str(ComicName) + ' (Alternate Issue #):' + str(altissuenumber) + ' to a status of ' + str(CStatus))
|
|
|
|
# here we update status of weekly table...
|
|
# added Issue to stop false hits on series' that have multiple releases in a week
|
|
# added CStatus to update status flags on Pullist screen
|
|
myDB = db.DBConnection()
|
|
issuecheck = myDB.selectone("SELECT * FROM weekly WHERE COMIC=? AND ISSUE=? and WEEKNUMBER=? AND YEAR=?", [ComicName, IssueNumber, int(weeknumber), year]).fetchone()
|
|
|
|
if issuecheck is not None:
|
|
controlValue = {"COMIC": str(ComicName),
|
|
"ISSUE": str(IssueNumber),
|
|
"WEEKNUMBER": int(weeknumber),
|
|
"YEAR": year}
|
|
|
|
logger.info('controlValue:' + str(controlValue))
|
|
try:
|
|
if CID['IssueID']:
|
|
cidissueid = CID['IssueID']
|
|
else:
|
|
cidissueid = None
|
|
except:
|
|
cidissueid = None
|
|
|
|
logger.info('CStatus:' + str(CStatus))
|
|
|
|
if CStatus:
|
|
newValue = {"STATUS": CStatus}
|
|
|
|
else:
|
|
if mylar.CONFIG.AUTOWANT_UPCOMING:
|
|
newValue = {"STATUS": "Wanted"}
|
|
else:
|
|
newValue = {"STATUS": "Skipped"}
|
|
|
|
#setting this here regardless, as it will be a match for a watchlist hit at this point anyways - so link it here what's availalbe.
|
|
newValue['ComicID'] = CID['ComicID']
|
|
newValue['IssueID'] = cidissueid
|
|
|
|
logger.info('newValue:' + str(newValue))
|
|
|
|
myDB.upsert("weekly", newValue, controlValue)
|
|
|
|
def newpullcheck(ComicName, ComicID, issue=None):
|
|
# When adding a new comic, let's check for new issues on this week's pullist and update.
|
|
if mylar.CONFIG.ALT_PULL != 2 or mylar.PULLBYFILE is True:
|
|
mylar.weeklypull.pullitcheck(comic1off_name=ComicName, comic1off_id=ComicID, issue=issue)
|
|
else:
|
|
mylar.weeklypull.new_pullcheck(weeknumber=mylar.CURRENT_WEEKNUMBER, pullyear=mylar.CURRENT_YEAR, comic1off_name=ComicName, comic1off_id=ComicID, issue=issue)
|
|
return
|
|
|
|
def no_searchresults(ComicID):
|
|
# when there's a mismatch between CV & GCD - let's change the status to
|
|
# something other than 'Loaded'
|
|
myDB = db.DBConnection()
|
|
controlValue = {"ComicID": ComicID}
|
|
newValue = {"Status": "Error",
|
|
"LatestDate": "Error",
|
|
"LatestIssue": "Error"}
|
|
myDB.upsert("comics", newValue, controlValue)
|
|
|
|
def nzblog(IssueID, NZBName, ComicName, SARC=None, IssueArcID=None, id=None, prov=None, alt_nzbname=None, oneoff=False):
|
|
myDB = db.DBConnection()
|
|
|
|
newValue = {'NZBName': NZBName}
|
|
|
|
if SARC:
|
|
logger.fdebug("Story Arc (SARC) detected as: " + str(SARC))
|
|
IssueID = 'S' + str(IssueArcID)
|
|
newValue['SARC'] = SARC
|
|
|
|
if oneoff is True:
|
|
logger.fdebug('One-Off download detected when updating - crossing the t\'s and dotting the i\'s so things work...')
|
|
newValue['OneOff'] = True
|
|
|
|
if IssueID is None or IssueID == 'None':
|
|
#if IssueID is None, it's a one-off download from the pull-list.
|
|
#give it a generic ID above the last one so it doesn't throw an error later.
|
|
if any([mylar.CONFIG.HIGHCOUNT == 0, mylar.CONFIG.HIGHCOUNT is None]):
|
|
mylar.CONFIG.HIGHCOUNT = 900000
|
|
else:
|
|
mylar.CONFIG.HIGHCOUNT+=1
|
|
|
|
IssueID = mylar.CONFIG.HIGHCOUNT
|
|
#mylar.config_write()
|
|
|
|
controlValue = {"IssueID": IssueID,
|
|
"Provider": prov}
|
|
|
|
|
|
if id:
|
|
logger.info('setting the nzbid for this download grabbed by ' + prov + ' in the nzblog to : ' + str(id))
|
|
newValue['ID'] = id
|
|
|
|
if alt_nzbname:
|
|
logger.info('setting the alternate nzbname for this download grabbed by ' + prov + ' in the nzblog to : ' + alt_nzbname)
|
|
newValue['AltNZBName'] = alt_nzbname
|
|
|
|
#check if it exists already in the log.
|
|
chkd = myDB.selectone('SELECT * FROM nzblog WHERE IssueID=? and Provider=?', [IssueID, prov]).fetchone()
|
|
if chkd is None:
|
|
pass
|
|
else:
|
|
altnames = chkd['AltNZBName']
|
|
if any([altnames is None, altnames == '']):
|
|
#we need to wipe the entry so we can re-update with the alt-nzbname if required
|
|
myDB.action('DELETE FROM nzblog WHERE IssueID=? and Provider=?', [IssueID, prov])
|
|
logger.fdebug('Deleted stale entry from nzblog for IssueID: ' + str(IssueID) + ' [' + prov + ']')
|
|
myDB.upsert("nzblog", newValue, controlValue)
|
|
|
|
|
|
def foundsearch(ComicID, IssueID, mode=None, down=None, provider=None, SARC=None, IssueArcID=None, module=None, hash=None, crc=None, comicname=None, issuenumber=None, pullinfo=None):
|
|
# When doing a Force Search (Wanted tab), the resulting search calls this to update.
|
|
|
|
# this is all redudant code that forceRescan already does.
|
|
# should be redone at some point so that instead of rescanning entire
|
|
# series directory, it just scans for the issue it just downloaded and
|
|
# and change the status to Snatched accordingly. It is not to increment the have count
|
|
# at this stage as it's not downloaded - just the .nzb has been snatched and sent to SAB.
|
|
if module is None:
|
|
module = ''
|
|
module += '[UPDATER]'
|
|
|
|
myDB = db.DBConnection()
|
|
modcomicname = False
|
|
|
|
logger.fdebug(module + ' comicid: ' + str(ComicID))
|
|
logger.fdebug(module + ' issueid: ' + str(IssueID))
|
|
if mode != 'pullwant':
|
|
if mode != 'story_arc':
|
|
comic = myDB.selectone('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone()
|
|
ComicName = comic['ComicName']
|
|
if mode == 'want_ann':
|
|
issue = myDB.selectone('SELECT * FROM annuals WHERE IssueID=?', [IssueID]).fetchone()
|
|
if ComicName != issue['ReleaseComicName'] + ' Annual':
|
|
ComicName = issue['ReleaseComicName']
|
|
modcomicname = True
|
|
else:
|
|
issue = myDB.selectone('SELECT * FROM issues WHERE IssueID=?', [IssueID]).fetchone()
|
|
CYear = issue['IssueDate'][:4]
|
|
IssueNum = issue['Issue_Number']
|
|
|
|
else:
|
|
issue = myDB.selectone('SELECT * FROM storyarcs WHERE IssueArcID=?', [IssueArcID]).fetchone()
|
|
ComicName = issue['ComicName']
|
|
CYear = issue['IssueYEAR']
|
|
IssueNum = issue['IssueNumber']
|
|
else:
|
|
oneinfo = myDB.selectone('SELECT * FROM weekly WHERE IssueID=?', [IssueID]).fetchone()
|
|
if oneinfo is None:
|
|
ComicName = comicname
|
|
IssueNum = issuenumber
|
|
onefail = True
|
|
else:
|
|
ComicName = oneinfo['COMIC']
|
|
IssueNum = oneinfo['ISSUE']
|
|
onefail = False
|
|
|
|
if down is None:
|
|
# update the status to Snatched (so it won't keep on re-downloading!)
|
|
logger.info(module + ' Updating status to snatched')
|
|
logger.fdebug(module + ' Provider is ' + provider)
|
|
if hash:
|
|
logger.fdebug(module + ' Hash set to : ' + hash)
|
|
newValue = {"Status": "Snatched"}
|
|
if mode == 'story_arc':
|
|
cValue = {"IssueArcID": IssueArcID}
|
|
snatchedupdate = {"IssueArcID": IssueArcID}
|
|
myDB.upsert("storyarcs", newValue, cValue)
|
|
# update the snatched DB
|
|
snatchedupdate = {"IssueID": IssueArcID,
|
|
"Status": "Snatched",
|
|
"Provider": provider
|
|
}
|
|
|
|
else:
|
|
if mode == 'want_ann':
|
|
controlValue = {"IssueID": IssueID}
|
|
myDB.upsert("annuals", newValue, controlValue)
|
|
else:
|
|
controlValue = {"IssueID": IssueID}
|
|
if mode != 'pullwant':
|
|
myDB.upsert("issues", newValue, controlValue)
|
|
|
|
# update the snatched DB
|
|
snatchedupdate = {"IssueID": IssueID,
|
|
"Status": "Snatched",
|
|
"Provider": provider
|
|
}
|
|
|
|
if mode == 'story_arc':
|
|
IssueNum = issue['IssueNumber']
|
|
newsnatchValues = {"ComicName": ComicName,
|
|
"ComicID": 'None',
|
|
"Issue_Number": IssueNum,
|
|
"DateAdded": helpers.now(),
|
|
"Status": "Snatched",
|
|
"Hash": hash
|
|
}
|
|
|
|
myDB.upsert("snatched", newsnatchValues, snatchedupdate)
|
|
|
|
elif mode != 'pullwant':
|
|
if modcomicname:
|
|
IssueNum = issue['Issue_Number']
|
|
else:
|
|
if mode == 'want_ann':
|
|
IssueNum = "Annual " + issue['Issue_Number']
|
|
else:
|
|
IssueNum = issue['Issue_Number']
|
|
|
|
newsnatchValues = {"ComicName": ComicName,
|
|
"ComicID": ComicID,
|
|
"Issue_Number": IssueNum,
|
|
"DateAdded": helpers.now(),
|
|
"Status": "Snatched",
|
|
"Hash": hash
|
|
}
|
|
|
|
myDB.upsert("snatched", newsnatchValues, snatchedupdate)
|
|
|
|
else:
|
|
#updating snatched table with one-off is abit difficult due to lack of complete information in some instances
|
|
#ie. alt_pull 2 not populated yet, alt_pull 0 method in general doesn't have enough info....
|
|
|
|
newsnatchValues = {"ComicName": ComicName,
|
|
"ComicID": ComicID,
|
|
"IssueID": IssueID,
|
|
"Issue_Number": IssueNum,
|
|
"DateAdded": helpers.now(),
|
|
"Status": "Snatched",
|
|
"Hash": hash
|
|
}
|
|
|
|
myDB.upsert("snatched", newsnatchValues, snatchedupdate)
|
|
|
|
#this will update the weeklypull list immediately after snatching to reflect the new status.
|
|
#-is ugly, should be linked directly to other table (IssueID should be populated in weekly pull at this point hopefully).
|
|
chkit = myDB.selectone("SELECT * FROM weekly WHERE ComicID=? AND IssueID=?", [ComicID, IssueID]).fetchone()
|
|
|
|
if chkit is not None:
|
|
comicname = chkit['COMIC']
|
|
issue = chkit['ISSUE']
|
|
|
|
ctlVal = {"ComicID": ComicID,
|
|
"IssueID": IssueID}
|
|
myDB.upsert("weekly", newValue, ctlVal)
|
|
|
|
newValue['IssueNumber'] = issue
|
|
newValue['ComicName'] = comicname
|
|
newValue['Status'] = "Snatched"
|
|
if pullinfo is not None:
|
|
newValue['weeknumber'] = pullinfo['weeknumber']
|
|
newValue['year'] = pullinfo['year']
|
|
else:
|
|
try:
|
|
newValue['weeknumber'] = chkit['weeknumber']
|
|
newValue['year'] = chkit['year']
|
|
except:
|
|
pass
|
|
|
|
myDB.upsert("oneoffhistory", newValue, ctlVal)
|
|
logger.info('%s Updated the status (Snatched) complete for %s Issue: %s' % (module, ComicName, IssueNum))
|
|
else:
|
|
if down == 'PP':
|
|
logger.info(module + ' Setting status to Post-Processed in history.')
|
|
downstatus = 'Post-Processed'
|
|
else:
|
|
logger.info(module + ' Setting status to Downloaded in history.')
|
|
downstatus = 'Downloaded'
|
|
if mode == 'want_ann':
|
|
if not modcomicname:
|
|
IssueNum = "Annual " + IssueNum
|
|
elif mode == 'story_arc':
|
|
IssueID = IssueArcID
|
|
|
|
snatchedupdate = {"IssueID": IssueID,
|
|
"Status": downstatus,
|
|
"Provider": provider
|
|
}
|
|
newsnatchValues = {"ComicName": ComicName,
|
|
"ComicID": ComicID,
|
|
"Issue_Number": IssueNum,
|
|
"DateAdded": helpers.now(),
|
|
"Status": downstatus,
|
|
"crc": crc
|
|
}
|
|
myDB.upsert("snatched", newsnatchValues, snatchedupdate)
|
|
|
|
if mode == 'story_arc':
|
|
cValue = {"IssueArcID": IssueArcID}
|
|
nValue = {"Status": "Downloaded"}
|
|
myDB.upsert("storyarcs", nValue, cValue)
|
|
|
|
elif mode != 'pullwant':
|
|
controlValue = {"IssueID": IssueID}
|
|
newValue = {"Status": "Downloaded"}
|
|
if mode == 'want_ann':
|
|
myDB.upsert("annuals", newValue, controlValue)
|
|
else:
|
|
myDB.upsert("issues", newValue, controlValue)
|
|
|
|
#this will update the weeklypull list immediately after post-processing to reflect the new status.
|
|
chkit = myDB.selectone("SELECT * FROM weekly WHERE ComicID=? AND IssueID=? AND Status='Snatched'", [ComicID, IssueID]).fetchone()
|
|
|
|
if chkit is not None:
|
|
comicname = chkit['COMIC']
|
|
issue = chkit['ISSUE']
|
|
|
|
ctlVal = {"ComicID": ComicID,
|
|
"IssueID": IssueID}
|
|
newVal = {"Status": "Downloaded"}
|
|
myDB.upsert("weekly", newVal, ctlVal)
|
|
|
|
newVal['IssueNumber'] = issue
|
|
newVal['ComicName'] = comicname
|
|
newVal['Status'] = "Downloaded"
|
|
if pullinfo is not None:
|
|
newVal['weeknumber'] = pullinfo['weeknumber']
|
|
newVal['year'] = pullinfo['year']
|
|
myDB.upsert("oneoffhistory", newVal, ctlVal)
|
|
|
|
logger.info('%s Updating Status (%s) now completed for %s issue: %s' % (module, downstatus, ComicName, IssueNum))
|
|
return
|
|
|
|
def forceRescan(ComicID, archive=None, module=None, recheck=False):
|
|
if module is None:
|
|
module = ''
|
|
module += '[FILE-RESCAN]'
|
|
myDB = db.DBConnection()
|
|
# file check to see if issue exists
|
|
rescan = myDB.selectone('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone()
|
|
if rescan['AlternateSearch'] is not None:
|
|
altnames = rescan['AlternateSearch'] + '##'
|
|
else:
|
|
altnames = ''
|
|
|
|
if (all([rescan['Type'] != 'Print', rescan['Type'] != 'Digital', rescan['Type'] != 'None', rescan['Type'] is not None]) and rescan['Corrected_Type'] != 'Print') or rescan['Corrected_Type'] == 'TPB':
|
|
if rescan['Type'] == 'One-Shot' and rescan['Corrected_Type'] is None:
|
|
booktype = 'One-Shot'
|
|
else:
|
|
booktype = 'TPB'
|
|
else:
|
|
booktype = None
|
|
|
|
annscan = myDB.select('SELECT * FROM annuals WHERE ComicID=?', [ComicID])
|
|
if annscan is None:
|
|
pass
|
|
else:
|
|
for ascan in annscan:
|
|
#logger.info('ReleaseComicName: ' + ascan['ReleaseComicName'])
|
|
if ascan['ReleaseComicName'] not in altnames:
|
|
altnames += ascan['ReleaseComicName'] + '!!' + ascan['ReleaseComicID'] + '##'
|
|
altnames = altnames[:-2]
|
|
logger.info(module + ' Now checking files for ' + rescan['ComicName'] + ' (' + str(rescan['ComicYear']) + ') in ' + rescan['ComicLocation'])
|
|
fca = []
|
|
if archive is None:
|
|
tval = filechecker.FileChecker(dir=rescan['ComicLocation'], watchcomic=rescan['ComicName'], Publisher=rescan['ComicPublisher'], AlternateSearch=altnames)
|
|
tmpval = tval.listFiles()
|
|
#tmpval = filechecker.listFiles(dir=rescan['ComicLocation'], watchcomic=rescan['ComicName'], Publisher=rescan['ComicPublisher'], AlternateSearch=altnames)
|
|
comiccnt = int(tmpval['comiccount'])
|
|
#logger.fdebug(module + 'comiccnt is:' + str(comiccnt))
|
|
fca.append(tmpval)
|
|
try:
|
|
if all([mylar.CONFIG.MULTIPLE_DEST_DIRS is not None, mylar.CONFIG.MULTIPLE_DEST_DIRS != 'None', os.path.join(mylar.CONFIG.MULTIPLE_DEST_DIRS, os.path.basename(rescan['ComicLocation'])) != rescan['ComicLocation'], os.path.exists(os.path.join(mylar.CONFIG.MULTIPLE_DEST_DIRS, os.path.basename(rescan['ComicLocation'])))]):
|
|
logger.fdebug(module + 'multiple_dest_dirs:' + mylar.CONFIG.MULTIPLE_DEST_DIRS)
|
|
logger.fdebug(module + 'dir: ' + rescan['ComicLocation'])
|
|
logger.fdebug(module + 'os.path.basename: ' + os.path.basename(rescan['ComicLocation']))
|
|
pathdir = os.path.join(mylar.CONFIG.MULTIPLE_DEST_DIRS, os.path.basename(rescan['ComicLocation']))
|
|
logger.info(module + ' Now checking files for ' + rescan['ComicName'] + ' (' + str(rescan['ComicYear']) + ') in :' + pathdir)
|
|
mvals = filechecker.FileChecker(dir=pathdir, watchcomic=rescan['ComicName'], Publisher=rescan['ComicPublisher'], AlternateSearch=altnames)
|
|
tmpv = mvals.listFiles()
|
|
#tmpv = filechecker.listFiles(dir=pathdir, watchcomic=rescan['ComicName'], Publisher=rescan['ComicPublisher'], AlternateSearch=altnames)
|
|
logger.fdebug(module + 'tmpv filecount: ' + str(tmpv['comiccount']))
|
|
comiccnt += int(tmpv['comiccount'])
|
|
fca.append(tmpv)
|
|
except:
|
|
pass
|
|
else:
|
|
# files_arc = filechecker.listFiles(dir=archive, watchcomic=rescan['ComicName'], Publisher=rescan['ComicPublisher'], AlternateSearch=rescan['AlternateSearch'])
|
|
arcval = filechecker.FileChecker(dir=archive, watchcomic=rescan['ComicName'], Publisher=rescan['ComicPublisher'], AlternateSearch=rescan['AlternateSearch'])
|
|
files_arc = arcval.listFiles()
|
|
fca.append(files_arc)
|
|
comiccnt = int(files_arc['comiccount'])
|
|
|
|
fcb = []
|
|
fc = {}
|
|
|
|
is_cnt = myDB.select("SELECT COUNT(*) FROM issues WHERE ComicID=?", [ComicID])
|
|
iscnt = is_cnt[0][0]
|
|
|
|
for ca in fca:
|
|
i = 0
|
|
while True:
|
|
try:
|
|
cla = ca['comiclist'][i]
|
|
except (IndexError, KeyError) as e:
|
|
break
|
|
|
|
try:
|
|
if all([booktype == 'TPB', iscnt > 1]) or all([booktype == 'One-Shot', iscnt == 1, cla['JusttheDigits'] is None]):
|
|
if cla['SeriesVolume'] is not None:
|
|
just_the_digits = re.sub('[^0-9]', '', cla['SeriesVolume']).strip()
|
|
else:
|
|
just_the_digits = re.sub('[^0-9]', '', cla['JusttheDigits']).strip()
|
|
else:
|
|
just_the_digits = cla['JusttheDigits']
|
|
except Exception as e:
|
|
logger.warn('[Exception: %s] Unable to properly match up/retrieve issue number (or volume) for this [CS: %s]' % (e,cla))
|
|
else:
|
|
fcb.append({"ComicFilename": cla['ComicFilename'],
|
|
"ComicLocation": cla['ComicLocation'],
|
|
"ComicSize": cla['ComicSize'],
|
|
"JusttheDigits": just_the_digits,
|
|
"AnnualComicID": cla['AnnualComicID']})
|
|
i+=1
|
|
|
|
fc['comiclist'] = fcb
|
|
|
|
havefiles = 0
|
|
if mylar.CONFIG.ANNUALS_ON:
|
|
an_cnt = myDB.select("SELECT COUNT(*) FROM annuals WHERE ComicID=?", [ComicID])
|
|
anncnt = an_cnt[0][0]
|
|
else:
|
|
anncnt = 0
|
|
fccnt = comiccnt #int(fc['comiccount'])
|
|
issnum = 1
|
|
fcnew = []
|
|
fn = 0
|
|
issuedupechk = []
|
|
annualdupechk = []
|
|
issueexceptdupechk = []
|
|
mc_issue = []
|
|
mc_issuenumber = []
|
|
multiple_check = myDB.select('SELECT * FROM issues WHERE ComicID=? GROUP BY Int_IssueNumber HAVING (COUNT(Int_IssueNumber) > 1)', [ComicID])
|
|
|
|
if len(multiple_check) == 0:
|
|
logger.fdebug('No issues with identical issue numbering were detected for this series')
|
|
mc_issuenumber = None
|
|
else:
|
|
logger.fdebug('Multiple issues with identical numbering were detected. Attempting to accomodate.')
|
|
for mc in multiple_check:
|
|
mc_issuenumber.append({"Int_IssueNumber": mc['Int_IssueNumber']})
|
|
|
|
if not mc_issuenumber is None:
|
|
for mciss in mc_issuenumber:
|
|
mchk = myDB.select('SELECT * FROM issues WHERE ComicID=? AND Int_IssueNumber=?', [ComicID, mciss['Int_IssueNumber']])
|
|
for mck in mchk:
|
|
mc_issue.append({"Int_IssueNumber": mck['Int_IssueNumber'],
|
|
"IssueYear": mck['IssueDate'][:4],
|
|
"IssueID": mck['IssueID']})
|
|
|
|
mc_annual = []
|
|
mc_annualnumber = []
|
|
|
|
if mylar.CONFIG.ANNUALS_ON:
|
|
mult_ann_check = myDB.select('SELECT * FROM annuals WHERE ComicID=? GROUP BY Int_IssueNumber HAVING (COUNT(Int_IssueNumber) > 1)', [ComicID])
|
|
|
|
if len(mult_ann_check) == 0:
|
|
logger.fdebug('[ANNUAL-CHK] No annuals with identical issue numbering across annual volumes were detected for this series')
|
|
mc_annualnumber = None
|
|
else:
|
|
logger.fdebug('[ANNUAL-CHK] Multiple issues with identical numbering were detected across multiple annual volumes. Attempting to accomodate.')
|
|
for mc in mult_ann_check:
|
|
mc_annualnumber.append({"Int_IssueNumber": mc['Int_IssueNumber']})
|
|
|
|
if not mc_annualnumber is None:
|
|
for mcann in mc_annualnumber:
|
|
achk = myDB.select('SELECT * FROM annuals WHERE ComicID=? AND Int_IssueNumber=?', [ComicID, mcann['Int_IssueNumber']])
|
|
for ack in achk:
|
|
mc_annual.append({"Int_IssueNumber": ack['Int_IssueNumber'],
|
|
"IssueYear": ack['IssueDate'][:4],
|
|
"IssueID": ack['IssueID'],
|
|
"ReleaseComicID": ack['ReleaseComicID']})
|
|
|
|
#logger.fdebug('mc_issue:' + str(mc_issue))
|
|
#logger.fdebug('mc_annual:' + str(mc_annual))
|
|
|
|
issID_to_ignore = []
|
|
issID_to_ignore.append(str(ComicID))
|
|
issID_to_write = []
|
|
ANNComicID = None
|
|
|
|
reissues = myDB.select('SELECT * FROM issues WHERE ComicID=?', [ComicID])
|
|
|
|
while (fn < fccnt):
|
|
haveissue = "no"
|
|
issuedupe = "no"
|
|
annualdupe = "no"
|
|
try:
|
|
tmpfc = fc['comiclist'][fn]
|
|
except IndexError:
|
|
logger.fdebug(module + ' Unable to properly retrieve a file listing for the given series.')
|
|
logger.fdebug(module + ' Probably because the filenames being scanned are not in a parseable format')
|
|
if fn == 0:
|
|
return
|
|
else:
|
|
break
|
|
|
|
if tmpfc['JusttheDigits'] is not None:
|
|
temploc= tmpfc['JusttheDigits'].replace('_', ' ')
|
|
temploc = re.sub('[\#\']', '', temploc)
|
|
logger.fdebug('temploc: %s' % temploc)
|
|
else:
|
|
#assume 1 if not given
|
|
if any([booktype == 'TPB', booktype == 'One-Shot']):
|
|
temploc = '1'
|
|
else:
|
|
temploc = None
|
|
logger.warn('The filename [%s] does not have a valid issue number, and the Edition of the series is %s. You might need to Forcibly Mark the Series as TPB/GN and try this again.' % (tmpfc['ComicFilename'], rescan['Type']))
|
|
return
|
|
|
|
if all(['annual' not in temploc.lower(), 'special' not in temploc.lower()]):
|
|
#remove the extension here
|
|
extensions = ('.cbr', '.cbz', '.cb7')
|
|
if temploc.lower().endswith(extensions):
|
|
logger.fdebug(module + ' Removed extension for issue: ' + temploc)
|
|
temploc = temploc[:-4]
|
|
fcnew_af = re.findall('[^\()]+', temploc)
|
|
fcnew = shlex.split(fcnew_af[0])
|
|
|
|
fcn = len(fcnew)
|
|
n = 0
|
|
while True:
|
|
try:
|
|
reiss = reissues[n]
|
|
int_iss = None
|
|
except IndexError:
|
|
break
|
|
int_iss = helpers.issuedigits(reiss['Issue_Number'])
|
|
issyear = reiss['IssueDate'][:4]
|
|
old_status = reiss['Status']
|
|
issname = reiss['IssueName']
|
|
|
|
fnd_iss_except = 'None'
|
|
|
|
if temploc is not None:
|
|
fcdigit = helpers.issuedigits(temploc)
|
|
elif any([booktype == 'TPB', booktype == 'One-Shot']) and temploc is None:
|
|
fcdigit = helpers.issuedigits('1')
|
|
|
|
if int(fcdigit) == int_iss:
|
|
logger.fdebug(module + ' [' + str(reiss['IssueID']) + '] Issue match - fcdigit: ' + str(fcdigit) + ' ... int_iss: ' + str(int_iss))
|
|
|
|
if '-' in temploc and temploc.find(reiss['Issue_Number']) > temploc.find('-'):
|
|
logger.fdebug(module + ' I have detected a possible Title in the filename')
|
|
logger.fdebug(module + ' the issue # has occured after the -, so I assume that it is part of the Title')
|
|
break
|
|
|
|
#baseline these to default to normal scanning
|
|
multiplechk = False
|
|
issuedupe = "no"
|
|
foundchk = False
|
|
|
|
#check here if muliple identical numbering issues exist for the series
|
|
if len(mc_issue) > 1:
|
|
for mi in mc_issue:
|
|
if mi['Int_IssueNumber'] == int_iss:
|
|
if mi['IssueID'] == reiss['IssueID']:
|
|
logger.fdebug(module + ' IssueID matches to multiple issues : ' + str(mi['IssueID']) + '. Checking dupe.')
|
|
logger.fdebug(module + ' miISSUEYEAR: ' + str(mi['IssueYear']) + ' -- issyear : ' + str(issyear))
|
|
if any(mi['IssueID'] == d['issueid'] for d in issuedupechk):
|
|
logger.fdebug(module + ' IssueID already within dupe. Checking next if available.')
|
|
multiplechk = True
|
|
break
|
|
if (mi['IssueYear'] in tmpfc['ComicFilename']) and (issyear == mi['IssueYear']):
|
|
logger.fdebug(module + ' Matched to year within filename : ' + str(issyear))
|
|
multiplechk = False
|
|
break
|
|
else:
|
|
logger.fdebug(module + ' Did not match to year within filename : ' + str(issyear))
|
|
multiplechk = True
|
|
if multiplechk == True:
|
|
n+=1
|
|
continue
|
|
|
|
#this will detect duplicate filenames within the same directory.
|
|
for di in issuedupechk:
|
|
if di['fcdigit'] == fcdigit and di['issueid'] == reiss['IssueID']:
|
|
#base off of config - base duplication keep on filesize or file-type (or both)
|
|
logger.fdebug('[DUPECHECK] Duplicate issue detected [' + di['filename'] + '] [' + tmpfc['ComicFilename'] + ']')
|
|
# mylar.CONFIG.DUPECONSTRAINT = 'filesize' / 'filetype-cbr' / 'filetype-cbz'
|
|
logger.fdebug('[DUPECHECK] Based on duplication preferences I will retain based on : ' + mylar.CONFIG.DUPECONSTRAINT)
|
|
removedupe = False
|
|
if 'cbr' in mylar.CONFIG.DUPECONSTRAINT or 'cbz' in mylar.CONFIG.DUPECONSTRAINT:
|
|
if 'cbr' in mylar.CONFIG.DUPECONSTRAINT:
|
|
#this has to be configured in config - either retain cbr or cbz.
|
|
if tmpfc['ComicFilename'].endswith('.cbz'):
|
|
#keep di['filename']
|
|
logger.fdebug('[DUPECHECK-CBR PRIORITY] [#' + reiss['Issue_Number'] + '] Retaining currently scanned in file : ' + di['filename'])
|
|
issuedupe = "yes"
|
|
break
|
|
else:
|
|
#keep tmpfc['ComicFilename']
|
|
logger.fdebug('[DUPECHECK-CBR PRIORITY] [#' + reiss['Issue_Number'] + '] Retaining newly scanned in file : ' + tmpfc['ComicFilename'])
|
|
removedupe = True
|
|
elif 'cbz' in mylar.CONFIG.DUPECONSTRAINT:
|
|
if tmpfc['ComicFilename'].endswith('.cbr'):
|
|
#keep di['filename']
|
|
logger.fdebug('[DUPECHECK-CBZ PRIORITY] [#' + reiss['Issue_Number'] + '] Retaining currently scanned in filename : ' + di['filename'])
|
|
issuedupe = "yes"
|
|
break
|
|
else:
|
|
#keep tmpfc['ComicFilename']
|
|
logger.fdebug('[DUPECHECK-CBZ PRIORITY] [#' + reiss['Issue_Number'] + '] Retaining newly scanned in filename : ' + tmpfc['ComicFilename'])
|
|
removedupe = True
|
|
|
|
if mylar.CONFIG.DUPECONSTRAINT == 'filesize':
|
|
if tmpfc['ComicSize'] <= di['filesize']:
|
|
logger.fdebug('[DUPECHECK-FILESIZE PRIORITY] [#' + reiss['Issue_Number'] + '] Retaining currently scanned in filename : ' + di['filename'])
|
|
issuedupe = "yes"
|
|
break
|
|
else:
|
|
logger.fdebug('[DUPECHECK-FILESIZE PRIORITY] [#' + reiss['Issue_Number'] + '] Retaining newly scanned in filename : ' + tmpfc['ComicFilename'])
|
|
removedupe = True
|
|
|
|
if removedupe:
|
|
#need to remove the entry from issuedupechk so can add new one.
|
|
#tuple(y for y in x if y) for x in a
|
|
issuedupe_temp = []
|
|
tmphavefiles = 0
|
|
for x in issuedupechk:
|
|
#logger.fdebug('Comparing x: ' + x['filename'] + ' to di:' + di['filename'])
|
|
if x['filename'] != di['filename']:
|
|
#logger.fdebug('Matched.')
|
|
issuedupe_temp.append(x)
|
|
tmphavefiles+=1
|
|
issuedupechk = issuedupe_temp
|
|
havefiles = tmphavefiles + len(annualdupechk)
|
|
foundchk = False
|
|
break
|
|
|
|
if issuedupe == "no":
|
|
|
|
if foundchk == False:
|
|
logger.fdebug(module + ' Matched...issue: ' + rescan['ComicName'] + '#' + reiss['Issue_Number'] + ' --- ' + str(int_iss))
|
|
havefiles+=1
|
|
haveissue = "yes"
|
|
isslocation = helpers.conversion(tmpfc['ComicFilename'])
|
|
issSize = str(tmpfc['ComicSize'])
|
|
logger.fdebug(module + ' .......filename: ' + isslocation)
|
|
logger.fdebug(module + ' .......filesize: ' + str(tmpfc['ComicSize']))
|
|
# to avoid duplicate issues which screws up the count...let's store the filename issues then
|
|
# compare earlier...
|
|
issuedupechk.append({'fcdigit': fcdigit,
|
|
'filename': tmpfc['ComicFilename'],
|
|
'filesize': tmpfc['ComicSize'],
|
|
'issueyear': issyear,
|
|
'issueid': reiss['IssueID']})
|
|
break
|
|
|
|
if issuedupe == "yes":
|
|
logger.fdebug(module + ' I should break out here because of a dupe.')
|
|
break
|
|
|
|
if haveissue == "yes" or issuedupe == "yes": break
|
|
n+=1
|
|
else:
|
|
if tmpfc['AnnualComicID']:
|
|
ANNComicID = tmpfc['AnnualComicID']
|
|
logger.fdebug(module + ' Forcing ComicID to ' + str(ANNComicID) + ' in case of duplicate numbering across volumes.')
|
|
reannuals = myDB.select('SELECT * FROM annuals WHERE ComicID=? AND ReleaseComicID=?', [ComicID, ANNComicID])
|
|
else:
|
|
reannuals = myDB.select('SELECT * FROM annuals WHERE ComicID=?', [ComicID])
|
|
ANNComicID = ComicID
|
|
|
|
if len(reannuals) == 0:
|
|
#it's possible if annual integration is enabled, and an annual series is added directly to the wachlist,
|
|
#not as part of a series, that the above won't work since it's looking in the wrong table.
|
|
reannuals = myDB.select('SELECT * FROM issues WHERE ComicID=?', [ComicID])
|
|
ANNComicID = None #need to set this to None so we write to the issues table and not the annuals
|
|
|
|
|
|
# annual inclusion here.
|
|
#logger.fdebug("checking " + str(temploc))
|
|
fcnew = shlex.split(str(temploc))
|
|
fcn = len(fcnew)
|
|
n = 0
|
|
reann = None
|
|
while True:
|
|
try:
|
|
reann = reannuals[n]
|
|
except IndexError:
|
|
break
|
|
int_iss = helpers.issuedigits(reann['Issue_Number'])
|
|
#logger.fdebug(module + ' int_iss:' + str(int_iss))
|
|
|
|
issyear = reann['IssueDate'][:4]
|
|
old_status = reann['Status']
|
|
|
|
fcdigit = helpers.issuedigits(re.sub('annual', '', temploc.lower()).strip())
|
|
if fcdigit == 999999999999999:
|
|
fcdigit = helpers.issuedigits(re.sub('special', '', temploc.lower()).strip())
|
|
|
|
if int(fcdigit) == int_iss and ANNComicID is not None:
|
|
logger.fdebug(module + ' [' + str(ANNComicID) + '] Annual match - issue : ' + str(int_iss))
|
|
|
|
#baseline these to default to normal scanning
|
|
multiplechk = False
|
|
annualdupe = "no"
|
|
foundchk = False
|
|
|
|
#check here if muliple identical numbering issues exist for the series
|
|
if len(mc_annual) > 1:
|
|
for ma in mc_annual:
|
|
if ma['Int_IssueNumber'] == int_iss:
|
|
if ma['IssueID'] == reann['IssueID']:
|
|
logger.fdebug(module + ' IssueID matches to multiple issues : ' + str(ma['IssueID']) + '. Checking dupe.')
|
|
logger.fdebug(module + ' maISSUEYEAR: ' + str(ma['IssueYear']) + ' -- issyear : ' + str(issyear))
|
|
if any(ma['IssueID'] == d['issueid'] for d in annualdupechk):
|
|
logger.fdebug(module + ' IssueID already within dupe. Checking next if available.')
|
|
multiplechk = True
|
|
break
|
|
if (ma['IssueYear'] in tmpfc['ComicFilename']) and (issyear == ma['IssueYear']):
|
|
logger.fdebug(module + ' Matched to year within filename : ' + str(issyear))
|
|
multiplechk = False
|
|
ANNComicID = ack['ReleaseComicID']
|
|
break
|
|
else:
|
|
logger.fdebug(module + ' Did not match to year within filename : ' + str(issyear))
|
|
multiplechk = True
|
|
if multiplechk == True:
|
|
n+=1
|
|
continue
|
|
|
|
#this will detect duplicate filenames within the same directory.
|
|
for di in annualdupechk:
|
|
if di['fcdigit'] == fcdigit and di['issueid'] == reann['IssueID']:
|
|
#base off of config - base duplication keep on filesize or file-type (or both)
|
|
logger.fdebug('[DUPECHECK] Duplicate issue detected [' + di['filename'] + '] [' + tmpfc['ComicFilename'] + ']')
|
|
# mylar.CONFIG.DUPECONSTRAINT = 'filesize' / 'filetype-cbr' / 'filetype-cbz'
|
|
logger.fdebug('[DUPECHECK] Based on duplication preferences I will retain based on : ' + mylar.CONFIG.DUPECONSTRAINT)
|
|
removedupe = False
|
|
if 'cbr' in mylar.CONFIG.DUPECONSTRAINT or 'cbz' in mylar.CONFIG.DUPECONSTRAINT:
|
|
if 'cbr' in mylar.CONFIG.DUPECONSTRAINT:
|
|
#this has to be configured in config - either retain cbr or cbz.
|
|
if tmpfc['ComicFilename'].endswith('.cbz'):
|
|
#keep di['filename']
|
|
logger.fdebug('[DUPECHECK-CBR PRIORITY] [#' + reann['Issue_Number'] + '] Retaining currently scanned in file : ' + di['filename'])
|
|
annualdupe = "yes"
|
|
break
|
|
else:
|
|
#keep tmpfc['ComicFilename']
|
|
logger.fdebug('[DUPECHECK-CBR PRIORITY] [#' + reann['Issue_Number'] + '] Retaining newly scanned in file : ' + tmpfc['ComicFilename'])
|
|
removedupe = True
|
|
elif 'cbz' in mylar.CONFIG.DUPECONSTRAINT:
|
|
if tmpfc['ComicFilename'].endswith('.cbr'):
|
|
#keep di['filename']
|
|
logger.fdebug('[DUPECHECK-CBZ PRIORITY] [#' + reann['Issue_Number'] + '] Retaining currently scanned in filename : ' + di['filename'])
|
|
annualdupe = "yes"
|
|
break
|
|
else:
|
|
#keep tmpfc['ComicFilename']
|
|
logger.fdebug('[DUPECHECK-CBZ PRIORITY] [#' + reann['Issue_Number'] + '] Retaining newly scanned in filename : ' + tmpfc['ComicFilename'])
|
|
removedupe = True
|
|
|
|
if mylar.CONFIG.DUPECONSTRAINT == 'filesize':
|
|
if tmpfc['ComicSize'] <= di['filesize']:
|
|
logger.fdebug('[DUPECHECK-FILESIZE PRIORITY] [#' + reann['Issue_Number'] + '] Retaining currently scanned in filename : ' + di['filename'])
|
|
annualdupe = "yes"
|
|
break
|
|
else:
|
|
logger.fdebug('[DUPECHECK-FILESIZE PRIORITY] [#' + reann['Issue_Number'] + '] Retaining newly scanned in filename : ' + tmpfc['ComicFilename'])
|
|
removedupe = True
|
|
|
|
if removedupe:
|
|
#need to remove the entry from issuedupechk so can add new one.
|
|
#tuple(y for y in x if y) for x in a
|
|
annualdupe_temp = []
|
|
tmphavefiles = 0
|
|
for x in annualdupechk:
|
|
logger.fdebug('Comparing x: ' + x['filename'] + ' to di:' + di['filename'])
|
|
if x['filename'] != di['filename']:
|
|
annualdupe_temp.append(x)
|
|
tmphavefiles+=1
|
|
annualdupechk = annualdupe_temp
|
|
havefiles = tmphavefiles + len(issuedupechk)
|
|
foundchk = False
|
|
break
|
|
|
|
|
|
if annualdupe == "no":
|
|
if foundchk == False:
|
|
logger.fdebug(module + ' Matched...annual issue: ' + rescan['ComicName'] + '#' + str(reann['Issue_Number']) + ' --- ' + str(int_iss))
|
|
havefiles+=1
|
|
haveissue = "yes"
|
|
isslocation = helpers.conversion(tmpfc['ComicFilename'])
|
|
issSize = str(tmpfc['ComicSize'])
|
|
logger.fdebug(module + ' .......filename: ' + isslocation)
|
|
logger.fdebug(module + ' .......filesize: ' + str(tmpfc['ComicSize']))
|
|
# to avoid duplicate issues which screws up the count...let's store the filename issues then
|
|
# compare earlier...
|
|
annualdupechk.append({'fcdigit': int(fcdigit),
|
|
'anncomicid': ANNComicID,
|
|
'filename': tmpfc['ComicFilename'],
|
|
'filesize': tmpfc['ComicSize'],
|
|
'issueyear': issyear,
|
|
'issueid': reann['IssueID']})
|
|
break
|
|
|
|
if annualdupe == "yes":
|
|
logger.fdebug(module + ' I should break out here because of a dupe.')
|
|
break
|
|
|
|
if haveissue == "yes" or annualdupe == "yes": break
|
|
n+=1
|
|
|
|
if issuedupe == "yes" or annualdupe == "yes": pass
|
|
else:
|
|
#we have the # of comics, now let's update the db.
|
|
#even if we couldn't find the physical issue, check the status.
|
|
#-- if annuals aren't enabled, this will bugger out.
|
|
writeit = True
|
|
try:
|
|
if mylar.CONFIG.ANNUALS_ON:
|
|
if any(['annual' in temploc.lower(), 'special' in temploc.lower()]):
|
|
if reann is None:
|
|
logger.fdebug(module + ' Annual/Special present in location, but series does not have any annuals attached to it - Ignoring')
|
|
writeit = False
|
|
else:
|
|
iss_id = reann['IssueID']
|
|
else:
|
|
iss_id = reiss['IssueID']
|
|
else:
|
|
if any(['annual' in temploc.lower(), 'special' in temploc.lower()]):
|
|
logger.fdebug(module + ' Annual support not enabled, but annual/special issue present within directory. Ignoring issue.')
|
|
writeit = False
|
|
else:
|
|
iss_id = reiss['IssueID']
|
|
except:
|
|
logger.warn(module + ' An error occured trying to get the relevant issue data. This is probably due to the series not having proper issue data.')
|
|
logger.warn(module + ' you should either Refresh the series, and/or submit an issue on github in regards to the series and the error.')
|
|
return
|
|
|
|
if writeit == True and haveissue == 'yes':
|
|
#logger.fdebug(module + ' issueID to write to db:' + str(iss_id))
|
|
controlValueDict = {"IssueID": str(iss_id)}
|
|
|
|
#if Archived, increase the 'Have' count.
|
|
if archive:
|
|
issStatus = "Archived"
|
|
else:
|
|
issStatus = "Downloaded"
|
|
|
|
newValueDict = {"Location": isslocation,
|
|
"ComicSize": issSize,
|
|
"Status": issStatus
|
|
}
|
|
|
|
issID_to_ignore.append(str(iss_id))
|
|
|
|
if ANNComicID:
|
|
myDB.upsert("annuals", newValueDict, controlValueDict)
|
|
ANNComicID = None
|
|
else:
|
|
myDB.upsert("issues", newValueDict, controlValueDict)
|
|
else:
|
|
ANNComicID = None
|
|
fn+=1
|
|
|
|
#here we need to change the status of the ones we DIDN'T FIND above since the loop only hits on FOUND issues.
|
|
update_iss = []
|
|
#break this up in sequnces of 200 so it doesn't break the sql statement.
|
|
cnt = 0
|
|
for genlist in helpers.chunker(issID_to_ignore, 200):
|
|
tmpsql = "SELECT * FROM issues WHERE ComicID=? AND IssueID not in ({seq})".format(seq=','.join(['?'] *(len(genlist) -1)))
|
|
chkthis = myDB.select(tmpsql, genlist)
|
|
if chkthis is None:
|
|
pass
|
|
else:
|
|
for chk in chkthis:
|
|
if chk['IssueID'] in issID_to_ignore:
|
|
continue
|
|
|
|
old_status = chk['Status']
|
|
|
|
if old_status == "Skipped":
|
|
if mylar.CONFIG.AUTOWANT_ALL:
|
|
issStatus = "Wanted"
|
|
else:
|
|
issStatus = "Skipped"
|
|
#elif old_status == "Archived":
|
|
# issStatus = "Archived"
|
|
elif old_status == "Downloaded":
|
|
issStatus = "Archived"
|
|
else:
|
|
continue
|
|
#elif old_status == "Wanted":
|
|
# issStatus = "Wanted"
|
|
#elif old_status == "Ignored":
|
|
# issStatus = "Ignored"
|
|
#elif old_status == "Snatched": #this is needed for torrents, or else it'll keep on queuing..
|
|
# issStatus = "Snatched"
|
|
#else:
|
|
# issStatus = "Skipped"
|
|
update_iss.append({"IssueID": chk['IssueID'],
|
|
"Status": issStatus})
|
|
|
|
if len(update_iss) > 0:
|
|
i = 0
|
|
#do it like this to avoid DB locks...
|
|
for ui in update_iss:
|
|
controlValueDict = {"IssueID": ui['IssueID']}
|
|
newStatusValue = {"Status": ui['Status']}
|
|
myDB.upsert("issues", newStatusValue, controlValueDict)
|
|
i+=1
|
|
logger.info(module + ' Updated the status of ' + str(i) + ' issues for ' + rescan['ComicName'] + ' (' + str(rescan['ComicYear']) + ') that were not found.')
|
|
|
|
logger.info(module + ' Total files located: ' + str(havefiles))
|
|
|
|
foundcount = havefiles
|
|
arcfiles = 0
|
|
arcanns = 0
|
|
# if filechecker returns 0 files (it doesn't find any), but some issues have a status of 'Archived'
|
|
# the loop below won't work...let's adjust :)
|
|
arcissues = myDB.select("SELECT count(*) FROM issues WHERE ComicID=? and Status='Archived'", [ComicID])
|
|
if int(arcissues[0][0]) > 0:
|
|
arcfiles = arcissues[0][0]
|
|
arcannuals = myDB.select("SELECT count(*) FROM annuals WHERE ComicID=? and Status='Archived'", [ComicID])
|
|
if int(arcannuals[0][0]) > 0:
|
|
arcanns = arcannuals[0][0]
|
|
|
|
if havefiles == 0:
|
|
if arcfiles > 0 or arcanns > 0:
|
|
arcfiles = arcfiles + arcanns
|
|
havefiles = havefiles + arcfiles
|
|
logger.fdebug(module + ' Adjusting have total to ' + str(havefiles) + ' because of this many archive files already in Archive status :' + str(arcfiles))
|
|
else:
|
|
#if files exist in the given directory, but are in an archived state - the numbers will get botched up here.
|
|
if (arcfiles + arcanns) > 0:
|
|
logger.fdebug(module + ' ' + str(int(arcfiles + arcanns)) + ' issue(s) are in an Archive status already. Increasing Have total from ' + str(havefiles) + ' to include these archives.')
|
|
havefiles = havefiles + (arcfiles + arcanns)
|
|
|
|
ignorecount = 0
|
|
if mylar.CONFIG.IGNORE_HAVETOTAL: # if this is enabled, will increase Have total as if in Archived Status
|
|
ignoresi = myDB.select("SELECT count(*) FROM issues WHERE ComicID=? AND Status='Ignored'", [ComicID])
|
|
ignoresa = myDB.select("SELECT count(*) FROM annuals WHERE ComicID=? AND Status='Ignored'", [ComicID])
|
|
ignorecount = int(ignoresi[0][0]) + int(ignoresa[0][0])
|
|
if ignorecount > 0:
|
|
havefiles = havefiles + ignorecount
|
|
logger.fdebug(module + ' Adjusting have total to ' + str(havefiles) + ' because of this many Ignored files:' + str(ignorecount))
|
|
|
|
|
|
snatchedcount = 0
|
|
if mylar.CONFIG.SNATCHED_HAVETOTAL: # if this is enabled, will increase Have total as if in Archived Status
|
|
snatches = myDB.select("SELECT count(*) FROM issues WHERE ComicID=? AND Status='Snatched'", [ComicID])
|
|
if int(snatches[0][0]) > 0:
|
|
snatchedcount = snatches[0][0]
|
|
havefiles = havefiles + snatchedcount
|
|
logger.fdebug(module + ' Adjusting have total to ' + str(havefiles) + ' because of this many Snatched files:' + str(snatchedcount))
|
|
|
|
#now that we are finished...
|
|
#adjust for issues that have been marked as Downloaded, but aren't found/don't exist.
|
|
#do it here, because above loop only cycles though found comics using filechecker.
|
|
downissues = myDB.select("SELECT * FROM issues WHERE ComicID=? and Status='Downloaded'", [ComicID])
|
|
downissues += myDB.select("SELECT * FROM annuals WHERE ComicID=? and Status='Downloaded'", [ComicID])
|
|
if downissues is None:
|
|
pass
|
|
else:
|
|
archivedissues = 0 #set this to 0 so it tallies correctly.
|
|
for down in downissues:
|
|
#print "downlocation:" + str(down['Location'])
|
|
#remove special characters from
|
|
#temploc = rescan['ComicLocation'].replace('_', ' ')
|
|
#temploc = re.sub('[\#\'\/\.]', '', temploc)
|
|
#print ("comiclocation: " + str(rescan['ComicLocation']))
|
|
#print ("downlocation: " + str(down['Location']))
|
|
if down['Location'] is None:
|
|
logger.fdebug(module + ' Location does not exist which means file was not downloaded successfully, or was moved.')
|
|
controlValue = {"IssueID": down['IssueID']}
|
|
newValue = {"Status": "Archived"}
|
|
myDB.upsert("issues", newValue, controlValue)
|
|
archivedissues+=1
|
|
pass
|
|
else:
|
|
comicpath = os.path.join(rescan['ComicLocation'], down['Location'])
|
|
if os.path.exists(comicpath):
|
|
continue
|
|
logger.fdebug('Issue exists - no need to change status.')
|
|
else:
|
|
if mylar.CONFIG.MULTIPLE_DEST_DIRS is not None and mylar.CONFIG.MULTIPLE_DEST_DIRS != 'None':
|
|
if os.path.exists(os.path.join(mylar.CONFIG.MULTIPLE_DEST_DIRS, os.path.basename(rescan['ComicLocation']))):
|
|
#logger.fdebug('Issue(s) currently exist and found within multiple destination directory location')
|
|
continue
|
|
#print "Changing status from Downloaded to Archived - cannot locate file"
|
|
controlValue = {"IssueID": down['IssueID']}
|
|
newValue = {"Status": "Archived"}
|
|
myDB.upsert("issues", newValue, controlValue)
|
|
archivedissues+=1
|
|
if archivedissues > 0:
|
|
logger.fdebug(module + ' I have changed the status of ' + str(archivedissues) + ' issues to a status of Archived, as I now cannot locate them in the series directory.')
|
|
|
|
havefiles = havefiles + archivedissues #arcfiles already tallied in havefiles in above segment
|
|
|
|
#combined total for dispay total purposes only.
|
|
combined_total = iscnt + anncnt
|
|
if mylar.CONFIG.IGNORE_TOTAL: # if this is enabled, will increase Have total as if in Archived Status
|
|
ignoresa = myDB.select("SELECT count(*) FROM issues WHERE ComicID=? AND Status='Ignored'", [ComicID])
|
|
ignoresb = myDB.select("SELECT count(*) FROM annuals WHERE ComicID=? AND Status='Ignored'", [ComicID])
|
|
ignorecnt = ignoresa[0][0] + ignoresb[0][0]
|
|
|
|
if ignorecnt > 0:
|
|
combined_total -= ignorecnt
|
|
logger.fdebug('%s Reducing total comics in series from %s to %s because of %s ignored files.' % (module, (iscnt+anncnt), combined_total, ignorecnt))
|
|
|
|
#quick check
|
|
if havefiles > combined_total:
|
|
logger.warn(module + ' It looks like you have physical issues in the series directory, but are forcing these issues to an Archived Status. Adjusting have counts.')
|
|
havefiles = havefiles - arcfiles
|
|
|
|
thetotals = totals(ComicID, havefiles, combined_total, module, recheck=recheck)
|
|
totalarc = arcfiles + archivedissues
|
|
|
|
#enforce permissions
|
|
if mylar.CONFIG.ENFORCE_PERMS:
|
|
logger.fdebug(module + ' Ensuring permissions/ownership enforced for series: ' + rescan['ComicName'])
|
|
filechecker.setperms(rescan['ComicLocation'])
|
|
logger.info(module + ' I have physically found ' + str(foundcount) + ' issues, ignored ' + str(ignorecount) + ' issues, snatched ' + str(snatchedcount) + ' issues, and accounted for ' + str(totalarc) + ' in an Archived state [ Total Issue Count: ' + str(havefiles) + ' / ' + str(combined_total) + ' ]')
|
|
|
|
def totals(ComicID, havefiles=None, totalfiles=None, module=None, issueid=None, file=None, recheck=False):
|
|
if module is None:
|
|
module = '[FILE-RESCAN]'
|
|
myDB = db.DBConnection()
|
|
filetable = 'issues'
|
|
if any([havefiles is None, havefiles == '+1']):
|
|
if havefiles is None:
|
|
hf = myDB.selectone("SELECT Have, Total FROM comics WHERE ComicID=?", [ComicID]).fetchone()
|
|
havefiles = int(hf['Have'])
|
|
totalfiles = int(hf['Total'])
|
|
else:
|
|
hf = myDB.selectone("SELECT a.Have, a.Total, b.Status as IssStatus FROM comics AS a INNER JOIN issues as b ON a.ComicID=b.ComicID WHERE b.IssueID=?", [issueid]).fetchone()
|
|
if hf is None:
|
|
hf = myDB.selectone("SELECT a.Have, a.Total, b.Status as IssStatus FROM comics AS a INNER JOIN annuals as b ON a.ComicID=b.ComicID WHERE b.IssueID=?", [issueid]).fetchone()
|
|
filetable = 'annuals'
|
|
totalfiles = int(hf['Total'])
|
|
logger.fdebug('totalfiles: %s' % totalfiles)
|
|
logger.fdebug('status: %s' % hf['IssStatus'])
|
|
if hf['IssStatus'] != 'Downloaded':
|
|
try:
|
|
havefiles = int(hf['Have']) +1
|
|
if havefiles > totalfiles and recheck is False:
|
|
recheck = True
|
|
return forceRescan(ComicID, recheck=recheck)
|
|
except TypeError:
|
|
if totalfiles == 1:
|
|
havefiles = 1
|
|
else:
|
|
logger.warn('Total issues for this series [ComiciD:%s/IssueID:%s] is not 1 when it should be. This is probably a mistake and the series should be refreshed.' % (ComicID, issueid))
|
|
havefiles = 0
|
|
logger.fdebug('incremented havefiles: %s' % havefiles)
|
|
else:
|
|
havefiles = int(hf['Have'])
|
|
logger.fdebug('untouched havefiles: %s' % havefiles)
|
|
#let's update the total count of comics that was found.
|
|
#store just the total of issues, since annuals gets tracked seperately.
|
|
controlValueStat = {"ComicID": ComicID}
|
|
newValueStat = {"Have": havefiles,
|
|
"Total": totalfiles}
|
|
|
|
myDB.upsert("comics", newValueStat, controlValueStat)
|
|
if file is not None:
|
|
controlValueStat = {"IssueID": issueid,
|
|
"ComicID": ComicID}
|
|
newValueStat = {"ComicSize": os.path.getsize(file)}
|
|
myDB.upsert(filetable, newValueStat, controlValueStat)
|