mirror of https://github.com/evilhero/mylar
1621 lines
79 KiB
Python
1621 lines
79 KiB
Python
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
|
|
# This file is part of Mylar.
|
|
#
|
|
# Mylar is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# Mylar is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with Mylar. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import time
|
|
import os, errno
|
|
import sys
|
|
import shlex
|
|
import datetime
|
|
import re
|
|
import json
|
|
import urllib
|
|
import urllib2
|
|
import shutil
|
|
import imghdr
|
|
import sqlite3
|
|
import cherrypy
|
|
import requests
|
|
|
|
import mylar
|
|
from mylar import logger, filers, helpers, db, mb, cv, parseit, filechecker, search, updater, moveit, comicbookdb
|
|
|
|
|
|
def is_exists(comicid):
|
|
|
|
myDB = db.DBConnection()
|
|
|
|
# See if the artist is already in the database
|
|
comiclist = myDB.select('SELECT ComicID, ComicName from comics WHERE ComicID=?', [comicid])
|
|
|
|
if any(comicid in x for x in comiclist):
|
|
logger.info(comiclist[0][1] + ' is already in the database.')
|
|
return False
|
|
else:
|
|
return False
|
|
|
|
|
|
def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=None, calledfrom=None, annload=None, chkwant=None, issuechk=None, issuetype=None, latestissueinfo=None, csyear=None, fixed_type=None):
|
|
myDB = db.DBConnection()
|
|
|
|
controlValueDict = {"ComicID": comicid}
|
|
|
|
dbcomic = myDB.selectone('SELECT * FROM comics WHERE ComicID=?', [comicid]).fetchone()
|
|
if dbcomic is None:
|
|
newValueDict = {"ComicName": "Comic ID: %s" % (comicid),
|
|
"Status": "Loading"}
|
|
if all([imported is not None, mylar.CONFIG.IMP_PATHS is True]):
|
|
comlocation = os.path.dirname(imported['filelisting'][0]['comiclocation'])
|
|
else:
|
|
comlocation = None
|
|
oldcomversion = None
|
|
series_status = 'Loading'
|
|
lastissueid = None
|
|
aliases = None
|
|
else:
|
|
if chkwant is not None:
|
|
logger.fdebug('ComicID: ' + str(comicid) + ' already exists. Not adding from the future pull list at this time.')
|
|
return 'Exists'
|
|
if dbcomic['Status'] == 'Active':
|
|
series_status = 'Active'
|
|
elif dbcomic['Status'] == 'Paused':
|
|
series_status = 'Paused'
|
|
else:
|
|
series_status = 'Loading'
|
|
|
|
newValueDict = {"Status": "Loading"}
|
|
comlocation = dbcomic['ComicLocation']
|
|
lastissueid = dbcomic['LatestIssueID']
|
|
aliases = dbcomic['AlternateSearch']
|
|
logger.info('aliases currently: %s' % aliases)
|
|
|
|
if not latestissueinfo:
|
|
latestissueinfo = []
|
|
latestissueinfo.append({"latestiss": dbcomic['LatestIssue'],
|
|
"latestdate": dbcomic['LatestDate']})
|
|
|
|
if mylar.CONFIG.CREATE_FOLDERS is True:
|
|
checkdirectory = filechecker.validateAndCreateDirectory(comlocation, True)
|
|
if not checkdirectory:
|
|
logger.warn('Error trying to validate/create directory. Aborting this process at this time.')
|
|
return
|
|
oldcomversion = dbcomic['ComicVersion'] #store the comicversion and chk if it exists before hammering.
|
|
myDB.upsert("comics", newValueDict, controlValueDict)
|
|
|
|
#run the re-sortorder here in order to properly display the page
|
|
if all([pullupd is None, calledfrom != 'maintenance']):
|
|
helpers.ComicSort(comicorder=mylar.COMICSORT, imported=comicid)
|
|
|
|
# we need to lookup the info for the requested ComicID in full now
|
|
comic = cv.getComic(comicid, 'comic')
|
|
|
|
if not comic:
|
|
logger.warn('Error fetching comic. ID for : ' + comicid)
|
|
if dbcomic is None:
|
|
newValueDict = {"ComicName": "Fetch failed, try refreshing. (%s)" % (comicid),
|
|
"Status": "Active"}
|
|
else:
|
|
if series_status == 'Active' or series_status == 'Loading':
|
|
newValueDict = {"Status": "Active"}
|
|
myDB.upsert("comics", newValueDict, controlValueDict)
|
|
return
|
|
|
|
if comic['ComicName'].startswith('The '):
|
|
sortname = comic['ComicName'][4:]
|
|
else:
|
|
sortname = comic['ComicName']
|
|
|
|
comic['Corrected_Type'] = fixed_type
|
|
if fixed_type is not None and fixed_type != comic['Type']:
|
|
logger.info('Forced Comic Type to : %s' % comic['Corrected_Type'])
|
|
|
|
logger.info('Now adding/updating: ' + comic['ComicName'])
|
|
#--Now that we know ComicName, let's try some scraping
|
|
#--Start
|
|
# gcd will return issue details (most importantly publishing date)
|
|
if not mylar.CONFIG.CV_ONLY:
|
|
if mismatch == "no" or mismatch is None:
|
|
gcdinfo=parseit.GCDScraper(comic['ComicName'], comic['ComicYear'], comic['ComicIssues'], comicid)
|
|
#print ("gcdinfo: " + str(gcdinfo))
|
|
mismatch_com = "no"
|
|
if gcdinfo == "No Match":
|
|
updater.no_searchresults(comicid)
|
|
nomatch = "true"
|
|
logger.info('There was an error when trying to add ' + comic['ComicName'] + ' (' + comic['ComicYear'] + ')')
|
|
return nomatch
|
|
else:
|
|
mismatch_com = "yes"
|
|
#print ("gcdinfo:" + str(gcdinfo))
|
|
|
|
elif mismatch == "yes":
|
|
CV_EXcomicid = myDB.selectone("SELECT * from exceptions WHERE ComicID=?", [comicid])
|
|
if CV_EXcomicid['variloop'] is None: pass
|
|
else:
|
|
vari_loop = CV_EXcomicid['variloop']
|
|
NewComicID = CV_EXcomicid['NewComicID']
|
|
gcomicid = CV_EXcomicid['GComicID']
|
|
resultURL = "/series/" + str(NewComicID) + "/"
|
|
#print ("variloop" + str(CV_EXcomicid['variloop']))
|
|
#if vari_loop == '99':
|
|
gcdinfo = parseit.GCDdetails(comseries=None, resultURL=resultURL, vari_loop=0, ComicID=comicid, TotalIssues=0, issvariation="no", resultPublished=None)
|
|
|
|
# print ("Series Published" + parseit.resultPublished)
|
|
|
|
CV_NoYearGiven = "no"
|
|
#if the SeriesYear returned by CV is blank or none (0000), let's use the gcd one.
|
|
if any([comic['ComicYear'] is None, comic['ComicYear'] == '0000', comic['ComicYear'][-1:] == '-']):
|
|
if mylar.CONFIG.CV_ONLY:
|
|
#we'll defer this until later when we grab all the issues and then figure it out
|
|
logger.info('Uh-oh. I cannot find a Series Year for this series. I am going to try analyzing deeper.')
|
|
SeriesYear = cv.getComic(comicid, 'firstissue', comic['FirstIssueID'])
|
|
if SeriesYear == '0000':
|
|
logger.info('Ok - I could not find a Series Year at all. Loading in the issue data now and will figure out the Series Year.')
|
|
CV_NoYearGiven = "yes"
|
|
issued = cv.getComic(comicid, 'issue')
|
|
SeriesYear = issued['firstdate'][:4]
|
|
else:
|
|
SeriesYear = gcdinfo['SeriesYear']
|
|
else:
|
|
SeriesYear = comic['ComicYear']
|
|
|
|
if any([int(SeriesYear) > int(datetime.datetime.now().year) + 1, int(SeriesYear) == 2099]) and csyear is not None:
|
|
logger.info('Corrected year of ' + str(SeriesYear) + ' to corrected year for series that was manually entered previously of ' + str(csyear))
|
|
SeriesYear = csyear
|
|
|
|
logger.info('Successfully retrieved details for ' + comic['ComicName'])
|
|
|
|
#since the weekly issue check could return either annuals or issues, let's initialize it here so it carries through properly.
|
|
weeklyissue_check = []
|
|
|
|
if any([oldcomversion is None, oldcomversion == "None"]):
|
|
logger.info('Previous version detected as None - seeing if update required')
|
|
if comic['ComicVersion'].isdigit():
|
|
comicVol = 'v' + comic['ComicVersion']
|
|
logger.info('Updated version to :' + str(comicVol))
|
|
if all([mylar.CONFIG.SETDEFAULTVOLUME is False, comicVol == 'v1']):
|
|
comicVol = None
|
|
else:
|
|
if mylar.CONFIG.SETDEFAULTVOLUME is True:
|
|
comicVol = 'v1'
|
|
else:
|
|
comicVol = None
|
|
else:
|
|
comicVol = oldcomversion
|
|
if all([mylar.CONFIG.SETDEFAULTVOLUME is True, comicVol is None]):
|
|
comicVol = 'v1'
|
|
|
|
|
|
|
|
# setup default location here
|
|
u_comicnm = comic['ComicName']
|
|
# let's remove the non-standard characters here that will break filenaming / searching.
|
|
comicname_filesafe = helpers.filesafe(u_comicnm)
|
|
|
|
if comlocation is None:
|
|
|
|
comic_values = {'ComicName': comic['ComicName'],
|
|
'ComicPublisher': comic['ComicPublisher'],
|
|
'ComicYear': SeriesYear,
|
|
'ComicVersion': comicVol,
|
|
'Type': comic['Type'],
|
|
'Corrected_Type': comic['Corrected_Type']}
|
|
|
|
dothedew = filers.FileHandlers(comic=comic_values)
|
|
comlocation = dothedew.folder_create()
|
|
|
|
#moved this out of the above loop so it will chk for existance of comlocation in case moved
|
|
#if it doesn't exist - create it (otherwise will bugger up later on)
|
|
if os.path.isdir(comlocation):
|
|
logger.info('Directory (' + comlocation + ') already exists! Continuing...')
|
|
else:
|
|
if mylar.CONFIG.CREATE_FOLDERS is True:
|
|
checkdirectory = filechecker.validateAndCreateDirectory(comlocation, True)
|
|
if not checkdirectory:
|
|
logger.warn('Error trying to validate/create directory. Aborting this process at this time.')
|
|
return
|
|
|
|
#try to account for CV not updating new issues as fast as GCD
|
|
#seems CV doesn't update total counts
|
|
#comicIssues = gcdinfo['totalissues']
|
|
comicIssues = comic['ComicIssues']
|
|
|
|
if not mylar.CONFIG.CV_ONLY:
|
|
if gcdinfo['gcdvariation'] == "cv":
|
|
comicIssues = str(int(comic['ComicIssues']) + 1)
|
|
|
|
if mylar.CONFIG.ALTERNATE_LATEST_SERIES_COVERS is False:
|
|
cimage = os.path.join(mylar.CONFIG.CACHE_DIR, str(comicid) + '.jpg')
|
|
PRComicImage = os.path.join('cache', str(comicid) + ".jpg")
|
|
ComicImage = helpers.replacetheslash(PRComicImage)
|
|
if os.path.isfile(cimage) is True:
|
|
logger.fdebug('Cover already exists for series. Not redownloading.')
|
|
else:
|
|
covercheck = helpers.getImage(comicid, comic['ComicImage'])
|
|
if covercheck == 'retry':
|
|
logger.info('Attempting to retrieve alternate comic image for the series.')
|
|
covercheck = helpers.getImage(comicid, comic['ComicImageALT'])
|
|
|
|
#if the comic cover local is checked, save a cover.jpg to the series folder.
|
|
if all([mylar.CONFIG.COMIC_COVER_LOCAL is True, os.path.isdir(comlocation) is True, os.path.isfile(os.path.join(comlocation, 'cover.jpg')) is False]):
|
|
try:
|
|
comiclocal = os.path.join(comlocation, 'cover.jpg')
|
|
shutil.copyfile(cimage, comiclocal)
|
|
if mylar.CONFIG.ENFORCE_PERMS:
|
|
filechecker.setperms(comiclocal)
|
|
except IOError as e:
|
|
logger.error('Unable to save cover (%s) into series directory (%s) at this time.' % (cimage, comiclocal))
|
|
else:
|
|
ComicImage = None
|
|
|
|
#for description ...
|
|
#Cdesc = helpers.cleanhtml(comic['ComicDescription'])
|
|
#cdes_find = Cdesc.find("Collected")
|
|
#cdes_removed = Cdesc[:cdes_find]
|
|
#logger.fdebug('description: ' + cdes_removed)
|
|
|
|
#dynamic-name generation here.
|
|
as_d = filechecker.FileChecker(watchcomic=comic['ComicName'])
|
|
as_dinfo = as_d.dynamic_replace(comic['ComicName'])
|
|
tmpseriesname = as_dinfo['mod_seriesname']
|
|
dynamic_seriesname = re.sub('[\|\s]','', tmpseriesname.lower()).strip()
|
|
|
|
if comic['Issue_List'] != 'None':
|
|
issue_list = json.dumps(comic['Issue_List'])
|
|
else:
|
|
issue_list = None
|
|
|
|
if comic['Aliases'] != 'None':
|
|
if all([aliases is not None, aliases != 'None']):
|
|
for x in aliases.split('##'):
|
|
aliaschk = [x for y in comic['Aliases'].split('##') if y == x]
|
|
if aliaschk and x not in aliases.split('##'):
|
|
aliases += '##' + ''.join(x)
|
|
else:
|
|
if x not in aliases.split('##'):
|
|
aliases += '##' + x
|
|
else:
|
|
aliases = comic['Aliases']
|
|
else:
|
|
aliases = aliases
|
|
|
|
logger.fdebug('comicIssues: %s' % comicIssues)
|
|
logger.fdebug('seriesyear: %s / currentyear: %s' % (SeriesYear, helpers.today()[:4]))
|
|
logger.fdebug('comicType: %s' % comic['Type'])
|
|
if all([int(comicIssues) == 1, SeriesYear < helpers.today()[:4], comic['Type'] != 'One-Shot', comic['Type'] != 'TPB']):
|
|
logger.info('Determined to be a one-shot issue. Forcing Edition to One-Shot')
|
|
booktype = 'One-Shot'
|
|
else:
|
|
booktype = comic['Type']
|
|
|
|
controlValueDict = {"ComicID": comicid}
|
|
newValueDict = {"ComicName": comic['ComicName'],
|
|
"ComicSortName": sortname,
|
|
"ComicName_Filesafe": comicname_filesafe,
|
|
"DynamicComicName": dynamic_seriesname,
|
|
"ComicYear": SeriesYear,
|
|
"ComicImage": ComicImage,
|
|
"ComicImageURL": comic.get("ComicImage", ""),
|
|
"ComicImageALTURL": comic.get("ComicImageALT", ""),
|
|
"Total": comicIssues,
|
|
"ComicVersion": comicVol,
|
|
"ComicLocation": comlocation,
|
|
"ComicPublisher": comic['ComicPublisher'],
|
|
# "Description": Cdesc, #.dencode('utf-8', 'replace'),
|
|
"DetailURL": comic['ComicURL'],
|
|
"AlternateSearch": aliases,
|
|
# "ComicPublished": gcdinfo['resultPublished'],
|
|
"ComicPublished": "Unknown",
|
|
"Type": booktype,
|
|
"Corrected_Type": comic['Corrected_Type'],
|
|
"Collects": issue_list,
|
|
"DateAdded": helpers.today(),
|
|
"Status": "Loading"}
|
|
|
|
myDB.upsert("comics", newValueDict, controlValueDict)
|
|
|
|
#comicsort here...
|
|
#run the re-sortorder here in order to properly display the page
|
|
if all([pullupd is None, calledfrom != 'maintenance']):
|
|
helpers.ComicSort(sequence='update')
|
|
|
|
if CV_NoYearGiven == 'no':
|
|
#if set to 'no' then we haven't pulled down the issues, otherwise we did it already
|
|
issued = cv.getComic(comicid, 'issue')
|
|
if issued is None:
|
|
logger.warn('Unable to retrieve data from ComicVine. Get your own API key already!')
|
|
return
|
|
logger.info('Sucessfully retrieved issue details for ' + comic['ComicName'])
|
|
|
|
#move to own function so can call independently to only refresh issue data
|
|
#issued is from cv.getComic, comic['ComicName'] & comicid would both be already known to do independent call.
|
|
updateddata = updateissuedata(comicid, comic['ComicName'], issued, comicIssues, calledfrom, SeriesYear=SeriesYear, latestissueinfo=latestissueinfo)
|
|
issuedata = updateddata['issuedata']
|
|
anndata = updateddata['annualchk']
|
|
nostatus = updateddata['nostatus']
|
|
importantdates = updateddata['importantdates']
|
|
if issuedata is None:
|
|
logger.warn('Unable to complete Refreshing / Adding issue data - this WILL create future problems if not addressed.')
|
|
return {'status': 'incomplete'}
|
|
|
|
if any([calledfrom is None, calledfrom == 'maintenance']):
|
|
issue_collection(issuedata, nostatus='False')
|
|
#need to update annuals at this point too....
|
|
if anndata:
|
|
manualAnnual(annchk=anndata)
|
|
|
|
if mylar.CONFIG.ALTERNATE_LATEST_SERIES_COVERS is True: #, lastissueid != importantdates['LatestIssueID']]):
|
|
if os.path.join(mylar.CONFIG.CACHE_DIR, comicid + '.jpg') is True:
|
|
cover_modtime = datetime.datetime.utcfromtimestamp(os.path.getmtime(os.path.join(mylar.CONFIG.CACHE_DIR, comicid + '.jpg')))
|
|
cover_mtime = datetime.datetime.strftime(cover_modtime, '%Y-%m-%d')
|
|
if importantdates['LatestStoreDate'] != '0000-00-00':
|
|
lsd = re.sub('-', '', importantdates['LatestStoreDate']).strip()
|
|
else:
|
|
lsd = re.sub('-', '', importantdates['LatestDate']).strip()
|
|
if re.sub('-', '', cover_mtime).strip() < lsd:
|
|
logger.info('Attempting to retrieve new issue cover for display')
|
|
image_it(comicid, importantdates['LatestIssueID'], comlocation, comic['ComicImage'])
|
|
else:
|
|
logger.fdebug('no update required - lastissueid [%s] = latestissueid [%s]' % (lastissueid, importantdates['LatestIssueID']))
|
|
else:
|
|
image_it(comicid, importantdates['LatestIssueID'], comlocation, comic['ComicImage'])
|
|
else:
|
|
logger.fdebug('no update required - lastissueid [%s] = latestissueid [%s]' % (lastissueid, importantdates['LatestIssueID']))
|
|
|
|
if (mylar.CONFIG.CVINFO or (mylar.CONFIG.CV_ONLY and mylar.CONFIG.CVINFO)) and os.path.isdir(comlocation):
|
|
if os.path.isfile(os.path.join(comlocation, "cvinfo")) is False:
|
|
with open(os.path.join(comlocation, "cvinfo"), "w") as text_file:
|
|
text_file.write(str(comic['ComicURL']))
|
|
|
|
|
|
if calledfrom == 'weekly':
|
|
logger.info('Successfully refreshed ' + comic['ComicName'] + ' (' + str(SeriesYear) + '). Returning to Weekly issue comparison.')
|
|
logger.info('Update issuedata for ' + str(issuechk) + ' of : ' + str(weeklyissue_check))
|
|
return {'status': 'complete',
|
|
'issuedata': issuedata} # this should be the weeklyissue_check data from updateissuedata function
|
|
|
|
elif calledfrom == 'dbupdate':
|
|
logger.info('returning to dbupdate module')
|
|
return {'status': 'complete',
|
|
'issuedata': issuedata,
|
|
'anndata': anndata } # this should be the issuedata data from updateissuedata function
|
|
|
|
elif calledfrom == 'weeklycheck':
|
|
logger.info('Successfully refreshed ' + comic['ComicName'] + ' (' + str(SeriesYear) + '). Returning to Weekly issue update.')
|
|
return #no need to return any data here.
|
|
|
|
|
|
logger.info('Updating complete for: ' + comic['ComicName'])
|
|
|
|
#if it made it here, then the issuedata contains dates, let's pull the data now.
|
|
latestiss = importantdates['LatestIssue']
|
|
latestdate = importantdates['LatestDate']
|
|
lastpubdate = importantdates['LastPubDate']
|
|
series_status = importantdates['SeriesStatus']
|
|
#move the files...if imported is not empty & not futurecheck (meaning it's not from the mass importer.)
|
|
#logger.info('imported is : ' + str(imported))
|
|
if imported is None or imported == 'None' or imported == 'futurecheck':
|
|
pass
|
|
else:
|
|
if mylar.CONFIG.IMP_MOVE:
|
|
logger.info('Mass import - Move files')
|
|
moveit.movefiles(comicid, comlocation, imported)
|
|
else:
|
|
logger.info('Mass import - Moving not Enabled. Setting Archived Status for import.')
|
|
moveit.archivefiles(comicid, comlocation, imported)
|
|
|
|
#check for existing files...
|
|
statbefore = myDB.selectone("SELECT Status FROM issues WHERE ComicID=? AND Int_IssueNumber=?", [comicid, helpers.issuedigits(latestiss)]).fetchone()
|
|
logger.fdebug('issue: ' + latestiss + ' status before chk :' + str(statbefore['Status']))
|
|
updater.forceRescan(comicid)
|
|
statafter = myDB.selectone("SELECT Status FROM issues WHERE ComicID=? AND Int_IssueNumber=?", [comicid, helpers.issuedigits(latestiss)]).fetchone()
|
|
logger.fdebug('issue: ' + latestiss + ' status after chk :' + str(statafter['Status']))
|
|
|
|
logger.fdebug('pullupd: ' + str(pullupd))
|
|
logger.fdebug('lastpubdate: ' + str(lastpubdate))
|
|
logger.fdebug('series_status: ' + str(series_status))
|
|
if pullupd is None:
|
|
# lets' check the pullist for anything at this time as well since we're here.
|
|
# do this for only Present comics....
|
|
if mylar.CONFIG.AUTOWANT_UPCOMING and lastpubdate == 'Present' and series_status == 'Active': #and 'Present' in gcdinfo['resultPublished']:
|
|
logger.fdebug('latestissue: #' + str(latestiss))
|
|
chkstats = myDB.selectone("SELECT * FROM issues WHERE ComicID=? AND Int_IssueNumber=?", [comicid, helpers.issuedigits(latestiss)]).fetchone()
|
|
if chkstats is None:
|
|
if mylar.CONFIG.ANNUALS_ON:
|
|
chkstats = myDB.selectone("SELECT * FROM annuals WHERE ComicID=? AND Int_IssueNumber=?", [comicid, helpers.issuedigits(latestiss)]).fetchone()
|
|
|
|
if chkstats:
|
|
logger.fdebug('latestissue status: ' + chkstats['Status'])
|
|
if chkstats['Status'] == 'Skipped' or chkstats['Status'] == 'Wanted' or chkstats['Status'] == 'Snatched':
|
|
logger.info('Checking this week pullist for new issues of ' + comic['ComicName'])
|
|
if comic['ComicName'] != comicname_filesafe:
|
|
cn_pull = comicname_filesafe
|
|
else:
|
|
cn_pull = comic['ComicName']
|
|
updater.newpullcheck(ComicName=cn_pull, ComicID=comicid, issue=latestiss)
|
|
|
|
#here we grab issues that have been marked as wanted above...
|
|
if calledfrom != 'maintenance':
|
|
results = []
|
|
issresults = myDB.select("SELECT * FROM issues where ComicID=? AND Status='Wanted'", [comicid])
|
|
if issresults:
|
|
for issr in issresults:
|
|
results.append({'IssueID': issr['IssueID'],
|
|
'Issue_Number': issr['Issue_Number'],
|
|
'Status': issr['Status']
|
|
})
|
|
if mylar.CONFIG.ANNUALS_ON:
|
|
an_results = myDB.select("SELECT * FROM annuals WHERE ComicID=? AND Status='Wanted'", [comicid])
|
|
if an_results:
|
|
for ar in an_results:
|
|
results.append({'IssueID': ar['IssueID'],
|
|
'Issue_Number': ar['Issue_Number'],
|
|
'Status': ar['Status']
|
|
})
|
|
|
|
|
|
if results:
|
|
logger.info('Attempting to grab wanted issues for : ' + comic['ComicName'])
|
|
|
|
for result in results:
|
|
logger.fdebug('Searching for : ' + str(result['Issue_Number']))
|
|
logger.fdebug('Status of : ' + str(result['Status']))
|
|
search.searchforissue(result['IssueID'])
|
|
else: logger.info('No issues marked as wanted for ' + comic['ComicName'])
|
|
|
|
logger.info('Finished grabbing what I could.')
|
|
else:
|
|
logger.info('Already have the latest issue : #' + str(latestiss))
|
|
|
|
if chkwant is not None:
|
|
#if this isn't None, this is being called from the futureupcoming list
|
|
#a new series was added automagically, but it has more than 1 issue (probably because it was a back-dated issue)
|
|
#the chkwant is a tuple containing all the data for the given series' issues that were marked as Wanted for futureupcoming dates.
|
|
chkresults = myDB.select("SELECT * FROM issues WHERE ComicID=? AND Status='Skipped'", [comicid])
|
|
if chkresults:
|
|
logger.info('[FROM THE FUTURE CHECKLIST] Attempting to grab wanted issues for : ' + comic['ComicName'])
|
|
for result in chkresults:
|
|
for chkit in chkwant:
|
|
logger.fdebug('checking ' + chkit['IssueNumber'] + ' against ' + result['Issue_Number'])
|
|
if chkit['IssueNumber'] == result['Issue_Number']:
|
|
logger.fdebug('Searching for : ' + result['Issue_Number'])
|
|
logger.fdebug('Status of : ' + str(result['Status']))
|
|
search.searchforissue(result['IssueID'])
|
|
else: logger.info('No issues marked as wanted for ' + comic['ComicName'])
|
|
|
|
logger.info('Finished grabbing what I could.')
|
|
|
|
if imported == 'futurecheck':
|
|
logger.info('Returning to Future-Check module to complete the add & remove entry.')
|
|
return
|
|
elif all([imported is not None, imported != 'None']):
|
|
logger.info('Successfully imported : ' + comic['ComicName'])
|
|
return
|
|
|
|
if calledfrom == 'addbyid':
|
|
logger.info('Sucessfully added %s (%s) to the watchlist by directly using the ComicVine ID' % (comic['ComicName'], SeriesYear))
|
|
return {'status': 'complete'}
|
|
elif calledfrom == 'maintenance':
|
|
logger.info('Sucessfully added %s (%s) to the watchlist' % (comic['ComicName'], SeriesYear))
|
|
return {'status': 'complete',
|
|
'comicname': comic['ComicName'],
|
|
'year': SeriesYear}
|
|
else:
|
|
logger.info('Sucessfully added %s (%s) to the watchlist' % (comic['ComicName'], SeriesYear))
|
|
return {'status': 'complete'}
|
|
|
|
# if imported['Volume'] is None or imported['Volume'] == 'None':
|
|
# results = myDB.select("SELECT * FROM importresults WHERE (WatchMatch is Null OR WatchMatch LIKE 'C%') AND DynamicName=? AND Volume IS NULL",[imported['DynamicName']])
|
|
# else:
|
|
# if not imported['Volume'].lower().startswith('v'):
|
|
# volume = 'v' + str(imported['Volume'])
|
|
# results = myDB.select("SELECT * FROM importresults WHERE (WatchMatch is Null OR WatchMatch LIKE 'C%') AND DynamicName=? AND Volume=?",[imported['DynamicName'],imported['Volume']])
|
|
#
|
|
# if results is not None:
|
|
# for result in results:
|
|
# controlValue = {"DynamicName": imported['DynamicName'],
|
|
# "Volume": imported['Volume']}
|
|
# newValue = {"Status": "Imported",
|
|
# "SRID": result['SRID'],
|
|
# "ComicID": comicid}
|
|
# myDB.upsert("importresults", newValue, controlValue)
|
|
|
|
|
|
def GCDimport(gcomicid, pullupd=None, imported=None, ogcname=None):
|
|
# this is for importing via GCD only and not using CV.
|
|
# used when volume spanning is discovered for a Comic (and can't be added using CV).
|
|
# Issue Counts are wrong (and can't be added).
|
|
|
|
# because Comicvine ComicID and GCD ComicID could be identical at some random point, let's distinguish.
|
|
# CV = comicid, GCD = gcomicid :) (ie. CV=2740, GCD=G3719)
|
|
|
|
gcdcomicid = gcomicid
|
|
myDB = db.DBConnection()
|
|
|
|
# We need the current minimal info in the database instantly
|
|
# so we don't throw a 500 error when we redirect to the artistPage
|
|
|
|
controlValueDict = {"ComicID": gcdcomicid}
|
|
|
|
comic = myDB.selectone('SELECT ComicName, ComicYear, Total, ComicPublished, ComicImage, ComicLocation, ComicPublisher FROM comics WHERE ComicID=?', [gcomicid]).fetchone()
|
|
ComicName = comic[0]
|
|
ComicYear = comic[1]
|
|
ComicIssues = comic[2]
|
|
ComicPublished = comic[3]
|
|
comlocation = comic[5]
|
|
ComicPublisher = comic[6]
|
|
#ComicImage = comic[4]
|
|
#print ("Comic:" + str(ComicName))
|
|
|
|
newValueDict = {"Status": "Loading"}
|
|
myDB.upsert("comics", newValueDict, controlValueDict)
|
|
|
|
# we need to lookup the info for the requested ComicID in full now
|
|
#comic = cv.getComic(comicid,'comic')
|
|
|
|
if not comic:
|
|
logger.warn('Error fetching comic. ID for : ' + gcdcomicid)
|
|
if dbcomic is None:
|
|
newValueDict = {"ComicName": "Fetch failed, try refreshing. (%s)" % (gcdcomicid),
|
|
"Status": "Active"}
|
|
else:
|
|
newValueDict = {"Status": "Active"}
|
|
myDB.upsert("comics", newValueDict, controlValueDict)
|
|
return
|
|
|
|
#run the re-sortorder here in order to properly display the page
|
|
if pullupd is None:
|
|
helpers.ComicSort(comicorder=mylar.COMICSORT, imported=gcomicid)
|
|
|
|
if ComicName.startswith('The '):
|
|
sortname = ComicName[4:]
|
|
else:
|
|
sortname = ComicName
|
|
|
|
|
|
logger.info(u"Now adding/updating: " + ComicName)
|
|
#--Now that we know ComicName, let's try some scraping
|
|
#--Start
|
|
# gcd will return issue details (most importantly publishing date)
|
|
comicid = gcomicid[1:]
|
|
resultURL = "/series/" + str(comicid) + "/"
|
|
gcdinfo=parseit.GCDdetails(comseries=None, resultURL=resultURL, vari_loop=0, ComicID=gcdcomicid, TotalIssues=ComicIssues, issvariation=None, resultPublished=None)
|
|
if gcdinfo == "No Match":
|
|
logger.warn("No matching result found for " + ComicName + " (" + ComicYear + ")")
|
|
updater.no_searchresults(gcomicid)
|
|
nomatch = "true"
|
|
return nomatch
|
|
logger.info(u"Sucessfully retrieved details for " + ComicName)
|
|
# print ("Series Published" + parseit.resultPublished)
|
|
#--End
|
|
|
|
ComicImage = gcdinfo['ComicImage']
|
|
|
|
#comic book location on machine
|
|
# setup default location here
|
|
if comlocation is None:
|
|
# let's remove the non-standard characters here.
|
|
u_comicnm = ComicName
|
|
u_comicname = u_comicnm.encode('ascii', 'ignore').strip()
|
|
if ':' in u_comicname or '/' in u_comicname or ',' in u_comicname:
|
|
comicdir = u_comicname
|
|
if ':' in comicdir:
|
|
comicdir = comicdir.replace(':', '')
|
|
if '/' in comicdir:
|
|
comicdir = comicdir.replace('/', '-')
|
|
if ',' in comicdir:
|
|
comicdir = comicdir.replace(',', '')
|
|
else: comicdir = u_comicname
|
|
|
|
series = comicdir
|
|
publisher = ComicPublisher
|
|
year = ComicYear
|
|
|
|
#do work to generate folder path
|
|
values = {'$Series': series,
|
|
'$Publisher': publisher,
|
|
'$Year': year,
|
|
'$series': series.lower(),
|
|
'$publisher': publisher.lower(),
|
|
'$Volume': year
|
|
}
|
|
|
|
if mylar.CONFIG.FOLDER_FORMAT == '':
|
|
comlocation = mylar.CONFIG.DESTINATION_DIR + "/" + comicdir + " (" + comic['ComicYear'] + ")"
|
|
else:
|
|
comlocation = mylar.CONFIG.DESTINATION_DIR + "/" + helpers.replace_all(mylar.CONFIG.FOLDER_FORMAT, values)
|
|
|
|
#comlocation = mylar.CONFIG.DESTINATION_DIR + "/" + comicdir + " (" + ComicYear + ")"
|
|
if mylar.CONFIG.DESTINATION_DIR == "":
|
|
logger.error(u"There is no general directory specified - please specify in Config/Post-Processing.")
|
|
return
|
|
if mylar.CONFIG.REPLACE_SPACES:
|
|
#mylar.CONFIG.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
|
|
comlocation = comlocation.replace(' ', mylar.CONFIG.REPLACE_CHAR)
|
|
|
|
#if it doesn't exist - create it (otherwise will bugger up later on)
|
|
if os.path.isdir(comlocation):
|
|
logger.info(u"Directory (" + comlocation + ") already exists! Continuing...")
|
|
else:
|
|
if mylar.CONFIG.CREATE_FOLDERS is True:
|
|
checkdirectory = filechecker.validateAndCreateDirectory(comlocation, True)
|
|
if not checkdirectory:
|
|
logger.warn('Error trying to validate/create directory. Aborting this process at this time.')
|
|
return
|
|
|
|
comicIssues = gcdinfo['totalissues']
|
|
|
|
#let's download the image...
|
|
if os.path.exists(mylar.CONFIG.CACHE_DIR): pass
|
|
else:
|
|
#let's make the dir.
|
|
try:
|
|
os.makedirs(str(mylar.CONFIG.CACHE_DIR))
|
|
logger.info(u"Cache Directory successfully created at: " + str(mylar.CONFIG.CACHE_DIR))
|
|
|
|
except OSError:
|
|
logger.error(u"Could not create cache dir : " + str(mylar.CONFIG.CACHE_DIR))
|
|
|
|
coverfile = os.path.join(mylar.CONFIG.CACHE_DIR, str(gcomicid) + ".jpg")
|
|
|
|
#new CV API restriction - one api request / second.
|
|
if mylar.CONFIG.CVAPI_RATE is None or mylar.CONFIG.CVAPI_RATE < 2:
|
|
time.sleep(2)
|
|
else:
|
|
time.sleep(mylar.CONFIG.CVAPI_RATE)
|
|
|
|
urllib.urlretrieve(str(ComicImage), str(coverfile))
|
|
try:
|
|
with open(str(coverfile)) as f:
|
|
ComicImage = os.path.join('cache', str(gcomicid) + ".jpg")
|
|
|
|
#this is for Firefox when outside the LAN...it works, but I don't know how to implement it
|
|
#without breaking the normal flow for inside the LAN (above)
|
|
#ComicImage = "http://" + str(mylar.CONFIG.HTTP_HOST) + ":" + str(mylar.CONFIG.HTTP_PORT) + "/cache/" + str(comi$
|
|
|
|
logger.info(u"Sucessfully retrieved cover for " + ComicName)
|
|
#if the comic cover local is checked, save a cover.jpg to the series folder.
|
|
if mylar.CONFIG.COMIC_COVER_LOCAL and os.path.isdir(comlocation):
|
|
comiclocal = os.path.join(comlocation + "/cover.jpg")
|
|
shutil.copy(ComicImage, comiclocal)
|
|
except IOError as e:
|
|
logger.error(u"Unable to save cover locally at this time.")
|
|
|
|
#if comic['ComicVersion'].isdigit():
|
|
# comicVol = "v" + comic['ComicVersion']
|
|
#else:
|
|
# comicVol = None
|
|
|
|
|
|
controlValueDict = {"ComicID": gcomicid}
|
|
newValueDict = {"ComicName": ComicName,
|
|
"ComicSortName": sortname,
|
|
"ComicYear": ComicYear,
|
|
"Total": comicIssues,
|
|
"ComicLocation": comlocation,
|
|
#"ComicVersion": comicVol,
|
|
"ComicImage": ComicImage,
|
|
"ComicImageURL": comic.get('ComicImage', ''),
|
|
"ComicImageALTURL": comic.get('ComicImageALT', ''),
|
|
#"ComicPublisher": comic['ComicPublisher'],
|
|
#"ComicPublished": comicPublished,
|
|
"DateAdded": helpers.today(),
|
|
"Status": "Loading"}
|
|
|
|
myDB.upsert("comics", newValueDict, controlValueDict)
|
|
|
|
#comicsort here...
|
|
#run the re-sortorder here in order to properly display the page
|
|
if pullupd is None:
|
|
helpers.ComicSort(sequence='update')
|
|
|
|
logger.info(u"Sucessfully retrieved issue details for " + ComicName)
|
|
n = 0
|
|
iscnt = int(comicIssues)
|
|
issnum = []
|
|
issname = []
|
|
issdate = []
|
|
int_issnum = []
|
|
#let's start issue #'s at 0 -- thanks to DC for the new 52 reboot! :)
|
|
latestiss = "0"
|
|
latestdate = "0000-00-00"
|
|
#print ("total issues:" + str(iscnt))
|
|
#---removed NEW code here---
|
|
logger.info(u"Now adding/updating issues for " + ComicName)
|
|
bb = 0
|
|
while (bb <= iscnt):
|
|
#---NEW.code
|
|
try:
|
|
gcdval = gcdinfo['gcdchoice'][bb]
|
|
#print ("gcdval: " + str(gcdval))
|
|
except IndexError:
|
|
#account for gcd variation here
|
|
if gcdinfo['gcdvariation'] == 'gcd':
|
|
#print ("gcd-variation accounted for.")
|
|
issdate = '0000-00-00'
|
|
int_issnum = int (issis / 1000)
|
|
break
|
|
if 'nn' in str(gcdval['GCDIssue']):
|
|
#no number detected - GN, TP or the like
|
|
logger.warn(u"Non Series detected (Graphic Novel, etc) - cannot proceed at this time.")
|
|
updater.no_searchresults(comicid)
|
|
return
|
|
elif '.' in str(gcdval['GCDIssue']):
|
|
issst = str(gcdval['GCDIssue']).find('.')
|
|
issb4dec = str(gcdval['GCDIssue'])[:issst]
|
|
#if the length of decimal is only 1 digit, assume it's a tenth
|
|
decis = str(gcdval['GCDIssue'])[issst +1:]
|
|
if len(decis) == 1:
|
|
decisval = int(decis) * 10
|
|
issaftdec = str(decisval)
|
|
if len(decis) == 2:
|
|
decisval = int(decis)
|
|
issaftdec = str(decisval)
|
|
if int(issaftdec) == 0: issaftdec = "00"
|
|
gcd_issue = issb4dec + "." + issaftdec
|
|
gcdis = (int(issb4dec) * 1000) + decisval
|
|
else:
|
|
gcdis = int(str(gcdval['GCDIssue'])) * 1000
|
|
gcd_issue = str(gcdval['GCDIssue'])
|
|
#get the latest issue / date using the date.
|
|
int_issnum = int(gcdis / 1000)
|
|
issdate = str(gcdval['GCDDate'])
|
|
issid = "G" + str(gcdval['IssueID'])
|
|
if gcdval['GCDDate'] > latestdate:
|
|
latestiss = str(gcd_issue)
|
|
latestdate = str(gcdval['GCDDate'])
|
|
#print("(" + str(bb) + ") IssueID: " + str(issid) + " IssueNo: " + str(gcd_issue) + " Date" + str(issdate) )
|
|
#---END.NEW.
|
|
|
|
# check if the issue already exists
|
|
iss_exists = myDB.selectone('SELECT * from issues WHERE IssueID=?', [issid]).fetchone()
|
|
|
|
|
|
# Only change the status & add DateAdded if the issue is not already in the database
|
|
if iss_exists is None:
|
|
newValueDict['DateAdded'] = helpers.today()
|
|
|
|
#adjust for inconsistencies in GCD date format - some dates have ? which borks up things.
|
|
if "?" in str(issdate):
|
|
issdate = "0000-00-00"
|
|
|
|
controlValueDict = {"IssueID": issid}
|
|
newValueDict = {"ComicID": gcomicid,
|
|
"ComicName": ComicName,
|
|
"Issue_Number": gcd_issue,
|
|
"IssueDate": issdate,
|
|
"Int_IssueNumber": int_issnum
|
|
}
|
|
|
|
#print ("issueid:" + str(controlValueDict))
|
|
#print ("values:" + str(newValueDict))
|
|
|
|
if mylar.CONFIG.AUTOWANT_ALL:
|
|
newValueDict['Status'] = "Wanted"
|
|
elif issdate > helpers.today() and mylar.CONFIG.AUTOWANT_UPCOMING:
|
|
newValueDict['Status'] = "Wanted"
|
|
else:
|
|
newValueDict['Status'] = "Skipped"
|
|
|
|
if iss_exists:
|
|
#print ("Existing status : " + str(iss_exists['Status']))
|
|
newValueDict['Status'] = iss_exists['Status']
|
|
|
|
|
|
myDB.upsert("issues", newValueDict, controlValueDict)
|
|
bb+=1
|
|
|
|
# logger.debug(u"Updating comic cache for " + ComicName)
|
|
# cache.getThumb(ComicID=issue['issueid'])
|
|
|
|
# logger.debug(u"Updating cache for: " + ComicName)
|
|
# cache.getThumb(ComicIDcomicid)
|
|
|
|
|
|
controlValueStat = {"ComicID": gcomicid}
|
|
newValueStat = {"Status": "Active",
|
|
"LatestIssue": latestiss,
|
|
"LatestDate": latestdate,
|
|
"LastUpdated": helpers.now()
|
|
}
|
|
|
|
myDB.upsert("comics", newValueStat, controlValueStat)
|
|
|
|
if mylar.CONFIG.CVINFO and os.path.isdir(comlocation):
|
|
if not os.path.exists(comlocation + "/cvinfo"):
|
|
with open(comlocation + "/cvinfo", "w") as text_file:
|
|
text_file.write("http://comicvine.gamespot.com/volume/49-" + str(comicid))
|
|
|
|
logger.info(u"Updating complete for: " + ComicName)
|
|
|
|
#move the files...if imported is not empty (meaning it's not from the mass importer.)
|
|
if imported is None or imported == 'None':
|
|
pass
|
|
else:
|
|
if mylar.CONFIG.IMP_MOVE:
|
|
logger.info("Mass import - Move files")
|
|
moveit.movefiles(gcomicid, comlocation, ogcname)
|
|
else:
|
|
logger.info("Mass import - Moving not Enabled. Setting Archived Status for import.")
|
|
moveit.archivefiles(gcomicid, ogcname)
|
|
|
|
#check for existing files...
|
|
updater.forceRescan(gcomicid)
|
|
|
|
|
|
if pullupd is None:
|
|
# lets' check the pullist for anyting at this time as well since we're here.
|
|
if mylar.CONFIG.AUTOWANT_UPCOMING and 'Present' in ComicPublished:
|
|
logger.info(u"Checking this week's pullist for new issues of " + ComicName)
|
|
updater.newpullcheck(comic['ComicName'], gcomicid)
|
|
|
|
#here we grab issues that have been marked as wanted above...
|
|
|
|
results = myDB.select("SELECT * FROM issues where ComicID=? AND Status='Wanted'", [gcomicid])
|
|
if results:
|
|
logger.info(u"Attempting to grab wanted issues for : " + ComicName)
|
|
|
|
for result in results:
|
|
foundNZB = "none"
|
|
if (mylar.CONFIG.NZBSU or mylar.CONFIG.DOGNZB or mylar.CONFIG.EXPERIMENTAL or mylar.CONFIG.NEWZNAB) and (mylar.CONFIG.SAB_HOST):
|
|
foundNZB = search.searchforissue(result['IssueID'])
|
|
if foundNZB == "yes":
|
|
updater.foundsearch(result['ComicID'], result['IssueID'])
|
|
else: logger.info(u"No issues marked as wanted for " + ComicName)
|
|
|
|
logger.info(u"Finished grabbing what I could.")
|
|
|
|
|
|
def issue_collection(issuedata, nostatus):
|
|
myDB = db.DBConnection()
|
|
nowdate = datetime.datetime.now()
|
|
now_week = datetime.datetime.strftime(nowdate, "%Y%U")
|
|
|
|
if issuedata:
|
|
for issue in issuedata:
|
|
|
|
|
|
controlValueDict = {"IssueID": issue['IssueID']}
|
|
newValueDict = {"ComicID": issue['ComicID'],
|
|
"ComicName": issue['ComicName'],
|
|
"IssueName": issue['IssueName'],
|
|
"Issue_Number": issue['Issue_Number'],
|
|
"IssueDate": issue['IssueDate'],
|
|
"ReleaseDate": issue['ReleaseDate'],
|
|
"DigitalDate": issue['DigitalDate'],
|
|
"Int_IssueNumber": issue['Int_IssueNumber'],
|
|
"ImageURL": issue['ImageURL'],
|
|
"ImageURL_ALT": issue['ImageURL_ALT']
|
|
#"Status": "Skipped" #set this to Skipped by default to avoid NULL entries.
|
|
}
|
|
|
|
# check if the issue already exists
|
|
iss_exists = myDB.selectone('SELECT * from issues WHERE IssueID=?', [issue['IssueID']]).fetchone()
|
|
dbwrite = "issues"
|
|
|
|
#if iss_exists is None:
|
|
# iss_exists = myDB.selectone('SELECT * from annuals WHERE IssueID=?', [issue['IssueID']]).fetchone()
|
|
# if iss_exists:
|
|
# dbwrite = "annuals"
|
|
|
|
if nostatus == 'False':
|
|
|
|
# Only change the status & add DateAdded if the issue is already in the database
|
|
if iss_exists is None:
|
|
newValueDict['DateAdded'] = helpers.today()
|
|
if issue['ReleaseDate'] == '0000-00-00':
|
|
dk = re.sub('-', '', issue['IssueDate']).strip()
|
|
else:
|
|
dk = re.sub('-', '', issue['ReleaseDate']).strip() # converts date to 20140718 format
|
|
if dk == '00000000':
|
|
logger.warn('Issue Data is invalid for Issue Number %s. Marking this issue as Skipped' % issue['Issue_Number'])
|
|
newValueDict['Status'] = "Skipped"
|
|
else:
|
|
datechk = datetime.datetime.strptime(dk, "%Y%m%d")
|
|
issue_week = datetime.datetime.strftime(datechk, "%Y%U")
|
|
if mylar.CONFIG.AUTOWANT_ALL:
|
|
newValueDict['Status'] = "Wanted"
|
|
#logger.fdebug('autowant all')
|
|
elif issue_week >= now_week and mylar.CONFIG.AUTOWANT_UPCOMING:
|
|
#logger.fdebug(str(datechk) + ' >= ' + str(nowtime))
|
|
newValueDict['Status'] = "Wanted"
|
|
else:
|
|
newValueDict['Status'] = "Skipped"
|
|
#logger.fdebug('status is : ' + str(newValueDict))
|
|
else:
|
|
#logger.fdebug('Existing status for issue #%s : %s' % (issue['Issue_Number'], iss_exists['Status']))
|
|
if any([iss_exists['Status'] is None, iss_exists['Status'] == 'None']):
|
|
is_status = 'Skipped'
|
|
else:
|
|
is_status = iss_exists['Status']
|
|
newValueDict['Status'] = is_status
|
|
|
|
else:
|
|
#logger.fdebug("Not changing the status at this time - reverting to previous module after to re-append existing status")
|
|
pass #newValueDict['Status'] = "Skipped"
|
|
|
|
try:
|
|
myDB.upsert(dbwrite, newValueDict, controlValueDict)
|
|
except sqlite3.InterfaceError, e:
|
|
#raise sqlite3.InterfaceError(e)
|
|
logger.error('Something went wrong - I cannot add the issue information into my DB.')
|
|
myDB.action("DELETE FROM comics WHERE ComicID=?", [issue['ComicID']])
|
|
return
|
|
|
|
|
|
def manualAnnual(manual_comicid=None, comicname=None, comicyear=None, comicid=None, annchk=None, manualupd=False):
|
|
#called when importing/refreshing an annual that was manually added.
|
|
myDB = db.DBConnection()
|
|
|
|
if annchk is None:
|
|
nowdate = datetime.datetime.now()
|
|
now_week = datetime.datetime.strftime(nowdate, "%Y%U")
|
|
annchk = []
|
|
issueid = manual_comicid
|
|
logger.fdebug(str(issueid) + ' added to series list as an Annual')
|
|
sr = cv.getComic(manual_comicid, 'comic')
|
|
logger.fdebug('Attempting to integrate ' + sr['ComicName'] + ' (' + str(issueid) + ') to the existing series of ' + comicname + '(' + str(comicyear) + ')')
|
|
if len(sr) is None or len(sr) == 0:
|
|
logger.fdebug('Could not find any information on the series indicated : ' + str(manual_comicid))
|
|
return
|
|
else:
|
|
n = 0
|
|
issued = cv.getComic(re.sub('4050-', '', manual_comicid).strip(), 'issue')
|
|
if int(sr['ComicIssues']) == 0 and len(issued['issuechoice']) == 1:
|
|
noissues = 1
|
|
else:
|
|
noissues = sr['ComicIssues']
|
|
logger.fdebug('there are ' + str(noissues) + ' annuals within this series.')
|
|
while (n < int(noissues)):
|
|
try:
|
|
firstval = issued['issuechoice'][n]
|
|
except IndexError:
|
|
break
|
|
try:
|
|
cleanname = helpers.cleanName(firstval['Issue_Name'])
|
|
except:
|
|
cleanname = 'None'
|
|
|
|
if firstval['Store_Date'] == '0000-00-00':
|
|
dk = re.sub('-', '', firstval['Issue_Date']).strip()
|
|
else:
|
|
dk = re.sub('-', '', firstval['Store_Date']).strip() # converts date to 20140718 format
|
|
if dk == '00000000':
|
|
logger.warn('Issue Data is invalid for Issue Number %s. Marking this issue as Skipped' % firstval['Issue_Number'])
|
|
astatus = "Skipped"
|
|
else:
|
|
datechk = datetime.datetime.strptime(dk, "%Y%m%d")
|
|
issue_week = datetime.datetime.strftime(datechk, "%Y%U")
|
|
if mylar.CONFIG.AUTOWANT_ALL:
|
|
astatus = "Wanted"
|
|
elif issue_week >= now_week and mylar.CONFIG.AUTOWANT_UPCOMING is True:
|
|
astatus = "Wanted"
|
|
else:
|
|
astatus = "Skipped"
|
|
|
|
annchk.append({'IssueID': str(firstval['Issue_ID']),
|
|
'ComicID': comicid,
|
|
'ReleaseComicID': re.sub('4050-', '', manual_comicid).strip(),
|
|
'ComicName': comicname,
|
|
'Issue_Number': str(firstval['Issue_Number']),
|
|
'IssueName': cleanname,
|
|
'IssueDate': str(firstval['Issue_Date']),
|
|
'ReleaseDate': str(firstval['Store_Date']),
|
|
'DigitalDate': str(firstval['Digital_Date']),
|
|
'Status': astatus,
|
|
'ReleaseComicName': sr['ComicName']})
|
|
n+=1
|
|
|
|
if manualupd is True:
|
|
return annchk
|
|
|
|
for ann in annchk:
|
|
newCtrl = {"IssueID": ann['IssueID']}
|
|
newVals = {"Issue_Number": ann['Issue_Number'],
|
|
"Int_IssueNumber": helpers.issuedigits(ann['Issue_Number']),
|
|
"IssueDate": ann['IssueDate'],
|
|
"ReleaseDate": ann['ReleaseDate'],
|
|
"DigitalDate": ann['DigitalDate'],
|
|
"IssueName": ann['IssueName'],
|
|
"ComicID": ann['ComicID'], #this is the series ID
|
|
"ReleaseComicID": ann['ReleaseComicID'], #this is the series ID for the annual(s)
|
|
"ComicName": ann['ComicName'], #series ComicName
|
|
"ReleaseComicName": ann['ReleaseComicName'], #series ComicName for the manual_comicid
|
|
"Status": ann['Status']}
|
|
#need to add in the values for the new series to be added.
|
|
#"M_ComicName": sr['ComicName'],
|
|
#"M_ComicID": manual_comicid}
|
|
myDB.upsert("annuals", newVals, newCtrl)
|
|
if len(annchk) > 0:
|
|
logger.info('Successfully integrated ' + str(len(annchk)) + ' annuals into the series: ' + annchk[0]['ComicName'])
|
|
return
|
|
|
|
|
|
def updateissuedata(comicid, comicname=None, issued=None, comicIssues=None, calledfrom=None, issuechk=None, issuetype=None, SeriesYear=None, latestissueinfo=None):
|
|
annualchk = []
|
|
weeklyissue_check = []
|
|
logger.fdebug('issuedata call references...')
|
|
logger.fdebug('comicid: %s' % comicid)
|
|
logger.fdebug('comicname: %s' % comicname)
|
|
logger.fdebug('comicissues: %s' % comicIssues)
|
|
logger.fdebug('calledfrom: %s' % calledfrom)
|
|
logger.fdebug('issuechk: %s' % issuechk)
|
|
logger.fdebug('latestissueinfo: %s' % latestissueinfo)
|
|
logger.fdebug('issuetype: %s' % issuetype)
|
|
#to facilitate independent calls to updateissuedata ONLY, account for data not available and get it.
|
|
#chkType comes from the weeklypulllist - either 'annual' or not to distinguish annuals vs. issues
|
|
if comicIssues is None:
|
|
comic = cv.getComic(comicid, 'comic')
|
|
if comic is None:
|
|
logger.warn('Error retrieving from ComicVine - either the site is down or you are not using your own CV API key')
|
|
return
|
|
if comicIssues is None:
|
|
comicIssues = comic['ComicIssues']
|
|
if SeriesYear is None:
|
|
SeriesYear = comic['ComicYear']
|
|
if comicname is None:
|
|
comicname = comic['ComicName']
|
|
if issued is None:
|
|
issued = cv.getComic(comicid, 'issue')
|
|
if issued is None:
|
|
logger.warn('Error retrieving from ComicVine - either the site is down or you are not using your own CV API key')
|
|
return
|
|
|
|
# poll against annuals here - to make sure annuals are uptodate.
|
|
annualchk = annual_check(comicname, SeriesYear, comicid, issuetype, issuechk, annualchk)
|
|
if annualchk is None:
|
|
annualchk = []
|
|
logger.fdebug('Finished Annual checking.')
|
|
|
|
n = 0
|
|
iscnt = int(comicIssues)
|
|
issid = []
|
|
issnum = []
|
|
issname = []
|
|
issdate = []
|
|
issuedata = []
|
|
#let's start issue #'s at 0 -- thanks to DC for the new 52 reboot! :)
|
|
latestiss = "0"
|
|
latestdate = "0000-00-00"
|
|
latest_stdate = "0000-00-00"
|
|
latestissueid = None
|
|
firstiss = "10000000"
|
|
firstdate = "2099-00-00"
|
|
#print ("total issues:" + str(iscnt))
|
|
logger.info('Now adding/updating issues for ' + comicname)
|
|
|
|
if iscnt > 0: #if a series is brand new, it wont have any issues/details yet so skip this part
|
|
while (n <= iscnt):
|
|
try:
|
|
firstval = issued['issuechoice'][n]
|
|
except IndexError:
|
|
break
|
|
try:
|
|
cleanname = helpers.cleanName(firstval['Issue_Name'])
|
|
except:
|
|
cleanname = 'None'
|
|
issid = str(firstval['Issue_ID'])
|
|
issnum = firstval['Issue_Number']
|
|
issname = cleanname
|
|
issdate = str(firstval['Issue_Date'])
|
|
storedate = str(firstval['Store_Date'])
|
|
digitaldate = str(firstval['Digital_Date'])
|
|
int_issnum = None
|
|
if issnum.isdigit():
|
|
int_issnum = int(issnum) * 1000
|
|
else:
|
|
if 'a.i.' in issnum.lower() or 'ai' in issnum.lower():
|
|
issnum = re.sub('\.', '', issnum)
|
|
#int_issnum = (int(issnum[:-2]) * 1000) + ord('a') + ord('i')
|
|
if 'au' in issnum.lower():
|
|
int_issnum = (int(issnum[:-2]) * 1000) + ord('a') + ord('u')
|
|
elif 'inh' in issnum.lower():
|
|
int_issnum = (int(issnum[:-4]) * 1000) + ord('i') + ord('n') + ord('h')
|
|
elif 'now' in issnum.lower():
|
|
int_issnum = (int(issnum[:-4]) * 1000) + ord('n') + ord('o') + ord('w')
|
|
elif 'mu' in issnum.lower():
|
|
int_issnum = (int(issnum[:-3]) * 1000) + ord('m') + ord('u')
|
|
elif 'hu' in issnum.lower():
|
|
int_issnum = (int(issnum[:-3]) * 1000) + ord('h') + ord('u')
|
|
elif u'\xbd' in issnum:
|
|
tmpiss = re.sub('[^0-9]', '', issnum).strip()
|
|
if len(tmpiss) > 0:
|
|
int_issnum = (int(tmpiss) + .5) * 1000
|
|
else:
|
|
int_issnum = .5 * 1000
|
|
logger.fdebug('1/2 issue detected :' + issnum + ' === ' + str(int_issnum))
|
|
elif u'\xbc' in issnum:
|
|
int_issnum = .25 * 1000
|
|
elif u'\xbe' in issnum:
|
|
int_issnum = .75 * 1000
|
|
elif u'\u221e' in issnum:
|
|
#issnum = utf-8 will encode the infinity symbol without any help
|
|
int_issnum = 9999999999 * 1000 # set 9999999999 for integer value of issue
|
|
elif '.' in issnum or ',' in issnum:
|
|
if ',' in issnum: issnum = re.sub(',', '.', issnum)
|
|
issst = str(issnum).find('.')
|
|
#logger.fdebug("issst:" + str(issst))
|
|
if issst == 0:
|
|
issb4dec = 0
|
|
else:
|
|
issb4dec = str(issnum)[:issst]
|
|
#logger.fdebug("issb4dec:" + str(issb4dec))
|
|
#if the length of decimal is only 1 digit, assume it's a tenth
|
|
decis = str(issnum)[issst +1:]
|
|
#logger.fdebug("decis:" + str(decis))
|
|
if len(decis) == 1:
|
|
decisval = int(decis) * 10
|
|
issaftdec = str(decisval)
|
|
elif len(decis) == 2:
|
|
decisval = int(decis)
|
|
issaftdec = str(decisval)
|
|
else:
|
|
decisval = decis
|
|
issaftdec = str(decisval)
|
|
#if there's a trailing decimal (ie. 1.50.) and it's either intentional or not, blow it away.
|
|
if issaftdec[-1:] == '.':
|
|
logger.fdebug('Trailing decimal located within issue number. Irrelevant to numbering. Obliterating.')
|
|
issaftdec = issaftdec[:-1]
|
|
try:
|
|
# int_issnum = str(issnum)
|
|
int_issnum = (int(issb4dec) * 1000) + (int(issaftdec) * 10)
|
|
except ValueError:
|
|
logger.error('This has no issue # for me to get - Either a Graphic Novel or one-shot.')
|
|
updater.no_searchresults(comicid)
|
|
return
|
|
else:
|
|
try:
|
|
x = float(issnum)
|
|
#validity check
|
|
if x < 0:
|
|
logger.fdebug('I have encountered a negative issue #: ' + str(issnum) + '. Trying to accomodate.')
|
|
logger.fdebug('value of x is : ' + str(x))
|
|
int_issnum = (int(x) *1000) - 1
|
|
else: raise ValueError
|
|
except ValueError, e:
|
|
x = 0
|
|
tstord = None
|
|
issno = None
|
|
invchk = "false"
|
|
if issnum.lower() != 'preview':
|
|
while (x < len(issnum)):
|
|
if issnum[x].isalpha():
|
|
#take first occurance of alpha in string and carry it through
|
|
tstord = issnum[x:].rstrip()
|
|
tstord = re.sub('[\-\,\.\+]', '', tstord).rstrip()
|
|
issno = issnum[:x].rstrip()
|
|
issno = re.sub('[\-\,\.\+]', '', issno).rstrip()
|
|
try:
|
|
isschk = float(issno)
|
|
except ValueError, e:
|
|
if len(issnum) == 1 and issnum.isalpha():
|
|
logger.fdebug('detected lone alpha issue. Attempting to figure this out.')
|
|
break
|
|
logger.fdebug('[' + issno + '] invalid numeric for issue - cannot be found. Ignoring.')
|
|
issno = None
|
|
tstord = None
|
|
invchk = "true"
|
|
break
|
|
x+=1
|
|
|
|
if all([tstord is not None, issno is not None, int_issnum is None]):
|
|
a = 0
|
|
ordtot = 0
|
|
if len(issnum) == 1 and issnum.isalpha():
|
|
int_issnum = ord(tstord.lower())
|
|
else:
|
|
while (a < len(tstord)):
|
|
ordtot += ord(tstord[a].lower()) #lower-case the letters for simplicty
|
|
a+=1
|
|
int_issnum = (int(issno) * 1000) + ordtot
|
|
elif invchk == "true":
|
|
if any([issnum.lower() == 'fall 2005', issnum.lower() == 'spring 2005', issnum.lower() == 'summer 2006', issnum.lower() == 'winter 2009']):
|
|
issnum = re.sub('[0-9]+', '', issnum).strip()
|
|
inu = 0
|
|
ordtot = 0
|
|
while (inu < len(issnum)):
|
|
ordtot += ord(issnum[inu].lower()) #lower-case the letters for simplicty
|
|
inu+=1
|
|
int_issnum = ordtot
|
|
else:
|
|
logger.fdebug('this does not have an issue # that I can parse properly.')
|
|
return
|
|
else:
|
|
if int_issnum is not None:
|
|
pass
|
|
elif issnum == '9-5':
|
|
issnum = u'9\xbd'
|
|
logger.fdebug('issue: 9-5 is an invalid entry. Correcting to : ' + issnum)
|
|
int_issnum = (9 * 1000) + (.5 * 1000)
|
|
elif issnum == '112/113':
|
|
int_issnum = (112 * 1000) + (.5 * 1000)
|
|
elif issnum == '14-16':
|
|
int_issnum = (15 * 1000) + (.5 * 1000)
|
|
elif issnum.lower() == 'preview':
|
|
inu = 0
|
|
ordtot = 0
|
|
while (inu < len(issnum)):
|
|
ordtot += ord(issnum[inu].lower()) #lower-case the letters for simplicty
|
|
inu+=1
|
|
int_issnum = ordtot
|
|
else:
|
|
logger.error(issnum + ' this has an alpha-numeric in the issue # which I cannot account for.')
|
|
return
|
|
#get the latest issue / date using the date.
|
|
#logger.fdebug('issue : ' + str(issnum))
|
|
#logger.fdebug('latest date: ' + str(latestdate))
|
|
#logger.fdebug('first date: ' + str(firstdate))
|
|
#logger.fdebug('issue date: ' + str(firstval['Issue_Date']))
|
|
#logger.fdebug('issue date: ' + storedate)
|
|
if any([firstval['Issue_Date'] >= latestdate, storedate >= latestdate]):
|
|
#logger.fdebug('date check hit for issue date > latestdate')
|
|
if int_issnum > helpers.issuedigits(latestiss):
|
|
#logger.fdebug('assigning latest issue to : ' + str(issnum))
|
|
latestiss = issnum
|
|
latestissueid = issid
|
|
if firstval['Issue_Date'] != '0000-00-00':
|
|
latestdate = str(firstval['Issue_Date'])
|
|
latest_stdate = storedate
|
|
else:
|
|
latestdate = storedate
|
|
latest_stdate = storedate
|
|
|
|
if firstval['Issue_Date'] < firstdate and firstval['Issue_Date'] != '0000-00-00':
|
|
firstiss = issnum
|
|
firstdate = str(firstval['Issue_Date'])
|
|
|
|
if issuechk is not None and issuetype == 'series':
|
|
logger.fdebug('comparing ' + str(issuechk) + ' .. to .. ' + str(int_issnum))
|
|
if issuechk == int_issnum:
|
|
weeklyissue_check.append({"Int_IssueNumber": int_issnum,
|
|
"Issue_Number": issnum,
|
|
"IssueDate": issdate,
|
|
"ReleaseDate": storedate,
|
|
"ComicID": comicid,
|
|
"IssueID": issid})
|
|
|
|
issuedata.append({"ComicID": comicid,
|
|
"IssueID": issid,
|
|
"ComicName": comicname,
|
|
"IssueName": issname,
|
|
"Issue_Number": issnum,
|
|
"IssueDate": issdate,
|
|
"ReleaseDate": storedate,
|
|
"DigitalDate": digitaldate,
|
|
"Int_IssueNumber": int_issnum,
|
|
"ImageURL": firstval['Image'],
|
|
"ImageURL_ALT": firstval['ImageALT']})
|
|
|
|
n+=1
|
|
|
|
if calledfrom == 'futurecheck' and len(issuedata) == 0:
|
|
logger.fdebug('This is a NEW series with no issue data - skipping issue updating for now, and assigning generic information so things don\'t break')
|
|
latestdate = latestissueinfo[0]['latestdate'] # if it's from futurecheck, issuechk holds the latestdate for the given issue
|
|
latestiss = latestissueinfo[0]['latestiss']
|
|
lastpubdate = 'Present'
|
|
publishfigure = str(SeriesYear) + ' - ' + str(lastpubdate)
|
|
else:
|
|
#if calledfrom == 'weeklycheck':
|
|
if len(issuedata) >= 1 and not calledfrom == 'dbupdate':
|
|
logger.fdebug('initiating issue updating - info & status')
|
|
issue_collection(issuedata, nostatus='False')
|
|
else:
|
|
logger.fdebug('initiating issue updating - just the info')
|
|
issue_collection(issuedata, nostatus='True')
|
|
|
|
styear = str(SeriesYear)
|
|
if firstdate is not None:
|
|
if SeriesYear != firstdate[:4]:
|
|
if firstdate[:4] == '2099':
|
|
logger.fdebug('Series start date (%s) differs from First Issue start date as First Issue date is unknown - assuming Series Year as Start Year (even though CV might say previous year - it\'s all gravy).' % (SeriesYear))
|
|
else:
|
|
logger.fdebug('Series start date (%s) cannot be properly determined and/or it might cross over into different year (%s) - assuming store date of first issue (%s) as Start Year (even though CV might say previous year - it\'s all gravy).' % (SeriesYear, firstdate[:4], firstdate))
|
|
if firstdate == '2099-00-00':
|
|
firstdate = '%s-01-01' % SeriesYear
|
|
styear = str(firstdate[:4])
|
|
|
|
if firstdate[5:7] == '00':
|
|
stmonth = "?"
|
|
else:
|
|
stmonth = helpers.fullmonth(firstdate[5:7])
|
|
|
|
ltyear = re.sub('/s', '', latestdate[:4])
|
|
if latestdate[5:7] == '00':
|
|
ltmonth = "?"
|
|
else:
|
|
ltmonth = helpers.fullmonth(latestdate[5:7])
|
|
|
|
#try to determine if it's an 'actively' published comic from above dates
|
|
#threshold is if it's within a month (<55 days) let's assume it's recent.
|
|
try:
|
|
c_date = datetime.date(int(latestdate[:4]), int(latestdate[5:7]), 1)
|
|
except:
|
|
logger.error('Cannot determine Latest Date for given series. This is most likely due to an issue having a date of : 0000-00-00')
|
|
latestdate = str(SeriesYear) + '-01-01'
|
|
logger.error('Setting Latest Date to be ' + str(latestdate) + '. You should inform CV that the issue data is stale.')
|
|
c_date = datetime.date(int(latestdate[:4]), int(latestdate[5:7]), 1)
|
|
|
|
n_date = datetime.date.today()
|
|
recentchk = (n_date - c_date).days
|
|
|
|
if recentchk <= 55:
|
|
lastpubdate = 'Present'
|
|
else:
|
|
if ltmonth == '?':
|
|
if ltyear == '0000':
|
|
lastpubdate = '?'
|
|
else:
|
|
lastpubdate = str(ltyear)
|
|
elif ltyear == '0000':
|
|
lastpubdate = '?'
|
|
else:
|
|
lastpubdate = str(ltmonth) + ' ' + str(ltyear)
|
|
|
|
if stmonth == '?' and ('?' in lastpubdate and '0000' in lastpubdate):
|
|
lastpubdate = 'Present'
|
|
newpublish = True
|
|
publishfigure = str(styear) + ' - ' + str(lastpubdate)
|
|
else:
|
|
newpublish = False
|
|
publishfigure = str(stmonth) + ' ' + str(styear) + ' - ' + str(lastpubdate)
|
|
|
|
if stmonth == '?' and styear == '?' and lastpubdate =='0000' and comicIssues == '0':
|
|
logger.info('No available issue data - I believe this is a NEW series.')
|
|
latestdate = latestissueinfo[0]['latestdate']
|
|
latestiss = latestissueinfo[0]['latestiss']
|
|
lastpubdate = 'Present'
|
|
publishfigure = str(SeriesYear) + ' - ' + str(lastpubdate)
|
|
|
|
|
|
controlValueStat = {"ComicID": comicid}
|
|
|
|
newValueStat = {"Status": "Active",
|
|
"Total": comicIssues,
|
|
"ComicPublished": publishfigure,
|
|
"NewPublish": newpublish,
|
|
"LatestIssue": latestiss,
|
|
"LatestIssueID": latestissueid,
|
|
"LatestDate": latestdate,
|
|
"LastUpdated": helpers.now()
|
|
}
|
|
|
|
myDB = db.DBConnection()
|
|
myDB.upsert("comics", newValueStat, controlValueStat)
|
|
|
|
importantdates = {}
|
|
importantdates['LatestIssue'] = latestiss
|
|
importantdates['LatestIssueID'] = latestissueid
|
|
importantdates['LatestDate'] = latestdate
|
|
importantdates['LatestStoreDate'] = latest_stdate
|
|
importantdates['LastPubDate'] = lastpubdate
|
|
importantdates['SeriesStatus'] = 'Active'
|
|
|
|
if calledfrom == 'weeklycheck':
|
|
return weeklyissue_check
|
|
|
|
elif len(issuedata) >= 1 and not calledfrom == 'dbupdate':
|
|
return {'issuedata': issuedata,
|
|
'annualchk': annualchk,
|
|
'importantdates': importantdates,
|
|
'nostatus': False}
|
|
|
|
elif calledfrom == 'dbupdate':
|
|
return {'issuedata': issuedata,
|
|
'annualchk': annualchk,
|
|
'importantdates': importantdates,
|
|
'nostatus': True}
|
|
|
|
else:
|
|
return importantdates
|
|
|
|
def annual_check(ComicName, SeriesYear, comicid, issuetype, issuechk, annualslist):
|
|
annualids = [] #to be used to make sure an ID isn't double-loaded
|
|
annload = []
|
|
anncnt = 0
|
|
|
|
nowdate = datetime.datetime.now()
|
|
now_week = datetime.datetime.strftime(nowdate, "%Y%U")
|
|
|
|
myDB = db.DBConnection()
|
|
|
|
annual_load = myDB.select('SELECT * FROM annuals WHERE ComicID=?', [comicid])
|
|
logger.fdebug('checking annual db')
|
|
for annthis in annual_load:
|
|
if not any(d['ReleaseComicID'] == annthis['ReleaseComicID'] for d in annload):
|
|
#print 'matched on annual'
|
|
annload.append({
|
|
'ReleaseComicID': annthis['ReleaseComicID'],
|
|
'ReleaseComicName': annthis['ReleaseComicName'],
|
|
'ComicID': annthis['ComicID'],
|
|
'ComicName': annthis['ComicName']
|
|
})
|
|
|
|
if annload is None:
|
|
pass
|
|
else:
|
|
for manchk in annload:
|
|
if manchk['ReleaseComicID'] is not None or manchk['ReleaseComicID'] is not None: #if it exists, then it's a pre-existing add
|
|
#print str(manchk['ReleaseComicID']), comic['ComicName'], str(SeriesYear), str(comicid)
|
|
annualslist += manualAnnual(manchk['ReleaseComicID'], ComicName, SeriesYear, comicid, manualupd=True)
|
|
annualids.append(manchk['ReleaseComicID'])
|
|
|
|
annualcomicname = re.sub('[\,\:]', '', ComicName)
|
|
|
|
if annualcomicname.lower().startswith('the'):
|
|
annComicName = annualcomicname[4:] + ' annual'
|
|
else:
|
|
annComicName = annualcomicname + ' annual'
|
|
mode = 'series'
|
|
|
|
annualyear = SeriesYear # no matter what, the year won't be less than this.
|
|
logger.fdebug('[IMPORTER-ANNUAL] - Annual Year:' + str(annualyear))
|
|
sresults = mb.findComic(annComicName, mode, issue=None)
|
|
type='comic'
|
|
|
|
annual_types_ignore = {'paperback', 'collecting', 'reprints', 'collected edition', 'print edition', 'tpb', 'available in print', 'collects'}
|
|
|
|
if len(sresults) > 0:
|
|
logger.fdebug('[IMPORTER-ANNUAL] - there are ' + str(len(sresults)) + ' results.')
|
|
num_res = 0
|
|
while (num_res < len(sresults)):
|
|
sr = sresults[num_res]
|
|
#logger.fdebug("description:" + sr['description'])
|
|
for x in annual_types_ignore:
|
|
if x in sr['description'].lower():
|
|
test_id_position = sr['description'].find(comicid)
|
|
if test_id_position >= sr['description'].lower().find(x) or test_id_position == -1:
|
|
logger.fdebug('[IMPORTER-ANNUAL] - tradeback/collected edition detected - skipping ' + str(sr['comicid']))
|
|
continue
|
|
|
|
if comicid in sr['description']:
|
|
logger.fdebug('[IMPORTER-ANNUAL] - ' + str(comicid) + ' found. Assuming it is part of the greater collection.')
|
|
issueid = sr['comicid']
|
|
logger.fdebug('[IMPORTER-ANNUAL] - ' + str(issueid) + ' added to series list as an Annual')
|
|
if issueid in annualids:
|
|
logger.fdebug('[IMPORTER-ANNUAL] - ' + str(issueid) + ' already exists within current annual list for series.')
|
|
num_res+=1 # need to manually increment since not a for-next loop
|
|
continue
|
|
issued = cv.getComic(issueid, 'issue')
|
|
if len(issued) is None or len(issued) == 0:
|
|
logger.fdebug('[IMPORTER-ANNUAL] - Could not find any annual information...')
|
|
pass
|
|
else:
|
|
n = 0
|
|
if int(sr['issues']) == 0 and len(issued['issuechoice']) == 1:
|
|
sr_issues = 1
|
|
else:
|
|
if int(sr['issues']) != len(issued['issuechoice']):
|
|
sr_issues = len(issued['issuechoice'])
|
|
else:
|
|
sr_issues = sr['issues']
|
|
logger.fdebug('[IMPORTER-ANNUAL] - There are ' + str(sr_issues) + ' annuals in this series.')
|
|
while (n < int(sr_issues)):
|
|
try:
|
|
firstval = issued['issuechoice'][n]
|
|
except IndexError:
|
|
break
|
|
try:
|
|
cleanname = helpers.cleanName(firstval['Issue_Name'])
|
|
except:
|
|
cleanname = 'None'
|
|
issid = str(firstval['Issue_ID'])
|
|
issnum = str(firstval['Issue_Number'])
|
|
issname = cleanname
|
|
issdate = str(firstval['Issue_Date'])
|
|
stdate = str(firstval['Store_Date'])
|
|
digdate = str(firstval['Digital_Date'])
|
|
int_issnum = helpers.issuedigits(issnum)
|
|
|
|
iss_exists = myDB.selectone('SELECT * from annuals WHERE IssueID=?', [issid]).fetchone()
|
|
if iss_exists is None:
|
|
if stdate == '0000-00-00':
|
|
dk = re.sub('-', '', issdate).strip()
|
|
else:
|
|
dk = re.sub('-', '', stdate).strip() # converts date to 20140718 format
|
|
if dk == '00000000':
|
|
logger.warn('Issue Data is invalid for Issue Number %s. Marking this issue as Skipped' % firstval['Issue_Number'])
|
|
astatus = "Skipped"
|
|
else:
|
|
datechk = datetime.datetime.strptime(dk, "%Y%m%d")
|
|
issue_week = datetime.datetime.strftime(datechk, "%Y%U")
|
|
if mylar.CONFIG.AUTOWANT_ALL:
|
|
astatus = "Wanted"
|
|
elif issue_week >= now_week and mylar.CONFIG.AUTOWANT_UPCOMING is True:
|
|
astatus = "Wanted"
|
|
else:
|
|
astatus = "Skipped"
|
|
else:
|
|
astatus = iss_exists['Status']
|
|
|
|
annualslist.append({"Issue_Number": issnum,
|
|
"Int_IssueNumber": int_issnum,
|
|
"IssueDate": issdate,
|
|
"ReleaseDate": stdate,
|
|
"DigitalDate": digdate,
|
|
"IssueName": issname,
|
|
"ComicID": comicid,
|
|
"IssueID": issid,
|
|
"ComicName": ComicName,
|
|
"ReleaseComicID": re.sub('4050-', '', firstval['Comic_ID']).strip(),
|
|
"ReleaseComicName": sr['name'],
|
|
"Status": astatus})
|
|
|
|
#myDB.upsert("annuals", newVals, newCtrl)
|
|
|
|
# --- don't think this does anything since the value isn't returned in this module
|
|
#if issuechk is not None and issuetype == 'annual':
|
|
# #logger.fdebug('[IMPORTER-ANNUAL] - Comparing annual ' + str(issuechk) + ' .. to .. ' + str(int_issnum))
|
|
# if issuechk == int_issnum:
|
|
# weeklyissue_check.append({"Int_IssueNumber": int_issnum,
|
|
# "Issue_Number": issnum,
|
|
# "IssueDate": issdate,
|
|
# "ReleaseDate": stdate})
|
|
|
|
n+=1
|
|
num_res+=1
|
|
manualAnnual(annchk=annualslist)
|
|
return annualslist
|
|
|
|
elif len(sresults) == 0 or len(sresults) is None:
|
|
logger.fdebug('[IMPORTER-ANNUAL] - No results, removing the year from the agenda and re-querying.')
|
|
sresults = mb.findComic(annComicName, mode, issue=None)
|
|
if len(sresults) == 1:
|
|
sr = sresults[0]
|
|
logger.fdebug('[IMPORTER-ANNUAL] - ' + str(comicid) + ' found. Assuming it is part of the greater collection.')
|
|
else:
|
|
resultset = 0
|
|
else:
|
|
logger.fdebug('[IMPORTER-ANNUAL] - Returning results to screen - more than one possibility')
|
|
for sr in sresults:
|
|
if annualyear < sr['comicyear']:
|
|
logger.fdebug('[IMPORTER-ANNUAL] - ' + str(annualyear) + ' is less than ' + str(sr['comicyear']))
|
|
if int(sr['issues']) > (2013 - int(sr['comicyear'])):
|
|
logger.fdebug('[IMPORTER-ANNUAL] - Issue count is wrong')
|
|
|
|
#if this is called from the importer module, return the weeklyissue_check
|
|
|
|
def image_it(comicid, latestissueid, comlocation, ComicImage):
|
|
#alternate series covers download latest image...
|
|
|
|
cimage = os.path.join(mylar.CONFIG.CACHE_DIR, str(comicid) + '.jpg')
|
|
imageurl = mylar.cv.getComic(comicid, 'image', issueid=latestissueid)
|
|
covercheck = helpers.getImage(comicid, imageurl['image'])
|
|
if covercheck == 'retry':
|
|
logger.fdebug('Attempting to retrieve a different comic image for this particular issue.')
|
|
if imageurl['image_alt'] is not None:
|
|
covercheck = helpers.getImage(comicid, imageurl['image_alt'])
|
|
else:
|
|
if not os.path.isfile(cimage):
|
|
logger.fdebug('Failed to retrieve issue image, possibly because not available. Reverting back to series image.')
|
|
covercheck = helpers.getImage(comicid, ComicImage)
|
|
PRComicImage = os.path.join('cache', str(comicid) + ".jpg")
|
|
ComicImage = helpers.replacetheslash(PRComicImage)
|
|
|
|
#if the comic cover local is checked, save a cover.jpg to the series folder.
|
|
if all([mylar.CONFIG.COMIC_COVER_LOCAL is True, os.path.isdir(comlocation) is True, os.path.isfile(os.path.join(comlocation, 'cover.jpg'))]):
|
|
try:
|
|
comiclocal = os.path.join(comlocation, 'cover.jpg')
|
|
shutil.copyfile(cimage, comiclocal)
|
|
if mylar.CONFIG.ENFORCE_PERMS:
|
|
filechecker.setperms(comiclocal)
|
|
except IOError as e:
|
|
logger.error('[%s] Error saving cover (%s) into series directory (%s) at this time' % (e, cimage, comiclocal))
|
|
except Exception as e:
|
|
logger.error('[%s] Unable to save cover (%s) into series directory (%s) at this time' % (e, cimage, comiclocal))
|
|
|
|
myDB = db.DBConnection()
|
|
myDB.upsert('comics', {'ComicImage': ComicImage}, {'ComicID': comicid})
|