FIX:(#1253) Story arc would fail on adding and error on post-processing due to Dynamic Names not being updated, FIX: Fixed some errant unicode conversion when dealing with Issue Numbering, FIX: Removed some str references when adding/refreshing a series that caused unicode characters to break, FIX: If issue number was unicoded, would error out during scan / actual import

This commit is contained in:
evilhero 2016-04-17 13:17:40 -04:00
parent 6a17c53ded
commit 64c9e88e18
6 changed files with 153 additions and 135 deletions

View File

@ -1075,15 +1075,16 @@ class PostProcessor(object):
#'dupe_file' - do not write new file as existing file is better quality
#'dupe_src' - write new file, as existing file is a lesser quality (dupe)
if mylar.DUPLICATE_DUMP:
dupchkit = self.duplicate_process(dupthis)
if dupchkit == False:
logger.warn('Unable to move duplicate file - skipping post-processing of this file.')
self.valreturn.append({"self.log": self.log,
"mode": 'stop',
"issueid": issueid,
"comicid": comicid})
if mylar.DDUMP and not all([mylar.DUPLICATE_DUMP is None, mylar.DUPLICATE_DUMP == '']):
dupchkit = self.duplicate_process(dupthis)
if dupchkit == False:
logger.warn('Unable to move duplicate file - skipping post-processing of this file.')
self.valreturn.append({"self.log": self.log,
"mode": 'stop',
"issueid": issueid,
"comicid": comicid})
return self.queue.put(self.valreturn)
return self.queue.put(self.valreturn)
if dupthis[0]['action'] == "write" or dupthis[0]['action'] == 'dupe_src':
return self.Process_next(comicid, issueid, issuenumOG)

View File

@ -917,99 +917,101 @@ def issuedigits(issnum):
#except:
# logger.fdebug('Unicode character detected: ' + issnum)
#else: issnum.decode(mylar.SYS_ENCODING).decode('utf-8')
if type(issnum) == str:
try:
issnum = issnum.decode('utf-8')
except:
issnum = issnum.decode('windows-1252')
if u'\xbd' in issnum:
int_issnum = .5 * 1000
elif u'\xbc' in issnum:
int_issnum = .25 * 1000
elif u'\xbe' in issnum:
int_issnum = .75 * 1000
elif u'\u221e' in issnum:
#issnum = utf-8 will encode the infinity symbol without any help
int_issnum = 9999999999 * 1000 # set 9999999999 for integer value of issue
elif '.' in issnum or ',' in issnum:
#logger.fdebug('decimal detected.')
if ',' in issnum: issnum = re.sub(',', '.', issnum)
issst = str(issnum).find('.')
if issst == 0:
issb4dec = 0
else:
issb4dec = str(issnum)[:issst]
decis = str(issnum)[issst +1:]
if len(decis) == 1:
decisval = int(decis) * 10
issaftdec = str(decisval)
elif len(decis) == 2:
decisval = int(decis)
issaftdec = str(decisval)
else:
decisval = decis
issaftdec = str(decisval)
#if there's a trailing decimal (ie. 1.50.) and it's either intentional or not, blow it away.
if issaftdec[-1:] == '.':
issaftdec = issaftdec[:-1]
try:
int_issnum = (int(issb4dec) * 1000) + (int(issaftdec) * 10)
except ValueError:
#logger.fdebug('This has no issue # for me to get - Either a Graphic Novel or one-shot.')
int_issnum = 999999999999999
if type(issnum) == unicode:
vals = {u'\xbd':.5,u'\xbc':.25,u'\xbe':.75,u'\u221e':9999999999,u'\xe2':9999999999}
else:
try:
x = float(issnum)
#validity check
if x < 0:
#logger.info("I've encountered a negative issue #: " + str(issnum) + ". Trying to accomodate.")
int_issnum = (int(x) *1000) - 1
else: raise ValueError
except ValueError, e:
#this will account for any alpha in a issue#, so long as it doesn't have decimals.
x = 0
tstord = None
issno = None
invchk = "false"
while (x < len(issnum)):
if issnum[x].isalpha():
#take first occurance of alpha in string and carry it through
tstord = issnum[x:].rstrip()
tstord = re.sub('[\-\,\.\+]', '', tstord).rstrip()
issno = issnum[:x].rstrip()
issno = re.sub('[\-\,\.\+]', '', issno).rstrip()
try:
isschk = float(issno)
except ValueError, e:
if len(issnum) == 1 and issnum.isalpha():
break
logger.fdebug('[' + issno + '] Invalid numeric for issue - cannot be found. Ignoring.')
issno = None
tstord = None
invchk = "true"
break
x+=1
if tstord is not None and issno is not None:
a = 0
ordtot = 0
if len(issnum) == 1 and issnum.isalpha():
int_issnum = ord(tstord.lower())
else:
while (a < len(tstord)):
try:
ordtot += ord(tstord[a].lower()) #lower-case the letters for simplicty
except ValueError:
break
a+=1
int_issnum = (int(issno) * 1000) + ordtot
elif invchk == "true":
logger.fdebug('this does not have an issue # that I can parse properly.')
int_issnum = 999999999999999
vals = {'\xbd':.5,'\xbc':.25,'\xbe':.75,'\u221e':9999999999,'\xe2':9999999999}
x = [vals[key] for key in vals if key in issnum]
if x:
logger.info('Unicode Issue present - adjusting.')
int_issnum = x[0] * 1000
logger.info('int_issnum: ' + str(int_issnum))
else:
if any(['.' in issnum, ',' in issnum]):
#logger.fdebug('decimal detected.')
if ',' in issnum: issnum = re.sub(',', '.', issnum)
issst = str(issnum).find('.')
if issst == 0:
issb4dec = 0
else:
logger.error(str(issnum) + 'this has an alpha-numeric in the issue # which I cannot account for.')
issb4dec = str(issnum)[:issst]
decis = str(issnum)[issst +1:]
if len(decis) == 1:
decisval = int(decis) * 10
issaftdec = str(decisval)
elif len(decis) == 2:
decisval = int(decis)
issaftdec = str(decisval)
else:
decisval = decis
issaftdec = str(decisval)
#if there's a trailing decimal (ie. 1.50.) and it's either intentional or not, blow it away.
if issaftdec[-1:] == '.':
issaftdec = issaftdec[:-1]
try:
int_issnum = (int(issb4dec) * 1000) + (int(issaftdec) * 10)
except ValueError:
#logger.fdebug('This has no issue # for me to get - Either a Graphic Novel or one-shot.')
int_issnum = 999999999999999
else:
try:
x = float(issnum)
#validity check
if x < 0:
#logger.info("I've encountered a negative issue #: " + str(issnum) + ". Trying to accomodate.")
int_issnum = (int(x) *1000) - 1
else: raise ValueError
except ValueError, e:
#this will account for any alpha in a issue#, so long as it doesn't have decimals.
x = 0
tstord = None
issno = None
invchk = "false"
while (x < len(issnum)):
if issnum[x].isalpha():
#take first occurance of alpha in string and carry it through
tstord = issnum[x:].rstrip()
tstord = re.sub('[\-\,\.\+]', '', tstord).rstrip()
issno = issnum[:x].rstrip()
issno = re.sub('[\-\,\.\+]', '', issno).rstrip()
try:
isschk = float(issno)
except ValueError, e:
if len(issnum) == 1 and issnum.isalpha():
break
logger.fdebug('[' + issno + '] Invalid numeric for issue - cannot be found. Ignoring.')
issno = None
tstord = None
invchk = "true"
break
x+=1
if tstord is not None and issno is not None:
a = 0
ordtot = 0
if len(issnum) == 1 and issnum.isalpha():
int_issnum = ord(tstord.lower())
else:
while (a < len(tstord)):
try:
ordtot += ord(tstord[a].lower()) #lower-case the letters for simplicty
except ValueError:
break
a+=1
int_issnum = (int(issno) * 1000) + ordtot
elif invchk == "true":
logger.fdebug('this does not have an issue # that I can parse properly.')
int_issnum = 999999999999999
else:
logger.error(str(issnum) + 'this has an alpha-numeric in the issue # which I cannot account for.')
int_issnum = 999999999999999
return int_issnum

View File

@ -599,11 +599,11 @@ def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=No
moveit.archivefiles(comicid, comlocation, ogcname)
#check for existing files...
statbefore = myDB.selectone("SELECT * FROM issues WHERE ComicID=? AND Issue_Number=?", [comicid, str(latestiss)]).fetchone()
logger.fdebug('issue: ' + str(latestiss) + ' status before chk :' + str(statbefore['Status']))
statbefore = myDB.selectone("SELECT * FROM issues WHERE ComicID=? AND Int_IssueNumber=?", [comicid, helpers.issuedigits(latestiss)]).fetchone()
logger.fdebug('issue: ' + latestiss + ' status before chk :' + str(statbefore['Status']))
updater.forceRescan(comicid)
statafter = myDB.selectone("SELECT * FROM issues WHERE ComicID=? AND Issue_Number=?", [comicid, str(latestiss)]).fetchone()
logger.fdebug('issue: ' + str(latestiss) + ' status after chk :' + str(statafter['Status']))
statafter = myDB.selectone("SELECT * FROM issues WHERE ComicID=? AND Int_IssueNumber=?", [comicid, helpers.issuedigits(latestiss)]).fetchone()
logger.fdebug('issue: ' + latestiss + ' status after chk :' + str(statafter['Status']))
logger.fdebug('pullupd: ' + str(pullupd))
logger.fdebug('lastpubdate: ' + str(lastpubdate))
@ -613,10 +613,10 @@ def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=No
# do this for only Present comics....
if mylar.AUTOWANT_UPCOMING and lastpubdate == 'Present' and series_status == 'Active': #and 'Present' in gcdinfo['resultPublished']:
logger.fdebug('latestissue: #' + str(latestiss))
chkstats = myDB.selectone("SELECT * FROM issues WHERE ComicID=? AND Issue_Number=?", [comicid, str(latestiss)]).fetchone()
chkstats = myDB.selectone("SELECT * FROM issues WHERE ComicID=? AND Int_IssueNumber=?", [comicid, helpers.issuedigits(latestiss)]).fetchone()
if chkstats is None:
if mylar.ANNUALS_ON:
chkstats = myDB.selectone("SELECT * FROM annuals WHERE ComicID=? AND Issue_Number=?", [comicid, latestiss]).fetchone()
chkstats = myDB.selectone("SELECT * FROM annuals WHERE ComicID=? AND Int_IssueNumber=?", [comicid, helpers.issuedigits(latestiss)]).fetchone()
if chkstats:
logger.fdebug('latestissue status: ' + chkstats['Status'])
@ -669,9 +669,9 @@ def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=No
logger.info('[FROM THE FUTURE CHECKLIST] Attempting to grab wanted issues for : ' + comic['ComicName'])
for result in chkresults:
for chkit in chkwant:
logger.fdebug('checking ' + str(chkit['IssueNumber']) + ' against ' + str(result['Issue_Number']))
logger.fdebug('checking ' + chkit['IssueNumber'] + ' against ' + result['Issue_Number'])
if chkit['IssueNumber'] == result['Issue_Number']:
logger.fdebug('Searching for : ' + str(result['Issue_Number']))
logger.fdebug('Searching for : ' + result['Issue_Number'])
logger.fdebug('Status of : ' + str(result['Status']))
search.searchforissue(result['IssueID'])
else: logger.info('No issues marked as wanted for ' + comic['ComicName'])

View File

@ -434,7 +434,7 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None,
cv_cid = None
if issuevolume is None:
logger.fdebug('issue volume is none : ' + str(issuevolume))
logger.fdebug('issue volume is : ' + str(issuevolume))
if i['parsedinfo']['series_volume'] is None:
issuevolume = None
else:
@ -460,7 +460,7 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None,
"comicname": i['parsedinfo']['series_name'],
"dynamicname": is_dyninfo['mod_seriesname'].lower(),
"comicyear": i['parsedinfo']['issue_year'],
"issuenumber": issuenumber,
"issuenumber": issuenumber, #issuenumber,
"volume": issuevolume,
"comfilename": comfilename,
"comlocation": comlocation.decode(mylar.SYS_ENCODING)
@ -628,6 +628,15 @@ def scanLibrary(scan=None, queue=None):
if int(soma['import_count']) > 0:
for ss in soma['import_by_comicids']:
if type(ss['issuenumber']) == str:
try:
theissuenumber = ss['issuenumber'].decode('utf-8')
except:
theissuenumber = ss['issuenumber'].decode('windows-1252').encode('utf-8')#mylar.SYS_ENCODING)
theissuenumber = unicode(theissuenumber, mylar.SYS_ENCODING)
else:
theissuenumber = ss['issuenumber']
nspace_dynamicname = re.sub('[\|\s]', '', ss['dynamicname'].lower()).strip()
controlValue = {"impID": ss['impid']}
newValue = {"ComicYear": ss['comicyear'],
@ -638,7 +647,7 @@ def scanLibrary(scan=None, queue=None):
"ComicID": ss['comicid'], #if it's been scanned in for cvinfo, this will be the CID - otherwise it's None
"IssueID": None,
"Volume": ss['volume'],
"IssueNumber": ss['issuenumber'].decode('utf-8'),
"IssueNumber": theissuenumber,
"ComicFilename": ss['comfilename'].decode('utf-8'), #ss['comfilename'].encode('utf-8'),
"ComicLocation": ss['comlocation'],
"ImportDate": helpers.today(),

View File

@ -12,7 +12,6 @@
#
# You should have received a copy of the GNU General Public License
# along with Mylar. If not, see <http://www.gnu.org/licenses/>.
import time
import datetime
from xml.dom.minidom import parseString
@ -880,15 +879,15 @@ def forceRescan(ComicID, archive=None, module=None):
return
else:
break
temploc= tmpfc['JusttheDigits'].replace('_', ' ')
temploc = tmpfc['JusttheDigits'].replace('_', ' ')
temploc = re.sub('[\#\']', '', temploc)
logger.fdebug(module + ' temploc: ' + str(temploc))
logger.fdebug(module + ' temploc: ' + temploc)
if 'annual' not in temploc.lower():
#remove the extension here
extensions = ('.cbr', '.cbz', '.cb7')
if temploc.lower().endswith(extensions):
logger.fdebug(module + ' Removed extension for issue: ' + str(temploc))
logger.fdebug(module + ' Removed extension for issue: ' + temploc)
temploc = temploc[:-4]
fcnew_af = re.findall('[^\()]+', temploc)
fcnew = shlex.split(fcnew_af[0])

View File

@ -442,6 +442,10 @@ class WebInterface(object):
except IndexError:
break
comicname = arcval['ComicName']
st_d = mylar.filechecker.FileChecker(watchcomic=comicname)
st_dyninfo = st_d.dynamic_replace(comicname)
dynamic_name = re.sub('[\|\s]','', st_dyninfo['mod_seriesname'].lower()).strip()
issname = arcval['Issue_Name']
issid = str(arcval['IssueID'])
comicid = str(arcval['ComicID'])
@ -484,6 +488,7 @@ class WebInterface(object):
"StoryArcID": storyarcid,
"IssueArcID": st_issueid,
"ComicName": comicname,
"DynamicName": dynamic_name,
"IssueName": issname,
"Issue_Number": issnum,
"IssueDate": issdate,
@ -510,23 +515,24 @@ class WebInterface(object):
issuePublisher = cid['Publisher']
break
newCtrl = {"IssueArcID": AD['IssueArcID'],
"StoryArcID": AD['StoryArcID']}
newVals = {"ComicID": AD['ComicID'],
"IssueID": AD['IssueID'],
"StoryArc": storyarcname,
"ComicName": AD['ComicName'],
"IssueName": IssueName,
"IssueNumber": AD['Issue_Number'],
"Publisher": storyarcpublisher,
"TotalIssues": storyarcissues,
"ReadingOrder": AD['ReadingOrder'],
"IssueDate": AD['IssueDate'],
"StoreDate": AD['ReleaseDate'],
"SeriesYear": seriesYear,
"IssuePublisher": issuePublisher,
"CV_ArcID": arcid,
"Int_IssueNumber": AD['Int_IssueNumber']}
newCtrl = {"IssueArcID": AD['IssueArcID'],
"StoryArcID": AD['StoryArcID']}
newVals = {"ComicID": AD['ComicID'],
"IssueID": AD['IssueID'],
"StoryArc": storyarcname,
"ComicName": AD['ComicName'],
"DynamicComicName": AD['DynamicName'],
"IssueName": IssueName,
"IssueNumber": AD['Issue_Number'],
"Publisher": storyarcpublisher,
"TotalIssues": storyarcissues,
"ReadingOrder": AD['ReadingOrder'],
"IssueDate": AD['IssueDate'],
"StoreDate": AD['ReleaseDate'],
"SeriesYear": seriesYear,
"IssuePublisher": issuePublisher,
"CV_ArcID": arcid,
"Int_IssueNumber": AD['Int_IssueNumber']}
myDB.upsert("readinglist", newVals, newCtrl)
@ -1372,13 +1378,13 @@ class WebInterface(object):
controlValueDict = {"IssueID": IssueID}
if mode == 'failed' and mylar.FAILED_DOWNLOAD_HANDLING:
logger.info(u"Marking " + ComicName + " issue # " + str(IssueNumber) + " as Failed...")
logger.info(u"Marking " + ComicName + " issue # " + IssueNumber + " as Failed...")
newValueDict = {"Status": "Failed"}
myDB.upsert("failed", newValueDict, controlValueDict)
yield cherrypy.HTTPRedirect("comicDetails?ComicID=%s" % ComicID)
self.failed_handling(ComicID=ComicID, IssueID=IssueID)
else:
logger.info(u"Marking " + ComicName + " issue # " + str(IssueNumber) + " as Skipped...")
logger.info(u"Marking " + ComicName + " issue # " + IssueNumber + " as Skipped...")
newValueDict = {"Status": "Skipped"}
if annchk == 'yes':
@ -1394,7 +1400,7 @@ class WebInterface(object):
thefuture = myDB.selectone('SELECT * FROM future WHERE ComicID=?', [ComicID]).fetchone()
else:
logger.info('FutureID: ' + str(FutureID))
logger.info('no comicid - ComicName: ' + str(ComicName) + ' -- Issue: #' + str(Issue))
logger.info('no comicid - ComicName: ' + str(ComicName) + ' -- Issue: #' + Issue)
thefuture = myDB.selectone('SELECT * FROM future WHERE FutureID=?', [FutureID]).fetchone()
if thefuture is None:
logger.info('Cannot find the corresponding issue in the Futures List for some reason. This is probably an Error.')
@ -2505,7 +2511,7 @@ class WebInterface(object):
for comic in comics:
mod_watch = comic['DynamicComicName'] #is from the comics db
if re.sub('[\|\s]','', mod_watch.lower()).strip() == re.sub('[\|\s]', '', mod_arc.lower()).strip():
if re.sub('[\|\s]','', mod_watch.lower()).strip() == re.sub('[\|\s]', '', arc['DynamicComicName'].lower()).strip():
logger.fdebug("initial name match - confirming issue # is present in series")
if comic['ComicID'][:1] == 'G':
# if it's a multi-volume series, it's decimalized - let's get rid of the decimal.
@ -3233,7 +3239,7 @@ class WebInterface(object):
else:
comicstoIMP.append(result['ComicLocation'])#.decode(mylar.SYS_ENCODING, 'replace'))
getiss = result['IssueNumber']
logger.info('getiss:' + str(getiss))
logger.info('getiss:' + getiss)
if 'annual' in getiss.lower():
tmpiss = re.sub('[^0-9]','', getiss).strip()
if any([tmpiss.startswith('19'), tmpiss.startswith('20')]) and len(tmpiss) == 4:
@ -3245,8 +3251,8 @@ class WebInterface(object):
yearRANGE.append(str(result['ComicYear']))
yearTOP = str(result['ComicYear'])
getiss_num = helpers.issuedigits(getiss)
miniss_num = helpers.issuedigits(str(minISSUE))
startiss_num = helpers.issuedigits(str(startISSUE))
miniss_num = helpers.issuedigits(minISSUE)
startiss_num = helpers.issuedigits(startISSUE)
if int(getiss_num) > int(miniss_num):
#logger.fdebug('Minimum issue now set to : ' + getiss + ' - it was : ' + minISSUE)
minISSUE = getiss
@ -3266,13 +3272,14 @@ class WebInterface(object):
#figure out # of issues and the year range allowable
logger.info('yearTOP: ' + str(yearTOP))
logger.info('minISSUE: ' + str(minISSUE))
logger.info('minISSUE: ' + minISSUE)
logger.info('yearRANGE: ' + str(yearRANGE))
if starttheyear is None:
if all([yearTOP != None, yearTOP != 'None']):
if int(str(yearTOP)) > 0:
minni = helpers.int_num(minISSUE)
if minni < 1:
minni = helpers.issuedigits(minISSUE)
logger.info(minni)
if minni < 1 or minni > 999999999:
maxyear = int(str(yearTOP))
else:
maxyear = int(str(yearTOP)) - (minni / 12)