From fad92c7745e7a8a7a0ed2ecf7e82547517e7edae Mon Sep 17 00:00:00 2001 From: evilhero Date: Mon, 11 Mar 2013 13:25:45 -0400 Subject: [PATCH] FIX: Ongoing Limited Series wouldn't have status of Continuing and wouldn't get marked on weekly pull list (ie.Alpha:Big Time), IMP: Comic Volume is now autopopulated (v2,v3,etc) - can also be edited --- data/interfaces/default/index.html | 2 +- mylar/cv.py | 52 +++++++++++++++++++++++------- mylar/helpers.py | 26 ++++++++------- mylar/importer.py | 6 +++- mylar/parseit.py | 7 ++-- 5 files changed, 67 insertions(+), 26 deletions(-) diff --git a/data/interfaces/default/index.html b/data/interfaces/default/index.html index 82a7a07e..afa6b954 100644 --- a/data/interfaces/default/index.html +++ b/data/interfaces/default/index.html @@ -73,7 +73,7 @@ %if comic['ComicPublished'] is None or comic['ComicPublished'] == '': Unknown - %elif 'present' in comic['ComicPublished'].lower(): + %elif 'present' in comic['ComicPublished'].lower() or ( helpers.today()[:4] in comic['LatestDate']): Continuing %else: Ended diff --git a/mylar/cv.py b/mylar/cv.py index 1880c5e2..6ec30e5e 100755 --- a/mylar/cv.py +++ b/mylar/cv.py @@ -16,6 +16,7 @@ import sys import os +import re import logger import string import urllib @@ -25,7 +26,7 @@ from bs4 import BeautifulSoup as Soup def getComic(comicid,type): comicapi='583939a3df0a25fc4e8b7a29934a13078002dc27' #api - PULLURL='http://api.comicvine.com/volume/' + str(comicid) + '/?api_key=' + str(comicapi) + '&format=xml&field_list=name,count_of_issues,start_year,last_issue,site_detail_url,image,publisher' + PULLURL='http://api.comicvine.com/volume/' + str(comicid) + '/?api_key=' + str(comicapi) + '&format=xml&field_list=name,count_of_issues,start_year,last_issue,site_detail_url,image,publisher,description' #import library to do http requests: import urllib2 @@ -90,6 +91,34 @@ def GetComicInfo(comicid,dom): comic['ComicName'] = comic['ComicName'].rstrip() comic['ComicYear'] = dom.getElementsByTagName('start_year')[0].firstChild.wholeText comic['ComicURL'] = dom.getElementsByTagName('site_detail_url')[0].firstChild.wholeText + #the description field actually holds the Volume# - so let's grab it + try: + comic['ComicDescription'] = dom.getElementsByTagName('description')[0].firstChild.wholeText + except: + comic['ComicDescription'] = 'None' + #extract the first 60 characters + comicDes = comic['ComicDescription'][:60] + if 'volume' in comicDes.lower(): + #found volume - let's grab it. + v_find = comicDes.lower().find('volume') + #arbitrarily grab the next 10 chars (6 for volume + 1 for space + 3 for the actual vol #) + #increased to 10 to allow for text numbering (+5 max) + vfind = comicDes[v_find:v_find+15] + volconv = '' + basenums = {'zero':'0','one':'1','two':'2','three':'3','four':'4','five':'5','six':'6','seven':'7','eight':'8'} + for nums in basenums: + if nums in vfind.lower(): + sconv = basenums[nums] + volconv = re.sub(nums, sconv, vfind.lower()) + break + if volconv != '': + vfind = volconv + + comic['ComicVersion'] = re.sub("[^0-9]", "", vfind) + logger.info("Volume information found! Adding to series record : volume " + comic['ComicVersion']) + else: + comic['ComicVersion'] = "noversion" + if vari == "yes": comic['ComicIssues'] = str(cntit) else: @@ -97,17 +126,18 @@ def GetComicInfo(comicid,dom): comic['ComicImage'] = dom.getElementsByTagName('super_url')[0].firstChild.wholeText comic['ComicPublisher'] = dom.getElementsByTagName('name')[trackcnt+1].firstChild.wholeText - comicchoice.append({ - 'ComicName': comic['ComicName'], - 'ComicYear': comic['ComicYear'], - 'Comicid': comicid, - 'ComicURL': comic['ComicURL'], - 'ComicIssues': comic['ComicIssues'], - 'ComicImage': comic['ComicImage'], - 'ComicPublisher': comic['ComicPublisher'] - }) +# comicchoice.append({ +# 'ComicName': comic['ComicName'], +# 'ComicYear': comic['ComicYear'], +# 'Comicid': comicid, +# 'ComicURL': comic['ComicURL'], +# 'ComicIssues': comic['ComicIssues'], +# 'ComicImage': comic['ComicImage'], +# 'ComicVolume': ParseVol, +# 'ComicPublisher': comic['ComicPublisher'] +# }) - comic['comicchoice'] = comicchoice +# comic['comicchoice'] = comicchoice return comic def GetIssuesInfo(comicid,dom): diff --git a/mylar/helpers.py b/mylar/helpers.py index aea1db5b..0f03309e 100755 --- a/mylar/helpers.py +++ b/mylar/helpers.py @@ -195,19 +195,23 @@ def is_number(s): def decimal_issue(iss): iss_find = iss.find('.') - iss_b4dec = iss[:iss_find] - iss_decval = iss[iss_find+1:] - if int(iss_decval) == 0: - iss = iss_b4dec - issdec = int(iss_decval) + if iss_find == -1: + #no matches for a decimal, assume we're converting from decimal to int. + deciss = int(iss) * 1000 else: - if len(iss_decval) == 1: - iss = iss_b4dec + "." + iss_decval - issdec = int(iss_decval) * 10 + iss_b4dec = iss[:iss_find] + iss_decval = iss[iss_find+1:] + if int(iss_decval) == 0: + iss = iss_b4dec + issdec = int(iss_decval) else: - iss = iss_b4dec + "." + iss_decval.rstrip('0') - issdec = int(iss_decval.rstrip('0')) * 10 - deciss = (int(iss_b4dec) * 1000) + issdec + if len(iss_decval) == 1: + iss = iss_b4dec + "." + iss_decval + issdec = int(iss_decval) * 10 + else: + iss = iss_b4dec + "." + iss_decval.rstrip('0') + issdec = int(iss_decval.rstrip('0')) * 10 + deciss = (int(iss_b4dec) * 1000) + issdec return deciss def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=None): diff --git a/mylar/importer.py b/mylar/importer.py index f9dea1d5..932f287e 100755 --- a/mylar/importer.py +++ b/mylar/importer.py @@ -208,7 +208,10 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None): except IOError as e: logger.error(u"Unable to save cover locally at this time.") - + if comic['ComicVersion'].isdigit(): + comicVol = "v" + comic['ComicVersion'] + else: + comicVol = None controlValueDict = {"ComicID": comicid} newValueDict = {"ComicName": comic['ComicName'], @@ -216,6 +219,7 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None): "ComicYear": comic['ComicYear'], "ComicImage": ComicImage, "Total": comicIssues, + "ComicVersion": comicVol, "ComicLocation": comlocation, "ComicPublisher": comic['ComicPublisher'], "ComicPublished": gcdinfo['resultPublished'], diff --git a/mylar/parseit.py b/mylar/parseit.py index 83ec7d19..87445748 100755 --- a/mylar/parseit.py +++ b/mylar/parseit.py @@ -39,7 +39,8 @@ def GCDScraper(ComicName, ComicYear, Total, ComicID, quickmatch=None): #print ( "comicyear: " + str(comicyr) ) #print ( "comichave: " + str(comicis) ) #print ( "comicid: " + str(comicid) ) - comicnm = re.sub(' ', '+', comicnm) + comicnm_1 = re.sub('\+', '%2B', comicnm) + comicnm = re.sub(' ', '+', comicnm_1) input = 'http://www.comics.org/search/advanced/process/?target=series&method=icontains&logic=False&order2=date&order3=&start_date=' + str(comicyr) + '-01-01&end_date=' + str(NOWyr) + '-12-31&series=' + str(comicnm) + '&is_indexed=None' response = urllib2.urlopen ( input ) soup = BeautifulSoup ( response) @@ -48,7 +49,7 @@ def GCDScraper(ComicName, ComicYear, Total, ComicID, quickmatch=None): cnt = int(cnt1 + cnt2) - #print (str(cnt) + " results") + print (str(cnt) + " results") resultName = [] resultID = [] @@ -224,6 +225,8 @@ def GCDdetails(comseries, resultURL, vari_loop, ComicID, TotalIssues, issvariati if resultFormat != '': if 'ongoing series' in resultFormat.lower() and 'was' not in resultFormat.lower() and 'present' not in resultPublished.lower(): resultPublished = resultPublished + " - Present" + if 'limited series' in resultFormat.lower() and '?' in resultPublished: + resultPublished = resultPublished + " (Limited Series)" coverst = soup.find("div", {"id" : "series_cover"}) if coverst < 0: gcdcover = "None"