mirror
/
mylar
mirror of https://github.com/evilhero/mylar
1
0
Fork 0
Code Issues Releases Wiki Activity

IMP: Added TPB/GN detection for file parser, IMP: Added TPB as a booktype to series detail page, IMP: Added to format options for Folder Formatting which would denote either TPB or GN, IMP: If detected as a TPB/GN volume, will display the series/issues that are contained within - if they exist on your watchlist, they will be clickable links, IMP: Weeklypull set to display TPB/GN's when option is configured (backend change), IMP: Added proper detection for filenames that have mini-series designations within them ie ( '(of 04)' ), IMP:(#2102) detect decimal issues that also contain unicoded numbering, IMP: Proper handling of filenames that have '38 p' or the like to denote number of pages in issue, FIX: Removed forceful reference to lib.requests when using WWT, IMP: If an issue has a digital release prior to the actual print release, will now look at either date when attempting to reference if a successful match when post-processing/searching, IMP: Issues that also have digital release dates will be indicated on the series detail page as such, IMP: Added job completion notifs for refreshing/adding story-arcs, deleting directories, IMP: Main Home page will display if a given volume is a TPB/GN if determined, IMP: Reworded 'Search for Watchlist matches' to 'Search for Watchlist Matches/Recheck Files' for more clarity, FIX: Fixed story-arc posters causing some text boxes to overflow

This commit is contained in:
evilhero 2018-11-29 14:59:34 -05:00
parent 3cdae9153a
commit f63e746280
17 changed files with 586 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
data/interfaces/default/comicdetails.html
View File

@ -1,7 +1,7 @@
<%inherit file="base.html"/>
<%!
import os
from mylar import db
import os, re
from mylar import db, helpers
import mylar
%>
@ -112,10 +112,41 @@
if comic['Type'] == 'None' or comic['Type'] is None or comic['Type'] == 'Print':
comictype = 'Print'
else:
comictype = 'Digital'
comictype = comic['Type']
%>
<div>
<label><big>Edition: </big><norm>${comictype}</norm></label>
<label><big>Edition: </big><norm>${comictype}</norm>
<%
if comicConfig['issue_list'] is not None:
cnt = 0
line = ''
for x in comicConfig['issue_list']:
if x['comicid'] is not None:
cid = helpers.listLibrary(x['comicid'])
try:
if cid[re.sub('4050-', '', x['comicid']).strip()]['comicid']:
watch = "<a href='comicDetails?ComicID=%s' title='In Watchlist'>%s</a>" % (cid[re.sub('4050-', '', x['comicid']).strip()]['comicid'], x['series'])
except:
watch = "%s" % (x['series'])
else:
watch = x['series']
if cnt == 0:
if x['issues'] is not None:
line += "<span style='display:inline-block;'>%s %s</span>" % (watch, x['issues'])
else:
line += "<span style='display:inline-block;'>%s</span>" % watch
else:
if x['issues'] is not None:
line += " / <span style='display:inline-block;'>%s %s</span>" % (watch, x['issues'])
else:
line += " / <span style='display:inline-block;'>%s</span>" % watch
cnt+=1
%>
%if comicConfig['issue_list'] is not None:
( Collects: ${line} )
%endif
</label>
</div>
<div>
<label><big>Last Updated: </big>
@ -326,7 +357,15 @@
endif
%>
<td id="issuename">${issuename}</td>
<td class="edit" title="Publication Date (click to edit)" id="${issue['ComicID']}.${issue['IssueID']}">${issue['IssueDate']}</td>
<%
issdate = issue['IssueDate']
dateline = "Publication Date (click to edit)"
if all([issue['DigitalDate'] != '0000-00-00', issue['DigitalDate'] is not None]):
dateline += "\nDigital Release: %s" % issue['DigitalDate']
issdate += "**"
endif
%>
<td class="edit" title="${dateline}" id="${issue['ComicID']}.${issue['IssueID']}">${issdate}</td>
<td id="status">${issue['Status']}
%if issue['Status'] == 'Downloaded' or issue['Status'] == 'Archived':
<%Csize = mylar.helpers.human_size(issue['ComicSize'])%>
@ -480,7 +519,15 @@
<td id="aint_issuenumber">${annual_Number}</td>
<td id="aissuenumber">${annual['Issue_Number']}</td>
<td id="aissuename">${annual['IssueName']}</td>
<td id="areldate">${annual['IssueDate']}</td>
<%
anndate = annual['IssueDate']
dateline = ""
if all([annual['DigitalDate'] != '0000-00-00', annual['DigitalDate'] is not None]):
dateline += "\nDigital Release: %s" % annual['DigitalDate']
anndate += "**"
endif
%>
<td title="${dateline}" id="areldate">${anndate}</td>
<td id="astatus">${annual['Status']}
%if annual['Status'] == 'Downloaded' or annual['Status'] == 'Archived':
<%Csize = mylar.helpers.human_size(annual['ComicSize'])%>

16
data/interfaces/default/index.html
View File

@ -69,10 +69,24 @@
except:
pass
comicline = comicname
comictype = comic['ComicType']
try:
if any([comictype == 'None', comictype is None, comictype == 'Print']):
comictype = None
else:
comictype = comictype
except:
comictype = None
if comictype is not None:
comicline += ' [%s]' % (comictype)
%>
<tr class="grade${grade}">
<td id="publisher"><span title="${comicpub}"></span>${comicpub}</td>
<td id="name"><span title="${comicname}"></span><a title="${comic['ComicName']}" href="comicDetails?ComicID=${comic['ComicID']}">${comicname}</a></td>
<td id="name"><span title="${comicname}"></span><a title="${comic['ComicName']}" href="comicDetails?ComicID=${comic['ComicID']}">${comicline}</a></td>
<td id="year"><span title="${comic['ComicYear']}"></span>${comic['ComicYear']}</td>
<td id="issue"><span title="${comic['LatestIssue']}"></span># ${comic['LatestIssue']}</td>
<td id="published">${comic['LatestDate']}</td>

5
data/interfaces/default/storyarc_detail.html
View File

@ -14,7 +14,7 @@
<a id="menu_link_delete" href="#">Remove Read</a>
<a id="menu_link_delete" href="#">Clear File Cache</a>
<a id="menu_link_refresh" onclick="doAjaxCall('SearchArcIssues?StoryArcID=${storyarcid}',$(this),'table')" data-success="Now searching for Missing StoryArc Issues">Search for Missing</a>
<a id="menu_link_refresh" onclick="doAjaxCall('ArcWatchlist?StoryArcID=${storyarcid}',$(this),'table')" data-success="Now searching for matches on Watchlist">Search for Watchlist matches</a>
<a id="menu_link_refresh" onclick="doAjaxCall('ArcWatchlist?StoryArcID=${storyarcid}',$(this),'table')" data-success="Now searching for matches on Watchlist & Rechecking files">Search for Watchlist matches/Recheck Files</a>
%if cvarcid:
<a id="menu_link_refresh" onclick="doAjaxCall('addStoryArc_thread?arcid=${storyarcid}&cvarcid=${cvarcid}&storyarcname=${storyarcname}&arcrefresh=True',$(this),'table')" data-success="Refreshed Story Arc">Refresh Story Arc</a>
%endif
@ -110,6 +110,9 @@
</table>
</div>
</div>
<button type="button" onclick="">Finalize & Rename</button>
<table class="display" id="arc_detail">
<thead>
<tr>

10
data/interfaces/default/storyarc_detail.poster.html
View File

@ -34,10 +34,8 @@
<tr/><tr>
<td id="mainimg">
<fieldset>
<div id="artistImg">
<div id="aristImg">
<img src="${storyarcbanner}" onload="resizeimage('${bannerwidth}')" height="400" width="${bannerwidth}" id="banner" style="text-decoration: none;position: relative;top:0px;right:0px;left:0px;"/>
</div>
</fieldset>
<%
if arcdetail['percent'] == 101:
css = '<div class=\"progress-container warning\">'
@ -48,9 +46,11 @@
%>
<div style="display:table;margin:auto;position:relative;top:0px;"><span title="${arcdetail['percent']}"></span>${css}<div style="width:${arcdetail['percent']}%"><span class="progressbar-front-text">${arcdetail['Have']}/${arcdetail['Total']}</span></div></div></div>
</div>
</fieldset>
</td>
<td width="100%" padding="5px" style="vertical-align:bottom;">
<div style="display:inline;position:relative;text-color:black;top:-200px;margin:0 auto;">
<div style="display:inline;position:relative;text-color:black;top:-150px;margin:0 auto;">
<center><h1>${storyarcname}</h1>(${spanyears})</center>
</div>
@ -81,7 +81,7 @@
</form>
</div>
<%
optpos = '<div style=\"display:block;float:right;position:relative;right:20px;top:50px;\">'
optpos = '<div style=\"display:block;float:right;position:relative;right:20px;top:-10px;\">'
%>
${optpos}
<form action="downloadBanner" method="GET">

4
data/interfaces/default/weeklypull.html
View File

@ -129,6 +129,10 @@
%if weekly['SERIESYEAR'] is not None:
&nbsp(${weekly['SERIESYEAR']})
%endif
%if weekly['FORMAT'] == 'Digital':
&nbsp[${weekly['FORMAT']}]
%endif
</td>
<td class="comicnumber">${weekly['ISSUE']}</td>
%if weekly['AUTOWANT']:

36
mylar/PostProcessor.py
View File

@ -569,11 +569,28 @@ class PostProcessor(object):
for isc in issuechk:
datematch = "True"
if isc['ReleaseDate'] is not None and isc['ReleaseDate'] != '0000-00-00':
monthval = isc['ReleaseDate']
watch_issueyear = isc['ReleaseDate'][:4]
try:
if isc['DigitalDate'] != '0000-00-00' and int(re.sub('-', '', isc['DigitalDate']).strip()) <= int(re.sub('-', '', isc['ReleaseDate']).strip()):
monthval = isc['DigitalDate']
watch_issueyear = isc['DigitalDate'][:4]
else:
monthval = isc['ReleaseDate']
watch_issueyear = isc['ReleaseDate'][:4]
except:
monthval = isc['ReleaseDate']
watch_issueyear = isc['ReleaseDate'][:4]
else:
monthval = isc['IssueDate']
watch_issueyear = isc['IssueDate'][:4]
try:
if isc['DigitalDate'] != '0000-00-00' and int(re.sub('-', '', isc['DigitalDate']).strip()) <= int(re.sub('-', '', isc['ReleaseDate']).strip()):
monthval = isc['DigitalDate']
watch_issueyear = isc['DigitalDate'][:4]
else:
monthval = isc['IssueDate']
watch_issueyear = isc['IssueDate'][:4]
except:
monthval = isc['IssueDate']
watch_issueyear = isc['IssueDate'][:4]
if len(watchmatch) >= 1 and watchmatch['issue_year'] is not None:
#if the # of matches is more than 1, we need to make sure we get the right series
@ -587,7 +604,12 @@ class PostProcessor(object):
#logger.info(module + ' ReleaseDate: ' + str(isc['ReleaseDate']))
#logger.info(module + ' IssueDate: ' + str(isc['IssueDate']))
if isc['ReleaseDate'] is not None and isc['ReleaseDate'] != '0000-00-00':
if isc['DigitalDate'] is not None and isc['DigitalDate'] != '0000-00-00':
if int(isc['DigitalDate'][:4]) < int(watchmatch['issue_year']):
logger.fdebug(module + '[ISSUE-VERIFY] ' + str(isc['DigitalDate']) + ' is before the issue year of ' + str(watchmatch['issue_year']) + ' that was discovered in the filename')
datematch = "False"
elif isc['ReleaseDate'] is not None and isc['ReleaseDate'] != '0000-00-00':
if int(isc['ReleaseDate'][:4]) < int(watchmatch['issue_year']):
logger.fdebug(module + '[ISSUE-VERIFY] ' + str(isc['ReleaseDate']) + ' is before the issue year of ' + str(watchmatch['issue_year']) + ' that was discovered in the filename')
datematch = "False"
@ -1285,9 +1307,13 @@ class PostProcessor(object):
#loop through the hits here.
if len(manual_list) == 0 and len(manual_arclist) == 0:
logger.info(module + ' No matches for Manual Run ... exiting.')
if mylar.APILOCK is True:
mylar.APILOCK = False
return
elif len(manual_arclist) > 0 and len(manual_list) == 0:
logger.info(module + ' Manual post-processing completed for ' + str(len(manual_arclist)) + ' story-arc issues.')
if mylar.APILOCK is True:
mylar.APILOCK = False
return
elif len(manual_arclist) > 0:
logger.info(module + ' Manual post-processing completed for ' + str(len(manual_arclist)) + ' story-arc issues.')

34
mylar/__init__.py
View File

@ -482,21 +482,21 @@ def dbcheck():
c.execute('SELECT ReleaseDate from storyarcs')
except sqlite3.OperationalError:
try:
c.execute('CREATE TABLE IF NOT EXISTS storyarcs(StoryArcID TEXT, ComicName TEXT, IssueNumber TEXT, SeriesYear TEXT, IssueYEAR TEXT, StoryArc TEXT, TotalIssues TEXT, Status TEXT, inCacheDir TEXT, Location TEXT, IssueArcID TEXT, ReadingOrder INT, IssueID TEXT, ComicID TEXT, ReleaseDate TEXT, IssueDate TEXT, Publisher TEXT, IssuePublisher TEXT, IssueName TEXT, CV_ArcID TEXT, Int_IssueNumber INT, DynamicComicName TEXT, Volume TEXT, Manual TEXT, DateAdded TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS storyarcs(StoryArcID TEXT, ComicName TEXT, IssueNumber TEXT, SeriesYear TEXT, IssueYEAR TEXT, StoryArc TEXT, TotalIssues TEXT, Status TEXT, inCacheDir TEXT, Location TEXT, IssueArcID TEXT, ReadingOrder INT, IssueID TEXT, ComicID TEXT, ReleaseDate TEXT, IssueDate TEXT, Publisher TEXT, IssuePublisher TEXT, IssueName TEXT, CV_ArcID TEXT, Int_IssueNumber INT, DynamicComicName TEXT, Volume TEXT, Manual TEXT, DateAdded TEXT, DigitalDate TEXT)')
c.execute('INSERT INTO storyarcs(StoryArcID, ComicName, IssueNumber, SeriesYear, IssueYEAR, StoryArc, TotalIssues, Status, inCacheDir, Location, IssueArcID, ReadingOrder, IssueID, ComicID, ReleaseDate, IssueDate, Publisher, IssuePublisher, IssueName, CV_ArcID, Int_IssueNumber, DynamicComicName, Volume, Manual) SELECT StoryArcID, ComicName, IssueNumber, SeriesYear, IssueYEAR, StoryArc, TotalIssues, Status, inCacheDir, Location, IssueArcID, ReadingOrder, IssueID, ComicID, StoreDate, IssueDate, Publisher, IssuePublisher, IssueName, CV_ArcID, Int_IssueNumber, DynamicComicName, Volume, Manual FROM readinglist')
c.execute('DROP TABLE readinglist')
except sqlite3.OperationalError:
logger.warn('Unable to update readinglist table to new storyarc table format.')
c.execute('CREATE TABLE IF NOT EXISTS comics (ComicID TEXT UNIQUE, ComicName TEXT, ComicSortName TEXT, ComicYear TEXT, DateAdded TEXT, Status TEXT, IncludeExtras INTEGER, Have INTEGER, Total INTEGER, ComicImage TEXT, ComicPublisher TEXT, ComicLocation TEXT, ComicPublished TEXT, NewPublish TEXT, LatestIssue TEXT, LatestDate TEXT, Description TEXT, QUALalt_vers TEXT, QUALtype TEXT, QUALscanner TEXT, QUALquality TEXT, LastUpdated TEXT, AlternateSearch TEXT, UseFuzzy TEXT, ComicVersion TEXT, SortOrder INTEGER, DetailURL TEXT, ForceContinuing INTEGER, ComicName_Filesafe TEXT, AlternateFileName TEXT, ComicImageURL TEXT, ComicImageALTURL TEXT, DynamicComicName TEXT, AllowPacks TEXT, Type TEXT, Corrected_SeriesYear TEXT, TorrentID_32P TEXT, LatestIssueID TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS issues (IssueID TEXT, ComicName TEXT, IssueName TEXT, Issue_Number TEXT, DateAdded TEXT, Status TEXT, Type TEXT, ComicID TEXT, ArtworkURL Text, ReleaseDate TEXT, Location TEXT, IssueDate TEXT, Int_IssueNumber INT, ComicSize TEXT, AltIssueNumber TEXT, IssueDate_Edit TEXT, ImageURL TEXT, ImageURL_ALT TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS issues (IssueID TEXT, ComicName TEXT, IssueName TEXT, Issue_Number TEXT, DateAdded TEXT, Status TEXT, Type TEXT, ComicID TEXT, ArtworkURL Text, ReleaseDate TEXT, Location TEXT, IssueDate TEXT, DigitalDate TEXT, Int_IssueNumber INT, ComicSize TEXT, AltIssueNumber TEXT, IssueDate_Edit TEXT, ImageURL TEXT, ImageURL_ALT TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS snatched (IssueID TEXT, ComicName TEXT, Issue_Number TEXT, Size INTEGER, DateAdded TEXT, Status TEXT, FolderName TEXT, ComicID TEXT, Provider TEXT, Hash TEXT, crc TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS upcoming (ComicName TEXT, IssueNumber TEXT, ComicID TEXT, IssueID TEXT, IssueDate TEXT, Status TEXT, DisplayComicName TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS nzblog (IssueID TEXT, NZBName TEXT, SARC TEXT, PROVIDER TEXT, ID TEXT, AltNZBName TEXT, OneOff TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS weekly (SHIPDATE TEXT, PUBLISHER TEXT, ISSUE TEXT, COMIC VARCHAR(150), EXTRA TEXT, STATUS TEXT, ComicID TEXT, IssueID TEXT, CV_Last_Update TEXT, DynamicName TEXT, weeknumber TEXT, year TEXT, volume TEXT, seriesyear TEXT, annuallink TEXT, rowid INTEGER PRIMARY KEY)')
c.execute('CREATE TABLE IF NOT EXISTS weekly (SHIPDATE TEXT, PUBLISHER TEXT, ISSUE TEXT, COMIC VARCHAR(150), EXTRA TEXT, STATUS TEXT, ComicID TEXT, IssueID TEXT, CV_Last_Update TEXT, DynamicName TEXT, weeknumber TEXT, year TEXT, volume TEXT, seriesyear TEXT, annuallink TEXT, format TEXT, rowid INTEGER PRIMARY KEY)')
c.execute('CREATE TABLE IF NOT EXISTS importresults (impID TEXT, ComicName TEXT, ComicYear TEXT, Status TEXT, ImportDate TEXT, ComicFilename TEXT, ComicLocation TEXT, WatchMatch TEXT, DisplayName TEXT, SRID TEXT, ComicID TEXT, IssueID TEXT, Volume TEXT, IssueNumber TEXT, DynamicName TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS readlist (IssueID TEXT, ComicName TEXT, Issue_Number TEXT, Status TEXT, DateAdded TEXT, Location TEXT, inCacheDir TEXT, SeriesYear TEXT, ComicID TEXT, StatusChange TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS annuals (IssueID TEXT, Issue_Number TEXT, IssueName TEXT, IssueDate TEXT, Status TEXT, ComicID TEXT, GCDComicID TEXT, Location TEXT, ComicSize TEXT, Int_IssueNumber INT, ComicName TEXT, ReleaseDate TEXT, ReleaseComicID TEXT, ReleaseComicName TEXT, IssueDate_Edit TEXT, DateAdded TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS annuals (IssueID TEXT, Issue_Number TEXT, IssueName TEXT, IssueDate TEXT, Status TEXT, ComicID TEXT, GCDComicID TEXT, Location TEXT, ComicSize TEXT, Int_IssueNumber INT, ComicName TEXT, ReleaseDate TEXT, DigitalDate TEXT, ReleaseComicID TEXT, ReleaseComicName TEXT, IssueDate_Edit TEXT, DateAdded TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS rssdb (Title TEXT UNIQUE, Link TEXT, Pubdate TEXT, Site TEXT, Size TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS futureupcoming (ComicName TEXT, IssueNumber TEXT, ComicID TEXT, IssueID TEXT, IssueDate TEXT, Publisher TEXT, Status TEXT, DisplayComicName TEXT, weeknumber TEXT, year TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS failed (ID TEXT, Status TEXT, ComicID TEXT, IssueID TEXT, Provider TEXT, ComicName TEXT, Issue_Number TEXT, NZBName TEXT, DateFailed TEXT)')
@ -505,6 +505,7 @@ def dbcheck():
c.execute('CREATE TABLE IF NOT EXISTS oneoffhistory (ComicName TEXT, IssueNumber TEXT, ComicID TEXT, IssueID TEXT, Status TEXT, weeknumber TEXT, year TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS jobhistory (JobName TEXT, prev_run_datetime timestamp, prev_run_timestamp REAL, next_run_datetime timestamp, next_run_timestamp REAL, last_run_completed TEXT, successful_completions TEXT, failed_completions TEXT, status TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS manualresults (provider TEXT, id TEXT, kind TEXT, comicname TEXT, volume TEXT, oneoff TEXT, fullprov TEXT, issuenumber TEXT, modcomicname TEXT, name TEXT, link TEXT, size TEXT, pack_numbers TEXT, pack_issuelist TEXT, comicyear TEXT, issuedate TEXT, tmpprov TEXT, pack TEXT, issueid TEXT, comicid TEXT, sarc TEXT, issuearcid TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS storyarcs(StoryArcID TEXT, ComicName TEXT, IssueNumber TEXT, SeriesYear TEXT, IssueYEAR TEXT, StoryArc TEXT, TotalIssues TEXT, Status TEXT, inCacheDir TEXT, Location TEXT, IssueArcID TEXT, ReadingOrder INT, IssueID TEXT, ComicID TEXT, ReleaseDate TEXT, IssueDate TEXT, Publisher TEXT, IssuePublisher TEXT, IssueName TEXT, CV_ArcID TEXT, Int_IssueNumber INT, DynamicComicName TEXT, Volume TEXT, Manual TEXT, DateAdded TEXT, DigitalDate TEXT)')
conn.commit
c.close
@ -619,6 +620,11 @@ def dbcheck():
except sqlite3.OperationalError:
c.execute('ALTER TABLE comics ADD COLUMN LatestIssueID TEXT')
try:
c.execute('SELECT Collects from comics')
except sqlite3.OperationalError:
c.execute('ALTER TABLE comics ADD COLUMN Collects CLOB')
try:
c.execute('SELECT DynamicComicName from comics')
if CONFIG.DYNAMIC_UPDATE < 3:
@ -661,6 +667,11 @@ def dbcheck():
except sqlite3.OperationalError:
c.execute('ALTER TABLE issues ADD COLUMN ImageURL_ALT TEXT')
try:
c.execute('SELECT DigitalDate from issues')
except sqlite3.OperationalError:
c.execute('ALTER TABLE issues ADD COLUMN DigitalDate TEXT')
## -- ImportResults Table --
try:
@ -812,6 +823,11 @@ def dbcheck():
except sqlite3.OperationalError:
c.execute('ALTER TABLE weekly ADD COLUMN annuallink TEXT')
try:
c.execute('SELECT format from weekly')
except sqlite3.OperationalError:
c.execute('ALTER TABLE weekly ADD COLUMN format TEXT')
## -- Nzblog Table --
try:
@ -892,6 +908,11 @@ def dbcheck():
except sqlite3.OperationalError:
c.execute('ALTER TABLE annuals ADD COLUMN DateAdded TEXT')
try:
c.execute('SELECT DigitalDate from annuals')
except sqlite3.OperationalError:
c.execute('ALTER TABLE annuals ADD COLUMN DigitalDate TEXT')
## -- Snatched Table --
try:
@ -984,6 +1005,11 @@ def dbcheck():
except sqlite3.OperationalError:
c.execute('ALTER TABLE storyarcs ADD COLUMN DateAdded TEXT')
try:
c.execute('SELECT DigitalDate from storyarcs')
except sqlite3.OperationalError:
c.execute('ALTER TABLE storyarcs ADD COLUMN DigitalDate TEXT')
## -- searchresults Table --
try:
c.execute('SELECT SRID from searchresults')

108
mylar/cv.py
View File

@ -190,7 +190,7 @@ def getComic(comicid, type, issueid=None, arc=None, arcid=None, arclist=None, co
else:
tmpidlist += '|' + str(comicidlist[i])
in_cnt +=1
logger.info('tmpidlist: ' + str(tmpidlist))
logger.fdebug('tmpidlist: ' + str(tmpidlist))
searched = pulldetails(None, 'import', offset=0, comicidlist=tmpidlist)
@ -287,6 +287,8 @@ def GetComicInfo(comicid, dom, safechk=None):
#the description field actually holds the Volume# - so let's grab it
try:
descchunk = dom.getElementsByTagName('description')[0].firstChild.wholeText
desc_soup = Soup(descchunk, "html.parser")
desclinks = desc_soup.findAll('a')
comic_desc = drophtml(descchunk)
desdeck +=1
except:
@ -312,26 +314,87 @@ def GetComicInfo(comicid, dom, safechk=None):
comic['Aliases'] = 'None'
comic['ComicVersion'] = 'None' #noversion'
#logger.info('comic_desc:' + comic_desc)
#logger.info('comic_deck:' + comic_deck)
#logger.info('desdeck: ' + str(desdeck))
#figure out if it's a print / digital edition.
comic['Type'] = 'None'
if comic_deck != 'None':
if any(['print' in comic_deck.lower(), 'digital' in comic_deck.lower()]):
if any(['print' in comic_deck.lower(), 'digital' in comic_deck.lower(), 'paperback' in comic_deck.lower(), 'hardcover' in comic_deck.lower()]):
if 'print' in comic_deck.lower():
comic['Type'] = 'Print'
elif 'digital' in comic_deck.lower():
comic['Type'] = 'Digital'
comic['Type'] = 'Digital'
elif 'paperback' in comic_deck.lower():
comic['Type'] = 'TPB'
elif 'hardcover' in comic_deck.lower():
comic['Type'] = 'HC'
if comic_desc != 'None' and comic['Type'] == 'None':
if 'print' in comic_desc[:60].lower() and 'print edition can be found' not in comic_desc.lower():
comic['Type'] = 'Print'
elif 'digital' in comic_desc[:60].lower() and 'digital edition can be found' not in comic_desc.lower():
comic['Type'] = 'Digital'
elif all(['paperback' in comic_desc[:60].lower(), 'paperback can be found' not in comic_desc.lower()]) or 'collects' in comic_desc[:60].lower():
comic['Type'] = 'TPB'
elif 'hardcover' in comic_desc[:60].lower() and 'hardcover can be found' not in comic_desc.lower():
comic['Type'] = 'HC'
else:
comic['Type'] = 'Print'
if all([comic_desc != 'None', 'trade paperback' in comic_desc[:30].lower(), 'collecting' in comic_desc[:40].lower()]):
#ie. Trade paperback collecting Marvel Team-Up #9-11, 48-51, 72, 110 & 145.
first_collect = comic_desc.lower().find('collecting')
#logger.info('first_collect: %s' % first_collect)
#logger.info('comic_desc: %s' % comic_desc)
#logger.info('desclinks: %s' % desclinks)
issue_list = []
for fc in desclinks:
#logger.info('fc: %s' % fc)
fc_id = fc['data-ref-id']
#logger.info('fc_id: %s' % fc_id)
fc_name = fc.findNext(text=True)
#logger.info('fc_name: %s' % fc_name)
if fc_id.startswith('4000'):
fc_cid = None
fc_isid = fc_id
iss_start = fc_name.find('#')
issuerun = fc_name[iss_start:].strip()
fc_name = fc_name[:iss_start].strip()
elif fc_id.startswith('4050'):
fc_cid = fc_id
fc_isid = None
issuerun = fc.next_sibling
lines = re.sub("[^0-9]", ' ', issuerun).strip().split(' ')
if len(lines) > 0:
for x in sorted(lines, reverse=True):
srchline = issuerun.rfind(x)
if srchline != -1:
try:
if issuerun[srchline+len(x)] == ',' or issuerun[srchline+len(x)] == '.' or issuerun[srchline+len(x)] == ' ':
issuerun = issuerun[:srchline+len(x)]
break
except:
continue
if issuerun.endswith('.') or issuerun.endswith(','):
#logger.fdebug('Changed issuerun from %s to %s' % (issuerun, issuerun[:-1]))
issuerun = issuerun[:-1]
if issuerun.endswith(' and '):
issuerun = issuerun[:-4].strip()
elif issuerun.endswith(' and'):
issuerun = issuerun[:-3].strip()
# except:
# pass
issue_list.append({'series': fc_name,
'comicid': fc_cid,
'issueid': fc_isid,
'issues': issuerun})
#first_collect = cis
logger.info('Collected issues in volume: %s' % issue_list)
comic['Issue_List'] = issue_list
else:
comic['Issue_List'] = 'None'
while (desdeck > 0):
if desdeck == 1:
if comic_desc == 'None':
@ -412,19 +475,7 @@ def GetComicInfo(comicid, dom, safechk=None):
comic['FirstIssueID'] = dom.getElementsByTagName('id')[0].firstChild.wholeText
# print ("fistIss:" + str(comic['FirstIssueID']))
# comicchoice.append({
# 'ComicName': comic['ComicName'],
# 'ComicYear': comic['ComicYear'],
# 'Comicid': comicid,
# 'ComicURL': comic['ComicURL'],
# 'ComicIssues': comic['ComicIssues'],
# 'ComicImage': comic['ComicImage'],
# 'ComicVolume': ParseVol,
# 'ComicPublisher': comic['ComicPublisher']
# })
# comic['comicchoice'] = comicchoice
#logger.info('comic: %s' % comic)
return comic
def GetIssuesInfo(comicid, dom, arcid=None):
@ -495,6 +546,19 @@ def GetIssuesInfo(comicid, dom, arcid=None):
tempissue['StoreDate'] = subtrack.getElementsByTagName('store_date')[0].firstChild.wholeText
except:
tempissue['StoreDate'] = '0000-00-00'
try:
digital_desc = subtrack.getElementsByTagName('description')[0].firstChild.wholeText
except:
tempissue['DigitalDate'] = '0000-00-00'
else:
tempissue['DigitalDate'] = '0000-00-00'
if all(['digital' in digital_desc.lower()[-90:], 'print' in digital_desc.lower()[-90:]]):
#get the digital date of issue here...
mff = mylar.filechecker.FileChecker()
vlddate = mff.checkthedate(digital_desc[-90:], fulldate=True)
#logger.fdebug('vlddate: %s' % vlddate)
if vlddate:
tempissue['DigitalDate'] = vlddate
try:
tempissue['Issue_Number'] = subtrack.getElementsByTagName('issue_number')[0].firstChild.wholeText
except:
@ -517,6 +581,7 @@ def GetIssuesInfo(comicid, dom, arcid=None):
'Issue_Number': tempissue['Issue_Number'],
'Issue_Date': tempissue['CoverDate'],
'Store_Date': tempissue['StoreDate'],
'Digital_Date': tempissue['DigitalDate'],
'Issue_Name': tempissue['Issue_Name'],
'Image': tempissue['ComicImage'],
'ImageALT': tempissue['ComicImageALT']
@ -531,6 +596,7 @@ def GetIssuesInfo(comicid, dom, arcid=None):
'Issue_Number': tempissue['Issue_Number'],
'Issue_Date': tempissue['CoverDate'],
'Store_Date': tempissue['StoreDate'],
'Digital_Date': tempissue['DigitalDate'],
'Issue_Name': tempissue['Issue_Name']
})
@ -538,6 +604,7 @@ def GetIssuesInfo(comicid, dom, arcid=None):
firstdate = tempissue['CoverDate']
n-= 1
#logger.fdebug('issue_info: %s' % issuech)
#issue['firstdate'] = firstdate
return issuech, firstdate
@ -817,8 +884,7 @@ def GetImportList(results):
return serieslist
def drophtml(html):
from bs4 import BeautifulSoup
soup = BeautifulSoup(html, "html.parser")
soup = Soup(html, "html.parser")
text_parts = soup.findAll(text=True)
#print ''.join(text_parts)

236
mylar/filechecker.py
View File

@ -1,4 +1,5 @@
#/usr/bin/env python
# -*- coding: utf-8 -*-
# This file is part of Mylar.
#
# Mylar is free software: you can redistribute it and/or modify
@ -325,6 +326,56 @@ class FileChecker(object):
#split_file = re.findall('\([\w\s-]+\)|[-+]?\d*\.\d+|\d+|[\w-]+|#?\d\.\d+|#(?<![\w\d])XCV(?![\w\d])+|\)', ret_sf1, re.UNICODE)
split_file = re.findall('(?imu)\([\w\s-]+\)|[-+]?\d*\.\d+|\d+|[\w-]+|#?\d\.\d+|#(?<![\w\d])XCV(?![\w\d])+|\)', ret_sf1, re.UNICODE)
#10-20-2018 ---START -- attempt to detect '01 (of 7.3)'
#10-20-2018 -- attempt to detect '36p ctc' as one element
spf = []
mini = False
wrdcnt = 0
for x in split_file:
if x == 'of':
mini = True
spf.append(x)
elif mini is True:
mini = False
try:
logger.fdebug('checking now: %s' % x)
if x.isdigit():
logger.fdebug('[MINI-SERIES] MAX ISSUES IN SERIES: %s' % x)
spf.append('(of %s)' % x)
elif float(x) > 0:
logger.fdebug('[MINI-DECIMAL SERIES] MAX ISSUES IN SERIES: %s' % x)
spf.append('(of %s)' % x)
except Exception as e:
logger.error('Exception: %s' % e)
spf.append(x)
elif x == ')':
pass
elif x == 'p' or x == 'ctc':
try:
if spf[wrdcnt-1].isdigit():
logger.debug('THIS SHOULD BE : %s%s' % (spf[wrdcnt-1], x))
newline = '%s%s' % (spf[wrdcnt-1], x)
spf[wrdcnt -1] = newline
#wrdcnt =-1
elif spf[wrdcnt-1][-1] == 'p' and spf[wrdcnt-1][:-1].isdigit() and x == 'ctc':
logger.fdebug('THIS SHOULD BE : %s%s' % (spf[wrdcnt-1], x))
newline = '%s%s' % (spf[wrdcnt-1], x)
spf[wrdcnt -1] = newline
#wrdcnt =-1
except Exception as e:
logger.error('[ERROR] %s' % e)
logger.warn('this should not be passed: %s' % x)
spf.append(x)
else:
spf.append(x)
wrdcnt +=1
if len(spf) > 0:
split_file = spf
logger.fdebug('NEWLY SPLIT REORGD: %s' % split_file)
#10-20-2018 ---END
if len(split_file) == 1:
logger.fdebug('Improperly formatted filename - there is no seperation using appropriate characters between wording.')
ret_sf1 = re.sub('\-',' ', ret_sf1).strip()
@ -339,7 +390,7 @@ class FileChecker(object):
lastissue_label = None
lastissue_position = 0
lastmod_position = 0
booktype = 'issue'
#exceptions that are considered alpha-numeric issue numbers
exceptions = ('NOW', 'AI', 'AU', 'X', 'A', 'B', 'C', 'INH', 'MU')
@ -416,7 +467,7 @@ class FileChecker(object):
count = match.group()
found = True
if not found:
if found is False:
match = re.search('(?<=\(of\s)\d+(?=\))', sf, re.IGNORECASE)
if match:
count = match.group()
@ -431,7 +482,7 @@ class FileChecker(object):
logger.fdebug('Issue Number SHOULD BE: ' + str(lastissue_label))
validcountchk = True
if all([lastissue_position == (split_file.index(sf) -1), lastissue_label is not None and '#' not in sf]):
if all([lastissue_position == (split_file.index(sf) -1), lastissue_label is not None, '#' not in sf, sf != 'p']):
#find it in the original file to see if there's a decimal between.
findst = lastissue_mod_position+1
if findst > len(modfilename):
@ -439,11 +490,17 @@ class FileChecker(object):
if modfilename[findst] != '.' or modfilename[findst] != '#': #findst != '.' and findst != '#':
if sf.isdigit():
logger.fdebug('2 seperate numbers detected. Assuming 2nd number is the actual issue')
possible_issuenumbers.append({'number': sf,
'position': split_file.index(sf, lastissue_position), #modfilename.find(sf)})
'mod_position': self.char_file_position(modfilename, sf, lastmod_position),
'validcountchk': validcountchk})
seper_num = False
for x in datecheck:
if x['position'] == split_file.index(sf, lastissue_position):
seper_num = True
if seper_num is False:
logger.fdebug('2 seperate numbers detected. Assuming 2nd number is the actual issue')
#possible_issuenumbers.append({'number': sf,
# 'position': split_file.index(sf, lastissue_position), #modfilename.find(sf)})
# 'mod_position': self.char_file_position(modfilename, sf, lastmod_position),
# 'validcountchk': validcountchk})
#used to see if the issue is an alpha-numeric (ie. 18.NOW, 50-X, etc)
lastissue_position = split_file.index(sf, lastissue_position)
@ -481,7 +538,6 @@ class FileChecker(object):
else:
if ('#' in sf or sf.isdigit()) or validcountchk:
logger.fdebug('validated: ' + sf)
if validcountchk:
#if it's not a decimal but the digits are back-to-back, then it's something else.
possible_issuenumbers.append({'number': lastissue_label,
@ -593,11 +649,57 @@ class FileChecker(object):
else:
raise ValueError
except ValueError, e:
#10-20-2018 - to detect issue numbers such as #000.0000½
if lastissue_label is not None and lastissue_position == int(split_file.index(sf))-1 and sf == 'XCV':
logger.info('this should be: %s%s' % (lastissue_label, sf))
pi = []
for x in possible_issuenumbers:
if (x['number'] == lastissue_label and x['position'] == lastissue_position) or (x['number'] == sf and x['position'] == split_file.index(sf, lastissue_position)):
pass
else:
pi.append({'number': x['number'],
'position': x['position'],
'mod_position': x['mod_position'],
'validcountchk': x['validcountchk']})
lastissue_label = '%s%s' % (lastissue_label, sf)
pi.append({'number': lastissue_label,
'position': lastissue_position,
'mod_position': lastmod_position,
'validcountchk': validcountchk})
if len(pi) > 0:
possible_issuenumbers = pi
elif sf.lower() == 'of' and lastissue_label is not None and lastissue_position == int(split_file.index(sf))-1:
logger.info('MINI-SERIES DETECTED')
else:
if any([re.sub('[\(\)]', '', sf.lower()).strip() == 'tpb', re.sub('[\(\)]', '', sf.lower()).strip() == 'digital tpb']):
logger.info('TRADE PAPERBACK DETECTED. NOT DETECTING ISSUE NUMBER - ASSUMING VOLUME')
booktype = 'TPB'
try:
if volume_found['volume'] is not None:
possible_issuenumbers.append({'number': volume_found['volume'],
'position': volume_found['position'],
'mod_position': self.char_file_position(modfilename, volume_found['volume'], lastmod_position),
'validcountchk': validcountchk})
except:
possible_issuenumbers.append({'number': '1',
'position': split_file.index(sf, lastissue_position), #modfilename.find(sf)})
'mod_position': self.char_file_position(modfilename, sf, lastmod_position),
'validcountchk': validcountchk})
elif any([sf.lower() == 'gn', sf.lower() == 'graphic novel']):
logger.info('GRAPHIC NOVEL DETECTED. NOT DETECTING ISSUE NUMBER - ASSUMING VOLUME')
booktype = 'GN'
else:
logger.info('[%s] Error detecting issue # - ignoring this result : %s' % (e, sf))
volumeprior = False
volumeprior_label = None
sep_volume = False
pass
#logger.fdebug('Error detecting issue # - ignoring this result : ' + str(sf))
#keep track of where in the original modfilename the positions are in order to check against it for decimal places, etc.
file_length += len(sf) + 1 #1 for space
@ -742,13 +844,19 @@ class FileChecker(object):
issue_number_position -=1
if issue_number is None:
logger.fdebug('No issue number present in filename.')
if any([booktype == 'TPB', booktype == 'GN']):
logger.info('%s detected. Volume assumption is number: %s' % (booktype, volume_found))
else:
if len(volume_found) > 0:
logger.info('UNKNOWN TPB/GN detected. Volume assumption is number: %s' % (volume_found))
else:
logger.info('No issue number present in filename.')
else:
logger.fdebug('issue verified as : ' + issue_number)
issue_volume = None
if len(volume_found) > 0:
issue_volume = 'v' + str(volume_found['volume'])
if all([highest_series_pos + 1 != volume_found['position'], highest_series_pos != volume_found['position'] + 1, sep_volume == False]):
if all([highest_series_pos + 1 != volume_found['position'], highest_series_pos != volume_found['position'] + 1, sep_volume == False, booktype == 'issue', len(possible_issuenumbers) > 0]):
logger.fdebug('Extra item(s) are present between the volume label and the issue number. Checking..')
split_file.insert(int(issue_number_position), split_file.pop(volume_found['position'])) #highest_series_pos-1, split_file.pop(volume_found['position']))
logger.fdebug('new split: ' + str(split_file))
@ -761,10 +869,13 @@ class FileChecker(object):
else:
highest_series_pos = volume_found['position']
logger.fdebug('Volume detected as : ' + issue_volume)
if all([len(volume_found) == 0, booktype != 'issue']) or all([len(volume_found) == 0, issue_number_position == len(split_file)]):
issue_volume = 'v1'
#at this point it should be in a SERIES ISSUE VOLUME YEAR kind of format
#if the position of the issue number is greater than the highest series position, make it the highest series position.
if issue_number_position > highest_series_pos:
if issue_number_position != len(split_file) and issue_number_position > highest_series_pos:
if not volume_found:
highest_series_pos = issue_number_position
else:
@ -807,16 +918,13 @@ class FileChecker(object):
tmpval = yearposition - issue_number_position
else:
tmpval = 1
#logger.fdebug('TMPVAL: %s' % tmpval)
except:
pass
else:
if tmpval > 2:
logger.fdebug('There are %s extra words between the issue # and the year position. Deciphering if issue title or part of series title.' % tmpval)
tmpval1 = ' '.join(split_file[issue_number_position+1:yearposition])
#logger.info('%s' % tmpval)
if split_file[issue_number_position+1] == '-':
#logger.info('dash detected')
usevalue = ' '.join(split_file[issue_number_position+2:yearposition])
splitv = split_file[issue_number_position+2:yearposition]
else:
@ -904,8 +1012,15 @@ class FileChecker(object):
if splitvalue is not None:
logger.fdebug('[SPLITVALUE] possible issue title: %s' % splitvalue)
alt_series = '%s %s' % (series_name, splitvalue)
alt_issue = splitvalue
if booktype != 'issue':
if alt_issue is not None:
alt_issue = re.sub('tpb', '', splitvalue, flags=re.I).strip()
if alt_series is not None:
alt_series = re.sub('tpb', '', alt_series, flags=re.I).strip()
if alt_series is not None:
if booktype != 'issue':
if alt_series is not None:
alt_series = re.sub('tpb', '', alt_series, flags=re.I).strip()
logger.fdebug('Alternate series / issue title: %s [%s]' % (alt_series, alt_issue))
#if the filename is unicoded, it won't match due to the unicode translation. Keep the unicode as well as the decoded.
@ -921,28 +1036,33 @@ class FileChecker(object):
series_name = re.sub('special', '', series_name, flags=re.I).strip()
series_name_decoded = re.sub('special', '', series_name_decoded, flags=re.I).strip()
if issue_number is None or series_name is None:
logger.fdebug('Cannot parse the filename properly. I\'m going to make note of this filename so that my evil ruler can make it work.')
if series_name is not None:
dreplace = self.dynamic_replace(series_name)['mod_seriesname']
if (any([issue_number is None, series_name is None]) and booktype == 'issue'):
if all([issue_number is None, booktype == 'issue', issue_volume is not None]):
logger.info('Possible UKNOWN TPB/GN detected - no issue number present, no clarification in filename, but volume present with series title')
else:
dreplace = None
return {'parse_status': 'failure',
'sub': path_list,
'comicfilename': filename,
'comiclocation': self.dir,
'series_name': series_name,
'series_name_decoded': series_name_decoded,
'alt_series': alt_series,
'alt_issue': alt_issue,
'dynamic_name': dreplace,
'issue_number': issue_number,
'justthedigits': issue_number, #redundant but it's needed atm
'series_volume': issue_volume,
'issue_year': issue_year,
'annual_comicid': None,
'scangroup': scangroup,
'reading_order': None}
logger.fdebug('Cannot parse the filename properly. I\'m going to make note of this filename so that my evil ruler can make it work.')
if series_name is not None:
dreplace = self.dynamic_replace(series_name)['mod_seriesname']
else:
dreplace = None
return {'parse_status': 'failure',
'sub': path_list,
'comicfilename': filename,
'comiclocation': self.dir,
'series_name': series_name,
'series_name_decoded': series_name_decoded,
'alt_series': alt_series,
'alt_issue': alt_issue,
'dynamic_name': dreplace,
'issue_number': issue_number,
'justthedigits': issue_number, #redundant but it's needed atm
'series_volume': issue_volume,
'issue_year': issue_year,
'annual_comicid': None,
'scangroup': scangroup,
'reading_order': None}
if self.justparse:
return {'parse_status': 'success',
@ -1323,7 +1443,7 @@ class FileChecker(object):
return {'AS_Alt': AS_Alt,
'AS_Tuple': AS_Tuple}
def checkthedate(self, txt):
def checkthedate(self, txt, fulldate=False):
# txt='''\
# Jan 19, 1990
# January 19, 1990
@ -1334,17 +1454,28 @@ class FileChecker(object):
# Jan 1990
# January1990'''
fmts = ('%Y','%b %d, %Y','%B %d, %Y','%B %d %Y','%m/%d/%Y','%m/%d/%y','%b %Y','%B%Y','%b %d,%Y','%m-%Y','%B %Y','%Y-%m-%d','%Y-%m','%Y%m')
fmts = ('%Y','%b %d, %Y','%B %d, %Y','%B %d %Y','%m/%d/%Y','%m/%d/%y','(%m/%d/%Y)','%b %Y','%B%Y','%b %d,%Y','%m-%Y','%B %Y','%Y-%m-%d','%Y-%m','%Y%m')
parsed=[]
for e in txt.splitlines():
for fmt in fmts:
try:
t = dt.datetime.strptime(e, fmt)
parsed.append((e, fmt, t))
break
except ValueError as err:
pass
if fulldate is False:
for e in txt.splitlines():
for fmt in fmts:
try:
t = dt.datetime.strptime(e, fmt)
parsed.append((e, fmt, t))
break
except ValueError as err:
pass
else:
for e in txt.split():
logger.info('word: %s' % e)
for fmt in fmts:
try:
t = dt.datetime.strptime(e, fmt)
parsed.append((e, fmt, t))
break
except ValueError as err:
pass
# check that all the cases are handled
success={t[0] for t in parsed}
@ -1354,12 +1485,17 @@ class FileChecker(object):
dateyear = None
#logger.info('parsed: %s' % parsed)
for t in parsed:
# logger.fdebug('"{:20}" => "{:20}" => {}'.format(*t)
dateyear = t[2].year
if fulldate is False:
dateline = t[2].year
else:
dateline = t[2].strftime('%Y-%m-%d')
break
return dateyear
return dateline
def validateAndCreateDirectory(dir, create=False, module=None):
if module is None:

12
mylar/helpers.py
View File

@ -1424,7 +1424,8 @@ def havetotals(refreshit=None):
"percent": percent,
"totalissues": totalissues,
"haveissues": haveissues,
"DateAdded": comic['LastUpdated']})
"DateAdded": comic['LastUpdated'],
"ComicType": comic['Type']})
return comics
@ -1842,17 +1843,22 @@ def listPull(weeknumber, year):
library[row['ComicID']] = row['ComicID']
return library
def listLibrary():
def listLibrary(comicid=None):
import db
library = {}
myDB = db.DBConnection()
list = myDB.select("SELECT a.comicid, b.releasecomicid, a.status FROM Comics AS a LEFT JOIN annuals AS b on a.comicid=b.comicid group by a.comicid")
if comicid is None:
list = myDB.select("SELECT a.comicid, b.releasecomicid, a.status FROM Comics AS a LEFT JOIN annuals AS b on a.comicid=b.comicid group by a.comicid")
else:
list = myDB.select("SELECT a.comicid, b.releasecomicid, a.status FROM Comics AS a LEFT JOIN annuals AS b on a.comicid=b.comicid WHERE a.comicid=? group by a.comicid", [re.sub('4050-', '', comicid).strip()])
for row in list:
library[row['ComicID']] = {'comicid': row['ComicID'],
'status': row['Status']}
if row['ReleaseComicID'] is not None:
library[row['ReleaseComicID']] = {'comicid': row['ComicID'],
'status': row['Status']}
return library
def listStoryArcs():

32
mylar/importer.py
View File

@ -22,6 +22,7 @@ import sys
import shlex
import datetime
import re
import json
import urllib
import urllib2
import shutil
@ -204,17 +205,23 @@ def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=No
publisher = re.sub('!', '', comic['ComicPublisher']) # thanks Boom!
publisher = helpers.filesafe(publisher)
year = SeriesYear
if comicVol is None:
booktype = comic['Type']
if booktype == 'Print' or all([comic['Type'] != 'Print', mylar.CONFIG.FORMAT_BOOKTYPE is False]):
chunk_fb = re.sub('\$Type', '', mylar.CONFIG.FOLDER_FORMAT)
chunk_b = re.compile(r'\s+')
chunk_folder_format = chunk_b.sub(' ', chunk_fb)
else:
chunk_folder_format = mylar.CONFIG.FOLDER_FORMAT
if any([comicVol is None, comic['Type'] != 'Print']):
comicVol = 'None'
#if comversion is None, remove it so it doesn't populate with 'None'
if comicVol == 'None':
chunk_f_f = re.sub('\$VolumeN', '', mylar.CONFIG.FOLDER_FORMAT)
chunk_f_f = re.sub('\$VolumeN', '', chunk_folder_format)
chunk_f = re.compile(r'\s+')
chunk_folder_format = chunk_f.sub(' ', chunk_f_f)
logger.fdebug('No version # found for series, removing from folder format')
logger.fdebug("new folder format: " + str(chunk_folder_format))
else:
chunk_folder_format = mylar.CONFIG.FOLDER_FORMAT
#do work to generate folder path
@ -225,12 +232,14 @@ def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=No
'$publisher': publisher.lower(),
'$VolumeY': 'V' + str(year),
'$VolumeN': comicVol.upper(),
'$Annual': 'Annual'
'$Annual': 'Annual',
'$Type': booktype
}
try:
if mylar.CONFIG.FOLDER_FORMAT == '':
comlocation = os.path.join(mylar.CONFIG.DESTINATION_DIR, comicdir, " (" + SeriesYear + ")")
else:
chunk_folder_format = re.sub('[()|[]]', '', chunk_folder_format).strip()
comlocation = os.path.join(mylar.CONFIG.DESTINATION_DIR, helpers.replace_all(chunk_folder_format, values))
except Exception as e:
if 'TypeError' in e:
@ -308,6 +317,11 @@ def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=No
tmpseriesname = as_dinfo['mod_seriesname']
dynamic_seriesname = re.sub('[\|\s]','', tmpseriesname.lower()).strip()
if comic['Issue_List'] != 'None':
issue_list = json.dumps(comic['Issue_List'])
else:
issue_list = None
controlValueDict = {"ComicID": comicid}
newValueDict = {"ComicName": comic['ComicName'],
"ComicSortName": sortname,
@ -327,6 +341,7 @@ def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=No
# "ComicPublished": gcdinfo['resultPublished'],
"ComicPublished": "Unknown",
"Type": comic['Type'],
"Collects": issue_list,
"DateAdded": helpers.today(),
"Status": "Loading"}
@ -887,6 +902,7 @@ def issue_collection(issuedata, nostatus):
"Issue_Number": issue['Issue_Number'],
"IssueDate": issue['IssueDate'],
"ReleaseDate": issue['ReleaseDate'],
"DigitalDate": issue['DigitalDate'],
"Int_IssueNumber": issue['Int_IssueNumber'],
"ImageURL": issue['ImageURL'],
"ImageURL_ALT": issue['ImageURL_ALT']
@ -1005,6 +1021,7 @@ def manualAnnual(manual_comicid=None, comicname=None, comicyear=None, comicid=No
'IssueName': cleanname,
'IssueDate': str(firstval['Issue_Date']),
'ReleaseDate': str(firstval['Store_Date']),
'DigitalDate': str(firstval['Digital_Date']),
'Status': astatus,
'ReleaseComicName': sr['ComicName']})
n+=1
@ -1018,6 +1035,7 @@ def manualAnnual(manual_comicid=None, comicname=None, comicyear=None, comicid=No
"Int_IssueNumber": helpers.issuedigits(ann['Issue_Number']),
"IssueDate": ann['IssueDate'],
"ReleaseDate": ann['ReleaseDate'],
"DigitalDate": ann['DigitalDate'],
"IssueName": ann['IssueName'],
"ComicID": ann['ComicID'], #this is the series ID
"ReleaseComicID": ann['ReleaseComicID'], #this is the series ID for the annual(s)
@ -1100,6 +1118,7 @@ def updateissuedata(comicid, comicname=None, issued=None, comicIssues=None, call
issname = cleanname
issdate = str(firstval['Issue_Date'])
storedate = str(firstval['Store_Date'])
digitaldate = str(firstval['Digital_Date'])
int_issnum = None
if issnum.isdigit():
int_issnum = int(issnum) * 1000
@ -1264,6 +1283,7 @@ def updateissuedata(comicid, comicname=None, issued=None, comicIssues=None, call
"Issue_Number": issnum,
"IssueDate": issdate,
"ReleaseDate": storedate,
"DigitalDate": digitaldate,
"Int_IssueNumber": int_issnum,
"ImageURL": firstval['Image'],
"ImageURL_ALT": firstval['ImageALT']})
@ -1467,6 +1487,7 @@ def annual_check(ComicName, SeriesYear, comicid, issuetype, issuechk, annualslis
issname = cleanname
issdate = str(firstval['Issue_Date'])
stdate = str(firstval['Store_Date'])
digdate = str(firstval['Digital_Date'])
int_issnum = helpers.issuedigits(issnum)
iss_exists = myDB.selectone('SELECT * from annuals WHERE IssueID=?', [issid]).fetchone()
@ -1494,6 +1515,7 @@ def annual_check(ComicName, SeriesYear, comicid, issuetype, issuechk, annualslis
"Int_IssueNumber": int_issnum,
"IssueDate": issdate,
"ReleaseDate": stdate,
"DigitalDate": digdate,
"IssueName": issname,
"ComicID": comicid,
"IssueID": issid,

8
mylar/locg.py
View File

@ -83,12 +83,13 @@ def locg(pulldate=None,weeknumber=None,year=None):
'annuallink': x['link'],
'year': x['year'],
'volume': x['volume'],
'seriesyear': x['seriesyear']})
'seriesyear': x['seriesyear'],
'format': x['type']})
shipdate = x['shipdate']
myDB = db.DBConnection()
myDB.action("CREATE TABLE IF NOT EXISTS weekly (SHIPDATE, PUBLISHER text, ISSUE text, COMIC VARCHAR(150), EXTRA text, STATUS text, ComicID text, IssueID text, CV_Last_Update text, DynamicName text, weeknumber text, year text, volume text, seriesyear text, annuallink text, rowid INTEGER PRIMARY KEY)")
myDB.action("CREATE TABLE IF NOT EXISTS weekly (SHIPDATE, PUBLISHER text, ISSUE text, COMIC VARCHAR(150), EXTRA text, STATUS text, ComicID text, IssueID text, CV_Last_Update text, DynamicName text, weeknumber text, year text, volume text, seriesyear text, annuallink text, format text, rowid INTEGER PRIMARY KEY)")
#clear out the upcoming table here so they show the new values properly.
if pulldate == '00000000':
@ -123,7 +124,8 @@ def locg(pulldate=None,weeknumber=None,year=None):
'ANNUALLINK': x['annuallink'],
'YEAR': x['year'],
'VOLUME': x['volume'],
'SERIESYEAR': x['seriesyear']}
'SERIESYEAR': x['seriesyear'],
'FORMAT': x['format']}
myDB.upsert("weekly", newValueDict, controlValueDict)
logger.info('[PULL-LIST] Successfully populated pull-list into Mylar for the week of: ' + str(weeknumber))

4
mylar/mb.py
View File

@ -373,7 +373,7 @@ def findComic(name, mode, issue, limityear=None, type=None):
xmltype = 'Print'
elif 'digital' in xmldesc[:60].lower() and 'digital edition can be found' not in xmldesc.lower():
xmltype = 'Digital'
elif 'paperback' in xmldesc[:60].lower() and 'paperback can be found' not in xmldesc.lower():
elif all(['paperback' in xmldesc[:60].lower(), 'paperback can be found' not in xmldesc.lower()]) or 'collects' in xmldesc.lower():
xmltype = 'TPB'
elif 'hardcover' in xmldesc[:60].lower() and 'hardcover can be found' not in xmldesc.lower():
xmltype = 'HC'
@ -458,8 +458,8 @@ def storyarcinfo(xmlid):
try:
logger.fdebug('story_arc ascension')
issuecount = len( arcdom.getElementsByTagName('issue') )
issuedom = arcdom.getElementsByTagName('issue')
issuecount = len( issuedom ) #arcdom.getElementsByTagName('issue') )
isc = 0
arclist = ''
ordernum = 1

120
mylar/search.py
View File

@ -38,7 +38,7 @@ from base64 import b16encode, b32decode
from operator import itemgetter
from wsgiref.handlers import format_date_time
def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, IssueID, AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=None, IssueArcID=None, mode=None, rsscheck=None, ComicID=None, manualsearch=None, filesafe=None, allow_packs=None, oneoff=False, manual=False, torrentid_32p=None):
def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, IssueID, AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=None, IssueArcID=None, mode=None, rsscheck=None, ComicID=None, manualsearch=None, filesafe=None, allow_packs=None, oneoff=False, manual=False, torrentid_32p=None, digitaldate=None):
mylar.COMICINFO = []
unaltered_ComicName = None
@ -313,7 +313,7 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueD
prov_count+=1
continue
if searchmode == 'rss':
findit = NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, searchprov, send_prov_count, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="yes", ComicID=ComicID, issuetitle=issuetitle, unaltered_ComicName=unaltered_ComicName, oneoff=oneoff, cmloopit=cmloopit, manual=manual, torznab_host=torznab_host)
findit = NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, searchprov, send_prov_count, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="yes", ComicID=ComicID, issuetitle=issuetitle, unaltered_ComicName=unaltered_ComicName, oneoff=oneoff, cmloopit=cmloopit, manual=manual, torznab_host=torznab_host, digitaldate=digitaldate)
if findit['status'] is False:
if AlternateSearch is not None and AlternateSearch != "None":
chkthealt = AlternateSearch.split('##')
@ -323,7 +323,7 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueD
for calt in chkthealt:
AS_Alternate = re.sub('##', '', calt)
logger.info(u"Alternate Search pattern detected...re-adjusting to : " + str(AS_Alternate))
findit = NZB_SEARCH(AS_Alternate, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, searchprov, send_prov_count, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="yes", ComicID=ComicID, issuetitle=issuetitle, unaltered_ComicName=AS_Alternate, allow_packs=allow_packs, oneoff=oneoff, cmloopit=cmloopit, manual=manual, torznab_host=torznab_host)
findit = NZB_SEARCH(AS_Alternate, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, searchprov, send_prov_count, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="yes", ComicID=ComicID, issuetitle=issuetitle, unaltered_ComicName=AS_Alternate, allow_packs=allow_packs, oneoff=oneoff, cmloopit=cmloopit, manual=manual, torznab_host=torznab_host, digitaldate=digitaldate)
if findit['status'] is True:
break
if findit['status'] is True:
@ -333,7 +333,7 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueD
break
else:
findit = NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, searchprov, send_prov_count, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="no", ComicID=ComicID, issuetitle=issuetitle, unaltered_ComicName=unaltered_ComicName, allow_packs=allow_packs, oneoff=oneoff, cmloopit=cmloopit, manual=manual, torznab_host=torznab_host, torrentid_32p=torrentid_32p)
findit = NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, searchprov, send_prov_count, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="no", ComicID=ComicID, issuetitle=issuetitle, unaltered_ComicName=unaltered_ComicName, allow_packs=allow_packs, oneoff=oneoff, cmloopit=cmloopit, manual=manual, torznab_host=torznab_host, torrentid_32p=torrentid_32p, digitaldate=digitaldate)
if all([searchprov == '32P', checked_once is False]) or all([searchprov == 'Public Torrents', checked_once is False]):
checked_once = True
if findit['status'] is False:
@ -345,7 +345,7 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueD
for calt in chkthealt:
AS_Alternate = re.sub('##', '', calt)
logger.info(u"Alternate Search pattern detected...re-adjusting to : " + str(AS_Alternate))
findit = NZB_SEARCH(AS_Alternate, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, searchprov, send_prov_count, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="no", ComicID=ComicID, issuetitle=issuetitle, unaltered_ComicName=unaltered_ComicName, allow_packs=allow_packs, oneoff=oneoff, cmloopit=cmloopit, manual=manual, torznab_host=torznab_host, torrentid_32p=torrentid_32p)
findit = NZB_SEARCH(AS_Alternate, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, searchprov, send_prov_count, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="no", ComicID=ComicID, issuetitle=issuetitle, unaltered_ComicName=unaltered_ComicName, allow_packs=allow_packs, oneoff=oneoff, cmloopit=cmloopit, manual=manual, torznab_host=torznab_host, torrentid_32p=torrentid_32p, digitaldate=digitaldate)
if findit['status'] is True:
break
if findit['status'] is True:
@ -409,7 +409,7 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueD
return findit, 'None'
def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, nzbprov, prov_count, IssDateFix, IssueID, UseFuzzy, newznab_host=None, ComicVersion=None, SARC=None, IssueArcID=None, RSS=None, ComicID=None, issuetitle=None, unaltered_ComicName=None, allow_packs=None, oneoff=False, cmloopit=None, manual=False, torznab_host=None, torrentid_32p=None):
def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, nzbprov, prov_count, IssDateFix, IssueID, UseFuzzy, newznab_host=None, ComicVersion=None, SARC=None, IssueArcID=None, RSS=None, ComicID=None, issuetitle=None, unaltered_ComicName=None, allow_packs=None, oneoff=False, cmloopit=None, manual=False, torznab_host=None, torrentid_32p=None, digitaldate=None):
if any([allow_packs is None, allow_packs == 'None', allow_packs == 0, allow_packs == '0']) and all([mylar.CONFIG.ENABLE_TORRENT_SEARCH, mylar.CONFIG.ENABLE_32P]):
allow_packs = False
@ -927,8 +927,11 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDa
continue
else:
stdate = IssueDate
logger.fdebug('issue date used is : %s' % stdate)
else:
stdate = StoreDate
logger.fdebug('store date used is : %s' % stdate)
logger.fdebug('date used is : %s' % stdate)
postdate_int = None
if nzbprov == '32P' and RSS == 'no':
@ -959,42 +962,70 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDa
logger.warn('Unable to parse posting date from provider result set for :' + entry['title'])
continue
#convert it to a Thu, 06 Feb 2014 00:00:00 format
issue_converted = datetime.datetime.strptime(stdate.rstrip(), '%Y-%m-%d')
issue_convert = issue_converted + datetime.timedelta(days=-1)
# to get past different locale's os-dependent dates, let's convert it to a generic datetime format
try:
stamp = time.mktime(issue_convert.timetuple())
issconv = format_date_time(stamp)
except OverflowError:
logger.fdebug('Error attempting to convert the timestamp into a generic format. Probably due to the epoch limiation.')
issconv = issue_convert.strftime('%a, %d %b %Y %H:%M:%S')
#convert it to a tuple
econv = email.utils.parsedate_tz(issconv)
econv2 = datetime.datetime(*econv[:6])
#convert it to a numeric and drop the GMT/Timezone
try:
issuedate_int = time.mktime(econv[:len(econv) -1])
except OverflowError:
logger.fdebug('Unable to convert timestamp to integer format. Forcing things through.')
isyear = econv[1]
epochyr = '1970'
if int(isyear) <= int(epochyr):
tm = datetime.datetime(1970, 1, 1)
issuedate_int = int(time.mktime(tm.timetuple()))
if all([digitaldate != '0000-00-00', digitaldate is not None]):
i = 0
else:
digitaldate_int = '00000000'
i = 1
while i <= 1:
logger.info('i: %s' % i)
if i == 0:
usedate = digitaldate
else:
continue
usedate = stdate
logger.fdebug('usedate: %s' % usedate)
#convert it to a Thu, 06 Feb 2014 00:00:00 format
issue_converted = datetime.datetime.strptime(usedate.rstrip(), '%Y-%m-%d')
issue_convert = issue_converted + datetime.timedelta(days=-1)
# to get past different locale's os-dependent dates, let's convert it to a generic datetime format
try:
stamp = time.mktime(issue_convert.timetuple())
issconv = format_date_time(stamp)
except OverflowError:
logger.fdebug('Error attempting to convert the timestamp into a generic format. Probably due to the epoch limiation.')
issconv = issue_convert.strftime('%a, %d %b %Y %H:%M:%S')
#convert it to a tuple
econv = email.utils.parsedate_tz(issconv)
econv2 = datetime.datetime(*econv[:6])
#convert it to a numeric and drop the GMT/Timezone
try:
usedate_int = time.mktime(econv[:len(econv) -1])
except OverflowError:
logger.fdebug('Unable to convert timestamp to integer format. Forcing things through.')
isyear = econv[1]
epochyr = '1970'
if int(isyear) <= int(epochyr):
tm = datetime.datetime(1970, 1, 1)
usedate_int = int(time.mktime(tm.timetuple()))
else:
continue
if i == 0:
digitaldate_int = usedate_int
digconv2 = econv2
else:
issuedate_int = usedate_int
issconv2 = econv2
i+=1
try:
#try new method to get around issues populating in a diff timezone thereby putting them in a different day.
if dateconv2.date() < econv2.date():
logger.fdebug('[CONV]pubdate: %s < storedate: %s' % (dateconv2.date(), econv2.date()))
#logger.info('digitaldate: %s' % digitaldate)
#logger.info('dateconv2: %s' % dateconv2.date())
#logger.info('digconv2: %s' % digconv2.date())
if digitaldate != '0000-00-00' and dateconv2.date() >= digconv2.date():
logger.fdebug(str(pubdate) + ' is after DIGITAL store date of ' + str(digitaldate))
elif dateconv2.date() < issconv2.date():
logger.fdebug('[CONV]pubdate: %s < storedate: %s' % (dateconv2.date(), issconv2.date()))
logger.fdebug(str(pubdate) + ' is before store date of ' + str(stdate) + '. Ignoring search result as this is not the right issue.')
continue
else:
logger.fdebug(str(pubdate) + ' is after store date of ' + str(stdate))
except:
#if the above fails, drop down to the integer compare method as a failsafe.
if postdate_int < issuedate_int:
if digitaldate != '0000-00-00' and postdate_int >= digitaldate_int:
logger.fdebug(str(pubdate) + ' is after DIGITAL store date of ' + str(digitaldate))
elif postdate_int < issuedate_int:
logger.fdebug('[INT]pubdate: %s < storedate: %s' % (postdate_int, issuedate_int))
logger.fdebug(str(pubdate) + ' is before store date of ' + str(stdate) + '. Ignoring search result as this is not the right issue.')
continue
@ -1311,10 +1342,14 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDa
#need to convert dates to just be yyyy-mm-dd and do comparison, time operator in the below calc as well which probably throws off$
F_ComicVersion = '1'
if postdate_int is not None:
if postdate_int >= issuedate_int and nzbprov == '32P':
if digitaldate != '0000-00-00' and all([postdate_int >= digitaldate_int, nzbprov == '32P']):
logger.fdebug('32P torrent discovery. Posting date (' + str(pubdate) + ') is after DIGITAL store date (' + str(digitaldate) + '), forcing volume label to be the same as series label (0-Day Enforcement): v' + str(F_ComicVersion) + ' --> v' + str(S_ComicVersion))
F_ComicVersion = D_ComicVersion
elif all([postdate_int >= issuedate_int, nzbprov == '32P']):
logger.fdebug('32P torrent discovery. Posting date (' + str(pubdate) + ') is after store date (' + str(stdate) + '), forcing volume label to be the same as series label (0-Day Enforcement): v' + str(F_ComicVersion) + ' --> v' + str(S_ComicVersion))
F_ComicVersion = D_ComicVersion
else:
pass
logger.fdebug("FCVersion: " + str(F_ComicVersion))
logger.fdebug("DCVersion: " + str(D_ComicVersion))
logger.fdebug("SCVersion: " + str(S_ComicVersion))
@ -1914,6 +1949,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None, manual=False):
'Issue_Number': iss['Issue_Number'],
'IssueDate': iss['IssueDate'],
'StoreDate': iss['ReleaseDate'],
'DigitalDate': iss['DigitalDate'],
'SARC': None,
'StoryArcID': None,
'IssueArcID': None,
@ -1933,6 +1969,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None, manual=False):
'Issue_Number': iss['IssueNumber'],
'IssueDate': iss['IssueDate'],
'StoreDate': iss['ReleaseDate'],
'DigitalDate': iss['DigitalDate'],
'SARC': iss['StoryArc'],
'StoryArcID': iss['StoryArcID'],
'IssueArcID': iss['IssueArcID'],
@ -1952,6 +1989,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None, manual=False):
'Issue_Number': iss['Issue_Number'],
'IssueDate': iss['IssueDate'],
'StoreDate': iss['ReleaseDate'], #need to replace with Store date
'DigitalDate': iss['DigitalDate'],
'SARC': None,
'StoryArcID': None,
'IssueArcID': None,
@ -1991,7 +2029,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None, manual=False):
else:
storyarc_watchlist = True
if result['StoreDate'] == '0000-00-00' or result['StoreDate'] is None:
if result['IssueDate'] is None or result['IssueDate'] == '0000-00-00':
if any([result['IssueDate'] is None, result['IssueDate'] == '0000-00-00']) and result['DigitalDate'] == '0000-00-00':
logger.fdebug('ComicID: ' + str(result['ComicID']) + ' has invalid Date data. Skipping searching for this series.')
continue
@ -2018,6 +2056,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None, manual=False):
IssueDate = result['IssueDate']
StoreDate = result['StoreDate']
DigitalDate = result['DigitalDate']
if result['IssueDate'] is None:
ComicYear = SeriesYear
@ -2046,7 +2085,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None, manual=False):
mode = result['mode']
foundNZB, prov = search_init(comic['ComicName'], result['Issue_Number'], str(ComicYear), SeriesYear, Publisher, IssueDate, StoreDate, result['IssueID'], AlternateSearch, UseFuzzy, ComicVersion, SARC=result['SARC'], IssueArcID=result['IssueArcID'], mode=mode, rsscheck=rsscheck, ComicID=result['ComicID'], filesafe=Comicname_filesafe, allow_packs=AllowPacks, oneoff=OneOff, torrentid_32p=TorrentID_32p)
foundNZB, prov = search_init(comic['ComicName'], result['Issue_Number'], str(ComicYear), SeriesYear, Publisher, IssueDate, StoreDate, result['IssueID'], AlternateSearch, UseFuzzy, ComicVersion, SARC=result['SARC'], IssueArcID=result['IssueArcID'], mode=mode, rsscheck=rsscheck, ComicID=result['ComicID'], filesafe=Comicname_filesafe, allow_packs=AllowPacks, oneoff=OneOff, torrentid_32p=TorrentID_32p, digitaldate=DigitalDate)
if foundNZB['status'] is True:
updater.foundsearch(result['ComicID'], result['IssueID'], mode=mode, provider=prov, SARC=result['SARC'], IssueArcID=result['IssueArcID'], hash=foundNZB['info']['t_hash'])
@ -2093,6 +2132,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None, manual=False):
actissueid = None
IssueDate = result['IssueDate']
StoreDate = result['ReleaseDate']
DigitalDate = result['DigitalDate']
TorrentID_32p = None
elif mode == 'pullwant':
ComicName = result['COMIC']
@ -2109,6 +2149,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None, manual=False):
TorrentID_32p = None
IssueDate = result['SHIPDATE']
StoreDate = IssueDate
DigitalDate = '0000-00-00'
else:
comic = myDB.selectone('SELECT * FROM comics where ComicID=?', [ComicID]).fetchone()
if mode == 'want_ann':
@ -2126,6 +2167,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None, manual=False):
ComicVersion = comic['ComicVersion']
IssueDate = result['IssueDate']
StoreDate = result['ReleaseDate']
DigitalDate = result['DigitalDate']
SARC = None
IssueArcID = None
actissueid = issueid
@ -2138,7 +2180,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None, manual=False):
else:
IssueYear = str(IssueDate)[:4]
foundNZB, prov = search_init(ComicName, IssueNumber, str(IssueYear), SeriesYear, Publisher, IssueDate, StoreDate, actissueid, AlternateSearch, UseFuzzy, ComicVersion, SARC=SARC, IssueArcID=IssueArcID, mode=mode, rsscheck=rsscheck, ComicID=ComicID, filesafe=Comicname_filesafe, allow_packs=allow_packs, oneoff=oneoff, manual=manual, torrentid_32p=TorrentID_32p)
foundNZB, prov = search_init(ComicName, IssueNumber, str(IssueYear), SeriesYear, Publisher, IssueDate, StoreDate, actissueid, AlternateSearch, UseFuzzy, ComicVersion, SARC=SARC, IssueArcID=IssueArcID, mode=mode, rsscheck=rsscheck, ComicID=ComicID, filesafe=Comicname_filesafe, allow_packs=allow_packs, oneoff=oneoff, manual=manual, torrentid_32p=TorrentID_32p, digitaldate=DigitalDate)
if manual is True:
return foundNZB
if foundNZB['status'] is True:
@ -2192,7 +2234,7 @@ def searchIssueIDList(issuelist):
else:
AllowPacks = False
foundNZB, prov = search_init(comic['ComicName'], issue['Issue_Number'], str(IssueYear), comic['ComicYear'], Publisher, issue['IssueDate'], issue['ReleaseDate'], issue['IssueID'], AlternateSearch, UseFuzzy, ComicVersion, SARC=None, IssueArcID=None, mode=mode, ComicID=issue['ComicID'], filesafe=comic['ComicName_Filesafe'], allow_packs=AllowPacks, torrentid_32p=TorrentID_32p)
foundNZB, prov = search_init(comic['ComicName'], issue['Issue_Number'], str(IssueYear), comic['ComicYear'], Publisher, issue['IssueDate'], issue['ReleaseDate'], issue['IssueID'], AlternateSearch, UseFuzzy, ComicVersion, SARC=None, IssueArcID=None, mode=mode, ComicID=issue['ComicID'], filesafe=comic['ComicName_Filesafe'], allow_packs=AllowPacks, torrentid_32p=TorrentID_32p, digitaldate=issue['DigitalDate'])
if foundNZB['status'] is True:
updater.foundsearch(ComicID=issue['ComicID'], IssueID=issue['IssueID'], mode=mode, provider=prov, hash=foundNZB['info']['t_hash'])
logger.info('Completed search request.')

53
mylar/webserve.py
View File

@ -26,6 +26,7 @@ from datetime import timedelta, date
import re
import json
import copy
import ntpath
from mako.template import Template
from mako.lookup import TemplateLookup
@ -179,6 +180,13 @@ class WebInterface(object):
else:
comicImage = comic['ComicImage']
comicpublisher = helpers.publisherImages(comic['ComicPublisher'])
if comic['Collects'] is not None:
issues_list = json.loads(comic['Collects'])
else:
issues_list = None
#logger.info('issues_list: %s' % issues_list)
comicConfig = {
"fuzzy_year0": helpers.radio(int(usethefuzzy), 0),
"fuzzy_year1": helpers.radio(int(usethefuzzy), 1),
@ -196,6 +204,7 @@ class WebInterface(object):
"publisher_image_alt": comicpublisher['publisher_image_alt'],
"publisher_imageH": comicpublisher['publisher_imageH'],
"publisher_imageW": comicpublisher['publisher_imageW'],
"issue_list": issues_list,
"ComicImage": comicImage + '?' + datetime.datetime.now().strftime('%y-%m-%d %H:%M:%S')
}
@ -223,6 +232,7 @@ class WebInterface(object):
"Int_IssueNumber": ann['Int_IssueNumber'],
"IssueName": issuename,
"IssueDate": ann['IssueDate'],
"DigitalDate": ann['DigitalDate'],
"Status": ann['Status'],
"Location": ann['Location'],
"ComicID": ann['ComicID'],
@ -574,6 +584,7 @@ class WebInterface(object):
st_issueid = str(storyarcid) + "_" + str(random.randint(1000,9999))
issnum = arcval['Issue_Number']
issdate = str(arcval['Issue_Date'])
digitaldate = str(arcval['Digital_Date'])
storedate = str(arcval['Store_Date'])
int_issnum = helpers.issuedigits(issnum)
@ -603,6 +614,7 @@ class WebInterface(object):
"Issue_Number": issnum,
"IssueDate": issdate,
"ReleaseDate": storedate,
"DigitalDate": digitaldate,
"ReadingOrder": readingorder, #n +1,
"Int_IssueNumber": int_issnum,
"Manual": manual_mod})
@ -644,7 +656,8 @@ class WebInterface(object):
"TotalIssues": storyarcissues,
"ReadingOrder": AD['ReadingOrder'],
"IssueDate": AD['IssueDate'],
"ReleaseDate": AD['ReleaseDate'],
"ReleaseDate": AD['ReleaseDate'],
"DigitalDate": AD['DigitalDate'],
"SeriesYear": seriesYear,
"IssuePublisher": issuePublisher,
"CV_ArcID": arcid,
@ -657,8 +670,10 @@ class WebInterface(object):
logger.fdebug(module + ' Now searching your watchlist for matches belonging to this story arc.')
self.ArcWatchlist(storyarcid)
if arcrefresh:
logger.info('%s Successfully Refreshed %s' % (module, storyarcname))
return
else:
logger.info('%s Successfully Added %s' % (module, storyarcname))
raise cherrypy.HTTPRedirect("detailStoryArc?StoryArcID=%s&StoryArcName=%s" % (storyarcid, storyarcname))
addStoryArc.exposed = True
@ -898,6 +913,8 @@ class WebInterface(object):
if comic['ComicName'] is None: ComicName = "None"
else: ComicName = comic['ComicName']
seriesdir = comic['ComicLocation']
seriesyear = comic['ComicYear']
seriesvol = comic['ComicVersion']
logger.info(u"Deleting all traces of Comic: " + ComicName)
myDB.action('DELETE from comics WHERE ComicID=?', [ComicID])
myDB.action('DELETE from issues WHERE ComicID=?', [ComicID])
@ -912,10 +929,12 @@ class WebInterface(object):
shutil.rmtree(seriesdir)
except:
logger.warn('Unable to remove directory after removing series from Mylar.')
else:
logger.info('Successfully removed directory: %s' % (seriesdir))
else:
logger.warn('Unable to remove directory as it does not exist in : ' + seriesdir)
myDB.action('DELETE from readlist WHERE ComicID=?', [ComicID])
logger.info('Successful deletion of %s %s (%s) from your watchlist' % (ComicName, seriesvol, seriesyear))
helpers.ComicSort(sequence='update')
raise cherrypy.HTTPRedirect("home")
deleteSeries.exposed = True
@ -1682,7 +1701,8 @@ class WebInterface(object):
"HAVEIT": haveit,
"LINK": linkit,
"HASH": None,
"AUTOWANT": False
"AUTOWANT": False,
"FORMAT": weekly['format']
})
else:
if any(x['ComicName'].lower() == weekly['COMIC'].lower() for x in autowant):
@ -1698,7 +1718,8 @@ class WebInterface(object):
"HAVEIT": haveit,
"LINK": linkit,
"HASH": None,
"AUTOWANT": True
"AUTOWANT": True,
"FORMAT": weekly['format']
})
else:
weeklyresults.append({
@ -1713,7 +1734,8 @@ class WebInterface(object):
"HAVEIT": haveit,
"LINK": linkit,
"HASH": None,
"AUTOWANT": False
"AUTOWANT": False,
"FORMAT": weekly['format']
})
if tmp_status == 'Wanted':
@ -2817,6 +2839,9 @@ class WebInterface(object):
logger.fdebug('[%s] Issue to renumber sequence from : %s' % (issuearcid, valid_readingorder))
reading_seq = 1
for rc in sorted(readchk, key=itemgetter('ReadingOrder'), reverse=False):
filename = None
if rc['Location'] is not None:
filename = ntpath.basename(rc['Location'])
if str(issuearcid) == str(rc['IssueArcID']):
logger.fdebug('new order sequence detected at #: %s' % valid_readingorder)
if valid_readingorder > int(rc['ReadingOrder']):
@ -2839,10 +2864,8 @@ class WebInterface(object):
else:
#valid_readingorder
if valid_readingorder < old_reading_seq:
logger.info('2')
reading_seq = int(rc['ReadingOrder'])
else:
logger.info('3')
reading_seq = oldreading_seq +1
logger.fdebug('old sequence discovered at %s to %s' % (oldreading_seq, reading_seq))
oldreading_seq = None
@ -2855,7 +2878,8 @@ class WebInterface(object):
logger.fdebug('reordering existing sequence as lower sequence has changed. Altering from %s to %s' % (rc['ReadingOrder'], reading_seq))
new_readorder.append({'IssueArcID': IssueArcID,
'IssueID': issueid,
'ReadingOrder': reading_seq})
'ReadingOrder': reading_seq,
'filename': filename})
#we resequence in the following way:
# everything before the new reading number stays the same
@ -2866,6 +2890,14 @@ class WebInterface(object):
#newrl = 0
for rl in sorted(new_readorder, key=itemgetter('ReadingOrder'), reverse=False):
if rl['filename'] is not None:
try:
if int(rl['ReadingOrder']) != int(rl['filename'][:rl['filename'].find('-')]) and mylar.CONFIG.READ2FILENAME is True:
logger.fdebug('Order-Change: %s TO %s' % (int(rl['filename'][:rl['filename'].find('-')]), int(rl['ReadingOrder'])))
logger.fdebug('%s to %s' % (rl['filename'], helpers.renamefile_readingorder(rl['ReadingOrder']) + '-' + rl['filename'][rl['filename'].find('-')+1:]))
except:
pass
rl_ctrl = {"IssueID": rl['IssueID'],
"IssueArcID": rl['IssueArcID'],
"StoryArcID": storyarcid}
@ -3156,7 +3188,8 @@ class WebInterface(object):
logger.info("No Story Arcs to search")
else:
#cycle through the story arcs here for matches on the watchlist
arcdir = helpers.filesafe(ArcWatch[0]['StoryArc'])
arcname = ArcWatch[0]['StoryArc']
arcdir = helpers.filesafe(arcname)
arcpub = ArcWatch[0]['Publisher']
if arcpub is None:
arcpub = ArcWatch[0]['IssuePublisher']
@ -3441,6 +3474,8 @@ class WebInterface(object):
myDB.upsert("storyarcs", newVal, ctrlVal)
logger.info("Marked " + issue['ComicName'] + " :# " + issue['Issue_Number'] + " as " + issue['Status'])
arcstats = self.storyarc_main(StoryArcID)
logger.info('[STORY-ARCS] Completed Missing/Recheck Files for %s [%s / %s]' % (arcname, arcstats['Have'], arcstats['TotalIssues']))
return
ArcWatchlist.exposed = True

4
mylar/weeklypull.py
View File

@ -46,7 +46,7 @@ def pullit(forcecheck=None, weeknumber=None, year=None):
except (sqlite3.OperationalError, TypeError), msg:
logger.info(u"Error Retrieving weekly pull list - attempting to adjust")
myDB.action("DROP TABLE weekly")
myDB.action("CREATE TABLE IF NOT EXISTS weekly (SHIPDATE TEXT, PUBLISHER TEXT, ISSUE TEXT, COMIC VARCHAR(150), EXTRA TEXT, STATUS TEXT, ComicID TEXT, IssueID TEXT, CV_Last_Update TEXT, DynamicName TEXT, weeknumber TEXT, year TEXT, volume TEXT, seriesyear TEXT, annuallink TEXT, rowid INTEGER PRIMARY KEY)")
myDB.action("CREATE TABLE IF NOT EXISTS weekly (SHIPDATE TEXT, PUBLISHER TEXT, ISSUE TEXT, COMIC VARCHAR(150), EXTRA TEXT, STATUS TEXT, ComicID TEXT, IssueID TEXT, CV_Last_Update TEXT, DynamicName TEXT, weeknumber TEXT, year TEXT, volume TEXT, seriesyear TEXT, annuallink TEXT, format TEXT, rowid INTEGER PRIMARY KEY)")
pulldate = '00000000'
logger.fdebug(u"Table re-created, trying to populate")
else:
@ -440,7 +440,7 @@ def pullit(forcecheck=None, weeknumber=None, year=None):
logger.info(u"Populating the NEW Weekly Pull list into Mylar for week " + str(weeknumber))
myDB.action("drop table if exists weekly")
myDB.action("CREATE TABLE IF NOT EXISTS weekly (SHIPDATE, PUBLISHER TEXT, ISSUE TEXT, COMIC VARCHAR(150), EXTRA TEXT, STATUS TEXT, ComicID TEXT, IssueID TEXT, CV_Last_Update TEXT, DynamicName TEXT, weeknumber TEXT, year TEXT, volume TEXT, seriesyear TEXT, annuallink TEXT, rowid INTEGER PRIMARY KEY)")
myDB.action("CREATE TABLE IF NOT EXISTS weekly (SHIPDATE, PUBLISHER TEXT, ISSUE TEXT, COMIC VARCHAR(150), EXTRA TEXT, STATUS TEXT, ComicID TEXT, IssueID TEXT, CV_Last_Update TEXT, DynamicName TEXT, weeknumber TEXT, year TEXT, volume TEXT, seriesyear TEXT, annuallink TEXT, format TEXT, rowid INTEGER PRIMARY KEY)")
csvfile = open(newfl, "rb")
creader = csv.reader(csvfile, delimiter='\t')

2
mylar/wwt.py
View File

@ -14,7 +14,7 @@
# You should have received a copy of the GNU General Public License
# along with Mylar. If not, see <http://www.gnu.org/licenses/>.
import lib.requests as requests
import requests
from bs4 import BeautifulSoup, UnicodeDammit
import urlparse
import re