mirror
/
mylar
mirror of https://github.com/evilhero/mylar
1
0
Fork 0
Code Issues Releases Wiki Activity

IMP: Added TPB/GN detection for file parser, IMP: Added TPB as a booktype to series detail page, IMP: Added to format options for Folder Formatting which would denote either TPB or GN, IMP: If detected as a TPB/GN volume, will display the series/issues that are contained within - if they exist on your watchlist, they will be clickable links, IMP: Weeklypull set to display TPB/GN's when option is configured (backend change), IMP: Added proper detection for filenames that have mini-series designations within them ie ( '(of 04)' ), IMP:(#2102) detect decimal issues that also contain unicoded numbering, IMP: Proper handling of filenames that have '38 p' or the like to denote number of pages in issue, FIX: Removed forceful reference to lib.requests when using WWT, IMP: If an issue has a digital release prior to the actual print release, will now look at either date when attempting to reference if a successful match when post-processing/searching, IMP: Issues that also have digital release dates will be indicated on the series detail page as such, IMP: Added job completion notifs for refreshing/adding story-arcs, deleting directories, IMP: Main Home page will display if a given volume is a TPB/GN if determined, IMP: Reworded 'Search for Watchlist matches' to 'Search for Watchlist Matches/Recheck Files' for more clarity, FIX: Fixed story-arc posters causing some text boxes to overflow

This commit is contained in:
evilhero 2018-11-29 14:59:34 -05:00
parent 3cdae9153a
commit f63e746280
17 changed files with 586 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
data/interfaces/default/comicdetails.html
View File

@ -1,7 +1,7 @@
<%inherit file="base.html"/> <%inherit file="base.html"/>
<%! <%!
import os import os, re
from mylar import db from mylar import db, helpers
import mylar import mylar
%> %>
@ -112,10 +112,41 @@
if comic['Type'] == 'None' or comic['Type'] is None or comic['Type'] == 'Print': if comic['Type'] == 'None' or comic['Type'] is None or comic['Type'] == 'Print':
comictype = 'Print' comictype = 'Print'
else: else:
comictype = 'Digital' comictype = comic['Type']
%> %>
<div> <div>
<label><big>Edition: </big><norm>${comictype}</norm></label> <label><big>Edition: </big><norm>${comictype}</norm>
<%
if comicConfig['issue_list'] is not None:
cnt = 0
line = ''
for x in comicConfig['issue_list']:
if x['comicid'] is not None:
cid = helpers.listLibrary(x['comicid'])
try:
if cid[re.sub('4050-', '', x['comicid']).strip()]['comicid']:
watch = "<a href='comicDetails?ComicID=%s' title='In Watchlist'>%s</a>" % (cid[re.sub('4050-', '', x['comicid']).strip()]['comicid'], x['series'])
except:
watch = "%s" % (x['series'])
else:
watch = x['series']
if cnt == 0:
if x['issues'] is not None:
line += "<span style='display:inline-block;'>%s %s</span>" % (watch, x['issues'])
else:
line += "<span style='display:inline-block;'>%s</span>" % watch
else:
if x['issues'] is not None:
line += " / <span style='display:inline-block;'>%s %s</span>" % (watch, x['issues'])
else:
line += " / <span style='display:inline-block;'>%s</span>" % watch
cnt+=1
%>
%if comicConfig['issue_list'] is not None:
( Collects: ${line} )
%endif
</label>
</div> </div>
<div> <div>
<label><big>Last Updated: </big> <label><big>Last Updated: </big>
@ -326,7 +357,15 @@
endif endif
%> %>
<td id="issuename">${issuename}</td> <td id="issuename">${issuename}</td>
<td class="edit" title="Publication Date (click to edit)" id="${issue['ComicID']}.${issue['IssueID']}">${issue['IssueDate']}</td> <%
issdate = issue['IssueDate']
dateline = "Publication Date (click to edit)"
if all([issue['DigitalDate'] != '0000-00-00', issue['DigitalDate'] is not None]):
dateline += "\nDigital Release: %s" % issue['DigitalDate']
issdate += "**"
endif
%>
<td class="edit" title="${dateline}" id="${issue['ComicID']}.${issue['IssueID']}">${issdate}</td>
<td id="status">${issue['Status']} <td id="status">${issue['Status']}
%if issue['Status'] == 'Downloaded' or issue['Status'] == 'Archived': %if issue['Status'] == 'Downloaded' or issue['Status'] == 'Archived':
<%Csize = mylar.helpers.human_size(issue['ComicSize'])%> <%Csize = mylar.helpers.human_size(issue['ComicSize'])%>
@ -480,7 +519,15 @@
<td id="aint_issuenumber">${annual_Number}</td> <td id="aint_issuenumber">${annual_Number}</td>
<td id="aissuenumber">${annual['Issue_Number']}</td> <td id="aissuenumber">${annual['Issue_Number']}</td>
<td id="aissuename">${annual['IssueName']}</td> <td id="aissuename">${annual['IssueName']}</td>
<td id="areldate">${annual['IssueDate']}</td> <%
anndate = annual['IssueDate']
dateline = ""
if all([annual['DigitalDate'] != '0000-00-00', annual['DigitalDate'] is not None]):
dateline += "\nDigital Release: %s" % annual['DigitalDate']
anndate += "**"
endif
%>
<td title="${dateline}" id="areldate">${anndate}</td>
<td id="astatus">${annual['Status']} <td id="astatus">${annual['Status']}
%if annual['Status'] == 'Downloaded' or annual['Status'] == 'Archived': %if annual['Status'] == 'Downloaded' or annual['Status'] == 'Archived':
<%Csize = mylar.helpers.human_size(annual['ComicSize'])%> <%Csize = mylar.helpers.human_size(annual['ComicSize'])%>

16
data/interfaces/default/index.html
View File

@ -69,10 +69,24 @@
except: except:
pass pass
comicline = comicname
comictype = comic['ComicType']
try:
if any([comictype == 'None', comictype is None, comictype == 'Print']):
comictype = None
else:
comictype = comictype
except:
comictype = None
if comictype is not None:
comicline += ' [%s]' % (comictype)
%> %>
<tr class="grade${grade}"> <tr class="grade${grade}">
<td id="publisher"><span title="${comicpub}"></span>${comicpub}</td> <td id="publisher"><span title="${comicpub}"></span>${comicpub}</td>
<td id="name"><span title="${comicname}"></span><a title="${comic['ComicName']}" href="comicDetails?ComicID=${comic['ComicID']}">${comicname}</a></td> <td id="name"><span title="${comicname}"></span><a title="${comic['ComicName']}" href="comicDetails?ComicID=${comic['ComicID']}">${comicline}</a></td>
<td id="year"><span title="${comic['ComicYear']}"></span>${comic['ComicYear']}</td> <td id="year"><span title="${comic['ComicYear']}"></span>${comic['ComicYear']}</td>
<td id="issue"><span title="${comic['LatestIssue']}"></span># ${comic['LatestIssue']}</td> <td id="issue"><span title="${comic['LatestIssue']}"></span># ${comic['LatestIssue']}</td>
<td id="published">${comic['LatestDate']}</td> <td id="published">${comic['LatestDate']}</td>

5
data/interfaces/default/storyarc_detail.html
View File

@ -14,7 +14,7 @@
<a id="menu_link_delete" href="#">Remove Read</a> <a id="menu_link_delete" href="#">Remove Read</a>
<a id="menu_link_delete" href="#">Clear File Cache</a> <a id="menu_link_delete" href="#">Clear File Cache</a>
<a id="menu_link_refresh" onclick="doAjaxCall('SearchArcIssues?StoryArcID=${storyarcid}',$(this),'table')" data-success="Now searching for Missing StoryArc Issues">Search for Missing</a> <a id="menu_link_refresh" onclick="doAjaxCall('SearchArcIssues?StoryArcID=${storyarcid}',$(this),'table')" data-success="Now searching for Missing StoryArc Issues">Search for Missing</a>
<a id="menu_link_refresh" onclick="doAjaxCall('ArcWatchlist?StoryArcID=${storyarcid}',$(this),'table')" data-success="Now searching for matches on Watchlist">Search for Watchlist matches</a> <a id="menu_link_refresh" onclick="doAjaxCall('ArcWatchlist?StoryArcID=${storyarcid}',$(this),'table')" data-success="Now searching for matches on Watchlist & Rechecking files">Search for Watchlist matches/Recheck Files</a>
%if cvarcid: %if cvarcid:
<a id="menu_link_refresh" onclick="doAjaxCall('addStoryArc_thread?arcid=${storyarcid}&cvarcid=${cvarcid}&storyarcname=${storyarcname}&arcrefresh=True',$(this),'table')" data-success="Refreshed Story Arc">Refresh Story Arc</a> <a id="menu_link_refresh" onclick="doAjaxCall('addStoryArc_thread?arcid=${storyarcid}&cvarcid=${cvarcid}&storyarcname=${storyarcname}&arcrefresh=True',$(this),'table')" data-success="Refreshed Story Arc">Refresh Story Arc</a>
%endif %endif
@ -110,6 +110,9 @@
</table> </table>
</div> </div>
</div> </div>
<button type="button" onclick="">Finalize & Rename</button>
<table class="display" id="arc_detail"> <table class="display" id="arc_detail">
<thead> <thead>
<tr> <tr>

10
data/interfaces/default/storyarc_detail.poster.html
View File

@ -34,10 +34,8 @@
<tr/><tr> <tr/><tr>
<td id="mainimg"> <td id="mainimg">
<fieldset> <fieldset>
<div id="artistImg"> <div id="aristImg">
<img src="${storyarcbanner}" onload="resizeimage('${bannerwidth}')" height="400" width="${bannerwidth}" id="banner" style="text-decoration: none;position: relative;top:0px;right:0px;left:0px;"/> <img src="${storyarcbanner}" onload="resizeimage('${bannerwidth}')" height="400" width="${bannerwidth}" id="banner" style="text-decoration: none;position: relative;top:0px;right:0px;left:0px;"/>
</div>
</fieldset>
<% <%
if arcdetail['percent'] == 101: if arcdetail['percent'] == 101:
css = '<div class=\"progress-container warning\">' css = '<div class=\"progress-container warning\">'
@ -48,9 +46,11 @@
%> %>
<div style="display:table;margin:auto;position:relative;top:0px;"><span title="${arcdetail['percent']}"></span>${css}<div style="width:${arcdetail['percent']}%"><span class="progressbar-front-text">${arcdetail['Have']}/${arcdetail['Total']}</span></div></div></div> <div style="display:table;margin:auto;position:relative;top:0px;"><span title="${arcdetail['percent']}"></span>${css}<div style="width:${arcdetail['percent']}%"><span class="progressbar-front-text">${arcdetail['Have']}/${arcdetail['Total']}</span></div></div></div>
</div>
</fieldset>
</td> </td>
<td width="100%" padding="5px" style="vertical-align:bottom;"> <td width="100%" padding="5px" style="vertical-align:bottom;">
<div style="display:inline;position:relative;text-color:black;top:-200px;margin:0 auto;"> <div style="display:inline;position:relative;text-color:black;top:-150px;margin:0 auto;">
<center><h1>${storyarcname}</h1>(${spanyears})</center> <center><h1>${storyarcname}</h1>(${spanyears})</center>
</div> </div>
@ -81,7 +81,7 @@
</form> </form>
</div> </div>
<% <%
optpos = '<div style=\"display:block;float:right;position:relative;right:20px;top:50px;\">' optpos = '<div style=\"display:block;float:right;position:relative;right:20px;top:-10px;\">'
%> %>
${optpos} ${optpos}
<form action="downloadBanner" method="GET"> <form action="downloadBanner" method="GET">

4
data/interfaces/default/weeklypull.html
View File

@ -129,6 +129,10 @@
%if weekly['SERIESYEAR'] is not None: %if weekly['SERIESYEAR'] is not None:
&nbsp(${weekly['SERIESYEAR']}) &nbsp(${weekly['SERIESYEAR']})
%endif %endif
%if weekly['FORMAT'] == 'Digital':
&nbsp[${weekly['FORMAT']}]
%endif
</td> </td>
<td class="comicnumber">${weekly['ISSUE']}</td> <td class="comicnumber">${weekly['ISSUE']}</td>
%if weekly['AUTOWANT']: %if weekly['AUTOWANT']:

36
mylar/PostProcessor.py
View File

@ -569,11 +569,28 @@ class PostProcessor(object):
for isc in issuechk: for isc in issuechk:
datematch = "True" datematch = "True"
if isc['ReleaseDate'] is not None and isc['ReleaseDate'] != '0000-00-00': if isc['ReleaseDate'] is not None and isc['ReleaseDate'] != '0000-00-00':
monthval = isc['ReleaseDate'] try:
watch_issueyear = isc['ReleaseDate'][:4] if isc['DigitalDate'] != '0000-00-00' and int(re.sub('-', '', isc['DigitalDate']).strip()) <= int(re.sub('-', '', isc['ReleaseDate']).strip()):
monthval = isc['DigitalDate']
watch_issueyear = isc['DigitalDate'][:4]
else:
monthval = isc['ReleaseDate']
watch_issueyear = isc['ReleaseDate'][:4]
except:
monthval = isc['ReleaseDate']
watch_issueyear = isc['ReleaseDate'][:4]
else: else:
monthval = isc['IssueDate'] try:
watch_issueyear = isc['IssueDate'][:4] if isc['DigitalDate'] != '0000-00-00' and int(re.sub('-', '', isc['DigitalDate']).strip()) <= int(re.sub('-', '', isc['ReleaseDate']).strip()):
monthval = isc['DigitalDate']
watch_issueyear = isc['DigitalDate'][:4]
else:
monthval = isc['IssueDate']
watch_issueyear = isc['IssueDate'][:4]
except:
monthval = isc['IssueDate']
watch_issueyear = isc['IssueDate'][:4]
if len(watchmatch) >= 1 and watchmatch['issue_year'] is not None: if len(watchmatch) >= 1 and watchmatch['issue_year'] is not None:
#if the # of matches is more than 1, we need to make sure we get the right series #if the # of matches is more than 1, we need to make sure we get the right series
@ -587,7 +604,12 @@ class PostProcessor(object):
#logger.info(module + ' ReleaseDate: ' + str(isc['ReleaseDate'])) #logger.info(module + ' ReleaseDate: ' + str(isc['ReleaseDate']))
#logger.info(module + ' IssueDate: ' + str(isc['IssueDate'])) #logger.info(module + ' IssueDate: ' + str(isc['IssueDate']))
if isc['ReleaseDate'] is not None and isc['ReleaseDate'] != '0000-00-00': if isc['DigitalDate'] is not None and isc['DigitalDate'] != '0000-00-00':
if int(isc['DigitalDate'][:4]) < int(watchmatch['issue_year']):
logger.fdebug(module + '[ISSUE-VERIFY] ' + str(isc['DigitalDate']) + ' is before the issue year of ' + str(watchmatch['issue_year']) + ' that was discovered in the filename')
datematch = "False"
elif isc['ReleaseDate'] is not None and isc['ReleaseDate'] != '0000-00-00':
if int(isc['ReleaseDate'][:4]) < int(watchmatch['issue_year']): if int(isc['ReleaseDate'][:4]) < int(watchmatch['issue_year']):
logger.fdebug(module + '[ISSUE-VERIFY] ' + str(isc['ReleaseDate']) + ' is before the issue year of ' + str(watchmatch['issue_year']) + ' that was discovered in the filename') logger.fdebug(module + '[ISSUE-VERIFY] ' + str(isc['ReleaseDate']) + ' is before the issue year of ' + str(watchmatch['issue_year']) + ' that was discovered in the filename')
datematch = "False" datematch = "False"
@ -1285,9 +1307,13 @@ class PostProcessor(object):
#loop through the hits here. #loop through the hits here.
if len(manual_list) == 0 and len(manual_arclist) == 0: if len(manual_list) == 0 and len(manual_arclist) == 0:
logger.info(module + ' No matches for Manual Run ... exiting.') logger.info(module + ' No matches for Manual Run ... exiting.')
if mylar.APILOCK is True:
mylar.APILOCK = False
return return
elif len(manual_arclist) > 0 and len(manual_list) == 0: elif len(manual_arclist) > 0 and len(manual_list) == 0:
logger.info(module + ' Manual post-processing completed for ' + str(len(manual_arclist)) + ' story-arc issues.') logger.info(module + ' Manual post-processing completed for ' + str(len(manual_arclist)) + ' story-arc issues.')
if mylar.APILOCK is True:
mylar.APILOCK = False
return return
elif len(manual_arclist) > 0: elif len(manual_arclist) > 0:
logger.info(module + ' Manual post-processing completed for ' + str(len(manual_arclist)) + ' story-arc issues.') logger.info(module + ' Manual post-processing completed for ' + str(len(manual_arclist)) + ' story-arc issues.')

34
mylar/__init__.py
View File

@ -482,21 +482,21 @@ def dbcheck():
c.execute('SELECT ReleaseDate from storyarcs') c.execute('SELECT ReleaseDate from storyarcs')
except sqlite3.OperationalError: except sqlite3.OperationalError:
try: try:
c.execute('CREATE TABLE IF NOT EXISTS storyarcs(StoryArcID TEXT, ComicName TEXT, IssueNumber TEXT, SeriesYear TEXT, IssueYEAR TEXT, StoryArc TEXT, TotalIssues TEXT, Status TEXT, inCacheDir TEXT, Location TEXT, IssueArcID TEXT, ReadingOrder INT, IssueID TEXT, ComicID TEXT, ReleaseDate TEXT, IssueDate TEXT, Publisher TEXT, IssuePublisher TEXT, IssueName TEXT, CV_ArcID TEXT, Int_IssueNumber INT, DynamicComicName TEXT, Volume TEXT, Manual TEXT, DateAdded TEXT)') c.execute('CREATE TABLE IF NOT EXISTS storyarcs(StoryArcID TEXT, ComicName TEXT, IssueNumber TEXT, SeriesYear TEXT, IssueYEAR TEXT, StoryArc TEXT, TotalIssues TEXT, Status TEXT, inCacheDir TEXT, Location TEXT, IssueArcID TEXT, ReadingOrder INT, IssueID TEXT, ComicID TEXT, ReleaseDate TEXT, IssueDate TEXT, Publisher TEXT, IssuePublisher TEXT, IssueName TEXT, CV_ArcID TEXT, Int_IssueNumber INT, DynamicComicName TEXT, Volume TEXT, Manual TEXT, DateAdded TEXT, DigitalDate TEXT)')
c.execute('INSERT INTO storyarcs(StoryArcID, ComicName, IssueNumber, SeriesYear, IssueYEAR, StoryArc, TotalIssues, Status, inCacheDir, Location, IssueArcID, ReadingOrder, IssueID, ComicID, ReleaseDate, IssueDate, Publisher, IssuePublisher, IssueName, CV_ArcID, Int_IssueNumber, DynamicComicName, Volume, Manual) SELECT StoryArcID, ComicName, IssueNumber, SeriesYear, IssueYEAR, StoryArc, TotalIssues, Status, inCacheDir, Location, IssueArcID, ReadingOrder, IssueID, ComicID, StoreDate, IssueDate, Publisher, IssuePublisher, IssueName, CV_ArcID, Int_IssueNumber, DynamicComicName, Volume, Manual FROM readinglist') c.execute('INSERT INTO storyarcs(StoryArcID, ComicName, IssueNumber, SeriesYear, IssueYEAR, StoryArc, TotalIssues, Status, inCacheDir, Location, IssueArcID, ReadingOrder, IssueID, ComicID, ReleaseDate, IssueDate, Publisher, IssuePublisher, IssueName, CV_ArcID, Int_IssueNumber, DynamicComicName, Volume, Manual) SELECT StoryArcID, ComicName, IssueNumber, SeriesYear, IssueYEAR, StoryArc, TotalIssues, Status, inCacheDir, Location, IssueArcID, ReadingOrder, IssueID, ComicID, StoreDate, IssueDate, Publisher, IssuePublisher, IssueName, CV_ArcID, Int_IssueNumber, DynamicComicName, Volume, Manual FROM readinglist')
c.execute('DROP TABLE readinglist') c.execute('DROP TABLE readinglist')
except sqlite3.OperationalError: except sqlite3.OperationalError:
logger.warn('Unable to update readinglist table to new storyarc table format.') logger.warn('Unable to update readinglist table to new storyarc table format.')
c.execute('CREATE TABLE IF NOT EXISTS comics (ComicID TEXT UNIQUE, ComicName TEXT, ComicSortName TEXT, ComicYear TEXT, DateAdded TEXT, Status TEXT, IncludeExtras INTEGER, Have INTEGER, Total INTEGER, ComicImage TEXT, ComicPublisher TEXT, ComicLocation TEXT, ComicPublished TEXT, NewPublish TEXT, LatestIssue TEXT, LatestDate TEXT, Description TEXT, QUALalt_vers TEXT, QUALtype TEXT, QUALscanner TEXT, QUALquality TEXT, LastUpdated TEXT, AlternateSearch TEXT, UseFuzzy TEXT, ComicVersion TEXT, SortOrder INTEGER, DetailURL TEXT, ForceContinuing INTEGER, ComicName_Filesafe TEXT, AlternateFileName TEXT, ComicImageURL TEXT, ComicImageALTURL TEXT, DynamicComicName TEXT, AllowPacks TEXT, Type TEXT, Corrected_SeriesYear TEXT, TorrentID_32P TEXT, LatestIssueID TEXT)') c.execute('CREATE TABLE IF NOT EXISTS comics (ComicID TEXT UNIQUE, ComicName TEXT, ComicSortName TEXT, ComicYear TEXT, DateAdded TEXT, Status TEXT, IncludeExtras INTEGER, Have INTEGER, Total INTEGER, ComicImage TEXT, ComicPublisher TEXT, ComicLocation TEXT, ComicPublished TEXT, NewPublish TEXT, LatestIssue TEXT, LatestDate TEXT, Description TEXT, QUALalt_vers TEXT, QUALtype TEXT, QUALscanner TEXT, QUALquality TEXT, LastUpdated TEXT, AlternateSearch TEXT, UseFuzzy TEXT, ComicVersion TEXT, SortOrder INTEGER, DetailURL TEXT, ForceContinuing INTEGER, ComicName_Filesafe TEXT, AlternateFileName TEXT, ComicImageURL TEXT, ComicImageALTURL TEXT, DynamicComicName TEXT, AllowPacks TEXT, Type TEXT, Corrected_SeriesYear TEXT, TorrentID_32P TEXT, LatestIssueID TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS issues (IssueID TEXT, ComicName TEXT, IssueName TEXT, Issue_Number TEXT, DateAdded TEXT, Status TEXT, Type TEXT, ComicID TEXT, ArtworkURL Text, ReleaseDate TEXT, Location TEXT, IssueDate TEXT, Int_IssueNumber INT, ComicSize TEXT, AltIssueNumber TEXT, IssueDate_Edit TEXT, ImageURL TEXT, ImageURL_ALT TEXT)') c.execute('CREATE TABLE IF NOT EXISTS issues (IssueID TEXT, ComicName TEXT, IssueName TEXT, Issue_Number TEXT, DateAdded TEXT, Status TEXT, Type TEXT, ComicID TEXT, ArtworkURL Text, ReleaseDate TEXT, Location TEXT, IssueDate TEXT, DigitalDate TEXT, Int_IssueNumber INT, ComicSize TEXT, AltIssueNumber TEXT, IssueDate_Edit TEXT, ImageURL TEXT, ImageURL_ALT TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS snatched (IssueID TEXT, ComicName TEXT, Issue_Number TEXT, Size INTEGER, DateAdded TEXT, Status TEXT, FolderName TEXT, ComicID TEXT, Provider TEXT, Hash TEXT, crc TEXT)') c.execute('CREATE TABLE IF NOT EXISTS snatched (IssueID TEXT, ComicName TEXT, Issue_Number TEXT, Size INTEGER, DateAdded TEXT, Status TEXT, FolderName TEXT, ComicID TEXT, Provider TEXT, Hash TEXT, crc TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS upcoming (ComicName TEXT, IssueNumber TEXT, ComicID TEXT, IssueID TEXT, IssueDate TEXT, Status TEXT, DisplayComicName TEXT)') c.execute('CREATE TABLE IF NOT EXISTS upcoming (ComicName TEXT, IssueNumber TEXT, ComicID TEXT, IssueID TEXT, IssueDate TEXT, Status TEXT, DisplayComicName TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS nzblog (IssueID TEXT, NZBName TEXT, SARC TEXT, PROVIDER TEXT, ID TEXT, AltNZBName TEXT, OneOff TEXT)') c.execute('CREATE TABLE IF NOT EXISTS nzblog (IssueID TEXT, NZBName TEXT, SARC TEXT, PROVIDER TEXT, ID TEXT, AltNZBName TEXT, OneOff TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS weekly (SHIPDATE TEXT, PUBLISHER TEXT, ISSUE TEXT, COMIC VARCHAR(150), EXTRA TEXT, STATUS TEXT, ComicID TEXT, IssueID TEXT, CV_Last_Update TEXT, DynamicName TEXT, weeknumber TEXT, year TEXT, volume TEXT, seriesyear TEXT, annuallink TEXT, rowid INTEGER PRIMARY KEY)') c.execute('CREATE TABLE IF NOT EXISTS weekly (SHIPDATE TEXT, PUBLISHER TEXT, ISSUE TEXT, COMIC VARCHAR(150), EXTRA TEXT, STATUS TEXT, ComicID TEXT, IssueID TEXT, CV_Last_Update TEXT, DynamicName TEXT, weeknumber TEXT, year TEXT, volume TEXT, seriesyear TEXT, annuallink TEXT, format TEXT, rowid INTEGER PRIMARY KEY)')
c.execute('CREATE TABLE IF NOT EXISTS importresults (impID TEXT, ComicName TEXT, ComicYear TEXT, Status TEXT, ImportDate TEXT, ComicFilename TEXT, ComicLocation TEXT, WatchMatch TEXT, DisplayName TEXT, SRID TEXT, ComicID TEXT, IssueID TEXT, Volume TEXT, IssueNumber TEXT, DynamicName TEXT)') c.execute('CREATE TABLE IF NOT EXISTS importresults (impID TEXT, ComicName TEXT, ComicYear TEXT, Status TEXT, ImportDate TEXT, ComicFilename TEXT, ComicLocation TEXT, WatchMatch TEXT, DisplayName TEXT, SRID TEXT, ComicID TEXT, IssueID TEXT, Volume TEXT, IssueNumber TEXT, DynamicName TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS readlist (IssueID TEXT, ComicName TEXT, Issue_Number TEXT, Status TEXT, DateAdded TEXT, Location TEXT, inCacheDir TEXT, SeriesYear TEXT, ComicID TEXT, StatusChange TEXT)') c.execute('CREATE TABLE IF NOT EXISTS readlist (IssueID TEXT, ComicName TEXT, Issue_Number TEXT, Status TEXT, DateAdded TEXT, Location TEXT, inCacheDir TEXT, SeriesYear TEXT, ComicID TEXT, StatusChange TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS annuals (IssueID TEXT, Issue_Number TEXT, IssueName TEXT, IssueDate TEXT, Status TEXT, ComicID TEXT, GCDComicID TEXT, Location TEXT, ComicSize TEXT, Int_IssueNumber INT, ComicName TEXT, ReleaseDate TEXT, ReleaseComicID TEXT, ReleaseComicName TEXT, IssueDate_Edit TEXT, DateAdded TEXT)') c.execute('CREATE TABLE IF NOT EXISTS annuals (IssueID TEXT, Issue_Number TEXT, IssueName TEXT, IssueDate TEXT, Status TEXT, ComicID TEXT, GCDComicID TEXT, Location TEXT, ComicSize TEXT, Int_IssueNumber INT, ComicName TEXT, ReleaseDate TEXT, DigitalDate TEXT, ReleaseComicID TEXT, ReleaseComicName TEXT, IssueDate_Edit TEXT, DateAdded TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS rssdb (Title TEXT UNIQUE, Link TEXT, Pubdate TEXT, Site TEXT, Size TEXT)') c.execute('CREATE TABLE IF NOT EXISTS rssdb (Title TEXT UNIQUE, Link TEXT, Pubdate TEXT, Site TEXT, Size TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS futureupcoming (ComicName TEXT, IssueNumber TEXT, ComicID TEXT, IssueID TEXT, IssueDate TEXT, Publisher TEXT, Status TEXT, DisplayComicName TEXT, weeknumber TEXT, year TEXT)') c.execute('CREATE TABLE IF NOT EXISTS futureupcoming (ComicName TEXT, IssueNumber TEXT, ComicID TEXT, IssueID TEXT, IssueDate TEXT, Publisher TEXT, Status TEXT, DisplayComicName TEXT, weeknumber TEXT, year TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS failed (ID TEXT, Status TEXT, ComicID TEXT, IssueID TEXT, Provider TEXT, ComicName TEXT, Issue_Number TEXT, NZBName TEXT, DateFailed TEXT)') c.execute('CREATE TABLE IF NOT EXISTS failed (ID TEXT, Status TEXT, ComicID TEXT, IssueID TEXT, Provider TEXT, ComicName TEXT, Issue_Number TEXT, NZBName TEXT, DateFailed TEXT)')
@ -505,6 +505,7 @@ def dbcheck():
c.execute('CREATE TABLE IF NOT EXISTS oneoffhistory (ComicName TEXT, IssueNumber TEXT, ComicID TEXT, IssueID TEXT, Status TEXT, weeknumber TEXT, year TEXT)') c.execute('CREATE TABLE IF NOT EXISTS oneoffhistory (ComicName TEXT, IssueNumber TEXT, ComicID TEXT, IssueID TEXT, Status TEXT, weeknumber TEXT, year TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS jobhistory (JobName TEXT, prev_run_datetime timestamp, prev_run_timestamp REAL, next_run_datetime timestamp, next_run_timestamp REAL, last_run_completed TEXT, successful_completions TEXT, failed_completions TEXT, status TEXT)') c.execute('CREATE TABLE IF NOT EXISTS jobhistory (JobName TEXT, prev_run_datetime timestamp, prev_run_timestamp REAL, next_run_datetime timestamp, next_run_timestamp REAL, last_run_completed TEXT, successful_completions TEXT, failed_completions TEXT, status TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS manualresults (provider TEXT, id TEXT, kind TEXT, comicname TEXT, volume TEXT, oneoff TEXT, fullprov TEXT, issuenumber TEXT, modcomicname TEXT, name TEXT, link TEXT, size TEXT, pack_numbers TEXT, pack_issuelist TEXT, comicyear TEXT, issuedate TEXT, tmpprov TEXT, pack TEXT, issueid TEXT, comicid TEXT, sarc TEXT, issuearcid TEXT)') c.execute('CREATE TABLE IF NOT EXISTS manualresults (provider TEXT, id TEXT, kind TEXT, comicname TEXT, volume TEXT, oneoff TEXT, fullprov TEXT, issuenumber TEXT, modcomicname TEXT, name TEXT, link TEXT, size TEXT, pack_numbers TEXT, pack_issuelist TEXT, comicyear TEXT, issuedate TEXT, tmpprov TEXT, pack TEXT, issueid TEXT, comicid TEXT, sarc TEXT, issuearcid TEXT)')
c.execute('CREATE TABLE IF NOT EXISTS storyarcs(StoryArcID TEXT, ComicName TEXT, IssueNumber TEXT, SeriesYear TEXT, IssueYEAR TEXT, StoryArc TEXT, TotalIssues TEXT, Status TEXT, inCacheDir TEXT, Location TEXT, IssueArcID TEXT, ReadingOrder INT, IssueID TEXT, ComicID TEXT, ReleaseDate TEXT, IssueDate TEXT, Publisher TEXT, IssuePublisher TEXT, IssueName TEXT, CV_ArcID TEXT, Int_IssueNumber INT, DynamicComicName TEXT, Volume TEXT, Manual TEXT, DateAdded TEXT, DigitalDate TEXT)')
conn.commit conn.commit
c.close c.close
@ -619,6 +620,11 @@ def dbcheck():
except sqlite3.OperationalError: except sqlite3.OperationalError:
c.execute('ALTER TABLE comics ADD COLUMN LatestIssueID TEXT') c.execute('ALTER TABLE comics ADD COLUMN LatestIssueID TEXT')
try:
c.execute('SELECT Collects from comics')
except sqlite3.OperationalError:
c.execute('ALTER TABLE comics ADD COLUMN Collects CLOB')
try: try:
c.execute('SELECT DynamicComicName from comics') c.execute('SELECT DynamicComicName from comics')
if CONFIG.DYNAMIC_UPDATE < 3: if CONFIG.DYNAMIC_UPDATE < 3:
@ -661,6 +667,11 @@ def dbcheck():
except sqlite3.OperationalError: except sqlite3.OperationalError:
c.execute('ALTER TABLE issues ADD COLUMN ImageURL_ALT TEXT') c.execute('ALTER TABLE issues ADD COLUMN ImageURL_ALT TEXT')
try:
c.execute('SELECT DigitalDate from issues')
except sqlite3.OperationalError:
c.execute('ALTER TABLE issues ADD COLUMN DigitalDate TEXT')
## -- ImportResults Table -- ## -- ImportResults Table --
try: try:
@ -812,6 +823,11 @@ def dbcheck():
except sqlite3.OperationalError: except sqlite3.OperationalError:
c.execute('ALTER TABLE weekly ADD COLUMN annuallink TEXT') c.execute('ALTER TABLE weekly ADD COLUMN annuallink TEXT')
try:
c.execute('SELECT format from weekly')
except sqlite3.OperationalError:
c.execute('ALTER TABLE weekly ADD COLUMN format TEXT')
## -- Nzblog Table -- ## -- Nzblog Table --
try: try:
@ -892,6 +908,11 @@ def dbcheck():
except sqlite3.OperationalError: except sqlite3.OperationalError:
c.execute('ALTER TABLE annuals ADD COLUMN DateAdded TEXT') c.execute('ALTER TABLE annuals ADD COLUMN DateAdded TEXT')
try:
c.execute('SELECT DigitalDate from annuals')
except sqlite3.OperationalError:
c.execute('ALTER TABLE annuals ADD COLUMN DigitalDate TEXT')
## -- Snatched Table -- ## -- Snatched Table --
try: try:
@ -984,6 +1005,11 @@ def dbcheck():
except sqlite3.OperationalError: except sqlite3.OperationalError:
c.execute('ALTER TABLE storyarcs ADD COLUMN DateAdded TEXT') c.execute('ALTER TABLE storyarcs ADD COLUMN DateAdded TEXT')
try:
c.execute('SELECT DigitalDate from storyarcs')
except sqlite3.OperationalError:
c.execute('ALTER TABLE storyarcs ADD COLUMN DigitalDate TEXT')
## -- searchresults Table -- ## -- searchresults Table --
try: try:
c.execute('SELECT SRID from searchresults') c.execute('SELECT SRID from searchresults')

108
mylar/cv.py
View File

@ -190,7 +190,7 @@ def getComic(comicid, type, issueid=None, arc=None, arcid=None, arclist=None, co
else: else:
tmpidlist += '|' + str(comicidlist[i]) tmpidlist += '|' + str(comicidlist[i])
in_cnt +=1 in_cnt +=1
logger.info('tmpidlist: ' + str(tmpidlist)) logger.fdebug('tmpidlist: ' + str(tmpidlist))
searched = pulldetails(None, 'import', offset=0, comicidlist=tmpidlist) searched = pulldetails(None, 'import', offset=0, comicidlist=tmpidlist)
@ -287,6 +287,8 @@ def GetComicInfo(comicid, dom, safechk=None):
#the description field actually holds the Volume# - so let's grab it #the description field actually holds the Volume# - so let's grab it
try: try:
descchunk = dom.getElementsByTagName('description')[0].firstChild.wholeText descchunk = dom.getElementsByTagName('description')[0].firstChild.wholeText
desc_soup = Soup(descchunk, "html.parser")
desclinks = desc_soup.findAll('a')
comic_desc = drophtml(descchunk) comic_desc = drophtml(descchunk)
desdeck +=1 desdeck +=1
except: except:
@ -312,26 +314,87 @@ def GetComicInfo(comicid, dom, safechk=None):
comic['Aliases'] = 'None' comic['Aliases'] = 'None'
comic['ComicVersion'] = 'None' #noversion' comic['ComicVersion'] = 'None' #noversion'
#logger.info('comic_desc:' + comic_desc)
#logger.info('comic_deck:' + comic_deck)
#logger.info('desdeck: ' + str(desdeck))
#figure out if it's a print / digital edition. #figure out if it's a print / digital edition.
comic['Type'] = 'None' comic['Type'] = 'None'
if comic_deck != 'None': if comic_deck != 'None':
if any(['print' in comic_deck.lower(), 'digital' in comic_deck.lower()]): if any(['print' in comic_deck.lower(), 'digital' in comic_deck.lower(), 'paperback' in comic_deck.lower(), 'hardcover' in comic_deck.lower()]):
if 'print' in comic_deck.lower(): if 'print' in comic_deck.lower():
comic['Type'] = 'Print' comic['Type'] = 'Print'
elif 'digital' in comic_deck.lower(): elif 'digital' in comic_deck.lower():
comic['Type'] = 'Digital' comic['Type'] = 'Digital'
elif 'paperback' in comic_deck.lower():
comic['Type'] = 'TPB'
elif 'hardcover' in comic_deck.lower():
comic['Type'] = 'HC'
if comic_desc != 'None' and comic['Type'] == 'None': if comic_desc != 'None' and comic['Type'] == 'None':
if 'print' in comic_desc[:60].lower() and 'print edition can be found' not in comic_desc.lower(): if 'print' in comic_desc[:60].lower() and 'print edition can be found' not in comic_desc.lower():
comic['Type'] = 'Print' comic['Type'] = 'Print'
elif 'digital' in comic_desc[:60].lower() and 'digital edition can be found' not in comic_desc.lower(): elif 'digital' in comic_desc[:60].lower() and 'digital edition can be found' not in comic_desc.lower():
comic['Type'] = 'Digital' comic['Type'] = 'Digital'
elif all(['paperback' in comic_desc[:60].lower(), 'paperback can be found' not in comic_desc.lower()]) or 'collects' in comic_desc[:60].lower():
comic['Type'] = 'TPB'
elif 'hardcover' in comic_desc[:60].lower() and 'hardcover can be found' not in comic_desc.lower():
comic['Type'] = 'HC'
else: else:
comic['Type'] = 'Print' comic['Type'] = 'Print'
if all([comic_desc != 'None', 'trade paperback' in comic_desc[:30].lower(), 'collecting' in comic_desc[:40].lower()]):
#ie. Trade paperback collecting Marvel Team-Up #9-11, 48-51, 72, 110 & 145.
first_collect = comic_desc.lower().find('collecting')
#logger.info('first_collect: %s' % first_collect)
#logger.info('comic_desc: %s' % comic_desc)
#logger.info('desclinks: %s' % desclinks)
issue_list = []
for fc in desclinks:
#logger.info('fc: %s' % fc)
fc_id = fc['data-ref-id']
#logger.info('fc_id: %s' % fc_id)
fc_name = fc.findNext(text=True)
#logger.info('fc_name: %s' % fc_name)
if fc_id.startswith('4000'):
fc_cid = None
fc_isid = fc_id
iss_start = fc_name.find('#')
issuerun = fc_name[iss_start:].strip()
fc_name = fc_name[:iss_start].strip()
elif fc_id.startswith('4050'):
fc_cid = fc_id
fc_isid = None
issuerun = fc.next_sibling
lines = re.sub("[^0-9]", ' ', issuerun).strip().split(' ')
if len(lines) > 0:
for x in sorted(lines, reverse=True):
srchline = issuerun.rfind(x)
if srchline != -1:
try:
if issuerun[srchline+len(x)] == ',' or issuerun[srchline+len(x)] == '.' or issuerun[srchline+len(x)] == ' ':
issuerun = issuerun[:srchline+len(x)]
break
except:
continue
if issuerun.endswith('.') or issuerun.endswith(','):
#logger.fdebug('Changed issuerun from %s to %s' % (issuerun, issuerun[:-1]))
issuerun = issuerun[:-1]
if issuerun.endswith(' and '):
issuerun = issuerun[:-4].strip()
elif issuerun.endswith(' and'):
issuerun = issuerun[:-3].strip()
# except:
# pass
issue_list.append({'series': fc_name,
'comicid': fc_cid,
'issueid': fc_isid,
'issues': issuerun})
#first_collect = cis
logger.info('Collected issues in volume: %s' % issue_list)
comic['Issue_List'] = issue_list
else:
comic['Issue_List'] = 'None'
while (desdeck > 0): while (desdeck > 0):
if desdeck == 1: if desdeck == 1:
if comic_desc == 'None': if comic_desc == 'None':
@ -412,19 +475,7 @@ def GetComicInfo(comicid, dom, safechk=None):
comic['FirstIssueID'] = dom.getElementsByTagName('id')[0].firstChild.wholeText comic['FirstIssueID'] = dom.getElementsByTagName('id')[0].firstChild.wholeText
# print ("fistIss:" + str(comic['FirstIssueID'])) #logger.info('comic: %s' % comic)
# comicchoice.append({
# 'ComicName': comic['ComicName'],
# 'ComicYear': comic['ComicYear'],
# 'Comicid': comicid,
# 'ComicURL': comic['ComicURL'],
# 'ComicIssues': comic['ComicIssues'],
# 'ComicImage': comic['ComicImage'],
# 'ComicVolume': ParseVol,
# 'ComicPublisher': comic['ComicPublisher']
# })
# comic['comicchoice'] = comicchoice
return comic return comic
def GetIssuesInfo(comicid, dom, arcid=None): def GetIssuesInfo(comicid, dom, arcid=None):
@ -495,6 +546,19 @@ def GetIssuesInfo(comicid, dom, arcid=None):
tempissue['StoreDate'] = subtrack.getElementsByTagName('store_date')[0].firstChild.wholeText tempissue['StoreDate'] = subtrack.getElementsByTagName('store_date')[0].firstChild.wholeText
except: except:
tempissue['StoreDate'] = '0000-00-00' tempissue['StoreDate'] = '0000-00-00'
try:
digital_desc = subtrack.getElementsByTagName('description')[0].firstChild.wholeText
except:
tempissue['DigitalDate'] = '0000-00-00'
else:
tempissue['DigitalDate'] = '0000-00-00'
if all(['digital' in digital_desc.lower()[-90:], 'print' in digital_desc.lower()[-90:]]):
#get the digital date of issue here...
mff = mylar.filechecker.FileChecker()
vlddate = mff.checkthedate(digital_desc[-90:], fulldate=True)
#logger.fdebug('vlddate: %s' % vlddate)
if vlddate:
tempissue['DigitalDate'] = vlddate
try: try:
tempissue['Issue_Number'] = subtrack.getElementsByTagName('issue_number')[0].firstChild.wholeText tempissue['Issue_Number'] = subtrack.getElementsByTagName('issue_number')[0].firstChild.wholeText
except: except:
@ -517,6 +581,7 @@ def GetIssuesInfo(comicid, dom, arcid=None):
'Issue_Number': tempissue['Issue_Number'], 'Issue_Number': tempissue['Issue_Number'],
'Issue_Date': tempissue['CoverDate'], 'Issue_Date': tempissue['CoverDate'],
'Store_Date': tempissue['StoreDate'], 'Store_Date': tempissue['StoreDate'],
'Digital_Date': tempissue['DigitalDate'],
'Issue_Name': tempissue['Issue_Name'], 'Issue_Name': tempissue['Issue_Name'],
'Image': tempissue['ComicImage'], 'Image': tempissue['ComicImage'],
'ImageALT': tempissue['ComicImageALT'] 'ImageALT': tempissue['ComicImageALT']
@ -531,6 +596,7 @@ def GetIssuesInfo(comicid, dom, arcid=None):
'Issue_Number': tempissue['Issue_Number'], 'Issue_Number': tempissue['Issue_Number'],
'Issue_Date': tempissue['CoverDate'], 'Issue_Date': tempissue['CoverDate'],
'Store_Date': tempissue['StoreDate'], 'Store_Date': tempissue['StoreDate'],
'Digital_Date': tempissue['DigitalDate'],
'Issue_Name': tempissue['Issue_Name'] 'Issue_Name': tempissue['Issue_Name']
}) })
@ -538,6 +604,7 @@ def GetIssuesInfo(comicid, dom, arcid=None):
firstdate = tempissue['CoverDate'] firstdate = tempissue['CoverDate']
n-= 1 n-= 1
#logger.fdebug('issue_info: %s' % issuech)
#issue['firstdate'] = firstdate #issue['firstdate'] = firstdate
return issuech, firstdate return issuech, firstdate
@ -817,8 +884,7 @@ def GetImportList(results):
return serieslist return serieslist
def drophtml(html): def drophtml(html):
from bs4 import BeautifulSoup soup = Soup(html, "html.parser")
soup = BeautifulSoup(html, "html.parser")
text_parts = soup.findAll(text=True) text_parts = soup.findAll(text=True)
#print ''.join(text_parts) #print ''.join(text_parts)

236
mylar/filechecker.py
View File

@ -1,4 +1,5 @@
#/usr/bin/env python #/usr/bin/env python
# -*- coding: utf-8 -*-
# This file is part of Mylar. # This file is part of Mylar.
# #
# Mylar is free software: you can redistribute it and/or modify # Mylar is free software: you can redistribute it and/or modify
@ -325,6 +326,56 @@ class FileChecker(object):
#split_file = re.findall('\([\w\s-]+\)|[-+]?\d*\.\d+|\d+|[\w-]+|#?\d\.\d+|#(?<![\w\d])XCV(?![\w\d])+|\)', ret_sf1, re.UNICODE) #split_file = re.findall('\([\w\s-]+\)|[-+]?\d*\.\d+|\d+|[\w-]+|#?\d\.\d+|#(?<![\w\d])XCV(?![\w\d])+|\)', ret_sf1, re.UNICODE)
split_file = re.findall('(?imu)\([\w\s-]+\)|[-+]?\d*\.\d+|\d+|[\w-]+|#?\d\.\d+|#(?<![\w\d])XCV(?![\w\d])+|\)', ret_sf1, re.UNICODE) split_file = re.findall('(?imu)\([\w\s-]+\)|[-+]?\d*\.\d+|\d+|[\w-]+|#?\d\.\d+|#(?<![\w\d])XCV(?![\w\d])+|\)', ret_sf1, re.UNICODE)
#10-20-2018 ---START -- attempt to detect '01 (of 7.3)'
#10-20-2018 -- attempt to detect '36p ctc' as one element
spf = []
mini = False
wrdcnt = 0
for x in split_file:
if x == 'of':
mini = True
spf.append(x)
elif mini is True:
mini = False
try:
logger.fdebug('checking now: %s' % x)
if x.isdigit():
logger.fdebug('[MINI-SERIES] MAX ISSUES IN SERIES: %s' % x)
spf.append('(of %s)' % x)
elif float(x) > 0:
logger.fdebug('[MINI-DECIMAL SERIES] MAX ISSUES IN SERIES: %s' % x)
spf.append('(of %s)' % x)
except Exception as e:
logger.error('Exception: %s' % e)
spf.append(x)
elif x == ')':
pass
elif x == 'p' or x == 'ctc':
try:
if spf[wrdcnt-1].isdigit():
logger.debug('THIS SHOULD BE : %s%s' % (spf[wrdcnt-1], x))
newline = '%s%s' % (spf[wrdcnt-1], x)
spf[wrdcnt -1] = newline
#wrdcnt =-1
elif spf[wrdcnt-1][-1] == 'p' and spf[wrdcnt-1][:-1].isdigit() and x == 'ctc':
logger.fdebug('THIS SHOULD BE : %s%s' % (spf[wrdcnt-1], x))
newline = '%s%s' % (spf[wrdcnt-1], x)
spf[wrdcnt -1] = newline
#wrdcnt =-1
except Exception as e:
logger.error('[ERROR] %s' % e)
logger.warn('this should not be passed: %s' % x)
spf.append(x)
else:
spf.append(x)
wrdcnt +=1
if len(spf) > 0:
split_file = spf
logger.fdebug('NEWLY SPLIT REORGD: %s' % split_file)
#10-20-2018 ---END
if len(split_file) == 1: if len(split_file) == 1:
logger.fdebug('Improperly formatted filename - there is no seperation using appropriate characters between wording.') logger.fdebug('Improperly formatted filename - there is no seperation using appropriate characters between wording.')
ret_sf1 = re.sub('\-',' ', ret_sf1).strip() ret_sf1 = re.sub('\-',' ', ret_sf1).strip()
@ -339,7 +390,7 @@ class FileChecker(object):
lastissue_label = None lastissue_label = None
lastissue_position = 0 lastissue_position = 0
lastmod_position = 0 lastmod_position = 0
booktype = 'issue'
#exceptions that are considered alpha-numeric issue numbers #exceptions that are considered alpha-numeric issue numbers
exceptions = ('NOW', 'AI', 'AU', 'X', 'A', 'B', 'C', 'INH', 'MU') exceptions = ('NOW', 'AI', 'AU', 'X', 'A', 'B', 'C', 'INH', 'MU')
@ -416,7 +467,7 @@ class FileChecker(object):
count = match.group() count = match.group()
found = True found = True
if not found: if found is False:
match = re.search('(?<=\(of\s)\d+(?=\))', sf, re.IGNORECASE) match = re.search('(?<=\(of\s)\d+(?=\))', sf, re.IGNORECASE)
if match: if match:
count = match.group() count = match.group()
@ -431,7 +482,7 @@ class FileChecker(object):
logger.fdebug('Issue Number SHOULD BE: ' + str(lastissue_label)) logger.fdebug('Issue Number SHOULD BE: ' + str(lastissue_label))
validcountchk = True validcountchk = True
if all([lastissue_position == (split_file.index(sf) -1), lastissue_label is not None and '#' not in sf]): if all([lastissue_position == (split_file.index(sf) -1), lastissue_label is not None, '#' not in sf, sf != 'p']):
#find it in the original file to see if there's a decimal between. #find it in the original file to see if there's a decimal between.
findst = lastissue_mod_position+1 findst = lastissue_mod_position+1
if findst > len(modfilename): if findst > len(modfilename):
@ -439,11 +490,17 @@ class FileChecker(object):
if modfilename[findst] != '.' or modfilename[findst] != '#': #findst != '.' and findst != '#': if modfilename[findst] != '.' or modfilename[findst] != '#': #findst != '.' and findst != '#':
if sf.isdigit(): if sf.isdigit():
logger.fdebug('2 seperate numbers detected. Assuming 2nd number is the actual issue') seper_num = False
possible_issuenumbers.append({'number': sf, for x in datecheck:
'position': split_file.index(sf, lastissue_position), #modfilename.find(sf)}) if x['position'] == split_file.index(sf, lastissue_position):
'mod_position': self.char_file_position(modfilename, sf, lastmod_position), seper_num = True
'validcountchk': validcountchk}) if seper_num is False:
logger.fdebug('2 seperate numbers detected. Assuming 2nd number is the actual issue')
#possible_issuenumbers.append({'number': sf,
# 'position': split_file.index(sf, lastissue_position), #modfilename.find(sf)})
# 'mod_position': self.char_file_position(modfilename, sf, lastmod_position),
# 'validcountchk': validcountchk})
#used to see if the issue is an alpha-numeric (ie. 18.NOW, 50-X, etc) #used to see if the issue is an alpha-numeric (ie. 18.NOW, 50-X, etc)
lastissue_position = split_file.index(sf, lastissue_position) lastissue_position = split_file.index(sf, lastissue_position)
@ -481,7 +538,6 @@ class FileChecker(object):
else: else:
if ('#' in sf or sf.isdigit()) or validcountchk: if ('#' in sf or sf.isdigit()) or validcountchk:
logger.fdebug('validated: ' + sf)
if validcountchk: if validcountchk:
#if it's not a decimal but the digits are back-to-back, then it's something else. #if it's not a decimal but the digits are back-to-back, then it's something else.
possible_issuenumbers.append({'number': lastissue_label, possible_issuenumbers.append({'number': lastissue_label,
@ -593,11 +649,57 @@ class FileChecker(object):
else: else:
raise ValueError raise ValueError
except ValueError, e: except ValueError, e:
#10-20-2018 - to detect issue numbers such as #000.0000½
if lastissue_label is not None and lastissue_position == int(split_file.index(sf))-1 and sf == 'XCV':
logger.info('this should be: %s%s' % (lastissue_label, sf))
pi = []
for x in possible_issuenumbers:
if (x['number'] == lastissue_label and x['position'] == lastissue_position) or (x['number'] == sf and x['position'] == split_file.index(sf, lastissue_position)):
pass
else:
pi.append({'number': x['number'],
'position': x['position'],
'mod_position': x['mod_position'],
'validcountchk': x['validcountchk']})
lastissue_label = '%s%s' % (lastissue_label, sf)
pi.append({'number': lastissue_label,
'position': lastissue_position,
'mod_position': lastmod_position,
'validcountchk': validcountchk})
if len(pi) > 0:
possible_issuenumbers = pi
elif sf.lower() == 'of' and lastissue_label is not None and lastissue_position == int(split_file.index(sf))-1:
logger.info('MINI-SERIES DETECTED')
else:
if any([re.sub('[\(\)]', '', sf.lower()).strip() == 'tpb', re.sub('[\(\)]', '', sf.lower()).strip() == 'digital tpb']):
logger.info('TRADE PAPERBACK DETECTED. NOT DETECTING ISSUE NUMBER - ASSUMING VOLUME')
booktype = 'TPB'
try:
if volume_found['volume'] is not None:
possible_issuenumbers.append({'number': volume_found['volume'],
'position': volume_found['position'],
'mod_position': self.char_file_position(modfilename, volume_found['volume'], lastmod_position),
'validcountchk': validcountchk})
except:
possible_issuenumbers.append({'number': '1',
'position': split_file.index(sf, lastissue_position), #modfilename.find(sf)})
'mod_position': self.char_file_position(modfilename, sf, lastmod_position),
'validcountchk': validcountchk})
elif any([sf.lower() == 'gn', sf.lower() == 'graphic novel']):
logger.info('GRAPHIC NOVEL DETECTED. NOT DETECTING ISSUE NUMBER - ASSUMING VOLUME')
booktype = 'GN'
else:
logger.info('[%s] Error detecting issue # - ignoring this result : %s' % (e, sf))
volumeprior = False volumeprior = False
volumeprior_label = None volumeprior_label = None
sep_volume = False sep_volume = False
pass pass
#logger.fdebug('Error detecting issue # - ignoring this result : ' + str(sf))
#keep track of where in the original modfilename the positions are in order to check against it for decimal places, etc. #keep track of where in the original modfilename the positions are in order to check against it for decimal places, etc.
file_length += len(sf) + 1 #1 for space file_length += len(sf) + 1 #1 for space
@ -742,13 +844,19 @@ class FileChecker(object):
issue_number_position -=1 issue_number_position -=1
if issue_number is None: if issue_number is None:
logger.fdebug('No issue number present in filename.') if any([booktype == 'TPB', booktype == 'GN']):
logger.info('%s detected. Volume assumption is number: %s' % (booktype, volume_found))
else:
if len(volume_found) > 0:
logger.info('UNKNOWN TPB/GN detected. Volume assumption is number: %s' % (volume_found))
else:
logger.info('No issue number present in filename.')
else: else:
logger.fdebug('issue verified as : ' + issue_number) logger.fdebug('issue verified as : ' + issue_number)
issue_volume = None issue_volume = None
if len(volume_found) > 0: if len(volume_found) > 0:
issue_volume = 'v' + str(volume_found['volume']) issue_volume = 'v' + str(volume_found['volume'])
if all([highest_series_pos + 1 != volume_found['position'], highest_series_pos != volume_found['position'] + 1, sep_volume == False]): if all([highest_series_pos + 1 != volume_found['position'], highest_series_pos != volume_found['position'] + 1, sep_volume == False, booktype == 'issue', len(possible_issuenumbers) > 0]):
logger.fdebug('Extra item(s) are present between the volume label and the issue number. Checking..') logger.fdebug('Extra item(s) are present between the volume label and the issue number. Checking..')
split_file.insert(int(issue_number_position), split_file.pop(volume_found['position'])) #highest_series_pos-1, split_file.pop(volume_found['position'])) split_file.insert(int(issue_number_position), split_file.pop(volume_found['position'])) #highest_series_pos-1, split_file.pop(volume_found['position']))
logger.fdebug('new split: ' + str(split_file)) logger.fdebug('new split: ' + str(split_file))
@ -761,10 +869,13 @@ class FileChecker(object):
else: else:
highest_series_pos = volume_found['position'] highest_series_pos = volume_found['position']
logger.fdebug('Volume detected as : ' + issue_volume) logger.fdebug('Volume detected as : ' + issue_volume)
if all([len(volume_found) == 0, booktype != 'issue']) or all([len(volume_found) == 0, issue_number_position == len(split_file)]):
issue_volume = 'v1'
#at this point it should be in a SERIES ISSUE VOLUME YEAR kind of format #at this point it should be in a SERIES ISSUE VOLUME YEAR kind of format
#if the position of the issue number is greater than the highest series position, make it the highest series position. #if the position of the issue number is greater than the highest series position, make it the highest series position.
if issue_number_position > highest_series_pos: if issue_number_position != len(split_file) and issue_number_position > highest_series_pos:
if not volume_found: if not volume_found:
highest_series_pos = issue_number_position highest_series_pos = issue_number_position
else: else:
@ -807,16 +918,13 @@ class FileChecker(object):
tmpval = yearposition - issue_number_position tmpval = yearposition - issue_number_position
else: else:
tmpval = 1 tmpval = 1
#logger.fdebug('TMPVAL: %s' % tmpval)
except: except:
pass pass
else: else:
if tmpval > 2: if tmpval > 2:
logger.fdebug('There are %s extra words between the issue # and the year position. Deciphering if issue title or part of series title.' % tmpval) logger.fdebug('There are %s extra words between the issue # and the year position. Deciphering if issue title or part of series title.' % tmpval)
tmpval1 = ' '.join(split_file[issue_number_position+1:yearposition]) tmpval1 = ' '.join(split_file[issue_number_position+1:yearposition])
#logger.info('%s' % tmpval)
if split_file[issue_number_position+1] == '-': if split_file[issue_number_position+1] == '-':
#logger.info('dash detected')
usevalue = ' '.join(split_file[issue_number_position+2:yearposition]) usevalue = ' '.join(split_file[issue_number_position+2:yearposition])
splitv = split_file[issue_number_position+2:yearposition] splitv = split_file[issue_number_position+2:yearposition]
else: else:
@ -904,8 +1012,15 @@ class FileChecker(object):
if splitvalue is not None: if splitvalue is not None:
logger.fdebug('[SPLITVALUE] possible issue title: %s' % splitvalue) logger.fdebug('[SPLITVALUE] possible issue title: %s' % splitvalue)
alt_series = '%s %s' % (series_name, splitvalue) alt_series = '%s %s' % (series_name, splitvalue)
alt_issue = splitvalue if booktype != 'issue':
if alt_issue is not None:
alt_issue = re.sub('tpb', '', splitvalue, flags=re.I).strip()
if alt_series is not None:
alt_series = re.sub('tpb', '', alt_series, flags=re.I).strip()
if alt_series is not None: if alt_series is not None:
if booktype != 'issue':
if alt_series is not None:
alt_series = re.sub('tpb', '', alt_series, flags=re.I).strip()
logger.fdebug('Alternate series / issue title: %s [%s]' % (alt_series, alt_issue)) logger.fdebug('Alternate series / issue title: %s [%s]' % (alt_series, alt_issue))
#if the filename is unicoded, it won't match due to the unicode translation. Keep the unicode as well as the decoded. #if the filename is unicoded, it won't match due to the unicode translation. Keep the unicode as well as the decoded.
@ -921,28 +1036,33 @@ class FileChecker(object):
series_name = re.sub('special', '', series_name, flags=re.I).strip() series_name = re.sub('special', '', series_name, flags=re.I).strip()
series_name_decoded = re.sub('special', '', series_name_decoded, flags=re.I).strip() series_name_decoded = re.sub('special', '', series_name_decoded, flags=re.I).strip()
if issue_number is None or series_name is None: if (any([issue_number is None, series_name is None]) and booktype == 'issue'):
logger.fdebug('Cannot parse the filename properly. I\'m going to make note of this filename so that my evil ruler can make it work.')
if series_name is not None: if all([issue_number is None, booktype == 'issue', issue_volume is not None]):
dreplace = self.dynamic_replace(series_name)['mod_seriesname'] logger.info('Possible UKNOWN TPB/GN detected - no issue number present, no clarification in filename, but volume present with series title')
else: else:
dreplace = None logger.fdebug('Cannot parse the filename properly. I\'m going to make note of this filename so that my evil ruler can make it work.')
return {'parse_status': 'failure',
'sub': path_list, if series_name is not None:
'comicfilename': filename, dreplace = self.dynamic_replace(series_name)['mod_seriesname']
'comiclocation': self.dir, else:
'series_name': series_name, dreplace = None
'series_name_decoded': series_name_decoded, return {'parse_status': 'failure',
'alt_series': alt_series, 'sub': path_list,
'alt_issue': alt_issue, 'comicfilename': filename,
'dynamic_name': dreplace, 'comiclocation': self.dir,
'issue_number': issue_number, 'series_name': series_name,
'justthedigits': issue_number, #redundant but it's needed atm 'series_name_decoded': series_name_decoded,
'series_volume': issue_volume, 'alt_series': alt_series,
'issue_year': issue_year, 'alt_issue': alt_issue,
'annual_comicid': None, 'dynamic_name': dreplace,
'scangroup': scangroup, 'issue_number': issue_number,
'reading_order': None} 'justthedigits': issue_number, #redundant but it's needed atm
'series_volume': issue_volume,
'issue_year': issue_year,
'annual_comicid': None,
'scangroup': scangroup,
'reading_order': None}
if self.justparse: if self.justparse:
return {'parse_status': 'success', return {'parse_status': 'success',
@ -1323,7 +1443,7 @@ class FileChecker(object):
return {'AS_Alt': AS_Alt, return {'AS_Alt': AS_Alt,
'AS_Tuple': AS_Tuple} 'AS_Tuple': AS_Tuple}
def checkthedate(self, txt): def checkthedate(self, txt, fulldate=False):
# txt='''\ # txt='''\
# Jan 19, 1990 # Jan 19, 1990
# January 19, 1990 # January 19, 1990
@ -1334,17 +1454,28 @@ class FileChecker(object):
# Jan 1990 # Jan 1990
# January1990''' # January1990'''
fmts = ('%Y','%b %d, %Y','%B %d, %Y','%B %d %Y','%m/%d/%Y','%m/%d/%y','%b %Y','%B%Y','%b %d,%Y','%m-%Y','%B %Y','%Y-%m-%d','%Y-%m','%Y%m') fmts = ('%Y','%b %d, %Y','%B %d, %Y','%B %d %Y','%m/%d/%Y','%m/%d/%y','(%m/%d/%Y)','%b %Y','%B%Y','%b %d,%Y','%m-%Y','%B %Y','%Y-%m-%d','%Y-%m','%Y%m')
parsed=[] parsed=[]
for e in txt.splitlines(): if fulldate is False:
for fmt in fmts: for e in txt.splitlines():
try: for fmt in fmts:
t = dt.datetime.strptime(e, fmt) try:
parsed.append((e, fmt, t)) t = dt.datetime.strptime(e, fmt)
break parsed.append((e, fmt, t))
except ValueError as err: break
pass except ValueError as err:
pass
else:
for e in txt.split():
logger.info('word: %s' % e)
for fmt in fmts:
try:
t = dt.datetime.strptime(e, fmt)
parsed.append((e, fmt, t))
break
except ValueError as err:
pass
# check that all the cases are handled # check that all the cases are handled
success={t[0] for t in parsed} success={t[0] for t in parsed}
@ -1354,12 +1485,17 @@ class FileChecker(object):
dateyear = None dateyear = None
#logger.info('parsed: %s' % parsed)
for t in parsed: for t in parsed:
# logger.fdebug('"{:20}" => "{:20}" => {}'.format(*t) # logger.fdebug('"{:20}" => "{:20}" => {}'.format(*t)
dateyear = t[2].year if fulldate is False:
dateline = t[2].year
else:
dateline = t[2].strftime('%Y-%m-%d')
break break
return dateyear return dateline
def validateAndCreateDirectory(dir, create=False, module=None): def validateAndCreateDirectory(dir, create=False, module=None):
if module is None: if module is None:

12
mylar/helpers.py
View File

@ -1424,7 +1424,8 @@ def havetotals(refreshit=None):
"percent": percent, "percent": percent,
"totalissues": totalissues, "totalissues": totalissues,
"haveissues": haveissues, "haveissues": haveissues,
"DateAdded": comic['LastUpdated']}) "DateAdded": comic['LastUpdated'],
"ComicType": comic['Type']})
return comics return comics
@ -1842,17 +1843,22 @@ def listPull(weeknumber, year):
library[row['ComicID']] = row['ComicID'] library[row['ComicID']] = row['ComicID']
return library return library
def listLibrary(): def listLibrary(comicid=None):
import db import db
library = {} library = {}
myDB = db.DBConnection() myDB = db.DBConnection()
list = myDB.select("SELECT a.comicid, b.releasecomicid, a.status FROM Comics AS a LEFT JOIN annuals AS b on a.comicid=b.comicid group by a.comicid") if comicid is None:
list = myDB.select("SELECT a.comicid, b.releasecomicid, a.status FROM Comics AS a LEFT JOIN annuals AS b on a.comicid=b.comicid group by a.comicid")
else:
list = myDB.select("SELECT a.comicid, b.releasecomicid, a.status FROM Comics AS a LEFT JOIN annuals AS b on a.comicid=b.comicid WHERE a.comicid=? group by a.comicid", [re.sub('4050-', '', comicid).strip()])
for row in list: for row in list:
library[row['ComicID']] = {'comicid': row['ComicID'], library[row['ComicID']] = {'comicid': row['ComicID'],
'status': row['Status']} 'status': row['Status']}
if row['ReleaseComicID'] is not None: if row['ReleaseComicID'] is not None:
library[row['ReleaseComicID']] = {'comicid': row['ComicID'], library[row['ReleaseComicID']] = {'comicid': row['ComicID'],
'status': row['Status']} 'status': row['Status']}
return library return library
def listStoryArcs(): def listStoryArcs():

32
mylar/importer.py
View File

@ -22,6 +22,7 @@ import sys
import shlex import shlex
import datetime import datetime
import re import re
import json
import urllib import urllib
import urllib2 import urllib2
import shutil import shutil
@ -204,17 +205,23 @@ def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=No
publisher = re.sub('!', '', comic['ComicPublisher']) # thanks Boom! publisher = re.sub('!', '', comic['ComicPublisher']) # thanks Boom!
publisher = helpers.filesafe(publisher) publisher = helpers.filesafe(publisher)
year = SeriesYear year = SeriesYear
if comicVol is None: booktype = comic['Type']
if booktype == 'Print' or all([comic['Type'] != 'Print', mylar.CONFIG.FORMAT_BOOKTYPE is False]):
chunk_fb = re.sub('\$Type', '', mylar.CONFIG.FOLDER_FORMAT)
chunk_b = re.compile(r'\s+')
chunk_folder_format = chunk_b.sub(' ', chunk_fb)
else:
chunk_folder_format = mylar.CONFIG.FOLDER_FORMAT
if any([comicVol is None, comic['Type'] != 'Print']):
comicVol = 'None' comicVol = 'None'
#if comversion is None, remove it so it doesn't populate with 'None' #if comversion is None, remove it so it doesn't populate with 'None'
if comicVol == 'None': if comicVol == 'None':
chunk_f_f = re.sub('\$VolumeN', '', mylar.CONFIG.FOLDER_FORMAT) chunk_f_f = re.sub('\$VolumeN', '', chunk_folder_format)
chunk_f = re.compile(r'\s+') chunk_f = re.compile(r'\s+')
chunk_folder_format = chunk_f.sub(' ', chunk_f_f) chunk_folder_format = chunk_f.sub(' ', chunk_f_f)
logger.fdebug('No version # found for series, removing from folder format') logger.fdebug('No version # found for series, removing from folder format')
logger.fdebug("new folder format: " + str(chunk_folder_format)) logger.fdebug("new folder format: " + str(chunk_folder_format))
else:
chunk_folder_format = mylar.CONFIG.FOLDER_FORMAT
#do work to generate folder path #do work to generate folder path
@ -225,12 +232,14 @@ def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=No
'$publisher': publisher.lower(), '$publisher': publisher.lower(),
'$VolumeY': 'V' + str(year), '$VolumeY': 'V' + str(year),
'$VolumeN': comicVol.upper(), '$VolumeN': comicVol.upper(),
'$Annual': 'Annual' '$Annual': 'Annual',
'$Type': booktype
} }
try: try:
if mylar.CONFIG.FOLDER_FORMAT == '': if mylar.CONFIG.FOLDER_FORMAT == '':
comlocation = os.path.join(mylar.CONFIG.DESTINATION_DIR, comicdir, " (" + SeriesYear + ")") comlocation = os.path.join(mylar.CONFIG.DESTINATION_DIR, comicdir, " (" + SeriesYear + ")")
else: else:
chunk_folder_format = re.sub('[()|[]]', '', chunk_folder_format).strip()
comlocation = os.path.join(mylar.CONFIG.DESTINATION_DIR, helpers.replace_all(chunk_folder_format, values)) comlocation = os.path.join(mylar.CONFIG.DESTINATION_DIR, helpers.replace_all(chunk_folder_format, values))
except Exception as e: except Exception as e:
if 'TypeError' in e: if 'TypeError' in e:
@ -308,6 +317,11 @@ def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=No
tmpseriesname = as_dinfo['mod_seriesname'] tmpseriesname = as_dinfo['mod_seriesname']
dynamic_seriesname = re.sub('[\|\s]','', tmpseriesname.lower()).strip() dynamic_seriesname = re.sub('[\|\s]','', tmpseriesname.lower()).strip()
if comic['Issue_List'] != 'None':
issue_list = json.dumps(comic['Issue_List'])
else:
issue_list = None
controlValueDict = {"ComicID": comicid} controlValueDict = {"ComicID": comicid}
newValueDict = {"ComicName": comic['ComicName'], newValueDict = {"ComicName": comic['ComicName'],
"ComicSortName": sortname, "ComicSortName": sortname,
@ -327,6 +341,7 @@ def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=No
# "ComicPublished": gcdinfo['resultPublished'], # "ComicPublished": gcdinfo['resultPublished'],
"ComicPublished": "Unknown", "ComicPublished": "Unknown",
"Type": comic['Type'], "Type": comic['Type'],
"Collects": issue_list,
"DateAdded": helpers.today(), "DateAdded": helpers.today(),
"Status": "Loading"} "Status": "Loading"}
@ -887,6 +902,7 @@ def issue_collection(issuedata, nostatus):
"Issue_Number": issue['Issue_Number'], "Issue_Number": issue['Issue_Number'],
"IssueDate": issue['IssueDate'], "IssueDate": issue['IssueDate'],
"ReleaseDate": issue['ReleaseDate'], "ReleaseDate": issue['ReleaseDate'],
"DigitalDate": issue['DigitalDate'],
"Int_IssueNumber": issue['Int_IssueNumber'], "Int_IssueNumber": issue['Int_IssueNumber'],
"ImageURL": issue['ImageURL'], "ImageURL": issue['ImageURL'],
"ImageURL_ALT": issue['ImageURL_ALT'] "ImageURL_ALT": issue['ImageURL_ALT']
@ -1005,6 +1021,7 @@ def manualAnnual(manual_comicid=None, comicname=None, comicyear=None, comicid=No
'IssueName': cleanname, 'IssueName': cleanname,
'IssueDate': str(firstval['Issue_Date']), 'IssueDate': str(firstval['Issue_Date']),
'ReleaseDate': str(firstval['Store_Date']), 'ReleaseDate': str(firstval['Store_Date']),
'DigitalDate': str(firstval['Digital_Date']),
'Status': astatus, 'Status': astatus,
'ReleaseComicName': sr['ComicName']}) 'ReleaseComicName': sr['ComicName']})
n+=1 n+=1
@ -1018,6 +1035,7 @@ def manualAnnual(manual_comicid=None, comicname=None, comicyear=None, comicid=No
"Int_IssueNumber": helpers.issuedigits(ann['Issue_Number']), "Int_IssueNumber": helpers.issuedigits(ann['Issue_Number']),
"IssueDate": ann['IssueDate'], "IssueDate": ann['IssueDate'],
"ReleaseDate": ann['ReleaseDate'], "ReleaseDate": ann['ReleaseDate'],
"DigitalDate": ann['DigitalDate'],
"IssueName": ann['IssueName'], "IssueName": ann['IssueName'],
"ComicID": ann['ComicID'], #this is the series ID "ComicID": ann['ComicID'], #this is the series ID
"ReleaseComicID": ann['ReleaseComicID'], #this is the series ID for the annual(s) "ReleaseComicID": ann['ReleaseComicID'], #this is the series ID for the annual(s)
@ -1100,6 +1118,7 @@ def updateissuedata(comicid, comicname=None, issued=None, comicIssues=None, call
issname = cleanname issname = cleanname
issdate = str(firstval['Issue_Date']) issdate = str(firstval['Issue_Date'])
storedate = str(firstval['Store_Date']) storedate = str(firstval['Store_Date'])
digitaldate = str(firstval['Digital_Date'])
int_issnum = None int_issnum = None
if issnum.isdigit(): if issnum.isdigit():
int_issnum = int(issnum) * 1000 int_issnum = int(issnum) * 1000
@ -1264,6 +1283,7 @@ def updateissuedata(comicid, comicname=None, issued=None, comicIssues=None, call
"Issue_Number": issnum, "Issue_Number": issnum,
"IssueDate": issdate, "IssueDate": issdate,
"ReleaseDate": storedate, "ReleaseDate": storedate,
"DigitalDate": digitaldate,
"Int_IssueNumber": int_issnum, "Int_IssueNumber": int_issnum,
"ImageURL": firstval['Image'], "ImageURL": firstval['Image'],
"ImageURL_ALT": firstval['ImageALT']}) "ImageURL_ALT": firstval['ImageALT']})
@ -1467,6 +1487,7 @@ def annual_check(ComicName, SeriesYear, comicid, issuetype, issuechk, annualslis
issname = cleanname issname = cleanname
issdate = str(firstval['Issue_Date']) issdate = str(firstval['Issue_Date'])
stdate = str(firstval['Store_Date']) stdate = str(firstval['Store_Date'])
digdate = str(firstval['Digital_Date'])
int_issnum = helpers.issuedigits(issnum) int_issnum = helpers.issuedigits(issnum)
iss_exists = myDB.selectone('SELECT * from annuals WHERE IssueID=?', [issid]).fetchone() iss_exists = myDB.selectone('SELECT * from annuals WHERE IssueID=?', [issid]).fetchone()
@ -1494,6 +1515,7 @@ def annual_check(ComicName, SeriesYear, comicid, issuetype, issuechk, annualslis
"Int_IssueNumber": int_issnum, "Int_IssueNumber": int_issnum,
"IssueDate": issdate, "IssueDate": issdate,
"ReleaseDate": stdate, "ReleaseDate": stdate,
"DigitalDate": digdate,
"IssueName": issname, "IssueName": issname,
"ComicID": comicid, "ComicID": comicid,
"IssueID": issid, "IssueID": issid,

8
mylar/locg.py
View File

@ -83,12 +83,13 @@ def locg(pulldate=None,weeknumber=None,year=None):
'annuallink': x['link'], 'annuallink': x['link'],
'year': x['year'], 'year': x['year'],
'volume': x['volume'], 'volume': x['volume'],
'seriesyear': x['seriesyear']}) 'seriesyear': x['seriesyear'],
'format': x['type']})
shipdate = x['shipdate'] shipdate = x['shipdate']
myDB = db.DBConnection() myDB = db.DBConnection()
myDB.action("CREATE TABLE IF NOT EXISTS weekly (SHIPDATE, PUBLISHER text, ISSUE text, COMIC VARCHAR(150), EXTRA text, STATUS text, ComicID text, IssueID text, CV_Last_Update text, DynamicName text, weeknumber text, year text, volume text, seriesyear text, annuallink text, rowid INTEGER PRIMARY KEY)") myDB.action("CREATE TABLE IF NOT EXISTS weekly (SHIPDATE, PUBLISHER text, ISSUE text, COMIC VARCHAR(150), EXTRA text, STATUS text, ComicID text, IssueID text, CV_Last_Update text, DynamicName text, weeknumber text, year text, volume text, seriesyear text, annuallink text, format text, rowid INTEGER PRIMARY KEY)")
#clear out the upcoming table here so they show the new values properly. #clear out the upcoming table here so they show the new values properly.
if pulldate == '00000000': if pulldate == '00000000':
@ -123,7 +124,8 @@ def locg(pulldate=None,weeknumber=None,year=None):
'ANNUALLINK': x['annuallink'], 'ANNUALLINK': x['annuallink'],
'YEAR': x['year'], 'YEAR': x['year'],
'VOLUME': x['volume'], 'VOLUME': x['volume'],
'SERIESYEAR': x['seriesyear']} 'SERIESYEAR': x['seriesyear'],
'FORMAT': x['format']}
myDB.upsert("weekly", newValueDict, controlValueDict) myDB.upsert("weekly", newValueDict, controlValueDict)
logger.info('[PULL-LIST] Successfully populated pull-list into Mylar for the week of: ' + str(weeknumber)) logger.info('[PULL-LIST] Successfully populated pull-list into Mylar for the week of: ' + str(weeknumber))

4
mylar/mb.py
View File

@ -373,7 +373,7 @@ def findComic(name, mode, issue, limityear=None, type=None):
xmltype = 'Print' xmltype = 'Print'
elif 'digital' in xmldesc[:60].lower() and 'digital edition can be found' not in xmldesc.lower(): elif 'digital' in xmldesc[:60].lower() and 'digital edition can be found' not in xmldesc.lower():
xmltype = 'Digital' xmltype = 'Digital'
elif 'paperback' in xmldesc[:60].lower() and 'paperback can be found' not in xmldesc.lower(): elif all(['paperback' in xmldesc[:60].lower(), 'paperback can be found' not in xmldesc.lower()]) or 'collects' in xmldesc.lower():
xmltype = 'TPB' xmltype = 'TPB'
elif 'hardcover' in xmldesc[:60].lower() and 'hardcover can be found' not in xmldesc.lower(): elif 'hardcover' in xmldesc[:60].lower() and 'hardcover can be found' not in xmldesc.lower():
xmltype = 'HC' xmltype = 'HC'
@ -458,8 +458,8 @@ def storyarcinfo(xmlid):
try: try:
logger.fdebug('story_arc ascension') logger.fdebug('story_arc ascension')
issuecount = len( arcdom.getElementsByTagName('issue') )
issuedom = arcdom.getElementsByTagName('issue') issuedom = arcdom.getElementsByTagName('issue')
issuecount = len( issuedom ) #arcdom.getElementsByTagName('issue') )
isc = 0 isc = 0
arclist = '' arclist = ''
ordernum = 1 ordernum = 1

120
mylar/search.py
View File

@ -38,7 +38,7 @@ from base64 import b16encode, b32decode
from operator import itemgetter from operator import itemgetter
from wsgiref.handlers import format_date_time from wsgiref.handlers import format_date_time
def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, IssueID, AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=None, IssueArcID=None, mode=None, rsscheck=None, ComicID=None, manualsearch=None, filesafe=None, allow_packs=None, oneoff=False, manual=False, torrentid_32p=None): def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, IssueID, AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=None, IssueArcID=None, mode=None, rsscheck=None, ComicID=None, manualsearch=None, filesafe=None, allow_packs=None, oneoff=False, manual=False, torrentid_32p=None, digitaldate=None):
mylar.COMICINFO = [] mylar.COMICINFO = []
unaltered_ComicName = None unaltered_ComicName = None
@ -313,7 +313,7 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueD
prov_count+=1 prov_count+=1
continue continue
if searchmode == 'rss': if searchmode == 'rss':
findit = NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, searchprov, send_prov_count, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="yes", ComicID=ComicID, issuetitle=issuetitle, unaltered_ComicName=unaltered_ComicName, oneoff=oneoff, cmloopit=cmloopit, manual=manual, torznab_host=torznab_host) findit = NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, searchprov, send_prov_count, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="yes", ComicID=ComicID, issuetitle=issuetitle, unaltered_ComicName=unaltered_ComicName, oneoff=oneoff, cmloopit=cmloopit, manual=manual, torznab_host=torznab_host, digitaldate=digitaldate)
if findit['status'] is False: if findit['status'] is False:
if AlternateSearch is not None and AlternateSearch != "None": if AlternateSearch is not None and AlternateSearch != "None":
chkthealt = AlternateSearch.split('##') chkthealt = AlternateSearch.split('##')
@ -323,7 +323,7 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueD
for calt in chkthealt: for calt in chkthealt:
AS_Alternate = re.sub('##', '', calt) AS_Alternate = re.sub('##', '', calt)
logger.info(u"Alternate Search pattern detected...re-adjusting to : " + str(AS_Alternate)) logger.info(u"Alternate Search pattern detected...re-adjusting to : " + str(AS_Alternate))
findit = NZB_SEARCH(AS_Alternate, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, searchprov, send_prov_count, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="yes", ComicID=ComicID, issuetitle=issuetitle, unaltered_ComicName=AS_Alternate, allow_packs=allow_packs, oneoff=oneoff, cmloopit=cmloopit, manual=manual, torznab_host=torznab_host) findit = NZB_SEARCH(AS_Alternate, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, searchprov, send_prov_count, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="yes", ComicID=ComicID, issuetitle=issuetitle, unaltered_ComicName=AS_Alternate, allow_packs=allow_packs, oneoff=oneoff, cmloopit=cmloopit, manual=manual, torznab_host=torznab_host, digitaldate=digitaldate)
if findit['status'] is True: if findit['status'] is True:
break break
if findit['status'] is True: if findit['status'] is True:
@ -333,7 +333,7 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueD
break break
else: else:
findit = NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, searchprov, send_prov_count, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="no", ComicID=ComicID, issuetitle=issuetitle, unaltered_ComicName=unaltered_ComicName, allow_packs=allow_packs, oneoff=oneoff, cmloopit=cmloopit, manual=manual, torznab_host=torznab_host, torrentid_32p=torrentid_32p) findit = NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, searchprov, send_prov_count, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="no", ComicID=ComicID, issuetitle=issuetitle, unaltered_ComicName=unaltered_ComicName, allow_packs=allow_packs, oneoff=oneoff, cmloopit=cmloopit, manual=manual, torznab_host=torznab_host, torrentid_32p=torrentid_32p, digitaldate=digitaldate)
if all([searchprov == '32P', checked_once is False]) or all([searchprov == 'Public Torrents', checked_once is False]): if all([searchprov == '32P', checked_once is False]) or all([searchprov == 'Public Torrents', checked_once is False]):
checked_once = True checked_once = True
if findit['status'] is False: if findit['status'] is False:
@ -345,7 +345,7 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueD
for calt in chkthealt: for calt in chkthealt:
AS_Alternate = re.sub('##', '', calt) AS_Alternate = re.sub('##', '', calt)
logger.info(u"Alternate Search pattern detected...re-adjusting to : " + str(AS_Alternate)) logger.info(u"Alternate Search pattern detected...re-adjusting to : " + str(AS_Alternate))
findit = NZB_SEARCH(AS_Alternate, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, searchprov, send_prov_count, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="no", ComicID=ComicID, issuetitle=issuetitle, unaltered_ComicName=unaltered_ComicName, allow_packs=allow_packs, oneoff=oneoff, cmloopit=cmloopit, manual=manual, torznab_host=torznab_host, torrentid_32p=torrentid_32p) findit = NZB_SEARCH(AS_Alternate, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, searchprov, send_prov_count, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="no", ComicID=ComicID, issuetitle=issuetitle, unaltered_ComicName=unaltered_ComicName, allow_packs=allow_packs, oneoff=oneoff, cmloopit=cmloopit, manual=manual, torznab_host=torznab_host, torrentid_32p=torrentid_32p, digitaldate=digitaldate)
if findit['status'] is True: if findit['status'] is True:
break break
if findit['status'] is True: if findit['status'] is True:
@ -409,7 +409,7 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueD
return findit, 'None' return findit, 'None'
def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, nzbprov, prov_count, IssDateFix, IssueID, UseFuzzy, newznab_host=None, ComicVersion=None, SARC=None, IssueArcID=None, RSS=None, ComicID=None, issuetitle=None, unaltered_ComicName=None, allow_packs=None, oneoff=False, cmloopit=None, manual=False, torznab_host=None, torrentid_32p=None): def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, nzbprov, prov_count, IssDateFix, IssueID, UseFuzzy, newznab_host=None, ComicVersion=None, SARC=None, IssueArcID=None, RSS=None, ComicID=None, issuetitle=None, unaltered_ComicName=None, allow_packs=None, oneoff=False, cmloopit=None, manual=False, torznab_host=None, torrentid_32p=None, digitaldate=None):
if any([allow_packs is None, allow_packs == 'None', allow_packs == 0, allow_packs == '0']) and all([mylar.CONFIG.ENABLE_TORRENT_SEARCH, mylar.CONFIG.ENABLE_32P]): if any([allow_packs is None, allow_packs == 'None', allow_packs == 0, allow_packs == '0']) and all([mylar.CONFIG.ENABLE_TORRENT_SEARCH, mylar.CONFIG.ENABLE_32P]):
allow_packs = False allow_packs = False
@ -927,8 +927,11 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDa
continue continue
else: else:
stdate = IssueDate stdate = IssueDate
logger.fdebug('issue date used is : %s' % stdate)
else: else:
stdate = StoreDate stdate = StoreDate
logger.fdebug('store date used is : %s' % stdate)
logger.fdebug('date used is : %s' % stdate)
postdate_int = None postdate_int = None
if nzbprov == '32P' and RSS == 'no': if nzbprov == '32P' and RSS == 'no':
@ -959,42 +962,70 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDa
logger.warn('Unable to parse posting date from provider result set for :' + entry['title']) logger.warn('Unable to parse posting date from provider result set for :' + entry['title'])
continue continue
#convert it to a Thu, 06 Feb 2014 00:00:00 format if all([digitaldate != '0000-00-00', digitaldate is not None]):
issue_converted = datetime.datetime.strptime(stdate.rstrip(), '%Y-%m-%d') i = 0
issue_convert = issue_converted + datetime.timedelta(days=-1) else:
# to get past different locale's os-dependent dates, let's convert it to a generic datetime format digitaldate_int = '00000000'
try: i = 1
stamp = time.mktime(issue_convert.timetuple())
issconv = format_date_time(stamp) while i <= 1:
except OverflowError: logger.info('i: %s' % i)
logger.fdebug('Error attempting to convert the timestamp into a generic format. Probably due to the epoch limiation.') if i == 0:
issconv = issue_convert.strftime('%a, %d %b %Y %H:%M:%S') usedate = digitaldate
#convert it to a tuple
econv = email.utils.parsedate_tz(issconv)
econv2 = datetime.datetime(*econv[:6])
#convert it to a numeric and drop the GMT/Timezone
try:
issuedate_int = time.mktime(econv[:len(econv) -1])
except OverflowError:
logger.fdebug('Unable to convert timestamp to integer format. Forcing things through.')
isyear = econv[1]
epochyr = '1970'
if int(isyear) <= int(epochyr):
tm = datetime.datetime(1970, 1, 1)
issuedate_int = int(time.mktime(tm.timetuple()))
else: else:
continue usedate = stdate
logger.fdebug('usedate: %s' % usedate)
#convert it to a Thu, 06 Feb 2014 00:00:00 format
issue_converted = datetime.datetime.strptime(usedate.rstrip(), '%Y-%m-%d')
issue_convert = issue_converted + datetime.timedelta(days=-1)
# to get past different locale's os-dependent dates, let's convert it to a generic datetime format
try:
stamp = time.mktime(issue_convert.timetuple())
issconv = format_date_time(stamp)
except OverflowError:
logger.fdebug('Error attempting to convert the timestamp into a generic format. Probably due to the epoch limiation.')
issconv = issue_convert.strftime('%a, %d %b %Y %H:%M:%S')
#convert it to a tuple
econv = email.utils.parsedate_tz(issconv)
econv2 = datetime.datetime(*econv[:6])
#convert it to a numeric and drop the GMT/Timezone
try:
usedate_int = time.mktime(econv[:len(econv) -1])
except OverflowError:
logger.fdebug('Unable to convert timestamp to integer format. Forcing things through.')
isyear = econv[1]
epochyr = '1970'
if int(isyear) <= int(epochyr):
tm = datetime.datetime(1970, 1, 1)
usedate_int = int(time.mktime(tm.timetuple()))
else:
continue
if i == 0:
digitaldate_int = usedate_int
digconv2 = econv2
else:
issuedate_int = usedate_int
issconv2 = econv2
i+=1
try: try:
#try new method to get around issues populating in a diff timezone thereby putting them in a different day. #try new method to get around issues populating in a diff timezone thereby putting them in a different day.
if dateconv2.date() < econv2.date(): #logger.info('digitaldate: %s' % digitaldate)
logger.fdebug('[CONV]pubdate: %s < storedate: %s' % (dateconv2.date(), econv2.date())) #logger.info('dateconv2: %s' % dateconv2.date())
#logger.info('digconv2: %s' % digconv2.date())
if digitaldate != '0000-00-00' and dateconv2.date() >= digconv2.date():
logger.fdebug(str(pubdate) + ' is after DIGITAL store date of ' + str(digitaldate))
elif dateconv2.date() < issconv2.date():
logger.fdebug('[CONV]pubdate: %s < storedate: %s' % (dateconv2.date(), issconv2.date()))
logger.fdebug(str(pubdate) + ' is before store date of ' + str(stdate) + '. Ignoring search result as this is not the right issue.') logger.fdebug(str(pubdate) + ' is before store date of ' + str(stdate) + '. Ignoring search result as this is not the right issue.')
continue continue
else: else:
logger.fdebug(str(pubdate) + ' is after store date of ' + str(stdate)) logger.fdebug(str(pubdate) + ' is after store date of ' + str(stdate))
except: except:
#if the above fails, drop down to the integer compare method as a failsafe. #if the above fails, drop down to the integer compare method as a failsafe.
if postdate_int < issuedate_int: if digitaldate != '0000-00-00' and postdate_int >= digitaldate_int:
logger.fdebug(str(pubdate) + ' is after DIGITAL store date of ' + str(digitaldate))
elif postdate_int < issuedate_int:
logger.fdebug('[INT]pubdate: %s < storedate: %s' % (postdate_int, issuedate_int)) logger.fdebug('[INT]pubdate: %s < storedate: %s' % (postdate_int, issuedate_int))
logger.fdebug(str(pubdate) + ' is before store date of ' + str(stdate) + '. Ignoring search result as this is not the right issue.') logger.fdebug(str(pubdate) + ' is before store date of ' + str(stdate) + '. Ignoring search result as this is not the right issue.')
continue continue
@ -1311,10 +1342,14 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDa
#need to convert dates to just be yyyy-mm-dd and do comparison, time operator in the below calc as well which probably throws off$ #need to convert dates to just be yyyy-mm-dd and do comparison, time operator in the below calc as well which probably throws off$
F_ComicVersion = '1' F_ComicVersion = '1'
if postdate_int is not None: if postdate_int is not None:
if postdate_int >= issuedate_int and nzbprov == '32P': if digitaldate != '0000-00-00' and all([postdate_int >= digitaldate_int, nzbprov == '32P']):
logger.fdebug('32P torrent discovery. Posting date (' + str(pubdate) + ') is after DIGITAL store date (' + str(digitaldate) + '), forcing volume label to be the same as series label (0-Day Enforcement): v' + str(F_ComicVersion) + ' --> v' + str(S_ComicVersion))
F_ComicVersion = D_ComicVersion
elif all([postdate_int >= issuedate_int, nzbprov == '32P']):
logger.fdebug('32P torrent discovery. Posting date (' + str(pubdate) + ') is after store date (' + str(stdate) + '), forcing volume label to be the same as series label (0-Day Enforcement): v' + str(F_ComicVersion) + ' --> v' + str(S_ComicVersion)) logger.fdebug('32P torrent discovery. Posting date (' + str(pubdate) + ') is after store date (' + str(stdate) + '), forcing volume label to be the same as series label (0-Day Enforcement): v' + str(F_ComicVersion) + ' --> v' + str(S_ComicVersion))
F_ComicVersion = D_ComicVersion F_ComicVersion = D_ComicVersion
else:
pass
logger.fdebug("FCVersion: " + str(F_ComicVersion)) logger.fdebug("FCVersion: " + str(F_ComicVersion))
logger.fdebug("DCVersion: " + str(D_ComicVersion)) logger.fdebug("DCVersion: " + str(D_ComicVersion))
logger.fdebug("SCVersion: " + str(S_ComicVersion)) logger.fdebug("SCVersion: " + str(S_ComicVersion))
@ -1914,6 +1949,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None, manual=False):
'Issue_Number': iss['Issue_Number'], 'Issue_Number': iss['Issue_Number'],
'IssueDate': iss['IssueDate'], 'IssueDate': iss['IssueDate'],
'StoreDate': iss['ReleaseDate'], 'StoreDate': iss['ReleaseDate'],
'DigitalDate': iss['DigitalDate'],
'SARC': None, 'SARC': None,
'StoryArcID': None, 'StoryArcID': None,
'IssueArcID': None, 'IssueArcID': None,
@ -1933,6 +1969,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None, manual=False):
'Issue_Number': iss['IssueNumber'], 'Issue_Number': iss['IssueNumber'],
'IssueDate': iss['IssueDate'], 'IssueDate': iss['IssueDate'],
'StoreDate': iss['ReleaseDate'], 'StoreDate': iss['ReleaseDate'],
'DigitalDate': iss['DigitalDate'],
'SARC': iss['StoryArc'], 'SARC': iss['StoryArc'],
'StoryArcID': iss['StoryArcID'], 'StoryArcID': iss['StoryArcID'],
'IssueArcID': iss['IssueArcID'], 'IssueArcID': iss['IssueArcID'],
@ -1952,6 +1989,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None, manual=False):
'Issue_Number': iss['Issue_Number'], 'Issue_Number': iss['Issue_Number'],
'IssueDate': iss['IssueDate'], 'IssueDate': iss['IssueDate'],
'StoreDate': iss['ReleaseDate'], #need to replace with Store date 'StoreDate': iss['ReleaseDate'], #need to replace with Store date
'DigitalDate': iss['DigitalDate'],
'SARC': None, 'SARC': None,
'StoryArcID': None, 'StoryArcID': None,
'IssueArcID': None, 'IssueArcID': None,
@ -1991,7 +2029,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None, manual=False):
else: else:
storyarc_watchlist = True storyarc_watchlist = True
if result['StoreDate'] == '0000-00-00' or result['StoreDate'] is None: if result['StoreDate'] == '0000-00-00' or result['StoreDate'] is None:
if result['IssueDate'] is None or result['IssueDate'] == '0000-00-00': if any([result['IssueDate'] is None, result['IssueDate'] == '0000-00-00']) and result['DigitalDate'] == '0000-00-00':
logger.fdebug('ComicID: ' + str(result['ComicID']) + ' has invalid Date data. Skipping searching for this series.') logger.fdebug('ComicID: ' + str(result['ComicID']) + ' has invalid Date data. Skipping searching for this series.')
continue continue
@ -2018,6 +2056,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None, manual=False):
IssueDate = result['IssueDate'] IssueDate = result['IssueDate']
StoreDate = result['StoreDate'] StoreDate = result['StoreDate']
DigitalDate = result['DigitalDate']
if result['IssueDate'] is None: if result['IssueDate'] is None:
ComicYear = SeriesYear ComicYear = SeriesYear
@ -2046,7 +2085,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None, manual=False):
mode = result['mode'] mode = result['mode']
foundNZB, prov = search_init(comic['ComicName'], result['Issue_Number'], str(ComicYear), SeriesYear, Publisher, IssueDate, StoreDate, result['IssueID'], AlternateSearch, UseFuzzy, ComicVersion, SARC=result['SARC'], IssueArcID=result['IssueArcID'], mode=mode, rsscheck=rsscheck, ComicID=result['ComicID'], filesafe=Comicname_filesafe, allow_packs=AllowPacks, oneoff=OneOff, torrentid_32p=TorrentID_32p) foundNZB, prov = search_init(comic['ComicName'], result['Issue_Number'], str(ComicYear), SeriesYear, Publisher, IssueDate, StoreDate, result['IssueID'], AlternateSearch, UseFuzzy, ComicVersion, SARC=result['SARC'], IssueArcID=result['IssueArcID'], mode=mode, rsscheck=rsscheck, ComicID=result['ComicID'], filesafe=Comicname_filesafe, allow_packs=AllowPacks, oneoff=OneOff, torrentid_32p=TorrentID_32p, digitaldate=DigitalDate)
if foundNZB['status'] is True: if foundNZB['status'] is True:
updater.foundsearch(result['ComicID'], result['IssueID'], mode=mode, provider=prov, SARC=result['SARC'], IssueArcID=result['IssueArcID'], hash=foundNZB['info']['t_hash']) updater.foundsearch(result['ComicID'], result['IssueID'], mode=mode, provider=prov, SARC=result['SARC'], IssueArcID=result['IssueArcID'], hash=foundNZB['info']['t_hash'])
@ -2093,6 +2132,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None, manual=False):
actissueid = None actissueid = None
IssueDate = result['IssueDate'] IssueDate = result['IssueDate']
StoreDate = result['ReleaseDate'] StoreDate = result['ReleaseDate']
DigitalDate = result['DigitalDate']
TorrentID_32p = None TorrentID_32p = None
elif mode == 'pullwant': elif mode == 'pullwant':
ComicName = result['COMIC'] ComicName = result['COMIC']
@ -2109,6 +2149,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None, manual=False):
TorrentID_32p = None TorrentID_32p = None
IssueDate = result['SHIPDATE'] IssueDate = result['SHIPDATE']
StoreDate = IssueDate StoreDate = IssueDate
DigitalDate = '0000-00-00'
else: else:
comic = myDB.selectone('SELECT * FROM comics where ComicID=?', [ComicID]).fetchone() comic = myDB.selectone('SELECT * FROM comics where ComicID=?', [ComicID]).fetchone()
if mode == 'want_ann': if mode == 'want_ann':
@ -2126,6 +2167,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None, manual=False):
ComicVersion = comic['ComicVersion'] ComicVersion = comic['ComicVersion']
IssueDate = result['IssueDate'] IssueDate = result['IssueDate']
StoreDate = result['ReleaseDate'] StoreDate = result['ReleaseDate']
DigitalDate = result['DigitalDate']
SARC = None SARC = None
IssueArcID = None IssueArcID = None
actissueid = issueid actissueid = issueid
@ -2138,7 +2180,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None, manual=False):
else: else:
IssueYear = str(IssueDate)[:4] IssueYear = str(IssueDate)[:4]
foundNZB, prov = search_init(ComicName, IssueNumber, str(IssueYear), SeriesYear, Publisher, IssueDate, StoreDate, actissueid, AlternateSearch, UseFuzzy, ComicVersion, SARC=SARC, IssueArcID=IssueArcID, mode=mode, rsscheck=rsscheck, ComicID=ComicID, filesafe=Comicname_filesafe, allow_packs=allow_packs, oneoff=oneoff, manual=manual, torrentid_32p=TorrentID_32p) foundNZB, prov = search_init(ComicName, IssueNumber, str(IssueYear), SeriesYear, Publisher, IssueDate, StoreDate, actissueid, AlternateSearch, UseFuzzy, ComicVersion, SARC=SARC, IssueArcID=IssueArcID, mode=mode, rsscheck=rsscheck, ComicID=ComicID, filesafe=Comicname_filesafe, allow_packs=allow_packs, oneoff=oneoff, manual=manual, torrentid_32p=TorrentID_32p, digitaldate=DigitalDate)
if manual is True: if manual is True:
return foundNZB return foundNZB
if foundNZB['status'] is True: if foundNZB['status'] is True:
@ -2192,7 +2234,7 @@ def searchIssueIDList(issuelist):
else: else:
AllowPacks = False AllowPacks = False
foundNZB, prov = search_init(comic['ComicName'], issue['Issue_Number'], str(IssueYear), comic['ComicYear'], Publisher, issue['IssueDate'], issue['ReleaseDate'], issue['IssueID'], AlternateSearch, UseFuzzy, ComicVersion, SARC=None, IssueArcID=None, mode=mode, ComicID=issue['ComicID'], filesafe=comic['ComicName_Filesafe'], allow_packs=AllowPacks, torrentid_32p=TorrentID_32p) foundNZB, prov = search_init(comic['ComicName'], issue['Issue_Number'], str(IssueYear), comic['ComicYear'], Publisher, issue['IssueDate'], issue['ReleaseDate'], issue['IssueID'], AlternateSearch, UseFuzzy, ComicVersion, SARC=None, IssueArcID=None, mode=mode, ComicID=issue['ComicID'], filesafe=comic['ComicName_Filesafe'], allow_packs=AllowPacks, torrentid_32p=TorrentID_32p, digitaldate=issue['DigitalDate'])
if foundNZB['status'] is True: if foundNZB['status'] is True:
updater.foundsearch(ComicID=issue['ComicID'], IssueID=issue['IssueID'], mode=mode, provider=prov, hash=foundNZB['info']['t_hash']) updater.foundsearch(ComicID=issue['ComicID'], IssueID=issue['IssueID'], mode=mode, provider=prov, hash=foundNZB['info']['t_hash'])
logger.info('Completed search request.') logger.info('Completed search request.')

53
mylar/webserve.py
View File

@ -26,6 +26,7 @@ from datetime import timedelta, date
import re import re
import json import json
import copy import copy
import ntpath
from mako.template import Template from mako.template import Template
from mako.lookup import TemplateLookup from mako.lookup import TemplateLookup
@ -179,6 +180,13 @@ class WebInterface(object):
else: else:
comicImage = comic['ComicImage'] comicImage = comic['ComicImage']
comicpublisher = helpers.publisherImages(comic['ComicPublisher']) comicpublisher = helpers.publisherImages(comic['ComicPublisher'])
if comic['Collects'] is not None:
issues_list = json.loads(comic['Collects'])
else:
issues_list = None
#logger.info('issues_list: %s' % issues_list)
comicConfig = { comicConfig = {
"fuzzy_year0": helpers.radio(int(usethefuzzy), 0), "fuzzy_year0": helpers.radio(int(usethefuzzy), 0),
"fuzzy_year1": helpers.radio(int(usethefuzzy), 1), "fuzzy_year1": helpers.radio(int(usethefuzzy), 1),
@ -196,6 +204,7 @@ class WebInterface(object):
"publisher_image_alt": comicpublisher['publisher_image_alt'], "publisher_image_alt": comicpublisher['publisher_image_alt'],
"publisher_imageH": comicpublisher['publisher_imageH'], "publisher_imageH": comicpublisher['publisher_imageH'],
"publisher_imageW": comicpublisher['publisher_imageW'], "publisher_imageW": comicpublisher['publisher_imageW'],
"issue_list": issues_list,
"ComicImage": comicImage + '?' + datetime.datetime.now().strftime('%y-%m-%d %H:%M:%S') "ComicImage": comicImage + '?' + datetime.datetime.now().strftime('%y-%m-%d %H:%M:%S')
} }
@ -223,6 +232,7 @@ class WebInterface(object):
"Int_IssueNumber": ann['Int_IssueNumber'], "Int_IssueNumber": ann['Int_IssueNumber'],
"IssueName": issuename, "IssueName": issuename,
"IssueDate": ann['IssueDate'], "IssueDate": ann['IssueDate'],
"DigitalDate": ann['DigitalDate'],
"Status": ann['Status'], "Status": ann['Status'],
"Location": ann['Location'], "Location": ann['Location'],
"ComicID": ann['ComicID'], "ComicID": ann['ComicID'],
@ -574,6 +584,7 @@ class WebInterface(object):
st_issueid = str(storyarcid) + "_" + str(random.randint(1000,9999)) st_issueid = str(storyarcid) + "_" + str(random.randint(1000,9999))
issnum = arcval['Issue_Number'] issnum = arcval['Issue_Number']
issdate = str(arcval['Issue_Date']) issdate = str(arcval['Issue_Date'])
digitaldate = str(arcval['Digital_Date'])
storedate = str(arcval['Store_Date']) storedate = str(arcval['Store_Date'])
int_issnum = helpers.issuedigits(issnum) int_issnum = helpers.issuedigits(issnum)
@ -603,6 +614,7 @@ class WebInterface(object):
"Issue_Number": issnum, "Issue_Number": issnum,
"IssueDate": issdate, "IssueDate": issdate,
"ReleaseDate": storedate, "ReleaseDate": storedate,
"DigitalDate": digitaldate,
"ReadingOrder": readingorder, #n +1, "ReadingOrder": readingorder, #n +1,
"Int_IssueNumber": int_issnum, "Int_IssueNumber": int_issnum,
"Manual": manual_mod}) "Manual": manual_mod})
@ -644,7 +656,8 @@ class WebInterface(object):
"TotalIssues": storyarcissues, "TotalIssues": storyarcissues,
"ReadingOrder": AD['ReadingOrder'], "ReadingOrder": AD['ReadingOrder'],
"IssueDate": AD['IssueDate'], "IssueDate": AD['IssueDate'],
"ReleaseDate": AD['ReleaseDate'], "ReleaseDate": AD['ReleaseDate'],
"DigitalDate": AD['DigitalDate'],
"SeriesYear": seriesYear, "SeriesYear": seriesYear,
"IssuePublisher": issuePublisher, "IssuePublisher": issuePublisher,
"CV_ArcID": arcid, "CV_ArcID": arcid,
@ -657,8 +670,10 @@ class WebInterface(object):
logger.fdebug(module + ' Now searching your watchlist for matches belonging to this story arc.') logger.fdebug(module + ' Now searching your watchlist for matches belonging to this story arc.')
self.ArcWatchlist(storyarcid) self.ArcWatchlist(storyarcid)
if arcrefresh: if arcrefresh:
logger.info('%s Successfully Refreshed %s' % (module, storyarcname))
return return
else: else:
logger.info('%s Successfully Added %s' % (module, storyarcname))
raise cherrypy.HTTPRedirect("detailStoryArc?StoryArcID=%s&StoryArcName=%s" % (storyarcid, storyarcname)) raise cherrypy.HTTPRedirect("detailStoryArc?StoryArcID=%s&StoryArcName=%s" % (storyarcid, storyarcname))
addStoryArc.exposed = True addStoryArc.exposed = True
@ -898,6 +913,8 @@ class WebInterface(object):
if comic['ComicName'] is None: ComicName = "None" if comic['ComicName'] is None: ComicName = "None"
else: ComicName = comic['ComicName'] else: ComicName = comic['ComicName']
seriesdir = comic['ComicLocation'] seriesdir = comic['ComicLocation']
seriesyear = comic['ComicYear']
seriesvol = comic['ComicVersion']
logger.info(u"Deleting all traces of Comic: " + ComicName) logger.info(u"Deleting all traces of Comic: " + ComicName)
myDB.action('DELETE from comics WHERE ComicID=?', [ComicID]) myDB.action('DELETE from comics WHERE ComicID=?', [ComicID])
myDB.action('DELETE from issues WHERE ComicID=?', [ComicID]) myDB.action('DELETE from issues WHERE ComicID=?', [ComicID])
@ -912,10 +929,12 @@ class WebInterface(object):
shutil.rmtree(seriesdir) shutil.rmtree(seriesdir)
except: except:
logger.warn('Unable to remove directory after removing series from Mylar.') logger.warn('Unable to remove directory after removing series from Mylar.')
else:
logger.info('Successfully removed directory: %s' % (seriesdir))
else: else:
logger.warn('Unable to remove directory as it does not exist in : ' + seriesdir) logger.warn('Unable to remove directory as it does not exist in : ' + seriesdir)
myDB.action('DELETE from readlist WHERE ComicID=?', [ComicID]) myDB.action('DELETE from readlist WHERE ComicID=?', [ComicID])
logger.info('Successful deletion of %s %s (%s) from your watchlist' % (ComicName, seriesvol, seriesyear))
helpers.ComicSort(sequence='update') helpers.ComicSort(sequence='update')
raise cherrypy.HTTPRedirect("home") raise cherrypy.HTTPRedirect("home")
deleteSeries.exposed = True deleteSeries.exposed = True
@ -1682,7 +1701,8 @@ class WebInterface(object):
"HAVEIT": haveit, "HAVEIT": haveit,
"LINK": linkit, "LINK": linkit,
"HASH": None, "HASH": None,
"AUTOWANT": False "AUTOWANT": False,
"FORMAT": weekly['format']
}) })
else: else:
if any(x['ComicName'].lower() == weekly['COMIC'].lower() for x in autowant): if any(x['ComicName'].lower() == weekly['COMIC'].lower() for x in autowant):
@ -1698,7 +1718,8 @@ class WebInterface(object):
"HAVEIT": haveit, "HAVEIT": haveit,
"LINK": linkit, "LINK": linkit,
"HASH": None, "HASH": None,
"AUTOWANT": True "AUTOWANT": True,
"FORMAT": weekly['format']
}) })
else: else:
weeklyresults.append({ weeklyresults.append({
@ -1713,7 +1734,8 @@ class WebInterface(object):
"HAVEIT": haveit, "HAVEIT": haveit,
"LINK": linkit, "LINK": linkit,
"HASH": None, "HASH": None,
"AUTOWANT": False "AUTOWANT": False,
"FORMAT": weekly['format']
}) })
if tmp_status == 'Wanted': if tmp_status == 'Wanted':
@ -2817,6 +2839,9 @@ class WebInterface(object):
logger.fdebug('[%s] Issue to renumber sequence from : %s' % (issuearcid, valid_readingorder)) logger.fdebug('[%s] Issue to renumber sequence from : %s' % (issuearcid, valid_readingorder))
reading_seq = 1 reading_seq = 1
for rc in sorted(readchk, key=itemgetter('ReadingOrder'), reverse=False): for rc in sorted(readchk, key=itemgetter('ReadingOrder'), reverse=False):
filename = None
if rc['Location'] is not None:
filename = ntpath.basename(rc['Location'])
if str(issuearcid) == str(rc['IssueArcID']): if str(issuearcid) == str(rc['IssueArcID']):
logger.fdebug('new order sequence detected at #: %s' % valid_readingorder) logger.fdebug('new order sequence detected at #: %s' % valid_readingorder)
if valid_readingorder > int(rc['ReadingOrder']): if valid_readingorder > int(rc['ReadingOrder']):
@ -2839,10 +2864,8 @@ class WebInterface(object):
else: else:
#valid_readingorder #valid_readingorder
if valid_readingorder < old_reading_seq: if valid_readingorder < old_reading_seq:
logger.info('2')
reading_seq = int(rc['ReadingOrder']) reading_seq = int(rc['ReadingOrder'])
else: else:
logger.info('3')
reading_seq = oldreading_seq +1 reading_seq = oldreading_seq +1
logger.fdebug('old sequence discovered at %s to %s' % (oldreading_seq, reading_seq)) logger.fdebug('old sequence discovered at %s to %s' % (oldreading_seq, reading_seq))
oldreading_seq = None oldreading_seq = None
@ -2855,7 +2878,8 @@ class WebInterface(object):
logger.fdebug('reordering existing sequence as lower sequence has changed. Altering from %s to %s' % (rc['ReadingOrder'], reading_seq)) logger.fdebug('reordering existing sequence as lower sequence has changed. Altering from %s to %s' % (rc['ReadingOrder'], reading_seq))
new_readorder.append({'IssueArcID': IssueArcID, new_readorder.append({'IssueArcID': IssueArcID,
'IssueID': issueid, 'IssueID': issueid,
'ReadingOrder': reading_seq}) 'ReadingOrder': reading_seq,
'filename': filename})
#we resequence in the following way: #we resequence in the following way:
# everything before the new reading number stays the same # everything before the new reading number stays the same
@ -2866,6 +2890,14 @@ class WebInterface(object):
#newrl = 0 #newrl = 0
for rl in sorted(new_readorder, key=itemgetter('ReadingOrder'), reverse=False): for rl in sorted(new_readorder, key=itemgetter('ReadingOrder'), reverse=False):
if rl['filename'] is not None:
try:
if int(rl['ReadingOrder']) != int(rl['filename'][:rl['filename'].find('-')]) and mylar.CONFIG.READ2FILENAME is True:
logger.fdebug('Order-Change: %s TO %s' % (int(rl['filename'][:rl['filename'].find('-')]), int(rl['ReadingOrder'])))
logger.fdebug('%s to %s' % (rl['filename'], helpers.renamefile_readingorder(rl['ReadingOrder']) + '-' + rl['filename'][rl['filename'].find('-')+1:]))
except:
pass
rl_ctrl = {"IssueID": rl['IssueID'], rl_ctrl = {"IssueID": rl['IssueID'],
"IssueArcID": rl['IssueArcID'], "IssueArcID": rl['IssueArcID'],
"StoryArcID": storyarcid} "StoryArcID": storyarcid}
@ -3156,7 +3188,8 @@ class WebInterface(object):
logger.info("No Story Arcs to search") logger.info("No Story Arcs to search")
else: else:
#cycle through the story arcs here for matches on the watchlist #cycle through the story arcs here for matches on the watchlist
arcdir = helpers.filesafe(ArcWatch[0]['StoryArc']) arcname = ArcWatch[0]['StoryArc']
arcdir = helpers.filesafe(arcname)
arcpub = ArcWatch[0]['Publisher'] arcpub = ArcWatch[0]['Publisher']
if arcpub is None: if arcpub is None:
arcpub = ArcWatch[0]['IssuePublisher'] arcpub = ArcWatch[0]['IssuePublisher']
@ -3441,6 +3474,8 @@ class WebInterface(object):
myDB.upsert("storyarcs", newVal, ctrlVal) myDB.upsert("storyarcs", newVal, ctrlVal)
logger.info("Marked " + issue['ComicName'] + " :# " + issue['Issue_Number'] + " as " + issue['Status']) logger.info("Marked " + issue['ComicName'] + " :# " + issue['Issue_Number'] + " as " + issue['Status'])
arcstats = self.storyarc_main(StoryArcID)
logger.info('[STORY-ARCS] Completed Missing/Recheck Files for %s [%s / %s]' % (arcname, arcstats['Have'], arcstats['TotalIssues']))
return return
ArcWatchlist.exposed = True ArcWatchlist.exposed = True

4
mylar/weeklypull.py
View File

@ -46,7 +46,7 @@ def pullit(forcecheck=None, weeknumber=None, year=None):
except (sqlite3.OperationalError, TypeError), msg: except (sqlite3.OperationalError, TypeError), msg:
logger.info(u"Error Retrieving weekly pull list - attempting to adjust") logger.info(u"Error Retrieving weekly pull list - attempting to adjust")
myDB.action("DROP TABLE weekly") myDB.action("DROP TABLE weekly")
myDB.action("CREATE TABLE IF NOT EXISTS weekly (SHIPDATE TEXT, PUBLISHER TEXT, ISSUE TEXT, COMIC VARCHAR(150), EXTRA TEXT, STATUS TEXT, ComicID TEXT, IssueID TEXT, CV_Last_Update TEXT, DynamicName TEXT, weeknumber TEXT, year TEXT, volume TEXT, seriesyear TEXT, annuallink TEXT, rowid INTEGER PRIMARY KEY)") myDB.action("CREATE TABLE IF NOT EXISTS weekly (SHIPDATE TEXT, PUBLISHER TEXT, ISSUE TEXT, COMIC VARCHAR(150), EXTRA TEXT, STATUS TEXT, ComicID TEXT, IssueID TEXT, CV_Last_Update TEXT, DynamicName TEXT, weeknumber TEXT, year TEXT, volume TEXT, seriesyear TEXT, annuallink TEXT, format TEXT, rowid INTEGER PRIMARY KEY)")
pulldate = '00000000' pulldate = '00000000'
logger.fdebug(u"Table re-created, trying to populate") logger.fdebug(u"Table re-created, trying to populate")
else: else:
@ -440,7 +440,7 @@ def pullit(forcecheck=None, weeknumber=None, year=None):
logger.info(u"Populating the NEW Weekly Pull list into Mylar for week " + str(weeknumber)) logger.info(u"Populating the NEW Weekly Pull list into Mylar for week " + str(weeknumber))
myDB.action("drop table if exists weekly") myDB.action("drop table if exists weekly")
myDB.action("CREATE TABLE IF NOT EXISTS weekly (SHIPDATE, PUBLISHER TEXT, ISSUE TEXT, COMIC VARCHAR(150), EXTRA TEXT, STATUS TEXT, ComicID TEXT, IssueID TEXT, CV_Last_Update TEXT, DynamicName TEXT, weeknumber TEXT, year TEXT, volume TEXT, seriesyear TEXT, annuallink TEXT, rowid INTEGER PRIMARY KEY)") myDB.action("CREATE TABLE IF NOT EXISTS weekly (SHIPDATE, PUBLISHER TEXT, ISSUE TEXT, COMIC VARCHAR(150), EXTRA TEXT, STATUS TEXT, ComicID TEXT, IssueID TEXT, CV_Last_Update TEXT, DynamicName TEXT, weeknumber TEXT, year TEXT, volume TEXT, seriesyear TEXT, annuallink TEXT, format TEXT, rowid INTEGER PRIMARY KEY)")
csvfile = open(newfl, "rb") csvfile = open(newfl, "rb")
creader = csv.reader(csvfile, delimiter='\t') creader = csv.reader(csvfile, delimiter='\t')

2
mylar/wwt.py
View File

@ -14,7 +14,7 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Mylar. If not, see <http://www.gnu.org/licenses/>. # along with Mylar. If not, see <http://www.gnu.org/licenses/>.
import lib.requests as requests import requests
from bs4 import BeautifulSoup, UnicodeDammit from bs4 import BeautifulSoup, UnicodeDammit
import urlparse import urlparse
import re import re