IMP: KAT torrent searches will now work (was a gzip issue), FIX:(#526) NzbGet support was broken (fixed - thanks ministoat), IMP: Merged Manual Run GUI option now in idirectory tabbed screen, FIX:(#524) ComicID not defined in updater.py

This commit is contained in:
evilhero 2013-09-04 22:36:41 -04:00
parent 67ca74fc65
commit 0db2968d36
7 changed files with 207 additions and 84 deletions

View File

@ -18,6 +18,8 @@
<div id="tabs">
<ul>
<li><a href="#tabs-1">Scan Comic Library</a></li>
<li><a href="#tabs-2">Manual Post-Processing</a></li>
<li><a href="#tabs-3">Advanced Options</a></li>
</ul>
<div id="tabs-1" class="configtable">
<form action="comicScan" method="GET" id="comicScan">
@ -31,7 +33,6 @@
as soon as you click 'Save changes'
</p>
<br/>
<p><strong>THIS IS CURRENTLY DISABLED UNTIL WORKING..</strong></p>
<div class="row">
<label for="">Path to directory</label>
%if mylar.COMIC_DIR:
@ -41,7 +42,7 @@
(this.value==this.defaultValue) this.value='';" name="path" size="70" />
%endif
</div>
<div class="row checkbox">
<!-- <div class="row checkbox">
<input type="checkbox" name="libraryscan" id="libraryscan" value="1" ${checked(mylar.LIBRARYSCAN)}><label>Automatically Scan Library</label>
</div>
<div class="row checkbox">
@ -51,22 +52,81 @@
<input type="checkbox" style="vertical-align: middle; margin: 3px; margin-top: -1px;" name="imp_metadata" id="imp_metadata" value="1" ${checked(mylar.IMP_METADATA)}><label>Use existing Metadata</label>
<small>Use existing Metadata to better locate series for import</small>
</div>
-->
<div class="row checkbox">
<input type="checkbox" style="vertical-align: middle; margin: 3px; margin-top: -1px;" name="imp_move" onclick="initConfigCheckbox($this));" id="imp_move" value="1" ${checked(mylar.IMP_MOVE)}><label>Move files into corresponding Series directory</label>
<small>Leaving this unchecked will not move anything, but will mark the issues as Archived</small>
</div>
<div class="config">
<div class="row checkbox">
<input type="checkbox" name="imp_rename" id="imp_rename" value="1" ${checked(mylar.IMP_RENAME)}><label>Rename Files </label>
<small>Rename files to configuration settings</small>
</div>
</div>
<br/>
<input type="button" value="Save Changes and Scan" onclick="addScanAction();doAjaxCall('comicScan',$(this),'tabs',true);return false;" data-success="Changes saved. Library will be scanned">
<input type="button" value="Save Changes without Scanning Library" onclick="doAjaxCall('comicScan',$(this),'tabs',true);return false;" data-success="Changes Saved Successfully">
<br/>
<input type="button" value="Save Changes and Scan" onclick="addScanAction();doAjaxCall('comicScan',$(this),'tabs',true);return false;" data-success="Changes saved. Library will be scanned">
<input type="button" value="Save Changes without Scanning Library" onclick="doAjaxCall('comicScan',$(this),'tabs',true);return false;" data-success="Changes Saved Successfully">
</fieldset>
</form>
</div>
<div id="tabs-2" class="configtable">
<tr>
<td>
<form action="post_process" method="GET" id="post_process">
<fieldset>
<legend>Manual Run</legend>
<p><strong>Manual Post-Processing</strong></p>
<p>You can put in any directory, and it will scan for comic files in that folder
(including all subdirectories) that exist on your watchlist.
<br/><small>For example: '/Users/name/Comics'</small></p>
<p>
It may take a while depending on how many files you have. You can navigate away from this
as soon as you click 'Go'.
</p>
<br/>
<p><strong>Only series' that exist on your watchlist will be post-processed with your current
post-processing options.<br/> The remainder will not be moved from the given directory</strong></p>
<div class="row">
<label for="">Path to Manually Run</label>
<input type="text" value="Enter the full path to post-process" name="nzb_folder" size="70" />
<input type="hidden" name="nzb_name" value='Manual Run' />
</div>
<input type="button" value="Go Manual Run!" onclick="doAjaxCall('post_process',$(this),'tabs',true);return false;">
</fieldset>
</form>
</td>
</tr>
</div>
<div id="tabs-3">
<table class="configtable" summary="Advanced Options">
<tr>
<td>
<fieldset>
<div class="links">
<legend>Force Options</legend>
<a href="#" onclick="doAjaxCall('forceSearch',$(this))" data-success="Checking for wanted issues successful" data-error="Error checking wanted issues"><span class="ui-icon ui-icon-search"></span>Force Check for Wanted Issues</a>
<a href="#" onclick="doAjaxCall('forceUpdate',$(this))" data-success="Update active comics successful" data-error="Error forcing update Comics"><span class="ui-icon ui-icon-heart"></span>Force Update Active Comics</a>
<a href="#" onclick="doAjaxCall('checkGithub',$(this))" data-success="Checking for update successful" data-error="Error checking for update"><span class="ui-icon ui-icon-refresh"></span>Check for mylar Updates</a>
</div>
</fieldset>
</td>
<td>
<fieldset>
<legend>Export</legend>
<div class="links">
<a href="#" onclick="doAjaxCall('wanted_Export',$(this))" data-sucess="Exported to Wanted list." data-error="Failed to export. Check logs"><span class="ui-icon ui-icon-refresh"></span>Export Wanted to CSV</a>
</div>
<br/><br/>
<legend>Hidden Options</legend>
<div classs="links">
<a href="readlist">Reading List Management</a><br/>
<a href="importResults">Import Results Management</a>
</div>
</fieldset>
</td>
</tr>
</table>
</div>
</div>
</%def>

View File

@ -280,7 +280,12 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None):
if justthedigits.isdigit():
digitsvalid = "true"
else:
digitsvalid = "false"
if '.' in justthedigits:
logger.fdebug("decimals")
digitsvalid = "true"
else:
logger.fdebug("no decimals")
digitsvalid = "false"
if justthedigits.lower() == 'annual':
logger.fdebug('ANNUAL [' + tmpthedigits.split(' ', 1)[1] + ']')
@ -291,6 +296,10 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None):
try:
if tmpthedigits.split(' ', 1)[1] is not None:
poss_alpha = tmpthedigits.split(' ', 1)[1]
if poss_alpha.isdigit():
logger.fdebug("decimal issue detected (filename space seperate most likely '.')")
digitsvalid = "true"
justthedigits += '.' + poss_alpha
for issexcept in issue_exceptions:
if issexcept.lower() in poss_alpha.lower() and len(poss_alpha) <= len(issexcept):
justthedigits += poss_alpha

View File

@ -675,11 +675,19 @@ def issuedigits(issnum):
x = 0
tstord = None
issno = None
invchk = "false"
while (x < len(issnum)):
if issnum[x].isalpha():
#take first occurance of alpha in string and carry it through
tstord = issnum[x:].rstrip()
issno = issnum[:x].rstrip()
try:
isschk = float(issno)
except ValueError, e:
logger.fdebug('invalid numeric for issue - cannot be found. Ignoring.')
issno = None
tstord = None
invchk = "true"
break
x+=1
if tstord is not None and issno is not None:
@ -695,6 +703,9 @@ def issuedigits(issnum):
logger.fdebug('issno: ' + str(issno))
int_issnum = (int(issno) * 1000) + ordtot
logger.fdebug('intissnum : ' + str(int_issnum))
elif invchk == "true":
logger.fdebug('this does not have an issue # that I can parse properly.')
int_issnum = 999999999999999
else:
logger.error(str(issnum) + 'this has an alpha-numeric in the issue # which I cannot account for.')
int_issnum = 999999999999999

View File

@ -643,11 +643,19 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None):
x = 0
tstord = None
issno = None
invchk = "false"
while (x < len(issnum)):
if issnum[x].isalpha():
#take first occurance of alpha in string and carry it through
tstord = issnum[x:].rstrip()
issno = issnum[:x].rstrip()
try:
isschk = float(issno)
except ValueError, e:
logger.fdebug('invalid numeric for issue - cannot be found. Ignoring.')
issno = None
tstord = None
invchk = "true"
break
x+=1
if tstord is not None and issno is not None:
@ -660,6 +668,9 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None):
logger.fdebug('issno: ' + str(issno))
int_issnum = (int(issno) * 1000) + ordtot
logger.fdebug('intissnum : ' + str(int_issnum))
elif invchk == "true":
logger.fdebug('this does not have an issue # that I can parse properly.')
return
else:
logger.error(str(issnum) + ' this has an alpha-numeric in the issue # which I cannot account for.')
return

View File

@ -6,6 +6,8 @@ import lib.feedparser as feedparser
import urllib2
import ftpsshup
import datetime
import gzip
from StringIO import StringIO
import mylar
from mylar import db, logger, ftpsshup, helpers
@ -83,7 +85,7 @@ def torrents(pickfeed=None,seriesname=None,issue=None):
else:
logger.error('invalid pickfeed denoted...')
return
logger.fdebug('feed #' + str(pickfeed) + ' chosen: ' + str(feed))
title = []
link = []
description = []
@ -330,7 +332,6 @@ def rssdbupdate(feeddata,i,type):
def torrentdbsearch(seriesname,issue,comicid=None,nzbprov=None):
myDB = db.DBConnection()
seriesname_alt = None
print "seriesname:" + str(seriesname)
if comicid is None or comicid == 'None':
pass
else:
@ -376,7 +377,13 @@ def torrentdbsearch(seriesname,issue,comicid=None,nzbprov=None):
AS_Altrem = re.sub("\\bthe\\b", "", AS_Altrem.lower())
AS_Alternate = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\+\'\&\?\@\s]', '%', AS_Altrem)
AS_Alt.append(AS_Alternate)
AS_Altrem_mod = re.sub('[\&]', ' ', AS_Altrem)
AS_formatrem_seriesname = re.sub('[\'\!\@\#\$\%\:\;\/\\=\?\.]', '',AS_Altrem_mod)
AS_formatrem_seriesname = re.sub('\s+', ' ', AS_formatrem_seriesname)
if AS_formatrem_seriesname[:1] == ' ': AS_formatrem_seriesname = AS_formatrem_seriesname[1:]
AS_Alt.append(AS_formatrem_seriesname)
AS_Alternate += '%'
if mylar.ENABLE_CBT:
@ -543,11 +550,9 @@ def torsend2client(seriesname, linkit, site):
logger.info(linkit)
linkit = str(linkit) + '&passkey=' + str(mylar.CBT_PASSKEY)
if linkit[-7:] != "torrent":
if linkit[-7:] != "torrent" and site != "KAT":
filename += ".torrent"
request = urllib2.Request(linkit)
request.add_header('User-Agent', str(mylar.USER_AGENT))
if mylar.TORRENT_LOCAL and mylar.LOCAL_WATCHDIR is not None:
filepath = os.path.join(mylar.LOCAL_WATCHDIR, filename)
logger.fdebug('filename for torrent set to : ' + filepath)
@ -559,12 +564,33 @@ def torsend2client(seriesname, linkit, site):
return "fail"
try:
opener = helpers.urlretrieve(urllib2.urlopen(request), filepath)
request = urllib2.Request(linkit)
#request.add_header('User-Agent', str(mylar.USER_AGENT))
request.add_header('Accept-encoding', 'gzip')
if site == 'KAT':
request.add_header('Referer', 'http://kat.ph/')
# response = helpers.urlretrieve(urllib2.urlopen(request), filepath)
response = urllib2.urlopen(request)
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO(response.read())
f = gzip.GzipFile(fileobj=buf)
torrent = f.read()
else:
torrent = response.read()
except Exception, e:
logger.warn('Error fetching data from %s: %s' % (site, e))
return "fail"
logger.fdebug('torrent file saved as : ' + str(filepath))
with open(filepath, 'w') as the_file:
the_file.write(torrent)
logger.info("saved.")
#logger.fdebug('torrent file saved as : ' + str(filepath))
if mylar.TORRENT_LOCAL:
return "pass"
#remote_file = urllib2.urlopen(linkit)
@ -579,6 +605,7 @@ def torsend2client(seriesname, linkit, site):
tssh = ftpsshup.putfile(filepath,filename)
return tssh
if __name__ == '__main__':
#torrents(sys.argv[1])
#torrentdbsearch(sys.argv[1], sys.argv[2], sys.argv[3])

View File

@ -505,7 +505,7 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is
logger.fdebug("Sending request to [KAT] for " + str(cmname) + " : " + str(mod_isssearch))
bb = rsscheck.torrents(pickfeed='2',seriesname=cmname,issue=mod_isssearch)
rss = "no"
if bb is not None: logger.fdebug("results: " + str(bb))
#if bb is not None: logger.fdebug("results: " + str(bb))
elif nzbprov != 'experimental':
if nzbprov == 'dognzb':
findurl = "http://dognzb.cr/api?t=search&q=" + str(comsearch) + "&o=xml&cat=7030"
@ -1115,7 +1115,7 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is
tmpapi = str(tmpapi) + str(mylar.NZBGET_USERNAME) + ":" + str(mylar.NZBGET_PASSWORD)
tmpapi = str(tmpapi) + "@" + str(nzbget_host) + ":" + str(mylar.NZBGET_PORT) + "/xmlrpc"
server = ServerProxy(tmpapi)
send_to_nzbget = server.appendurl(nzbname, str(mylar.NZBGET_CATEGORY), int(nzbgetpriority), True, linkapi)
send_to_nzbget = server.appendurl(nzbname + ".nzb", str(mylar.NZBGET_CATEGORY), int(nzbgetpriority), True, linkapi)
sent_to = "NZBGet"
if send_to_nzbget is True:
logger.info("Successfully sent nzb to NZBGet!")

View File

@ -73,7 +73,7 @@ def dbUpdate(ComicIDList=None):
annuals = []
ann_list = []
if mylar.ANNUALS_ON:
annuals_list = myDB.select('SELECT * FROM annuals WHERE ComicID=?', [ComicID])
annuals_list = myDB.select('SELECT * FROM annuals WHERE ComicID=?', [comicid])
ann_list += annuals_list
issues_new += annuals_list
@ -202,6 +202,10 @@ def upcoming_update(ComicID, ComicName, IssueNumber, IssueDate, forcecheck=None)
values = { "Status": "Skipped"}
newValue['Status'] = "Skipped"
#was in wrong place :(
else:
logger.fdebug('Issues do not match for some reason...weekly new issue: ' + str(IssueNumber))
return
if mylar.AUTOWANT_UPCOMING:
#for issues not in db - to be added to Upcoming table.
if issuechk is None:
@ -216,36 +220,37 @@ def upcoming_update(ComicID, ComicName, IssueNumber, IssueDate, forcecheck=None)
logger.fdebug('...Status already Wanted .. not changing.')
else:
logger.fdebug('...Already have issue - keeping existing status of : ' + issuechk['Status'])
if issuechk is None:
myDB.upsert("upcoming", newValue, controlValue)
else:
logger.fdebug('--attempt to find errant adds to Wanted list')
logger.fdebug('UpcomingNewValue: ' + str(newValue))
logger.fdebug('UpcomingcontrolValue: ' + str(controlValue))
if issuechk['IssueDate'] == '0000-00-00' and newValue['IssueDate'] != '0000-00-00':
logger.fdebug('Found a 0000-00-00 issue - force updating series to try and get it proper.')
dateVal = {"IssueDate": newValue['IssueDate'],
"ComicName": issuechk['ComicName'],
"Status": newValue['Status'],
"IssueNumber": issuechk['Issue_Number']}
logger.fdebug('updating date in upcoming table to : ' + str(newValue['IssueDate']))
logger.fdebug('ComicID:' + str(controlValue))
myDB.upsert("upcoming", dateVal, controlValue)
logger.fdebug('Temporarily putting the Issue Date for ' + str(issuechk['Issue_Number']) + ' to ' + str(newValue['IssueDate']))
values = {"IssueDate": newValue['IssueDate']}
#if ComicID[:1] == "G": mylar.importer.GCDimport(ComicID,pullupd='yes')
#else: mylar.importer.addComictoDB(ComicID,mismatch,pullupd='yes')
if 'annual' in ComicName.lower():
myDB.upsert("annuals", values, control)
else:
myDB.upsert("issues", values, control)
if issuechk['Status'] == 'Downloaded':
logger.fdebug('updating Pull-list to reflect status.')
downstats = {"Status": issuechk['Status'],
"ComicID": issuechk['ComicID']}
return downstats
if issuechk is None:
myDB.upsert("upcoming", newValue, controlValue)
else:
logger.fdebug('Issues do not match for some reason...weekly new issue: ' + str(IssueNumber))
logger.fdebug('--attempt to find errant adds to Wanted list')
logger.fdebug('UpcomingNewValue: ' + str(newValue))
logger.fdebug('UpcomingcontrolValue: ' + str(controlValue))
if issuechk['IssueDate'] == '0000-00-00' and newValue['IssueDate'] != '0000-00-00':
logger.fdebug('Found a 0000-00-00 issue - force updating series to try and get it proper.')
dateVal = {"IssueDate": newValue['IssueDate'],
"ComicName": issuechk['ComicName'],
"Status": newValue['Status'],
"IssueNumber": issuechk['Issue_Number']}
logger.fdebug('updating date in upcoming table to : ' + str(newValue['IssueDate']))
logger.fdebug('ComicID:' + str(controlValue))
myDB.upsert("upcoming", dateVal, controlValue)
logger.fdebug('Temporarily putting the Issue Date for ' + str(issuechk['Issue_Number']) + ' to ' + str(newValue['IssueDate']))
values = {"IssueDate": newValue['IssueDate']}
#if ComicID[:1] == "G": mylar.importer.GCDimport(ComicID,pullupd='yes')
#else: mylar.importer.addComictoDB(ComicID,mismatch,pullupd='yes')
if 'annual' in ComicName.lower():
myDB.upsert("annuals", values, control)
else:
myDB.upsert("issues", values, control)
if issuechk['Status'] == 'Downloaded':
logger.fdebug('updating Pull-list to reflect status.')
downstats = {"Status": issuechk['Status'],
"ComicID": issuechk['ComicID']}
return downstats
def weekly_update(ComicName,IssueNumber,CStatus,CID):
@ -419,43 +424,43 @@ def forceRescan(ComicID,archive=None):
if temploc.lower().endswith(extensions):
logger.fdebug('removed extension for issue: ' + str(temploc))
temploc = temploc[:-4]
deccnt = str(temploc).count('.')
if deccnt > 1:
# deccnt = str(temploc).count('.')
# if deccnt > 1:
#logger.fdebug('decimal counts are :' + str(deccnt))
#if the file is formatted with '.' in place of spaces we need to adjust.
#before replacing - check to see if digits on either side of decimal and if yes, DON'T REMOVE
occur=1
prevstart = 0
digitfound = "no"
decimalfound = "no"
tempreconstruct = ''
while (occur <= deccnt):
n = occur
start = temploc.find('.')
while start >=0 and n > 1:
start = temploc.find('.', start+len('.'))
n-=1
#logger.fdebug('occurance ' + str(occur) + ' of . at position: ' + str(start))
if temploc[prevstart:start].isdigit():
if digitfound == "yes":
#logger.fdebug('this is a decimal, assuming decimal issue.')
decimalfound = "yes"
reconst = "." + temploc[prevstart:start] + " "
else:
#logger.fdebug('digit detected.')
digitfound = "yes"
reconst = temploc[prevstart:start]
else:
reconst = temploc[prevstart:start] + " "
#logger.fdebug('word: ' + reconst)
tempreconstruct = tempreconstruct + reconst
#logger.fdebug('tempreconstruct is : ' + tempreconstruct)
prevstart = (start+1)
occur+=1
#logger.fdebug('word: ' + temploc[prevstart:])
tempreconstruct = tempreconstruct + " " + temploc[prevstart:]
#logger.fdebug('final filename to use is : ' + str(tempreconstruct))
temploc = tempreconstruct
# occur=1
# prevstart = 0
# digitfound = "no"
# decimalfound = "no"
# tempreconstruct = ''
# while (occur <= deccnt):
# n = occur
# start = temploc.find('.')
# while start >=0 and n > 1:
# start = temploc.find('.', start+len('.'))
# n-=1
# #logger.fdebug('occurance ' + str(occur) + ' of . at position: ' + str(start))
# if temploc[prevstart:start].isdigit():
# if digitfound == "yes":
# #logger.fdebug('this is a decimal, assuming decimal issue.')
# decimalfound = "yes"
# reconst = "." + temploc[prevstart:start] + " "
# else:
# #logger.fdebug('digit detected.')
# digitfound = "yes"
# reconst = temploc[prevstart:start]
# else:
# reconst = temploc[prevstart:start] + " "
# #logger.fdebug('word: ' + reconst)
# tempreconstruct = tempreconstruct + reconst
# #logger.fdebug('tempreconstruct is : ' + tempreconstruct)
# prevstart = (start+1)
# occur+=1
# #logger.fdebug('word: ' + temploc[prevstart:])
# tempreconstruct = tempreconstruct + " " + temploc[prevstart:]
# #logger.fdebug('final filename to use is : ' + str(tempreconstruct))
# temploc = tempreconstruct
#logger.fdebug("checking " + str(temploc))
#fcnew_b4 = shlex.split(str(temploc))
fcnew_af = re.findall('[^\()]+', temploc)
@ -515,8 +520,8 @@ def forceRescan(ComicID,archive=None):
fcdigit = helpers.issuedigits(fcnew[som])
#logger.fdebug("fcdigit: " + str(fcdigit))
#logger.fdebug("int_iss: " + str(int_iss))
logger.fdebug("fcdigit: " + str(fcdigit))
logger.fdebug("int_iss: " + str(int_iss))
if int(fcdigit) == int_iss:
logger.fdebug('issue match - fcdigit: ' + str(fcdigit) + ' ... int_iss: ' + str(int_iss))