diff --git a/data/interfaces/default/idirectory.html b/data/interfaces/default/idirectory.html index fe72b213..e3bbee74 100755 --- a/data/interfaces/default/idirectory.html +++ b/data/interfaces/default/idirectory.html @@ -18,6 +18,8 @@
@@ -31,7 +33,6 @@ as soon as you click 'Save changes'


-

THIS IS CURRENTLY DISABLED UNTIL WORKING..

%if mylar.COMIC_DIR: @@ -41,7 +42,7 @@ (this.value==this.defaultValue) this.value='';" name="path" size="70" /> %endif
-
+
Leaving this unchecked will not move anything, but will mark the issues as Archived
-
Rename files to configuration settings -
-
- - +
+ +
+
+ + +
+
+ Manual Run +

Manual Post-Processing

+

You can put in any directory, and it will scan for comic files in that folder + (including all subdirectories) that exist on your watchlist. +
For example: '/Users/name/Comics'

+

+ It may take a while depending on how many files you have. You can navigate away from this + as soon as you click 'Go'. +

+
+

Only series' that exist on your watchlist will be post-processed with your current + post-processing options.
The remainder will not be moved from the given directory

+
+ + + +
+ +
+
+ + +
+ +
+ + + + + + +
+
+
+ Force Options + Force Check for Wanted Issues + Force Update Active Comics + Check for mylar Updates +
+
+ 		+
+ Export +
+ Export Wanted to CSV +
+

+ Hidden Options + +
+
+
diff --git a/mylar/filechecker.py b/mylar/filechecker.py index 61c2b5f4..b3d9c8fe 100755 --- a/mylar/filechecker.py +++ b/mylar/filechecker.py @@ -280,7 +280,12 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None): if justthedigits.isdigit(): digitsvalid = "true" else: - digitsvalid = "false" + if '.' in justthedigits: + logger.fdebug("decimals") + digitsvalid = "true" + else: + logger.fdebug("no decimals") + digitsvalid = "false" if justthedigits.lower() == 'annual': logger.fdebug('ANNUAL [' + tmpthedigits.split(' ', 1)[1] + ']') @@ -291,6 +296,10 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None): try: if tmpthedigits.split(' ', 1)[1] is not None: poss_alpha = tmpthedigits.split(' ', 1)[1] + if poss_alpha.isdigit(): + logger.fdebug("decimal issue detected (filename space seperate most likely '.')") + digitsvalid = "true" + justthedigits += '.' + poss_alpha for issexcept in issue_exceptions: if issexcept.lower() in poss_alpha.lower() and len(poss_alpha) <= len(issexcept): justthedigits += poss_alpha diff --git a/mylar/helpers.py b/mylar/helpers.py index b68437ca..4712379e 100755 --- a/mylar/helpers.py +++ b/mylar/helpers.py @@ -675,11 +675,19 @@ def issuedigits(issnum): x = 0 tstord = None issno = None + invchk = "false" while (x < len(issnum)): if issnum[x].isalpha(): #take first occurance of alpha in string and carry it through tstord = issnum[x:].rstrip() issno = issnum[:x].rstrip() + try: + isschk = float(issno) + except ValueError, e: + logger.fdebug('invalid numeric for issue - cannot be found. Ignoring.') + issno = None + tstord = None + invchk = "true" break x+=1 if tstord is not None and issno is not None: @@ -695,6 +703,9 @@ def issuedigits(issnum): logger.fdebug('issno: ' + str(issno)) int_issnum = (int(issno) * 1000) + ordtot logger.fdebug('intissnum : ' + str(int_issnum)) + elif invchk == "true": + logger.fdebug('this does not have an issue # that I can parse properly.') + int_issnum = 999999999999999 else: logger.error(str(issnum) + 'this has an alpha-numeric in the issue # which I cannot account for.') int_issnum = 999999999999999 diff --git a/mylar/importer.py b/mylar/importer.py index 5db6c3cc..2575aa01 100755 --- a/mylar/importer.py +++ b/mylar/importer.py @@ -643,11 +643,19 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None): x = 0 tstord = None issno = None + invchk = "false" while (x < len(issnum)): if issnum[x].isalpha(): #take first occurance of alpha in string and carry it through tstord = issnum[x:].rstrip() issno = issnum[:x].rstrip() + try: + isschk = float(issno) + except ValueError, e: + logger.fdebug('invalid numeric for issue - cannot be found. Ignoring.') + issno = None + tstord = None + invchk = "true" break x+=1 if tstord is not None and issno is not None: @@ -660,6 +668,9 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None): logger.fdebug('issno: ' + str(issno)) int_issnum = (int(issno) * 1000) + ordtot logger.fdebug('intissnum : ' + str(int_issnum)) + elif invchk == "true": + logger.fdebug('this does not have an issue # that I can parse properly.') + return else: logger.error(str(issnum) + ' this has an alpha-numeric in the issue # which I cannot account for.') return diff --git a/mylar/rsscheck.py b/mylar/rsscheck.py index ca79951b..84963cd0 100755 --- a/mylar/rsscheck.py +++ b/mylar/rsscheck.py @@ -6,6 +6,8 @@ import lib.feedparser as feedparser import urllib2 import ftpsshup import datetime +import gzip +from StringIO import StringIO import mylar from mylar import db, logger, ftpsshup, helpers @@ -83,7 +85,7 @@ def torrents(pickfeed=None,seriesname=None,issue=None): else: logger.error('invalid pickfeed denoted...') return - logger.fdebug('feed #' + str(pickfeed) + ' chosen: ' + str(feed)) + title = [] link = [] description = [] @@ -330,7 +332,6 @@ def rssdbupdate(feeddata,i,type): def torrentdbsearch(seriesname,issue,comicid=None,nzbprov=None): myDB = db.DBConnection() seriesname_alt = None - print "seriesname:" + str(seriesname) if comicid is None or comicid == 'None': pass else: @@ -376,7 +377,13 @@ def torrentdbsearch(seriesname,issue,comicid=None,nzbprov=None): AS_Altrem = re.sub("\\bthe\\b", "", AS_Altrem.lower()) AS_Alternate = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\+\'\&\?\@\s]', '%', AS_Altrem) - AS_Alt.append(AS_Alternate) + + AS_Altrem_mod = re.sub('[\&]', ' ', AS_Altrem) + AS_formatrem_seriesname = re.sub('[\'\!\@\#\$\%\:\;\/\\=\?\.]', '',AS_Altrem_mod) + AS_formatrem_seriesname = re.sub('\s+', ' ', AS_formatrem_seriesname) + if AS_formatrem_seriesname[:1] == ' ': AS_formatrem_seriesname = AS_formatrem_seriesname[1:] + AS_Alt.append(AS_formatrem_seriesname) + AS_Alternate += '%' if mylar.ENABLE_CBT: @@ -543,11 +550,9 @@ def torsend2client(seriesname, linkit, site): logger.info(linkit) linkit = str(linkit) + '&passkey=' + str(mylar.CBT_PASSKEY) - if linkit[-7:] != "torrent": + if linkit[-7:] != "torrent" and site != "KAT": filename += ".torrent" - request = urllib2.Request(linkit) - request.add_header('User-Agent', str(mylar.USER_AGENT)) if mylar.TORRENT_LOCAL and mylar.LOCAL_WATCHDIR is not None: filepath = os.path.join(mylar.LOCAL_WATCHDIR, filename) logger.fdebug('filename for torrent set to : ' + filepath) @@ -559,12 +564,33 @@ def torsend2client(seriesname, linkit, site): return "fail" try: - opener = helpers.urlretrieve(urllib2.urlopen(request), filepath) + request = urllib2.Request(linkit) + #request.add_header('User-Agent', str(mylar.USER_AGENT)) + request.add_header('Accept-encoding', 'gzip') + + if site == 'KAT': + request.add_header('Referer', 'http://kat.ph/') + + +# response = helpers.urlretrieve(urllib2.urlopen(request), filepath) + response = urllib2.urlopen(request) + + if response.info().get('Content-Encoding') == 'gzip': + buf = StringIO(response.read()) + f = gzip.GzipFile(fileobj=buf) + torrent = f.read() + else: + torrent = response.read() + except Exception, e: logger.warn('Error fetching data from %s: %s' % (site, e)) return "fail" - logger.fdebug('torrent file saved as : ' + str(filepath)) + with open(filepath, 'w') as the_file: + the_file.write(torrent) + + logger.info("saved.") + #logger.fdebug('torrent file saved as : ' + str(filepath)) if mylar.TORRENT_LOCAL: return "pass" #remote_file = urllib2.urlopen(linkit) @@ -579,6 +605,7 @@ def torsend2client(seriesname, linkit, site): tssh = ftpsshup.putfile(filepath,filename) return tssh + if __name__ == '__main__': #torrents(sys.argv[1]) #torrentdbsearch(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/mylar/search.py b/mylar/search.py index edec31e0..6964ea9b 100755 --- a/mylar/search.py +++ b/mylar/search.py @@ -505,7 +505,7 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is logger.fdebug("Sending request to [KAT] for " + str(cmname) + " : " + str(mod_isssearch)) bb = rsscheck.torrents(pickfeed='2',seriesname=cmname,issue=mod_isssearch) rss = "no" - if bb is not None: logger.fdebug("results: " + str(bb)) + #if bb is not None: logger.fdebug("results: " + str(bb)) elif nzbprov != 'experimental': if nzbprov == 'dognzb': findurl = "http://dognzb.cr/api?t=search&q=" + str(comsearch) + "&o=xml&cat=7030" @@ -1115,7 +1115,7 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is tmpapi = str(tmpapi) + str(mylar.NZBGET_USERNAME) + ":" + str(mylar.NZBGET_PASSWORD) tmpapi = str(tmpapi) + "@" + str(nzbget_host) + ":" + str(mylar.NZBGET_PORT) + "/xmlrpc" server = ServerProxy(tmpapi) - send_to_nzbget = server.appendurl(nzbname, str(mylar.NZBGET_CATEGORY), int(nzbgetpriority), True, linkapi) + send_to_nzbget = server.appendurl(nzbname + ".nzb", str(mylar.NZBGET_CATEGORY), int(nzbgetpriority), True, linkapi) sent_to = "NZBGet" if send_to_nzbget is True: logger.info("Successfully sent nzb to NZBGet!") diff --git a/mylar/updater.py b/mylar/updater.py index 001efd3f..e06875ac 100755 --- a/mylar/updater.py +++ b/mylar/updater.py @@ -73,7 +73,7 @@ def dbUpdate(ComicIDList=None): annuals = [] ann_list = [] if mylar.ANNUALS_ON: - annuals_list = myDB.select('SELECT * FROM annuals WHERE ComicID=?', [ComicID]) + annuals_list = myDB.select('SELECT * FROM annuals WHERE ComicID=?', [comicid]) ann_list += annuals_list issues_new += annuals_list @@ -202,6 +202,10 @@ def upcoming_update(ComicID, ComicName, IssueNumber, IssueDate, forcecheck=None) values = { "Status": "Skipped"} newValue['Status'] = "Skipped" #was in wrong place :( + else: + logger.fdebug('Issues do not match for some reason...weekly new issue: ' + str(IssueNumber)) + return + if mylar.AUTOWANT_UPCOMING: #for issues not in db - to be added to Upcoming table. if issuechk is None: @@ -216,36 +220,37 @@ def upcoming_update(ComicID, ComicName, IssueNumber, IssueDate, forcecheck=None) logger.fdebug('...Status already Wanted .. not changing.') else: logger.fdebug('...Already have issue - keeping existing status of : ' + issuechk['Status']) - if issuechk is None: - myDB.upsert("upcoming", newValue, controlValue) - else: - logger.fdebug('--attempt to find errant adds to Wanted list') - logger.fdebug('UpcomingNewValue: ' + str(newValue)) - logger.fdebug('UpcomingcontrolValue: ' + str(controlValue)) - if issuechk['IssueDate'] == '0000-00-00' and newValue['IssueDate'] != '0000-00-00': - logger.fdebug('Found a 0000-00-00 issue - force updating series to try and get it proper.') - dateVal = {"IssueDate": newValue['IssueDate'], - "ComicName": issuechk['ComicName'], - "Status": newValue['Status'], - "IssueNumber": issuechk['Issue_Number']} - logger.fdebug('updating date in upcoming table to : ' + str(newValue['IssueDate'])) - logger.fdebug('ComicID:' + str(controlValue)) - myDB.upsert("upcoming", dateVal, controlValue) - logger.fdebug('Temporarily putting the Issue Date for ' + str(issuechk['Issue_Number']) + ' to ' + str(newValue['IssueDate'])) - values = {"IssueDate": newValue['IssueDate']} - #if ComicID[:1] == "G": mylar.importer.GCDimport(ComicID,pullupd='yes') - #else: mylar.importer.addComictoDB(ComicID,mismatch,pullupd='yes') - if 'annual' in ComicName.lower(): - myDB.upsert("annuals", values, control) - else: - myDB.upsert("issues", values, control) - if issuechk['Status'] == 'Downloaded': - logger.fdebug('updating Pull-list to reflect status.') - downstats = {"Status": issuechk['Status'], - "ComicID": issuechk['ComicID']} - return downstats + + if issuechk is None: + myDB.upsert("upcoming", newValue, controlValue) else: - logger.fdebug('Issues do not match for some reason...weekly new issue: ' + str(IssueNumber)) + logger.fdebug('--attempt to find errant adds to Wanted list') + logger.fdebug('UpcomingNewValue: ' + str(newValue)) + logger.fdebug('UpcomingcontrolValue: ' + str(controlValue)) + if issuechk['IssueDate'] == '0000-00-00' and newValue['IssueDate'] != '0000-00-00': + logger.fdebug('Found a 0000-00-00 issue - force updating series to try and get it proper.') + dateVal = {"IssueDate": newValue['IssueDate'], + "ComicName": issuechk['ComicName'], + "Status": newValue['Status'], + "IssueNumber": issuechk['Issue_Number']} + logger.fdebug('updating date in upcoming table to : ' + str(newValue['IssueDate'])) + logger.fdebug('ComicID:' + str(controlValue)) + myDB.upsert("upcoming", dateVal, controlValue) + logger.fdebug('Temporarily putting the Issue Date for ' + str(issuechk['Issue_Number']) + ' to ' + str(newValue['IssueDate'])) + values = {"IssueDate": newValue['IssueDate']} + #if ComicID[:1] == "G": mylar.importer.GCDimport(ComicID,pullupd='yes') + #else: mylar.importer.addComictoDB(ComicID,mismatch,pullupd='yes') + + if 'annual' in ComicName.lower(): + myDB.upsert("annuals", values, control) + else: + myDB.upsert("issues", values, control) + + if issuechk['Status'] == 'Downloaded': + logger.fdebug('updating Pull-list to reflect status.') + downstats = {"Status": issuechk['Status'], + "ComicID": issuechk['ComicID']} + return downstats def weekly_update(ComicName,IssueNumber,CStatus,CID): @@ -419,43 +424,43 @@ def forceRescan(ComicID,archive=None): if temploc.lower().endswith(extensions): logger.fdebug('removed extension for issue: ' + str(temploc)) temploc = temploc[:-4] - deccnt = str(temploc).count('.') - if deccnt > 1: +# deccnt = str(temploc).count('.') +# if deccnt > 1: #logger.fdebug('decimal counts are :' + str(deccnt)) #if the file is formatted with '.' in place of spaces we need to adjust. #before replacing - check to see if digits on either side of decimal and if yes, DON'T REMOVE - occur=1 - prevstart = 0 - digitfound = "no" - decimalfound = "no" - tempreconstruct = '' - while (occur <= deccnt): - n = occur - start = temploc.find('.') - while start >=0 and n > 1: - start = temploc.find('.', start+len('.')) - n-=1 - #logger.fdebug('occurance ' + str(occur) + ' of . at position: ' + str(start)) - if temploc[prevstart:start].isdigit(): - if digitfound == "yes": - #logger.fdebug('this is a decimal, assuming decimal issue.') - decimalfound = "yes" - reconst = "." + temploc[prevstart:start] + " " - else: - #logger.fdebug('digit detected.') - digitfound = "yes" - reconst = temploc[prevstart:start] - else: - reconst = temploc[prevstart:start] + " " - #logger.fdebug('word: ' + reconst) - tempreconstruct = tempreconstruct + reconst - #logger.fdebug('tempreconstruct is : ' + tempreconstruct) - prevstart = (start+1) - occur+=1 - #logger.fdebug('word: ' + temploc[prevstart:]) - tempreconstruct = tempreconstruct + " " + temploc[prevstart:] - #logger.fdebug('final filename to use is : ' + str(tempreconstruct)) - temploc = tempreconstruct +# occur=1 +# prevstart = 0 +# digitfound = "no" +# decimalfound = "no" +# tempreconstruct = '' +# while (occur <= deccnt): +# n = occur +# start = temploc.find('.') +# while start >=0 and n > 1: +# start = temploc.find('.', start+len('.')) +# n-=1 +# #logger.fdebug('occurance ' + str(occur) + ' of . at position: ' + str(start)) +# if temploc[prevstart:start].isdigit(): +# if digitfound == "yes": +# #logger.fdebug('this is a decimal, assuming decimal issue.') +# decimalfound = "yes" +# reconst = "." + temploc[prevstart:start] + " " +# else: +# #logger.fdebug('digit detected.') +# digitfound = "yes" +# reconst = temploc[prevstart:start] +# else: +# reconst = temploc[prevstart:start] + " " +# #logger.fdebug('word: ' + reconst) +# tempreconstruct = tempreconstruct + reconst +# #logger.fdebug('tempreconstruct is : ' + tempreconstruct) +# prevstart = (start+1) +# occur+=1 +# #logger.fdebug('word: ' + temploc[prevstart:]) +# tempreconstruct = tempreconstruct + " " + temploc[prevstart:] +# #logger.fdebug('final filename to use is : ' + str(tempreconstruct)) +# temploc = tempreconstruct #logger.fdebug("checking " + str(temploc)) #fcnew_b4 = shlex.split(str(temploc)) fcnew_af = re.findall('[^\()]+', temploc) @@ -515,8 +520,8 @@ def forceRescan(ComicID,archive=None): fcdigit = helpers.issuedigits(fcnew[som]) - #logger.fdebug("fcdigit: " + str(fcdigit)) - #logger.fdebug("int_iss: " + str(int_iss)) + logger.fdebug("fcdigit: " + str(fcdigit)) + logger.fdebug("int_iss: " + str(int_iss)) if int(fcdigit) == int_iss: logger.fdebug('issue match - fcdigit: ' + str(fcdigit) + ' ... int_iss: ' + str(int_iss))