IMP: KAT torrent searches will now work (was a gzip issue), FIX:(#526) NzbGet support was broken (fixed - thanks ministoat), IMP: Merged Manual Run GUI option now in idirectory tabbed screen, FIX:(#524) ComicID not defined in updater.py

data/interfaces/default/idirectory.html

@@ -18,6 +18,8 @@
 	<div id="tabs">
 		<ul>
 			<li><a href="#tabs-1">Scan Comic Library</a></li>
+                        <li><a href="#tabs-2">Manual Post-Processing</a></li>
+                        <li><a href="#tabs-3">Advanced Options</a></li>
 		</ul>
 		<div id="tabs-1" class="configtable">
 				<form action="comicScan" method="GET" id="comicScan">
@@ -31,7 +33,6 @@
 					as soon as you click 'Save changes'
 					</p>	
 					<br/>
-                                        <p><strong>THIS IS CURRENTLY DISABLED UNTIL WORKING..</strong></p>
 					<div class="row">
                                         <label for="">Path to directory</label>
                                         %if mylar.COMIC_DIR:
@@ -41,7 +42,7 @@
                                             (this.value==this.defaultValue) this.value='';" name="path" size="70" />
                                         %endif
                                         </div>
-                                        <div class="row checkbox">
+<!--                                        <div class="row checkbox">
                                            <input type="checkbox" name="libraryscan" id="libraryscan" value="1" ${checked(mylar.LIBRARYSCAN)}><label>Automatically Scan Library</label>
                                         </div>
                                         <div class="row checkbox">
@@ -51,22 +52,81 @@
                                              <input type="checkbox" style="vertical-align: middle; margin: 3px; margin-top: -1px;" name="imp_metadata" id="imp_metadata" value="1" ${checked(mylar.IMP_METADATA)}><label>Use existing Metadata</label>
                                              <small>Use existing Metadata to better locate series for import</small>
                                         </div>
+-->
                                         <div class="row checkbox">
                                            <input type="checkbox" style="vertical-align: middle; margin: 3px; margin-top: -1px;" name="imp_move" onclick="initConfigCheckbox($this));" id="imp_move" value="1" ${checked(mylar.IMP_MOVE)}><label>Move files into corresponding Series directory</label>
                                            <small>Leaving this unchecked will not move anything, but will mark the issues as Archived</small>
                                         </div>
                                         <div class="config">
-                                            <div class="row checkbox">
                                                <input type="checkbox" name="imp_rename" id="imp_rename" value="1" ${checked(mylar.IMP_RENAME)}><label>Rename Files </label>
                                                <small>Rename files to configuration settings</small>
                                         </div>
-                                        </div>
                                         <br/>
                                      <input type="button" value="Save Changes and Scan" onclick="addScanAction();doAjaxCall('comicScan',$(this),'tabs',true);return false;" data-success="Changes saved. Library will be scanned">
                                      <input type="button" value="Save Changes without Scanning Library" onclick="doAjaxCall('comicScan',$(this),'tabs',true);return false;" data-success="Changes Saved Successfully">
                                 </fieldset>
                        </form>
 		</div>
+                <div id="tabs-2" class="configtable">
+                  <tr>
+                    <td>
+                         <form action="post_process" method="GET" id="post_process">
+                                   <fieldset>
+                                        <legend>Manual Run</legend>
+                                        <p><strong>Manual Post-Processing</strong></p>
+                                        <p>You can put in any directory, and it will scan for comic files in that folder
+                                        (including all subdirectories) that exist on your watchlist.
+                                        <br/><small>For example: '/Users/name/Comics'</small></p>
+                                        <p>
+                                        It may take a while depending on how many files you have. You can navigate away from this
+                                        as soon as you click 'Go'.
+                                        </p>
+                                        <br/>
+                                        <p><strong>Only series' that exist on your watchlist will be post-processed with your current
+                                        post-processing options.<br/> The remainder will not be moved from the given directory</strong></p>
+                                        <div class="row">
+                                            <label for="">Path to Manually Run</label>
+                                            <input type="text" value="Enter the full path to post-process" name="nzb_folder" size="70" />
+                                            <input type="hidden" name="nzb_name" value='Manual Run' />
+                                        </div>
+                                      <input type="button" value="Go Manual Run!" onclick="doAjaxCall('post_process',$(this),'tabs',true);return false;">
+                                   </fieldset>
+                         </form>
+                    </td>
+                  </tr>
+                </div>
+
+        <div id="tabs-3">
+           <table class="configtable" summary="Advanced Options">
+
+               <tr>
+                 <td>
+                    <fieldset>
+                        <div class="links">
+                          <legend>Force Options</legend>
+                           <a href="#" onclick="doAjaxCall('forceSearch',$(this))" data-success="Checking for wanted issues successful" data-error="Error checking wanted issues"><span class="ui-icon ui-icon-search"></span>Force Check for Wanted Issues</a>
+                           <a href="#" onclick="doAjaxCall('forceUpdate',$(this))" data-success="Update active comics successful" data-error="Error forcing update Comics"><span class="ui-icon ui-icon-heart"></span>Force Update Active Comics</a>
+                           <a href="#" onclick="doAjaxCall('checkGithub',$(this))" data-success="Checking for update successful" data-error="Error checking for update"><span class="ui-icon ui-icon-refresh"></span>Check for mylar Updates</a>
+                        </div>
+                    </fieldset>
+                  </td>
+                  <td>
+                    <fieldset>
+                        <legend>Export</legend>
+                        <div class="links">
+                          <a href="#" onclick="doAjaxCall('wanted_Export',$(this))" data-sucess="Exported to Wanted list." data-error="Failed to export. Check logs"><span class="ui-icon ui-icon-refresh"></span>Export Wanted to CSV</a>
+                        </div>
+                        <br/><br/>
+                        <legend>Hidden Options</legend>
+                        <div classs="links">
+                          <a href="readlist">Reading List Management</a><br/>
+                          <a href="importResults">Import Results Management</a>
+                        </div>
+                    </fieldset>
+                 </td>
+               </tr>
+           </table>
+        </div>
 
 </div>
 </%def>

mylar/filechecker.py

@@ -280,6 +280,11 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None):
             if justthedigits.isdigit():
                 digitsvalid = "true"
             else:
+                if '.' in justthedigits:
+                    logger.fdebug("decimals")
+                    digitsvalid = "true"
+                else:
+                    logger.fdebug("no decimals")
                     digitsvalid = "false"
 
             if justthedigits.lower() == 'annual':
@@ -291,6 +296,10 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None):
                 try:
                     if tmpthedigits.split(' ', 1)[1] is not None:
                         poss_alpha = tmpthedigits.split(' ', 1)[1]
+                        if poss_alpha.isdigit():
+                            logger.fdebug("decimal issue detected (filename space seperate most likely '.')")
+                            digitsvalid = "true"
+                            justthedigits += '.' + poss_alpha
                         for issexcept in issue_exceptions:
                             if issexcept.lower() in poss_alpha.lower() and len(poss_alpha) <= len(issexcept):
                                 justthedigits += poss_alpha

mylar/helpers.py

@@ -675,11 +675,19 @@ def issuedigits(issnum):
                 x = 0
                 tstord = None
                 issno = None
+                invchk = "false"
                 while (x < len(issnum)):
                     if issnum[x].isalpha():
                     #take first occurance of alpha in string and carry it through
                         tstord = issnum[x:].rstrip()
                         issno = issnum[:x].rstrip()
+                        try:
+                            isschk = float(issno)
+                        except ValueError, e:
+                            logger.fdebug('invalid numeric for issue - cannot be found. Ignoring.')
+                            issno = None
+                            tstord = None
+                            invchk = "true"
                         break
                     x+=1
                 if tstord is not None and issno is not None:
@@ -695,6 +703,9 @@ def issuedigits(issnum):
                     logger.fdebug('issno: ' + str(issno))
                     int_issnum = (int(issno) * 1000) + ordtot
                     logger.fdebug('intissnum : ' + str(int_issnum))
+                elif invchk == "true":
+                    logger.fdebug('this does not have an issue # that I can parse properly.')
+                    int_issnum = 999999999999999
                 else:
                     logger.error(str(issnum) + 'this has an alpha-numeric in the issue # which I cannot account for.')
                     int_issnum = 999999999999999

mylar/importer.py

@@ -643,11 +643,19 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None):
                             x = 0
                             tstord = None
                             issno = None
+                            invchk = "false"
                             while (x < len(issnum)):
                                 if issnum[x].isalpha():
                                     #take first occurance of alpha in string and carry it through
                                     tstord = issnum[x:].rstrip()
                                     issno = issnum[:x].rstrip()
+                                    try:
+                                        isschk = float(issno)
+                                    except ValueError, e:
+                                        logger.fdebug('invalid numeric for issue - cannot be found. Ignoring.')
+                                        issno = None
+                                        tstord = None
+                                        invchk = "true"
                                     break
                                 x+=1
                             if tstord is not None and issno is not None:
@@ -660,6 +668,9 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None):
                                 logger.fdebug('issno: ' + str(issno))
                                 int_issnum = (int(issno) * 1000) + ordtot
                                 logger.fdebug('intissnum : ' + str(int_issnum))
+                            elif invchk == "true":
+                                logger.fdebug('this does not have an issue # that I can parse properly.')
+                                return
                             else:
                                 logger.error(str(issnum) + ' this has an alpha-numeric in the issue # which I cannot account for.')
                                 return    

mylar/rsscheck.py

@@ -6,6 +6,8 @@ import lib.feedparser as feedparser
 import urllib2
 import ftpsshup
 import datetime
+import gzip
+from StringIO import StringIO
 
 import mylar
 from mylar import db, logger, ftpsshup, helpers
@@ -83,7 +85,7 @@ def torrents(pickfeed=None,seriesname=None,issue=None):
     else:
         logger.error('invalid pickfeed denoted...')
         return
-    logger.fdebug('feed #' + str(pickfeed) + ' chosen: ' + str(feed))
+
     title = []
     link = []
     description = []
@@ -330,7 +332,6 @@ def rssdbupdate(feeddata,i,type):
 def torrentdbsearch(seriesname,issue,comicid=None,nzbprov=None):
     myDB = db.DBConnection()
     seriesname_alt = None
-    print "seriesname:" + str(seriesname)
     if comicid is None or comicid == 'None':
         pass
     else:
@@ -376,7 +377,13 @@ def torrentdbsearch(seriesname,issue,comicid=None,nzbprov=None):
             AS_Altrem = re.sub("\\bthe\\b", "", AS_Altrem.lower())
 
             AS_Alternate = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\+\'\&\?\@\s]', '%', AS_Altrem)
-            AS_Alt.append(AS_Alternate)
+
+            AS_Altrem_mod = re.sub('[\&]', ' ', AS_Altrem)
+            AS_formatrem_seriesname = re.sub('[\'\!\@\#\$\%\:\;\/\\=\?\.]', '',AS_Altrem_mod)
+            AS_formatrem_seriesname = re.sub('\s+', ' ', AS_formatrem_seriesname)
+            if AS_formatrem_seriesname[:1] == ' ': AS_formatrem_seriesname = AS_formatrem_seriesname[1:]
+            AS_Alt.append(AS_formatrem_seriesname)
+
             AS_Alternate += '%'
 
             if mylar.ENABLE_CBT:
@@ -543,11 +550,9 @@ def torsend2client(seriesname, linkit, site):
         logger.info(linkit)
         linkit = str(linkit) + '&passkey=' + str(mylar.CBT_PASSKEY)
 
-    if linkit[-7:] != "torrent":
+    if linkit[-7:] != "torrent" and site != "KAT":
         filename += ".torrent"
 
-    request = urllib2.Request(linkit)
-    request.add_header('User-Agent', str(mylar.USER_AGENT))
     if mylar.TORRENT_LOCAL and mylar.LOCAL_WATCHDIR is not None:
         filepath = os.path.join(mylar.LOCAL_WATCHDIR, filename)
         logger.fdebug('filename for torrent set to : ' + filepath)
@@ -559,12 +564,33 @@ def torsend2client(seriesname, linkit, site):
         return "fail"
 
     try:
-        opener = helpers.urlretrieve(urllib2.urlopen(request), filepath)
+        request = urllib2.Request(linkit)
+        #request.add_header('User-Agent', str(mylar.USER_AGENT))
+        request.add_header('Accept-encoding', 'gzip')
+
+        if site == 'KAT':
+            request.add_header('Referer', 'http://kat.ph/')
+
+
+#        response = helpers.urlretrieve(urllib2.urlopen(request), filepath)
+        response = urllib2.urlopen(request)
+
+        if response.info().get('Content-Encoding') == 'gzip':
+            buf = StringIO(response.read())
+            f = gzip.GzipFile(fileobj=buf)
+            torrent = f.read()
+        else:
+            torrent = response.read()
+
     except Exception, e:
         logger.warn('Error fetching data from %s: %s' % (site, e))
         return "fail"
 
-    logger.fdebug('torrent file saved as : ' + str(filepath))
+    with open(filepath, 'w') as the_file:
+        the_file.write(torrent)
+
+    logger.info("saved.")
+    #logger.fdebug('torrent file saved as : ' + str(filepath))
     if mylar.TORRENT_LOCAL:
         return "pass"
     #remote_file = urllib2.urlopen(linkit)
@@ -579,6 +605,7 @@ def torsend2client(seriesname, linkit, site):
         tssh = ftpsshup.putfile(filepath,filename)
         return tssh
 
+
 if __name__ == '__main__':
     #torrents(sys.argv[1])
     #torrentdbsearch(sys.argv[1], sys.argv[2], sys.argv[3])

mylar/search.py

@@ -505,7 +505,7 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is
                     logger.fdebug("Sending request to [KAT] for " + str(cmname) + " : " + str(mod_isssearch))
                     bb = rsscheck.torrents(pickfeed='2',seriesname=cmname,issue=mod_isssearch)
                     rss = "no"
-                    if bb is not None: logger.fdebug("results: " + str(bb))
+                    #if bb is not None: logger.fdebug("results: " + str(bb))
                 elif nzbprov != 'experimental':
                     if nzbprov == 'dognzb':
                         findurl = "http://dognzb.cr/api?t=search&q=" + str(comsearch) + "&o=xml&cat=7030"
@@ -1115,7 +1115,7 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is
                                     tmpapi = str(tmpapi) + str(mylar.NZBGET_USERNAME) + ":" + str(mylar.NZBGET_PASSWORD)
                                     tmpapi = str(tmpapi) + "@" + str(nzbget_host) + ":" + str(mylar.NZBGET_PORT) + "/xmlrpc"
                                     server = ServerProxy(tmpapi)
-                                    send_to_nzbget = server.appendurl(nzbname, str(mylar.NZBGET_CATEGORY), int(nzbgetpriority), True, linkapi)
+                                    send_to_nzbget = server.appendurl(nzbname + ".nzb", str(mylar.NZBGET_CATEGORY), int(nzbgetpriority), True, linkapi)
                                     sent_to = "NZBGet"
                                     if send_to_nzbget is True:
                                         logger.info("Successfully sent nzb to NZBGet!")

mylar/updater.py

@@ -73,7 +73,7 @@ def dbUpdate(ComicIDList=None):
                 annuals = []
                 ann_list = []
                 if mylar.ANNUALS_ON:
-                    annuals_list = myDB.select('SELECT * FROM annuals WHERE ComicID=?', [ComicID])
+                    annuals_list = myDB.select('SELECT * FROM annuals WHERE ComicID=?', [comicid])
                     ann_list += annuals_list
                     issues_new += annuals_list
 
@@ -202,6 +202,10 @@ def upcoming_update(ComicID, ComicName, IssueNumber, IssueDate, forcecheck=None)
             values = { "Status":    "Skipped"}
             newValue['Status'] = "Skipped"
         #was in wrong place :(
+    else:
+        logger.fdebug('Issues do not match for some reason...weekly new issue: ' + str(IssueNumber))
+        return
+
     if mylar.AUTOWANT_UPCOMING:
         #for issues not in db - to be added to Upcoming table.
         if issuechk is None:
@@ -216,6 +220,7 @@ def upcoming_update(ComicID, ComicName, IssueNumber, IssueDate, forcecheck=None)
             logger.fdebug('...Status already Wanted .. not changing.')
         else:
             logger.fdebug('...Already have issue - keeping existing status of : ' + issuechk['Status'])
+
     if issuechk is None:
         myDB.upsert("upcoming", newValue, controlValue)
     else:
@@ -235,17 +240,17 @@ def upcoming_update(ComicID, ComicName, IssueNumber, IssueDate, forcecheck=None)
             values = {"IssueDate":  newValue['IssueDate']}
             #if ComicID[:1] == "G": mylar.importer.GCDimport(ComicID,pullupd='yes')
             #else: mylar.importer.addComictoDB(ComicID,mismatch,pullupd='yes')
+
         if 'annual' in ComicName.lower():
             myDB.upsert("annuals", values, control)
         else:
             myDB.upsert("issues", values, control)
+
         if issuechk['Status'] == 'Downloaded': 
             logger.fdebug('updating Pull-list to reflect status.')
             downstats = {"Status":  issuechk['Status'],
                          "ComicID": issuechk['ComicID']}
             return downstats
-    else:
-        logger.fdebug('Issues do not match for some reason...weekly new issue: ' + str(IssueNumber))
 
 
 def weekly_update(ComicName,IssueNumber,CStatus,CID):
@@ -419,43 +424,43 @@ def forceRescan(ComicID,archive=None):
             if temploc.lower().endswith(extensions):
                 logger.fdebug('removed extension for issue: ' + str(temploc))
                 temploc = temploc[:-4]
-            deccnt = str(temploc).count('.')
+#            deccnt = str(temploc).count('.')
-            if deccnt > 1:
+#            if deccnt > 1:
                 #logger.fdebug('decimal counts are :' + str(deccnt))
                 #if the file is formatted with '.' in place of spaces we need to adjust.
                 #before replacing - check to see if digits on either side of decimal and if yes, DON'T REMOVE
-                occur=1
+#                occur=1
-                prevstart = 0
+#                prevstart = 0
-                digitfound = "no"
+#                digitfound = "no"
-                decimalfound = "no"
+#                decimalfound = "no"
-                tempreconstruct = ''
+#                tempreconstruct = ''
-                while (occur <= deccnt):
+#                while (occur <= deccnt):
-                    n = occur
+#                    n = occur
-                    start = temploc.find('.')
+#                    start = temploc.find('.')
-                    while start >=0 and n > 1:
+#                    while start >=0 and n > 1:
-                        start = temploc.find('.', start+len('.'))
+#                        start = temploc.find('.', start+len('.'))
-                        n-=1
+#                        n-=1
-                    #logger.fdebug('occurance ' + str(occur) + ' of . at position: ' + str(start))
+#                    #logger.fdebug('occurance ' + str(occur) + ' of . at position: ' + str(start))
-                    if temploc[prevstart:start].isdigit():
+#                    if temploc[prevstart:start].isdigit():
-                        if digitfound == "yes":
+#                        if digitfound == "yes":
-                            #logger.fdebug('this is a decimal, assuming decimal issue.')
+#                            #logger.fdebug('this is a decimal, assuming decimal issue.')
-                            decimalfound = "yes"
+#                            decimalfound = "yes"
-                            reconst = "." + temploc[prevstart:start] + " "
+#                            reconst = "." + temploc[prevstart:start] + " "
-                        else:
+#                        else:
-                            #logger.fdebug('digit detected.')
+#                            #logger.fdebug('digit detected.')
-                            digitfound = "yes"
+#                            digitfound = "yes"
-                            reconst = temploc[prevstart:start]
+#                            reconst = temploc[prevstart:start]
-                    else:
+#                    else:
-                        reconst = temploc[prevstart:start] + " "
+#                        reconst = temploc[prevstart:start] + " "
-                    #logger.fdebug('word: ' + reconst)
+#                    #logger.fdebug('word: ' + reconst)
-                    tempreconstruct = tempreconstruct + reconst 
+#                    tempreconstruct = tempreconstruct + reconst 
-                    #logger.fdebug('tempreconstruct is : ' + tempreconstruct)
+#                    #logger.fdebug('tempreconstruct is : ' + tempreconstruct)
-                    prevstart = (start+1)
+#                    prevstart = (start+1)
-                    occur+=1
+#                    occur+=1
-                #logger.fdebug('word: ' + temploc[prevstart:])
+#                #logger.fdebug('word: ' + temploc[prevstart:])
-                tempreconstruct = tempreconstruct + " " + temploc[prevstart:]
+#                tempreconstruct = tempreconstruct + " " + temploc[prevstart:]
-                #logger.fdebug('final filename to use is : ' + str(tempreconstruct))
+#                #logger.fdebug('final filename to use is : ' + str(tempreconstruct))
-                temploc = tempreconstruct            
+#                temploc = tempreconstruct            
             #logger.fdebug("checking " + str(temploc))
             #fcnew_b4 = shlex.split(str(temploc))            
             fcnew_af = re.findall('[^\()]+', temploc)
@@ -515,8 +520,8 @@ def forceRescan(ComicID,archive=None):
                     
                     fcdigit = helpers.issuedigits(fcnew[som])
 
-                    #logger.fdebug("fcdigit: " + str(fcdigit))
+                    logger.fdebug("fcdigit: " + str(fcdigit))
-                    #logger.fdebug("int_iss: " + str(int_iss))
+                    logger.fdebug("int_iss: " + str(int_iss))
 
                     if int(fcdigit) == int_iss:
                         logger.fdebug('issue match - fcdigit: ' + str(fcdigit) + ' ... int_iss: ' + str(int_iss))