IMP:(#559) Differences between pull-list naming and ComicVine - substitutions.csv usage, FIX:(#583) When Importing Series, if filenames had a volume label it would error out, FIX:(#597) When Importing Series, decimal in issue would cause error, FIX:(#600) Annuals file format now inluded as (thanks uspider7), IMP:(#571) centos-init.d and ubuntu-init.d added, Various other fixes....

centos-mylar.init.d

@@ -0,0 +1,93 @@
+#!/bin/sh
+#
+### BEGIN INIT INFO
+# Provides:          mylar
+# Required-Start:    $all
+# Required-Stop:     $all
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: starts Mylar
+# Description:       starts Mylar
+### END INIT INFO
+
+# Source function library.
+. /etc/init.d/functions
+
+# Source SickBeard configuration
+if [ -f /etc/sysconfig/mylar ]; then
+        . /etc/sysconfig/mylar
+fi
+
+prog=mylar
+lockfile=/var/lock/subsys/$prog
+
+## Edit user configuation in /etc/sysconfig/mylar to change
+## the defaults
+username=${MY_USER-mylar}
+homedir=${MY_HOME-/opt/mylar}
+datadir=${MY_DATA-/opt/mylar}
+pidfile=${MY_PIDFILE-/var/run/mylar/mylar.pid}
+nice=${MY_NICE-}
+##
+
+pidpath=`dirname ${pidfile}`
+options=" --daemon --nolaunch --pidfile=${pidfile} --datadir=${datadir}"
+
+# create PID directory if not exist and ensure the SickBeard user can write to it
+if [ ! -d $pidpath ]; then
+        mkdir -p $pidpath
+        chown $username $pidpath
+fi
+
+if [ ! -d $datadir ]; then
+        mkdir -p $datadir
+        chown $username $datadir
+fi
+
+start() {
+        # Start daemon.
+        echo -n $"Starting $prog: "
+        daemon --user=${username} --pidfile=${pidfile} ${nice} python ${homedir}/Mylar.py ${options}
+        RETVAL=$?
+        echo
+        [ $RETVAL -eq 0 ] && touch $lockfile
+        return $RETVAL
+}
+
+stop() {
+        echo -n $"Shutting down $prog: "
+        killproc -p ${pidfile} python
+        RETVAL=$?
+        echo
+        [ $RETVAL -eq 0 ] && rm -f $lockfile
+        return $RETVAL
+}
+
+# See how we were called.
+case "$1" in
+  start)
+        start
+        ;;
+  stop)
+        stop
+        ;;
+  status)
+        status $prog
+        ;;
+  restart|force-reload)
+        stop
+        start
+        ;;
+  try-restart|condrestart)
+        if status $prog > /dev/null; then
+            stop
+            start
+        fi
+        ;;
+  reload)
+        exit 3
+        ;;
+  *)
+        echo $"Usage: $0 {start|stop|status|restart|try-restart|force-reload}"
+        exit 2
+esac

data/interfaces/default/config.html

@@ -84,6 +84,9 @@
                 <fieldset>
                     <legend>Branch history</legend>
                           <label>This would be a nice place to see revision history...</label>
+<!--
+                          <div>${config['branch_history']}</div>
+-->
                 </fieldset>
                </td>
             </tr>
@@ -621,7 +624,7 @@
                                     <label> File Format</label>
                                     <input type="text" name="file_format" value="${config['file_format']}" size="43">
                                     <%
-                                         file_options = "$Series = SeriesName\n$Year = SeriesYear\n$Issue = IssueNumber\n$VolumeY = V{SeriesYear}\n$VolumeN = V{Volume#}"
+                                         file_options = "$Series = SeriesName\n$Year = SeriesYear\n$Annual = Annual (word)\n$Issue = IssueNumber\n$VolumeY = V{SeriesYear}\n$VolumeN = V{Volume#}"
                                     %>
                                     <a href="#" title="${file_options}"><img src="interfaces/default/images/info32.png" height="16" alt="" /></a>
                                     <small>Use:  $Series, $Year, $Issue<br />
@@ -732,13 +735,13 @@
                      <fieldset>
                         <h3><img src="interfaces/default/images/nma_logo.png" style="vertical-align: middle; margin: 3px; margin-top: -1px;" height="30" width="30"/>NotifyMyAndroid</h3>
                         <div class="checkbox row">
-<input type="checkbox" name="nma_enabled" id="nma" value="1" ${config['nma_enabled']} /><label>Enable NotifyMyAndroid</label>
+                             <input type="checkbox" name="nma_enabled" id="nma" value="1" ${config['nma_enabled']} /><label>Enable NotifyMyAndroid</label>
                         </div>
                         <div id="nmaoptions">
                             <div class="row">
                                     <div class="row checkbox">
-     <input type="checkbox" name="nma_onsnatch" value="1" ${config['nma_onsnatch']} /><label>Notify on snatch?</label>
+                                        <input type="checkbox" name="nma_onsnatch" value="1" ${config['nma_onsnatch']} /><label>Notify on snatch?</label>
-                                   </div>
+                                    </div>
                                     <label>NotifyMyAndroid API Key</label>
                                     <input type="text" name="nma_apikey" value="${config['nma_apikey']}" size="30">
                                     <small>Separate multiple api keys with commas</small>

mylar/PostProcessor.py

@@ -175,7 +175,7 @@ class PostProcessor(object):
             myDB = db.DBConnection()
 
             if self.nzb_name == 'Manual Run':
-                print ("manual run initiated")
+                logger.fdebug ("manual run initiated")
                 #Manual postprocessing on a folder.
                 #use the nzb_folder to determine every file
                 #walk the dir,
@@ -197,7 +197,7 @@ class PostProcessor(object):
                                      "ComicVersion": cs['ComicVersion'],
                                      "Total":        cs['Total']}
                         watchmatch = filechecker.listFiles(self.nzb_folder,cs['ComicName'],cs['AlternateSearch'], manual=watchvals)
-                        if watchmatch is None:
+                        if watchmatch['comiccount'] == 0: # is None:
                             nm+=1
                             continue
                         else:
@@ -267,10 +267,10 @@ class PostProcessor(object):
                                                             "IssueNumber":     issuechk['Issue_Number'],
                                                             "ComicName":       cs['ComicName']})
                                         ccnt+=1
-                                        print manual_list
+                                        #print manual_list
                                     wdc+=1
                                 fn+=1
-                    print("There are " + str(len(manual_list)) + " files found that match on your watchlist, " + str(nm) + " do not match anything and will be ignored.")    
+                    logger.fdebug("There are " + str(len(manual_list)) + " files found that match on your watchlist, " + str(nm) + " do not match anything and will be ignored.")    
                 
 
             else:
@@ -455,14 +455,11 @@ class PostProcessor(object):
             myDB = db.DBConnection()
             comicnzb = myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone()
             issuenzb = myDB.action("SELECT * from issues WHERE issueid=? AND comicid=? AND ComicName NOT NULL", [issueid,comicid]).fetchone()
-            print "issueid: " + str(issueid)
+            logger.fdebug('issueid: ' + str(issueid))
-            print "issuenumOG: " + str(issuenumOG)
+            logger.fdebug('issuenumOG: ' + str(issuenumOG))
             if issuenzb is None:
-                print "chk1"
                 issuenzb = myDB.action("SELECT * from annuals WHERE issueid=? and comicid=?", [issueid,comicid]).fetchone()
-                print "chk2"
                 annchk = "yes"
-            print issuenzb
             #issueno = str(issuenum).split('.')[0]
             #new CV API - removed all decimals...here we go AGAIN!
             issuenum = issuenzb['Issue_Number']
@@ -551,7 +548,6 @@ class PostProcessor(object):
                 self._log("issue length error - cannot determine length. Defaulting to None:  " + str(prettycomiss), logger.DEBUG)
 
             if annchk == "yes":
-                prettycomiss = "Annual " + str(prettycomiss)
                 self._log("Annual detected.")
             logger.fdebug("Pretty Comic Issue is : " + str(prettycomiss))
             issueyear = issuenzb['IssueDate'][:4]
@@ -587,6 +583,22 @@ class PostProcessor(object):
             else:
                 chunk_file_format = mylar.FILE_FORMAT
 
+            if annchk == "no":
+                chunk_f_f = re.sub('\$Annual','',chunk_file_format)
+                chunk_f = re.compile(r'\s+')
+                chunk_file_format = chunk_f.sub(' ', chunk_f_f)
+                logger.fdebug('not an annual - removing from filename paramaters')
+                logger.fdebug('new format: ' + str(chunk_file_format))
+
+            else:
+                logger.fdebug('chunk_file_format is: ' + str(chunk_file_format))
+                if '$Annual' not in chunk_file_format:
+                #if it's an annual, but $Annual isn't specified in file_format, we need to
+                #force it in there, by default in the format of $Annual $Issue
+                    prettycomiss = "Annual " + str(prettycomiss)
+                    logger.fdebug('prettycomiss: ' + str(prettycomiss))
+
+
             ofilename = None
 
             #if meta-tagging is not enabled, we need to declare the check as being fail
@@ -652,7 +664,8 @@ class PostProcessor(object):
                            '$Publisher': publisher,
                            '$publisher': publisher.lower(),
                            '$VolumeY':   'V' + str(seriesyear),
-                           '$VolumeN':   comversion
+                           '$VolumeN':   comversion,
+                           '$Annual':    'Annual'
                           }
 
 
@@ -667,12 +680,12 @@ class PostProcessor(object):
             else:
                 if pcheck == "fail":
                     otofilename = ml['ComicLocation']
-                print "otofilename:" + str(otofilename)
+                logger.fdebug('otofilename:' + str(otofilename))
                 odir, ofilename = os.path.split(otofilename)
-                print "ofilename: " + str(ofilename)
+                logger.fdebug('ofilename: ' + str(ofilename))
                 path, ext = os.path.splitext(ofilename)
-                print "path: " + str(path)
+                logger.fdebug('path: ' + str(path))
-                print "ext:" + str(ext)
+                logger.fdebug('ext:' + str(ext))
 
             if ofilename is None:
                 logger.error(u"Aborting PostProcessing - the filename doesn't exist in the location given. Make sure that " + str(self.nzb_folder) + " exists and is the correct location.")
@@ -696,6 +709,7 @@ class PostProcessor(object):
                     #mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
                     nfilename = nfilename.replace(' ', mylar.REPLACE_CHAR)
             nfilename = re.sub('[\,\:\?]', '', nfilename)
+            nfilename = re.sub('[\/]', '-', nfilename)
             self._log("New Filename: " + nfilename, logger.DEBUG)
             logger.fdebug("New Filename: " + str(nfilename))
 
@@ -714,6 +728,9 @@ class PostProcessor(object):
 
             if ml is None:
                 #non-manual run moving/deleting...
+                logger.fdebug('self.nzb_folder: ' + self.nzb_folder)
+                logger.fdebug('ofilename:' + str(ofilename))
+                logger.fdebug('nfilename:' + str(nfilename + ext))
                 os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder,str(nfilename + ext)))
                 src = os.path.join(self.nzb_folder, str(nfilename + ext))
                 try:

mylar/filechecker.py

@@ -144,6 +144,7 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None):
         nonocount = 0
         charpos = 0
         detneg = "no"
+        leavehyphen = False
         for nono in not_these:
             if nono in subname:
                 subcnt = subname.count(nono)
@@ -158,10 +159,11 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None):
                                 logger.fdebug('possible negative issue detected.')
                                 nonocount = nonocount + subcnt - 1
                                 detneg = "yes"                                
-                            if '-' in watchcomic and i < len(watchcomic):
+                            elif '-' in watchcomic and i < len(watchcomic):
                                 logger.fdebug('- appears in series title.')
+                                leavehyphen = True
                         i+=1
-                    if detneg == "no": 
+                    if detneg == "no" or leavehyphen == False: 
                         subname = re.sub(str(nono), ' ', subname)
                         nonocount = nonocount + subcnt
                 #logger.fdebug(str(nono) + " detected " + str(subcnt) + " times.")
@@ -197,7 +199,12 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None):
                     subname = re.sub(str(nono), ' ', subname)
                     nonocount = nonocount + subcnt + blspc
         #subname = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\+\'\?\@]',' ', subname)
-        modwatchcomic = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\'\?\@]', ' ', u_watchcomic)
+
+        modwatchcomic = re.sub('[\_\#\,\/\:\;\.\!\$\%\'\?\@\-]', ' ', u_watchcomic)
+        #if leavehyphen == False:
+        #    logger.fdebug('removing hyphen for comparisons')
+        #    modwatchcomic = re.sub('-', ' ', modwatchcomic)
+        #    subname = re.sub('-', ' ', subname)
         detectand = False
         detectthe = False
         modwatchcomic = re.sub('\&', ' and ', modwatchcomic)
@@ -513,6 +520,10 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None):
 
                 if yearmatch == "false": continue
 
+                if 'annual' in subname.lower():
+                    subname = re.sub('annual', '', subname.lower())
+                    subname = re.sub('\s+', ' ', subname)
+
                 #tmpitem = item[:jtd_len]
                 # if it's an alphanumeric with a space, rejoin, so we can remove it cleanly just below this.
                 substring_removal = None
@@ -575,9 +586,9 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None):
         else:
             pass
             #print ("directory found - ignoring")
+
     logger.fdebug('you have a total of ' + str(comiccnt) + ' ' + watchcomic + ' comics')
     watchmatch['comiccount'] = comiccnt
-    #print watchmatch
     return watchmatch
 
 def validateAndCreateDirectory(dir, create=False):

mylar/helpers.py

@@ -239,7 +239,7 @@ def decimal_issue(iss):
         deciss = (int(iss_b4dec) * 1000) + issdec
     return deciss, dec_except
 
-def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=None):
+def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=None, annualize=None):
             import db, logger
             myDB = db.DBConnection()
             logger.fdebug('comicid: ' + str(comicid))
@@ -262,22 +262,40 @@ def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=N
 #            issue = iss
 
 #            print ("converted issue#: " + str(issue))
+            logger.fdebug('issueid:' + str(issueid))
+
             if issueid is None:
-                chkissue = myDB.action("SELECT * from issues WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone()
+                logger.fdebug('annualize is ' + str(annualize))
+                if annualize is None:
+                    chkissue = myDB.action("SELECT * from issues WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone()
+                else:
+                    chkissue = myDB.action("SELECT * from annuals WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone()
+
                 if chkissue is None:
                     #rechk chkissue against int value of issue #
-                    chkissue = myDB.action("SELECT * from issues WHERE ComicID=? AND Issue_Number=?", [comicid, int(issue)]).fetchone()
+                    chkissue = myDB.action("SELECT * from issues WHERE ComicID=? AND Int_IssueNumber=?", [comicid, issuedigits(issue)]).fetchone()
                     if chkissue is None:
-                        logger.error('Invalid Issue_Number - please validate.')
+                        if chkissue is None:
-                        return
+                            logger.error('Invalid Issue_Number - please validate.')
+                            return
                     else:
                         logger.info('Int Issue_number compare found. continuing...')
-                        issueid = chkissue['IssueID']
+                        issueid = chkissue['IssueID']                       
                 else:
                     issueid = chkissue['IssueID']
 
             #use issueid to get publisher, series, year, issue number
-            issuenzb = myDB.action("SELECT * from issues WHERE issueid=?", [issueid]).fetchone()
+            logger.fdebug('issueid is now : ' + str(issueid))
+            issuenzb = myDB.action("SELECT * from issues WHERE ComicID=? AND IssueID=?", [comicid,issueid]).fetchone()
+            if issuenzb is None:
+                logger.fdebug('not an issue, checking against annuals')
+                issuenzb = myDB.action("SELECT * from annuals WHERE ComicID=? AND IssueID=?", [comicid,issueid]).fetchone()
+                if issuenzb is None:
+                    logger.fdebug('Unable to rename - cannot locate issue id within db')
+                    return
+                else:
+                    annualize = True
+            logger.fdebug('blah')
             #comicid = issuenzb['ComicID']
             issuenum = issuenzb['Issue_Number']
             #issueno = str(issuenum).split('.')[0]
@@ -306,6 +324,8 @@ def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=N
             else:
                 iss = issuenum
                 issueno = str(iss)
+            logger.fdebug('iss:' + str(iss))
+            logger.fdebug('issueno:' + str(issueno))
             # issue zero-suppression here
             if mylar.ZERO_LEVEL == "0":
                 zeroadd = ""
@@ -381,10 +401,25 @@ def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=N
                 chunk_f = re.compile(r'\s+')
                 chunk_file_format = chunk_f.sub(' ', chunk_f_f)
                 logger.fdebug('No version # found for series, removing from filename')
-                print ("new format: " + str(chunk_file_format))
+                logger.fdebug("new format: " + str(chunk_file_format))
             else:
                 chunk_file_format = mylar.FILE_FORMAT
 
+            if annualize is None:
+                chunk_f_f = re.sub('\$Annual','',chunk_file_format)
+                chunk_f = re.compile(r'\s+')
+                chunk_file_format = chunk_f.sub(' ', chunk_f_f)
+                logger.fdebug('not an annual - removing from filename paramaters')
+                logger.fdebug('new format: ' + str(chunk_file_format))
+
+            else:
+                logger.fdebug('chunk_file_format is: ' + str(chunk_file_format))
+                if '$Annual' not in chunk_file_format:
+                #if it's an annual, but $annual isn't specified in file_format, we need to
+                #force it in there, by default in the format of $Annual $Issue
+                    prettycomiss = "Annual " + str(prettycomiss)
+                    logger.fdebug('prettycomiss: ' + str(prettycomiss))
+
             file_values = {'$Series':    series,
                            '$Issue':     prettycomiss,
                            '$Year':      issueyear,
@@ -392,7 +427,8 @@ def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=N
                            '$Publisher': publisher,
                            '$publisher': publisher.lower(),
                            '$VolumeY':   'V' + str(seriesyear),
-                           '$VolumeN':   comversion
+                           '$VolumeN':   comversion,
+                           '$Annual':    'Annual'
                           }
 
             extensions = ('.cbr', '.cbz')

mylar/importer.py

@@ -296,7 +296,8 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None,c
                   '$series':        series.lower(),
                   '$publisher':     publisher.lower(),
                   '$VolumeY':       'V' + str(year),
-                  '$VolumeN':       comversion
+                  '$VolumeN':       comversion,
+                  '$Annual':        'Annual'
                   }
 
 

mylar/librarysync.py

@@ -138,9 +138,30 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None)
         cfilename = re.sub('[\_\#\,\/\:\;\-\!\$\%\&\+\'\?\@]', ' ', comfilename)
         #cfilename = re.sub('\s', '_', str(cfilename))
 
+        #versioning - remove it
+        subsplit = cfilename.replace('_', ' ').split()
+        volno = None
+        volyr = None
+        for subit in subsplit:
+            if subit[0].lower() == 'v':
+                vfull = 0
+                if subit[1:].isdigit():
+                    #if in format v1, v2009 etc...
+                    if len(subit) > 3:
+                        # if it's greater than 3 in length, then the format is Vyyyy
+                        vfull = 1 # add on 1 character length to account for extra space
+                    cfilename = re.sub(subit, '', cfilename)
+                    volno = re.sub("[^0-9]", " ", subit)
+                elif subit.lower()[:3] == 'vol':
+                    #if in format vol.2013 etc
+                    #because the '.' in Vol. gets removed, let's loop thru again after the Vol hit to remove it entirely
+                    logger.fdebug('volume indicator detected as version #:' + str(subit))
+                    cfilename = re.sub(subit, '', cfilename)
+                    volyr = re.sub("[^0-9]", " ", subit)
+
         cm_cn = 0
 
-        #we need to track the counter to make sure we are comparing the right array parts  
+        #we need to track the counter to make sure we are comparing the right array parts
         #this takes care of the brackets :)
         m = re.findall('[^()]+', cfilename)
         lenm = len(m)
@@ -150,6 +171,7 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None)
         foundonwatch = "False"
         issue = 999999
 
+
         while (cnt < lenm):
             if m[cnt] is None: break
             if m[cnt] == ' ':
@@ -239,11 +261,13 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None)
 
         splitit = []
         watchcomic_split = []
-        logger.fdebug("filename comic and issue: " + cfilename)
+        logger.fdebug("filename comic and issue: " + comic_andiss)
+
         #changed this from '' to ' '
-        comic_iss_b4 = re.sub('[\-\:\,]', ' ', com_NAME)
+        comic_iss_b4 = re.sub('[\-\:\,]', ' ', comic_andiss)
         comic_iss = comic_iss_b4.replace('.',' ')
-        logger.fdebug("adjusted  comic and issue: " + str(comic_iss))
+        comic_iss = re.sub('[\s+]', ' ', comic_iss).strip()
+        logger.fdebug("adjusted comic and issue: " + str(comic_iss))
         #remove 'the' from here for proper comparisons.
         if ' the ' in comic_iss.lower():
             comic_iss = comic_iss[-4:]
@@ -417,7 +441,13 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None)
 #                    cmnam+=1
 #                logger.fdebug("comic: " + str(com_NAME))
 #            n+=1
-        if result_comyear is None: result_comyear = '0000' #no year in filename basically.
+        if volyr is None:
+            if result_comyear is None: 
+                result_comyear = '0000' #no year in filename basically.
+        else:
+            if result_comyear is None:
+                result_comyear = volyr
+
         print ("adding " + com_NAME + " to the import-queue!")
         impid = com_NAME + "-" + str(result_comyear) + "-" + str(comiss)
         print ("impid: " + str(impid))

mylar/rsscheck.py

@@ -428,10 +428,16 @@ def torrentdbsearch(seriesname,issue,comicid=None,nzbprov=None):
         while (i < len(torsplit)):
             #we'll rebuild the string here so that it's formatted accordingly to be passed back to the parser.
             logger.fdebug('section(' + str(i) + '): ' + str(torsplit[i]))
+            #remove extensions
+            titletemp = torsplit[i]
+            titletemp = re.sub('cbr', '', str(titletemp))
+            titletemp = re.sub('cbz', '', str(titletemp))
+            titletemp = re.sub('none', '', str(titletemp))
+          
             if i == 0: 
-                rebuiltline = str(torsplit[i])
+                rebuiltline = str(titletemp)
             else:
-                rebuiltline = rebuiltline + ' (' + str(torsplit[i]) + ')'
+                rebuiltline = rebuiltline + ' (' + str(titletemp) + ')'
             i+=1
 
         logger.fdebug('rebuiltline is :' + str(rebuiltline))
@@ -462,9 +468,12 @@ def torrentdbsearch(seriesname,issue,comicid=None,nzbprov=None):
             logger.fdebug('matched on series title: ' + seriesname)
             titleend = formatrem_torsplit[len(formatrem_seriesname):]
             titleend = re.sub('\-', '', titleend)   #remove the '-' which is unnecessary
-
+            #remove extensions
-            titleend = re.sub('cbr', '', str(titleend)) #remove extensions
+            titleend = re.sub('cbr', '', str(titleend))
+            titleend = re.sub('cbz', '', str(titleend))
+            titleend = re.sub('none', '', str(titleend))
             logger.fdebug('titleend: ' + str(titleend))
+
             sptitle = titleend.split()
             extra = ''
 #            for sp in sptitle:
@@ -480,14 +489,16 @@ def torrentdbsearch(seriesname,issue,comicid=None,nzbprov=None):
             ctitle = tor['Title'].find('cbr')
             if ctitle == 0:
                 ctitle = tor['Title'].find('cbz')
-            if ctitle == 0:
+                if ctitle == 0:
-                logger.fdebug('cannot determine title properly - ignoring for now.')
+                    ctitle = tor['Title'].find('none')
-                continue
+                    if ctitle == 0:               
+                        logger.fdebug('cannot determine title properly - ignoring for now.')
+                        continue
             cttitle = tor['Title'][:ctitle]
             #print("change title to : " + str(cttitle))
 #           if extra == '':
             tortheinfo.append({
-                          'title':   cttitle, #tor['Title'],
+                          'title':   rebuiltline, #cttitle,
                           'link':    tor['Link'],
                           'pubdate': tor['Pubdate'],
                           'site':    tor['Site'],

mylar/search.py

@@ -33,6 +33,7 @@ import sys
 import getopt
 import re
 import time
+import urlparse
 from xml.dom.minidom import parseString
 import urllib2
 from datetime import datetime
@@ -389,6 +390,9 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is
             tmpprov = nzbprov
     logger.info(u"Shhh be very quiet...I'm looking for " + ComicName + " issue: " + str(IssueNumber) + " (" + str(ComicYear) + ") using " + str(tmpprov))
 
+    #load in do not download db here for given series
+    #myDB = db.DBConnection()
+    #nodown = myDB.action('SELECT * FROM nzblog')
 
     if mylar.PREFERRED_QUALITY == 0: filetype = ""
     elif mylar.PREFERRED_QUALITY == 1: filetype = ".cbr"
@@ -681,12 +685,12 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is
 
                     thisentry = entry['title']
                     logger.fdebug("Entry: " + thisentry)
-                    cleantitle = re.sub('[_/.]', ' ', entry['title'])
+                    cleantitle = re.sub('[\_\.]', ' ', entry['title'])
                     cleantitle = helpers.cleanName(cleantitle)
                     # this is new - if title contains a '&' in the title it will assume the filename has ended at that point
                     # which causes false positives (ie. wolverine & the x-men becomes the x-men, which matches on x-men.
                     # 'the' is removed for comparisons later on
-                    if '&' in cleantitle: cleantitle = re.sub('[/&]','and', cleantitle) 
+                    if '&' in cleantitle: cleantitle = re.sub('[\&]','and', cleantitle) 
 
                     nzbname = cleantitle
 
@@ -737,16 +741,20 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is
 
                         if vers4year == "no" and vers4vol == "no":
                             # if the series is a v1, let's remove the requirements for year and volume label
-                            if ComVersChk != 0:
+                            # even if it's a v1, the nzbname might not contain a valid year format (20xx) or v3,
+                            # and since it's already known that there is no (year) or vYEAR given
+                            # let's push it through (and edit out the following if constraint)...
+
+                            #if ComVersChk != 0:
                                 # if there are no () in the string, try to add them if it looks like a year (19xx or 20xx)
-                                if len(re.findall('[^()]+', cleantitle)):
+                            if len(re.findall('[^()]+', cleantitle)):
-                                    logger.fdebug("detected invalid nzb filename - attempting to detect year to continue")
+                                logger.fdebug("detected invalid nzb filename - attempting to detect year to continue")
-                                    cleantitle = re.sub('(.*)\s+(19\d{2}|20\d{2})(.*)', '\\1 (\\2) \\3', cleantitle)
+                                cleantitle = re.sub('(.*)\s+(19\d{2}|20\d{2})(.*)', '\\1 (\\2) \\3', cleantitle)
-                                    continue
+                                continue
-                                else:
+                            else:
-                                    logger.fdebug("invalid nzb and/or cover only - skipping.")
+                                logger.fdebug("invalid nzb and/or cover only - skipping.")
-                                    cleantitle = "abcdefghijk 0 (1901).cbz"
+                                cleantitle = "abcdefghijk 0 (1901).cbz"
-                                    continue
+                                continue
 
                     #adjust for covers only by removing them entirely...
                     logger.fdebug("Cleantitle: " + str(cleantitle))
@@ -1078,6 +1086,32 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is
                         
                         #issue comparison now as well
                         if int(intIss) == int(comintIss):
+                            #check if nzb is in do not download list ;)
+                            if nzbprov == 'experimental':
+                                #id is located after the /download/ portion
+                                url_parts = urlparse.urlparse(entry['link'])
+                                path_parts = url_parts[2].rpartition('/')
+                                nzbtempid = path_parts[0].rpartition('/')
+                                nzblen = len(nzbtempid)
+                                nzbid = nzbtempid[nzblen-1]
+                            elif nzbprov == 'CBT':
+                                url_parts = urlparse.urlparse(entry['link'])
+                                nzbtemp = url_parts[4] # get the query paramater string
+                                nzbtemp = re.sub('torrent=', '', nzbtemp).rstrip()
+                                nzbid = re.sub('.torrent', '', nzbtemp).rstrip()
+                            elif nzbprov == 'KAT':
+                                url_parts = urllib.parse.urlparse(entry['link'])
+                                path_parts = url_parts[2].rpartition('/')
+                                nzbtempid = pathparts[2]
+                                nzbid = re.sub('.torrent', '', nzbtempid).rstrip()
+                            elif nzbprov == 'nzb.su':
+                                pass
+                            elif nzbprov == 'dognzb':
+                                pass
+                            elif nzbprov == 'newznab':
+                                #if in format of http://newznab/getnzb/<id>.nzb&i=1&r=apikey
+                                nzbid = os.path.splitext(entry['link'])[0].rsplit('/', 1)[1]
+                                
                             logger.fdebug('issues match!')
                             logger.info(u"Found " + ComicName + " (" + str(comyear) + ") issue: " + str(IssueNumber) + " using " + str(tmpprov) )
                         ## -- inherit issue. Comic year is non-standard. nzb year is the year

mylar/solicit.py

@@ -80,7 +80,7 @@ def solicit(month, year):
 
         loopthis = (cnt-1)
         #this loops through each 'found' solicit page 
-        shipdate = str(month) + '-' + str(year)
+        shipdate = str(month_string) + '-' + str(year)
         while ( loopthis >= 0 ):
             upcoming += populate(resultURL[loopthis], publish[loopthis], shipdate)
             loopthis -=1
@@ -178,7 +178,7 @@ def populate(link,publisher,shipdate):
             get_next = False
         if prev_chk == True:
             tempName = titlet.findNext(text=True)
-            if ' TPB' not in tempName and ' HC' not in tempName and 'GN-TPB' not in tempName and 'for $1' not in tempName.lower() and 'subscription variant' not in tempName.lower():
+            if ' TPB' not in tempName and ' HC' not in tempName and 'GN-TPB' not in tempName and 'for $1' not in tempName.lower() and 'subscription variant' not in tempName.lower() and 'poster' not in tempName.lower():
                 #print publisher + ' found upcoming'
                 if '#' in tempName:
                     #tempName = tempName.replace(u'.',u"'")

mylar/webserve.py

@@ -859,7 +859,46 @@ class WebInterface(object):
     def upcoming(self):
         myDB = db.DBConnection()
         #upcoming = myDB.select("SELECT * from issues WHERE ReleaseDate > date('now') order by ReleaseDate DESC")
-        upcoming = myDB.select("SELECT * from upcoming WHERE IssueDate > date('now') AND IssueID is NULL order by IssueDate DESC")
+        upcomingdata = myDB.select("SELECT * from upcoming WHERE IssueID is NULL order by IssueDate DESC")
+        upcoming = []
+        for upc in upcomingdata:
+            if len(upc['IssueDate']) <= 7 :
+                #if it's less than or equal 7, then it's a future-pull so let's check the date and display
+                #tmpdate = datetime.datetime.com
+                tmpdatethis = upc['IssueDate']
+                if tmpdatethis[:2] == '20':
+                    tmpdate = tmpdatethis #in correct format of yyyymm
+                else:
+                    findst = tmpdatethis.find('-')  #find the '-'
+                    tmpdate = tmpdatethis[findst+1:] + tmpdatethis[:findst] #rebuild in format of yyyymm
+                timenow = datetime.datetime.now().strftime('%Y%m')
+                logger.fdebug('comparing pubdate of: ' + str(tmpdate) + ' to now date of: ' + str(timenow))
+                if int(tmpdate) >= int(timenow):
+                    if upc['Status'] == 'Wanted':
+                        upcoming.append({"ComicName":    upc['ComicName'],
+                                         "IssueNumber":  upc['IssueNumber'],
+                                         "IssueDate":    upc['IssueDate'],
+                                         "ComicID":      upc['ComicID'],
+                                         "IssueID":      upc['IssueID'],
+                                         "Status":       upc['Status'],
+                                         "DisplayComicName": upc['DisplayComicName']})
+            else:
+                #if it's greater than 7 it's a full date, and shouldn't be displayed ;)
+                timenow = datetime.datetime.now().strftime('%Y%m%d') #convert to yyyymmdd
+                tmpdate = re.sub("[^0-9]", "", upc['IssueDate'])  #convert date to numerics only (should be in yyyymmdd)
+
+                logger.fdebug('comparing pubdate of: ' + str(tmpdate) + ' to now date of: ' + str(timenow))
+
+                if int(tmpdate) >= int(timenow):
+                    if upc['Status'] == 'Wanted':
+                        upcoming.append({"ComicName":    upc['ComicName'],
+                                         "IssueNumber":  upc['IssueNumber'],
+                                         "IssueDate":    upc['IssueDate'],
+                                         "ComicID":      upc['ComicID'],
+                                         "IssueID":      upc['IssueID'],
+                                         "Status":       upc['Status'],
+                                         "DisplayComicName": upc['DisplayComicName']})
+
         issues = myDB.select("SELECT * from issues WHERE Status='Wanted'")
         ann_list = []
 
@@ -940,6 +979,8 @@ class WebInterface(object):
         comicname = comic['ComicName']
         extensions = ('.cbr', '.cbz')
         issues = myDB.action("SELECT * FROM issues WHERE ComicID=?", [comicid]).fetchall()
+        if mylar.ANNUALS_ON:
+            issues += myDB.action("SELECT * FROM annuals WHERE ComicID=?", [comicid]).fetchall()
         comfiles = []
         filefind = 0
         for root, dirnames, filenames in os.walk(comicdir):
@@ -949,7 +990,11 @@ class WebInterface(object):
                     for issue in issues:
                         if issue['Location'] == filename:
                             #logger.error("matched " + str(filename) + " to DB file " + str(issue['Location']))
-                            renameiss = helpers.rename_param(comicid, comicname, issue['Issue_Number'], filename, comicyear=None, issueid=None)
+                            if 'annual' in issue['Location'].lower():
+                                annualize = 'yes'
+                            else:
+                                annualize = None
+                            renameiss = helpers.rename_param(comicid, comicname, issue['Issue_Number'], filename, comicyear=None, issueid=None, annualize=annualize)
                             nfilename = renameiss['nfilename']
                             srciss = os.path.join(comicdir,filename)
                             if mylar.LOWERCASE_FILENAMES:
@@ -1901,14 +1946,58 @@ class WebInterface(object):
                 return serve_template(templatename="searchresults.html", title='Import Results for: "' + ComicName + '"',searchresults=sresults, type=type, imported='yes', ogcname=ogcname) #imported=comicstoIMP, ogcname=ogcname)
     preSearchit.exposed = True
 
+    def pretty_git(self, br_history):
+        #in order to 'prettify' the history log for display, we need to break it down so it's line by line.
+        br_split = br_history.split("\n")  #split it on each commit 
+        for br in br_split:
+            br_commit_st = br.find('-')  #first - will represent end of commit numeric
+            br_commit = br[:br_commit_st].strip()
+            br_time_en = br.replace('-', 'XXX', 1).find('-')  #2nd - is end of time datestamp
+            br_time = br[br_commit_st+1:br_time_en].strip()
+            print 'COMMIT:' + str(br_commit)
+            print 'TIME:' + str(br_time)
+            commit_split = br.split() #split it by space to break it further down..
+            tag_chk = False
+            statline = ''
+            commit = []
+            for cs in commit_split:
+                if tag_chk == True:
+                    if 'FIX:' in cs or 'IMP:' in cs:
+                        commit.append({"commit":    br_commit,
+                                       "time":      br_time,
+                                       "stat":      tag_status,
+                                       "line":      statline})
+                        print commit
+                        tag_chk == False
+                        statline = ''
+                    else:
+                        statline += str(cs) + ' '   
+                else:
+                    if 'FIX:' in cs:
+                        tag_status = 'FIX'
+                        tag_chk = True
+                        print 'status: ' + str(tag_status)
+                    elif 'IMP:' in cs:
+                        tag_status = 'IMPROVEMENT'
+                        tag_chk = True
+                        print 'status: ' + str(tag_status)
+
+    pretty_git.exposed = True
     #---
     def config(self):
     
         interface_dir = os.path.join(mylar.PROG_DIR, 'data/interfaces/')
         interface_list = [ name for name in os.listdir(interface_dir) if os.path.isdir(os.path.join(interface_dir, name)) ]
-
+#----
+# to be implemented in the future.
 #        branch_history, err = mylar.versioncheck.runGit("log --oneline --pretty=format:'%h - %ar - %s' -n 4")
-#        br_hist = branch_history.replace("\n", "<br />\n")
+#        #here we pass the branch_history to the pretty_git module to break it down
+#        if branch_history: 
+#            self.pretty_git(branch_history)
+#            br_hist = branch_history.replace("\n", "<br />\n")
+#        else:
+#            br_hist = err
+#----
         myDB = db.DBConnection()
         CCOMICS = myDB.action("SELECT COUNT(*) FROM comics").fetchall()
         CHAVES = myDB.action("SELECT COUNT(*) FROM issues WHERE Status='Downloaded' OR Status='Archived'").fetchall()
@@ -2040,7 +2129,8 @@ class WebInterface(object):
                     "prog_dir" : mylar.PROG_DIR,
                     "cache_dir" : mylar.CACHE_DIR,
                     "config_file" : mylar.CONFIG_FILE,
-#                    "branch_history" : br_hist
+                    "branch_history" : 'None',
+#                    "branch_history" : br_hist,
                     "enable_pre_scripts" : helpers.checked(mylar.ENABLE_PRE_SCRIPTS),
                     "pre_scripts" : mylar.PRE_SCRIPTS,
                     "log_dir" : mylar.LOG_DIR
@@ -2355,6 +2445,10 @@ class WebInterface(object):
             logger.info("CHMOD File value is not a valid numeric - please correct. Defaulting to 0660")
             mylar.CHMOD_FILE = '0660'
 
+        if mylar.SAB_HOST.endswith('/'):
+            logger.info("Auto-correcting trailing slash in SABnzbd url (not required)")
+            mylar.SAB_HOST = mylar.SAB_HOST[:-1]
+
         if mylar.ENABLE_META:
             if mylar.CMTAGGER_PATH is None or mylar.CMTAGGER_PATH == '':
                 logger.info("ComicTagger Path not set - defaulting to Mylar Program Directory : " + mylar.PROG_DIR)

mylar/weeklypull.py

@@ -56,6 +56,27 @@ def pullit(forcecheck=None):
     PULLURL = 'http://www.previewsworld.com/shipping/newreleases.txt'
     #PULLURL = 'http://www.previewsworld.com/Archive/GetFile/1/1/71/994/081512.txt'
 
+    #Prepare the Substitute name switch for pulllist to comic vine conversion
+    substitutes = os.path.join(mylar.DATA_DIR,"substitutes.csv")
+    if not os.path.exists(substitutes):
+        logger.debug('no substitues.csv file located - not performing substitutions on weekly pull list')
+        substitute_check = False
+    else:
+        substitute_check = True
+        #shortrep is the name to be replaced, longrep the replacement
+        shortrep=[]
+        longrep=[]
+        #open the file data
+        with open(substitutes) as f:
+            reader = csv.reader(f, delimiter='|')
+            for row in reader:
+                if not row.startswith('#'): 
+                    logger.debug ("Substitutes file read : "+str(row))
+                    shortrep.append(row[0])
+                    longrep.append(row[1])
+
+        f.close()
+
     not_these=['PREVIEWS',
                'Shipping',
                'Every Wednesday',
@@ -188,7 +209,7 @@ def pullit(forcecheck=None):
                 dupefound = "no"
                 if '#' in i:
                     issname = i.split()
-                    print (issname)
+                    #print (issname)
                     issnamec = len(issname)
                     n = 0
                     while (n < issnamec):
@@ -241,7 +262,7 @@ def pullit(forcecheck=None):
                     #if it doesn't have a '#' in the line, then we know it's either
                     #a special edition of some kind, or a non-comic
                     issname = i.split()
-                    print (issname)
+                    #print (issname)
                     issnamec = len(issname)
                     n = 1
                     issue = ''
@@ -301,6 +322,16 @@ def pullit(forcecheck=None):
                 # pullist has shortforms of a series' title sometimes and causes problems
                 if 'O/T' in comicnm:
                     comicnm = re.sub('O/T', 'OF THE', comicnm)
+
+                if substitute_check == True:
+                    #Step through the list - storing an index
+                    for repindex,repcheck in enumerate(shortrep):
+                        if len(comicnm)>= len(shortrep):
+                            #if the leftmost chars match the short text then replace them with the long text
+                            if comicnm[:len(repcheck)]==repcheck:
+                                logger.info("Switch worked on "+comicnm + " replacing " + str(repcheck) + " with " + str(longrep[repindex]))
+                                comicnm = re.sub(repcheck, longrep[repindex], comicnm)
+
                 for excl in excludes:
                     if excl in str(comicrm):
                         #duplicate comic / issue detected - don't add...

substitutes_sample.csv

@@ -0,0 +1,22 @@
+#--------------------------------------------
+# This is a substitutes file for the weekly pull list
+# (Thanks to IanHub for this)
+# --- 
+# What this does is it will abbreviations for series on the weekly pull list
+# with the full series title, so that Mylar can match it up properly against ComicVine.
+# This is due to the pull-list having abbreviations in some titles, and
+# ComicVine not using the same abbreviations.
+# ---- 
+# To use this you have to first either rename this file to substitutes.csv
+# Make sure it's in the the same directory as the Mylar.py file (root directory).
+#
+# Within the csv, you can add text in the format of:
+# pull-list text|replacement text
+# 
+# ie. GFT|GRIMM FAIRY TALES PRESENTS
+#--------------------------------------------
+GFT GRIMM FAIRY TALES|GRIMM FAIRY TALES PRESENTS
+GFT|GRIMM FAIRY TALES PRESENTS
+HELLRAISER|CLIVE BARKER'S HELLRAISER
+BTVS SEASON 9|BUFFY THE VAMPIRE SLAYER SEASON NINE
+SUPURBIA|GRACE RANDOLPH'S SUPURBIA