From aad93a23140f0dc0e1868370e321894eb3bb753a Mon Sep 17 00:00:00 2001 From: evilhero Date: Thu, 16 Jan 2014 15:25:02 -0500 Subject: [PATCH] IMP:(#559) Differences between pull-list naming and ComicVine - substitutions.csv usage, FIX:(#583) When Importing Series, if filenames had a volume label it would error out, FIX:(#597) When Importing Series, decimal in issue would cause error, FIX:(#600) Annuals file format now inluded as (thanks uspider7), IMP:(#571) centos-init.d and ubuntu-init.d added, Various other fixes.... --- centos-mylar.init.d | 93 +++++++++++++++++++++++++ data/interfaces/default/config.html | 11 +-- mylar/PostProcessor.py | 47 +++++++++---- mylar/filechecker.py | 19 +++-- mylar/helpers.py | 54 ++++++++++++--- mylar/importer.py | 3 +- mylar/librarysync.py | 40 +++++++++-- mylar/rsscheck.py | 27 +++++--- mylar/search.py | 56 ++++++++++++--- mylar/solicit.py | 4 +- mylar/webserve.py | 104 ++++++++++++++++++++++++++-- mylar/weeklypull.py | 35 +++++++++- substitutes_sample.csv | 22 ++++++ mylar.init.d => ubuntu-mylar.init.d | 0 14 files changed, 449 insertions(+), 66 deletions(-) create mode 100644 centos-mylar.init.d create mode 100644 substitutes_sample.csv rename mylar.init.d => ubuntu-mylar.init.d (100%) diff --git a/centos-mylar.init.d b/centos-mylar.init.d new file mode 100644 index 00000000..28784b3e --- /dev/null +++ b/centos-mylar.init.d @@ -0,0 +1,93 @@ +#!/bin/sh +# +### BEGIN INIT INFO +# Provides: mylar +# Required-Start: $all +# Required-Stop: $all +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: starts Mylar +# Description: starts Mylar +### END INIT INFO + +# Source function library. +. /etc/init.d/functions + +# Source SickBeard configuration +if [ -f /etc/sysconfig/mylar ]; then + . /etc/sysconfig/mylar +fi + +prog=mylar +lockfile=/var/lock/subsys/$prog + +## Edit user configuation in /etc/sysconfig/mylar to change +## the defaults +username=${MY_USER-mylar} +homedir=${MY_HOME-/opt/mylar} +datadir=${MY_DATA-/opt/mylar} +pidfile=${MY_PIDFILE-/var/run/mylar/mylar.pid} +nice=${MY_NICE-} +## + +pidpath=`dirname ${pidfile}` +options=" --daemon --nolaunch --pidfile=${pidfile} --datadir=${datadir}" + +# create PID directory if not exist and ensure the SickBeard user can write to it +if [ ! -d $pidpath ]; then + mkdir -p $pidpath + chown $username $pidpath +fi + +if [ ! -d $datadir ]; then + mkdir -p $datadir + chown $username $datadir +fi + +start() { + # Start daemon. + echo -n $"Starting $prog: " + daemon --user=${username} --pidfile=${pidfile} ${nice} python ${homedir}/Mylar.py ${options} + RETVAL=$? + echo + [ $RETVAL -eq 0 ] && touch $lockfile + return $RETVAL +} + +stop() { + echo -n $"Shutting down $prog: " + killproc -p ${pidfile} python + RETVAL=$? + echo + [ $RETVAL -eq 0 ] && rm -f $lockfile + return $RETVAL +} + +# See how we were called. +case "$1" in + start) + start + ;; + stop) + stop + ;; + status) + status $prog + ;; + restart|force-reload) + stop + start + ;; + try-restart|condrestart) + if status $prog > /dev/null; then + stop + start + fi + ;; + reload) + exit 3 + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|try-restart|force-reload}" + exit 2 +esac diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 35295ec1..7fbbd649 100755 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -84,6 +84,9 @@
Branch history +
@@ -621,7 +624,7 @@ <% - file_options = "$Series = SeriesName\n$Year = SeriesYear\n$Issue = IssueNumber\n$VolumeY = V{SeriesYear}\n$VolumeN = V{Volume#}" + file_options = "$Series = SeriesName\n$Year = SeriesYear\n$Annual = Annual (word)\n$Issue = IssueNumber\n$VolumeY = V{SeriesYear}\n$VolumeN = V{Volume#}" %> Use: $Series, $Year, $Issue
@@ -732,13 +735,13 @@

NotifyMyAndroid

- +
- -
+ +
Separate multiple api keys with commas diff --git a/mylar/PostProcessor.py b/mylar/PostProcessor.py index 487b5bf5..17dacebf 100755 --- a/mylar/PostProcessor.py +++ b/mylar/PostProcessor.py @@ -175,7 +175,7 @@ class PostProcessor(object): myDB = db.DBConnection() if self.nzb_name == 'Manual Run': - print ("manual run initiated") + logger.fdebug ("manual run initiated") #Manual postprocessing on a folder. #use the nzb_folder to determine every file #walk the dir, @@ -197,7 +197,7 @@ class PostProcessor(object): "ComicVersion": cs['ComicVersion'], "Total": cs['Total']} watchmatch = filechecker.listFiles(self.nzb_folder,cs['ComicName'],cs['AlternateSearch'], manual=watchvals) - if watchmatch is None: + if watchmatch['comiccount'] == 0: # is None: nm+=1 continue else: @@ -267,10 +267,10 @@ class PostProcessor(object): "IssueNumber": issuechk['Issue_Number'], "ComicName": cs['ComicName']}) ccnt+=1 - print manual_list + #print manual_list wdc+=1 fn+=1 - print("There are " + str(len(manual_list)) + " files found that match on your watchlist, " + str(nm) + " do not match anything and will be ignored.") + logger.fdebug("There are " + str(len(manual_list)) + " files found that match on your watchlist, " + str(nm) + " do not match anything and will be ignored.") else: @@ -455,14 +455,11 @@ class PostProcessor(object): myDB = db.DBConnection() comicnzb = myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone() issuenzb = myDB.action("SELECT * from issues WHERE issueid=? AND comicid=? AND ComicName NOT NULL", [issueid,comicid]).fetchone() - print "issueid: " + str(issueid) - print "issuenumOG: " + str(issuenumOG) + logger.fdebug('issueid: ' + str(issueid)) + logger.fdebug('issuenumOG: ' + str(issuenumOG)) if issuenzb is None: - print "chk1" issuenzb = myDB.action("SELECT * from annuals WHERE issueid=? and comicid=?", [issueid,comicid]).fetchone() - print "chk2" annchk = "yes" - print issuenzb #issueno = str(issuenum).split('.')[0] #new CV API - removed all decimals...here we go AGAIN! issuenum = issuenzb['Issue_Number'] @@ -551,7 +548,6 @@ class PostProcessor(object): self._log("issue length error - cannot determine length. Defaulting to None: " + str(prettycomiss), logger.DEBUG) if annchk == "yes": - prettycomiss = "Annual " + str(prettycomiss) self._log("Annual detected.") logger.fdebug("Pretty Comic Issue is : " + str(prettycomiss)) issueyear = issuenzb['IssueDate'][:4] @@ -587,6 +583,22 @@ class PostProcessor(object): else: chunk_file_format = mylar.FILE_FORMAT + if annchk == "no": + chunk_f_f = re.sub('\$Annual','',chunk_file_format) + chunk_f = re.compile(r'\s+') + chunk_file_format = chunk_f.sub(' ', chunk_f_f) + logger.fdebug('not an annual - removing from filename paramaters') + logger.fdebug('new format: ' + str(chunk_file_format)) + + else: + logger.fdebug('chunk_file_format is: ' + str(chunk_file_format)) + if '$Annual' not in chunk_file_format: + #if it's an annual, but $Annual isn't specified in file_format, we need to + #force it in there, by default in the format of $Annual $Issue + prettycomiss = "Annual " + str(prettycomiss) + logger.fdebug('prettycomiss: ' + str(prettycomiss)) + + ofilename = None #if meta-tagging is not enabled, we need to declare the check as being fail @@ -652,7 +664,8 @@ class PostProcessor(object): '$Publisher': publisher, '$publisher': publisher.lower(), '$VolumeY': 'V' + str(seriesyear), - '$VolumeN': comversion + '$VolumeN': comversion, + '$Annual': 'Annual' } @@ -667,12 +680,12 @@ class PostProcessor(object): else: if pcheck == "fail": otofilename = ml['ComicLocation'] - print "otofilename:" + str(otofilename) + logger.fdebug('otofilename:' + str(otofilename)) odir, ofilename = os.path.split(otofilename) - print "ofilename: " + str(ofilename) + logger.fdebug('ofilename: ' + str(ofilename)) path, ext = os.path.splitext(ofilename) - print "path: " + str(path) - print "ext:" + str(ext) + logger.fdebug('path: ' + str(path)) + logger.fdebug('ext:' + str(ext)) if ofilename is None: logger.error(u"Aborting PostProcessing - the filename doesn't exist in the location given. Make sure that " + str(self.nzb_folder) + " exists and is the correct location.") @@ -696,6 +709,7 @@ class PostProcessor(object): #mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot nfilename = nfilename.replace(' ', mylar.REPLACE_CHAR) nfilename = re.sub('[\,\:\?]', '', nfilename) + nfilename = re.sub('[\/]', '-', nfilename) self._log("New Filename: " + nfilename, logger.DEBUG) logger.fdebug("New Filename: " + str(nfilename)) @@ -714,6 +728,9 @@ class PostProcessor(object): if ml is None: #non-manual run moving/deleting... + logger.fdebug('self.nzb_folder: ' + self.nzb_folder) + logger.fdebug('ofilename:' + str(ofilename)) + logger.fdebug('nfilename:' + str(nfilename + ext)) os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder,str(nfilename + ext))) src = os.path.join(self.nzb_folder, str(nfilename + ext)) try: diff --git a/mylar/filechecker.py b/mylar/filechecker.py index da174ca9..e9471a50 100755 --- a/mylar/filechecker.py +++ b/mylar/filechecker.py @@ -144,6 +144,7 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): nonocount = 0 charpos = 0 detneg = "no" + leavehyphen = False for nono in not_these: if nono in subname: subcnt = subname.count(nono) @@ -158,10 +159,11 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): logger.fdebug('possible negative issue detected.') nonocount = nonocount + subcnt - 1 detneg = "yes" - if '-' in watchcomic and i < len(watchcomic): + elif '-' in watchcomic and i < len(watchcomic): logger.fdebug('- appears in series title.') + leavehyphen = True i+=1 - if detneg == "no": + if detneg == "no" or leavehyphen == False: subname = re.sub(str(nono), ' ', subname) nonocount = nonocount + subcnt #logger.fdebug(str(nono) + " detected " + str(subcnt) + " times.") @@ -197,7 +199,12 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): subname = re.sub(str(nono), ' ', subname) nonocount = nonocount + subcnt + blspc #subname = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\+\'\?\@]',' ', subname) - modwatchcomic = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\'\?\@]', ' ', u_watchcomic) + + modwatchcomic = re.sub('[\_\#\,\/\:\;\.\!\$\%\'\?\@\-]', ' ', u_watchcomic) + #if leavehyphen == False: + # logger.fdebug('removing hyphen for comparisons') + # modwatchcomic = re.sub('-', ' ', modwatchcomic) + # subname = re.sub('-', ' ', subname) detectand = False detectthe = False modwatchcomic = re.sub('\&', ' and ', modwatchcomic) @@ -513,6 +520,10 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): if yearmatch == "false": continue + if 'annual' in subname.lower(): + subname = re.sub('annual', '', subname.lower()) + subname = re.sub('\s+', ' ', subname) + #tmpitem = item[:jtd_len] # if it's an alphanumeric with a space, rejoin, so we can remove it cleanly just below this. substring_removal = None @@ -575,9 +586,9 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): else: pass #print ("directory found - ignoring") + logger.fdebug('you have a total of ' + str(comiccnt) + ' ' + watchcomic + ' comics') watchmatch['comiccount'] = comiccnt - #print watchmatch return watchmatch def validateAndCreateDirectory(dir, create=False): diff --git a/mylar/helpers.py b/mylar/helpers.py index 56d991b9..c756f980 100755 --- a/mylar/helpers.py +++ b/mylar/helpers.py @@ -239,7 +239,7 @@ def decimal_issue(iss): deciss = (int(iss_b4dec) * 1000) + issdec return deciss, dec_except -def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=None): +def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=None, annualize=None): import db, logger myDB = db.DBConnection() logger.fdebug('comicid: ' + str(comicid)) @@ -262,22 +262,40 @@ def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=N # issue = iss # print ("converted issue#: " + str(issue)) + logger.fdebug('issueid:' + str(issueid)) + if issueid is None: - chkissue = myDB.action("SELECT * from issues WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone() + logger.fdebug('annualize is ' + str(annualize)) + if annualize is None: + chkissue = myDB.action("SELECT * from issues WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone() + else: + chkissue = myDB.action("SELECT * from annuals WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone() + if chkissue is None: #rechk chkissue against int value of issue # - chkissue = myDB.action("SELECT * from issues WHERE ComicID=? AND Issue_Number=?", [comicid, int(issue)]).fetchone() + chkissue = myDB.action("SELECT * from issues WHERE ComicID=? AND Int_IssueNumber=?", [comicid, issuedigits(issue)]).fetchone() if chkissue is None: - logger.error('Invalid Issue_Number - please validate.') - return + if chkissue is None: + logger.error('Invalid Issue_Number - please validate.') + return else: logger.info('Int Issue_number compare found. continuing...') - issueid = chkissue['IssueID'] + issueid = chkissue['IssueID'] else: issueid = chkissue['IssueID'] #use issueid to get publisher, series, year, issue number - issuenzb = myDB.action("SELECT * from issues WHERE issueid=?", [issueid]).fetchone() + logger.fdebug('issueid is now : ' + str(issueid)) + issuenzb = myDB.action("SELECT * from issues WHERE ComicID=? AND IssueID=?", [comicid,issueid]).fetchone() + if issuenzb is None: + logger.fdebug('not an issue, checking against annuals') + issuenzb = myDB.action("SELECT * from annuals WHERE ComicID=? AND IssueID=?", [comicid,issueid]).fetchone() + if issuenzb is None: + logger.fdebug('Unable to rename - cannot locate issue id within db') + return + else: + annualize = True + logger.fdebug('blah') #comicid = issuenzb['ComicID'] issuenum = issuenzb['Issue_Number'] #issueno = str(issuenum).split('.')[0] @@ -306,6 +324,8 @@ def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=N else: iss = issuenum issueno = str(iss) + logger.fdebug('iss:' + str(iss)) + logger.fdebug('issueno:' + str(issueno)) # issue zero-suppression here if mylar.ZERO_LEVEL == "0": zeroadd = "" @@ -381,10 +401,25 @@ def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=N chunk_f = re.compile(r'\s+') chunk_file_format = chunk_f.sub(' ', chunk_f_f) logger.fdebug('No version # found for series, removing from filename') - print ("new format: " + str(chunk_file_format)) + logger.fdebug("new format: " + str(chunk_file_format)) else: chunk_file_format = mylar.FILE_FORMAT + if annualize is None: + chunk_f_f = re.sub('\$Annual','',chunk_file_format) + chunk_f = re.compile(r'\s+') + chunk_file_format = chunk_f.sub(' ', chunk_f_f) + logger.fdebug('not an annual - removing from filename paramaters') + logger.fdebug('new format: ' + str(chunk_file_format)) + + else: + logger.fdebug('chunk_file_format is: ' + str(chunk_file_format)) + if '$Annual' not in chunk_file_format: + #if it's an annual, but $annual isn't specified in file_format, we need to + #force it in there, by default in the format of $Annual $Issue + prettycomiss = "Annual " + str(prettycomiss) + logger.fdebug('prettycomiss: ' + str(prettycomiss)) + file_values = {'$Series': series, '$Issue': prettycomiss, '$Year': issueyear, @@ -392,7 +427,8 @@ def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=N '$Publisher': publisher, '$publisher': publisher.lower(), '$VolumeY': 'V' + str(seriesyear), - '$VolumeN': comversion + '$VolumeN': comversion, + '$Annual': 'Annual' } extensions = ('.cbr', '.cbz') diff --git a/mylar/importer.py b/mylar/importer.py index 748a0b84..d5623609 100755 --- a/mylar/importer.py +++ b/mylar/importer.py @@ -296,7 +296,8 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None,c '$series': series.lower(), '$publisher': publisher.lower(), '$VolumeY': 'V' + str(year), - '$VolumeN': comversion + '$VolumeN': comversion, + '$Annual': 'Annual' } diff --git a/mylar/librarysync.py b/mylar/librarysync.py index cd7ca9ac..7b14f952 100755 --- a/mylar/librarysync.py +++ b/mylar/librarysync.py @@ -138,9 +138,30 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None) cfilename = re.sub('[\_\#\,\/\:\;\-\!\$\%\&\+\'\?\@]', ' ', comfilename) #cfilename = re.sub('\s', '_', str(cfilename)) + #versioning - remove it + subsplit = cfilename.replace('_', ' ').split() + volno = None + volyr = None + for subit in subsplit: + if subit[0].lower() == 'v': + vfull = 0 + if subit[1:].isdigit(): + #if in format v1, v2009 etc... + if len(subit) > 3: + # if it's greater than 3 in length, then the format is Vyyyy + vfull = 1 # add on 1 character length to account for extra space + cfilename = re.sub(subit, '', cfilename) + volno = re.sub("[^0-9]", " ", subit) + elif subit.lower()[:3] == 'vol': + #if in format vol.2013 etc + #because the '.' in Vol. gets removed, let's loop thru again after the Vol hit to remove it entirely + logger.fdebug('volume indicator detected as version #:' + str(subit)) + cfilename = re.sub(subit, '', cfilename) + volyr = re.sub("[^0-9]", " ", subit) + cm_cn = 0 - #we need to track the counter to make sure we are comparing the right array parts + #we need to track the counter to make sure we are comparing the right array parts #this takes care of the brackets :) m = re.findall('[^()]+', cfilename) lenm = len(m) @@ -150,6 +171,7 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None) foundonwatch = "False" issue = 999999 + while (cnt < lenm): if m[cnt] is None: break if m[cnt] == ' ': @@ -239,11 +261,13 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None) splitit = [] watchcomic_split = [] - logger.fdebug("filename comic and issue: " + cfilename) + logger.fdebug("filename comic and issue: " + comic_andiss) + #changed this from '' to ' ' - comic_iss_b4 = re.sub('[\-\:\,]', ' ', com_NAME) + comic_iss_b4 = re.sub('[\-\:\,]', ' ', comic_andiss) comic_iss = comic_iss_b4.replace('.',' ') - logger.fdebug("adjusted comic and issue: " + str(comic_iss)) + comic_iss = re.sub('[\s+]', ' ', comic_iss).strip() + logger.fdebug("adjusted comic and issue: " + str(comic_iss)) #remove 'the' from here for proper comparisons. if ' the ' in comic_iss.lower(): comic_iss = comic_iss[-4:] @@ -417,7 +441,13 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None) # cmnam+=1 # logger.fdebug("comic: " + str(com_NAME)) # n+=1 - if result_comyear is None: result_comyear = '0000' #no year in filename basically. + if volyr is None: + if result_comyear is None: + result_comyear = '0000' #no year in filename basically. + else: + if result_comyear is None: + result_comyear = volyr + print ("adding " + com_NAME + " to the import-queue!") impid = com_NAME + "-" + str(result_comyear) + "-" + str(comiss) print ("impid: " + str(impid)) diff --git a/mylar/rsscheck.py b/mylar/rsscheck.py index 1f241c16..c9f7e146 100755 --- a/mylar/rsscheck.py +++ b/mylar/rsscheck.py @@ -428,10 +428,16 @@ def torrentdbsearch(seriesname,issue,comicid=None,nzbprov=None): while (i < len(torsplit)): #we'll rebuild the string here so that it's formatted accordingly to be passed back to the parser. logger.fdebug('section(' + str(i) + '): ' + str(torsplit[i])) + #remove extensions + titletemp = torsplit[i] + titletemp = re.sub('cbr', '', str(titletemp)) + titletemp = re.sub('cbz', '', str(titletemp)) + titletemp = re.sub('none', '', str(titletemp)) + if i == 0: - rebuiltline = str(torsplit[i]) + rebuiltline = str(titletemp) else: - rebuiltline = rebuiltline + ' (' + str(torsplit[i]) + ')' + rebuiltline = rebuiltline + ' (' + str(titletemp) + ')' i+=1 logger.fdebug('rebuiltline is :' + str(rebuiltline)) @@ -462,9 +468,12 @@ def torrentdbsearch(seriesname,issue,comicid=None,nzbprov=None): logger.fdebug('matched on series title: ' + seriesname) titleend = formatrem_torsplit[len(formatrem_seriesname):] titleend = re.sub('\-', '', titleend) #remove the '-' which is unnecessary - - titleend = re.sub('cbr', '', str(titleend)) #remove extensions + #remove extensions + titleend = re.sub('cbr', '', str(titleend)) + titleend = re.sub('cbz', '', str(titleend)) + titleend = re.sub('none', '', str(titleend)) logger.fdebug('titleend: ' + str(titleend)) + sptitle = titleend.split() extra = '' # for sp in sptitle: @@ -480,14 +489,16 @@ def torrentdbsearch(seriesname,issue,comicid=None,nzbprov=None): ctitle = tor['Title'].find('cbr') if ctitle == 0: ctitle = tor['Title'].find('cbz') - if ctitle == 0: - logger.fdebug('cannot determine title properly - ignoring for now.') - continue + if ctitle == 0: + ctitle = tor['Title'].find('none') + if ctitle == 0: + logger.fdebug('cannot determine title properly - ignoring for now.') + continue cttitle = tor['Title'][:ctitle] #print("change title to : " + str(cttitle)) # if extra == '': tortheinfo.append({ - 'title': cttitle, #tor['Title'], + 'title': rebuiltline, #cttitle, 'link': tor['Link'], 'pubdate': tor['Pubdate'], 'site': tor['Site'], diff --git a/mylar/search.py b/mylar/search.py index 72ab2069..14c8fbbc 100755 --- a/mylar/search.py +++ b/mylar/search.py @@ -33,6 +33,7 @@ import sys import getopt import re import time +import urlparse from xml.dom.minidom import parseString import urllib2 from datetime import datetime @@ -389,6 +390,9 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is tmpprov = nzbprov logger.info(u"Shhh be very quiet...I'm looking for " + ComicName + " issue: " + str(IssueNumber) + " (" + str(ComicYear) + ") using " + str(tmpprov)) + #load in do not download db here for given series + #myDB = db.DBConnection() + #nodown = myDB.action('SELECT * FROM nzblog') if mylar.PREFERRED_QUALITY == 0: filetype = "" elif mylar.PREFERRED_QUALITY == 1: filetype = ".cbr" @@ -681,12 +685,12 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is thisentry = entry['title'] logger.fdebug("Entry: " + thisentry) - cleantitle = re.sub('[_/.]', ' ', entry['title']) + cleantitle = re.sub('[\_\.]', ' ', entry['title']) cleantitle = helpers.cleanName(cleantitle) # this is new - if title contains a '&' in the title it will assume the filename has ended at that point # which causes false positives (ie. wolverine & the x-men becomes the x-men, which matches on x-men. # 'the' is removed for comparisons later on - if '&' in cleantitle: cleantitle = re.sub('[/&]','and', cleantitle) + if '&' in cleantitle: cleantitle = re.sub('[\&]','and', cleantitle) nzbname = cleantitle @@ -737,16 +741,20 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is if vers4year == "no" and vers4vol == "no": # if the series is a v1, let's remove the requirements for year and volume label - if ComVersChk != 0: + # even if it's a v1, the nzbname might not contain a valid year format (20xx) or v3, + # and since it's already known that there is no (year) or vYEAR given + # let's push it through (and edit out the following if constraint)... + + #if ComVersChk != 0: # if there are no () in the string, try to add them if it looks like a year (19xx or 20xx) - if len(re.findall('[^()]+', cleantitle)): - logger.fdebug("detected invalid nzb filename - attempting to detect year to continue") - cleantitle = re.sub('(.*)\s+(19\d{2}|20\d{2})(.*)', '\\1 (\\2) \\3', cleantitle) - continue - else: - logger.fdebug("invalid nzb and/or cover only - skipping.") - cleantitle = "abcdefghijk 0 (1901).cbz" - continue + if len(re.findall('[^()]+', cleantitle)): + logger.fdebug("detected invalid nzb filename - attempting to detect year to continue") + cleantitle = re.sub('(.*)\s+(19\d{2}|20\d{2})(.*)', '\\1 (\\2) \\3', cleantitle) + continue + else: + logger.fdebug("invalid nzb and/or cover only - skipping.") + cleantitle = "abcdefghijk 0 (1901).cbz" + continue #adjust for covers only by removing them entirely... logger.fdebug("Cleantitle: " + str(cleantitle)) @@ -1078,6 +1086,32 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is #issue comparison now as well if int(intIss) == int(comintIss): + #check if nzb is in do not download list ;) + if nzbprov == 'experimental': + #id is located after the /download/ portion + url_parts = urlparse.urlparse(entry['link']) + path_parts = url_parts[2].rpartition('/') + nzbtempid = path_parts[0].rpartition('/') + nzblen = len(nzbtempid) + nzbid = nzbtempid[nzblen-1] + elif nzbprov == 'CBT': + url_parts = urlparse.urlparse(entry['link']) + nzbtemp = url_parts[4] # get the query paramater string + nzbtemp = re.sub('torrent=', '', nzbtemp).rstrip() + nzbid = re.sub('.torrent', '', nzbtemp).rstrip() + elif nzbprov == 'KAT': + url_parts = urllib.parse.urlparse(entry['link']) + path_parts = url_parts[2].rpartition('/') + nzbtempid = pathparts[2] + nzbid = re.sub('.torrent', '', nzbtempid).rstrip() + elif nzbprov == 'nzb.su': + pass + elif nzbprov == 'dognzb': + pass + elif nzbprov == 'newznab': + #if in format of http://newznab/getnzb/.nzb&i=1&r=apikey + nzbid = os.path.splitext(entry['link'])[0].rsplit('/', 1)[1] + logger.fdebug('issues match!') logger.info(u"Found " + ComicName + " (" + str(comyear) + ") issue: " + str(IssueNumber) + " using " + str(tmpprov) ) ## -- inherit issue. Comic year is non-standard. nzb year is the year diff --git a/mylar/solicit.py b/mylar/solicit.py index ee26b5ed..fb9f485a 100755 --- a/mylar/solicit.py +++ b/mylar/solicit.py @@ -80,7 +80,7 @@ def solicit(month, year): loopthis = (cnt-1) #this loops through each 'found' solicit page - shipdate = str(month) + '-' + str(year) + shipdate = str(month_string) + '-' + str(year) while ( loopthis >= 0 ): upcoming += populate(resultURL[loopthis], publish[loopthis], shipdate) loopthis -=1 @@ -178,7 +178,7 @@ def populate(link,publisher,shipdate): get_next = False if prev_chk == True: tempName = titlet.findNext(text=True) - if ' TPB' not in tempName and ' HC' not in tempName and 'GN-TPB' not in tempName and 'for $1' not in tempName.lower() and 'subscription variant' not in tempName.lower(): + if ' TPB' not in tempName and ' HC' not in tempName and 'GN-TPB' not in tempName and 'for $1' not in tempName.lower() and 'subscription variant' not in tempName.lower() and 'poster' not in tempName.lower(): #print publisher + ' found upcoming' if '#' in tempName: #tempName = tempName.replace(u'.',u"'") diff --git a/mylar/webserve.py b/mylar/webserve.py index 0ce72e92..42d0e319 100755 --- a/mylar/webserve.py +++ b/mylar/webserve.py @@ -859,7 +859,46 @@ class WebInterface(object): def upcoming(self): myDB = db.DBConnection() #upcoming = myDB.select("SELECT * from issues WHERE ReleaseDate > date('now') order by ReleaseDate DESC") - upcoming = myDB.select("SELECT * from upcoming WHERE IssueDate > date('now') AND IssueID is NULL order by IssueDate DESC") + upcomingdata = myDB.select("SELECT * from upcoming WHERE IssueID is NULL order by IssueDate DESC") + upcoming = [] + for upc in upcomingdata: + if len(upc['IssueDate']) <= 7 : + #if it's less than or equal 7, then it's a future-pull so let's check the date and display + #tmpdate = datetime.datetime.com + tmpdatethis = upc['IssueDate'] + if tmpdatethis[:2] == '20': + tmpdate = tmpdatethis #in correct format of yyyymm + else: + findst = tmpdatethis.find('-') #find the '-' + tmpdate = tmpdatethis[findst+1:] + tmpdatethis[:findst] #rebuild in format of yyyymm + timenow = datetime.datetime.now().strftime('%Y%m') + logger.fdebug('comparing pubdate of: ' + str(tmpdate) + ' to now date of: ' + str(timenow)) + if int(tmpdate) >= int(timenow): + if upc['Status'] == 'Wanted': + upcoming.append({"ComicName": upc['ComicName'], + "IssueNumber": upc['IssueNumber'], + "IssueDate": upc['IssueDate'], + "ComicID": upc['ComicID'], + "IssueID": upc['IssueID'], + "Status": upc['Status'], + "DisplayComicName": upc['DisplayComicName']}) + else: + #if it's greater than 7 it's a full date, and shouldn't be displayed ;) + timenow = datetime.datetime.now().strftime('%Y%m%d') #convert to yyyymmdd + tmpdate = re.sub("[^0-9]", "", upc['IssueDate']) #convert date to numerics only (should be in yyyymmdd) + + logger.fdebug('comparing pubdate of: ' + str(tmpdate) + ' to now date of: ' + str(timenow)) + + if int(tmpdate) >= int(timenow): + if upc['Status'] == 'Wanted': + upcoming.append({"ComicName": upc['ComicName'], + "IssueNumber": upc['IssueNumber'], + "IssueDate": upc['IssueDate'], + "ComicID": upc['ComicID'], + "IssueID": upc['IssueID'], + "Status": upc['Status'], + "DisplayComicName": upc['DisplayComicName']}) + issues = myDB.select("SELECT * from issues WHERE Status='Wanted'") ann_list = [] @@ -940,6 +979,8 @@ class WebInterface(object): comicname = comic['ComicName'] extensions = ('.cbr', '.cbz') issues = myDB.action("SELECT * FROM issues WHERE ComicID=?", [comicid]).fetchall() + if mylar.ANNUALS_ON: + issues += myDB.action("SELECT * FROM annuals WHERE ComicID=?", [comicid]).fetchall() comfiles = [] filefind = 0 for root, dirnames, filenames in os.walk(comicdir): @@ -949,7 +990,11 @@ class WebInterface(object): for issue in issues: if issue['Location'] == filename: #logger.error("matched " + str(filename) + " to DB file " + str(issue['Location'])) - renameiss = helpers.rename_param(comicid, comicname, issue['Issue_Number'], filename, comicyear=None, issueid=None) + if 'annual' in issue['Location'].lower(): + annualize = 'yes' + else: + annualize = None + renameiss = helpers.rename_param(comicid, comicname, issue['Issue_Number'], filename, comicyear=None, issueid=None, annualize=annualize) nfilename = renameiss['nfilename'] srciss = os.path.join(comicdir,filename) if mylar.LOWERCASE_FILENAMES: @@ -1901,14 +1946,58 @@ class WebInterface(object): return serve_template(templatename="searchresults.html", title='Import Results for: "' + ComicName + '"',searchresults=sresults, type=type, imported='yes', ogcname=ogcname) #imported=comicstoIMP, ogcname=ogcname) preSearchit.exposed = True + def pretty_git(self, br_history): + #in order to 'prettify' the history log for display, we need to break it down so it's line by line. + br_split = br_history.split("\n") #split it on each commit + for br in br_split: + br_commit_st = br.find('-') #first - will represent end of commit numeric + br_commit = br[:br_commit_st].strip() + br_time_en = br.replace('-', 'XXX', 1).find('-') #2nd - is end of time datestamp + br_time = br[br_commit_st+1:br_time_en].strip() + print 'COMMIT:' + str(br_commit) + print 'TIME:' + str(br_time) + commit_split = br.split() #split it by space to break it further down.. + tag_chk = False + statline = '' + commit = [] + for cs in commit_split: + if tag_chk == True: + if 'FIX:' in cs or 'IMP:' in cs: + commit.append({"commit": br_commit, + "time": br_time, + "stat": tag_status, + "line": statline}) + print commit + tag_chk == False + statline = '' + else: + statline += str(cs) + ' ' + else: + if 'FIX:' in cs: + tag_status = 'FIX' + tag_chk = True + print 'status: ' + str(tag_status) + elif 'IMP:' in cs: + tag_status = 'IMPROVEMENT' + tag_chk = True + print 'status: ' + str(tag_status) + + pretty_git.exposed = True #--- def config(self): interface_dir = os.path.join(mylar.PROG_DIR, 'data/interfaces/') interface_list = [ name for name in os.listdir(interface_dir) if os.path.isdir(os.path.join(interface_dir, name)) ] - +#---- +# to be implemented in the future. # branch_history, err = mylar.versioncheck.runGit("log --oneline --pretty=format:'%h - %ar - %s' -n 4") -# br_hist = branch_history.replace("\n", "
\n") +# #here we pass the branch_history to the pretty_git module to break it down +# if branch_history: +# self.pretty_git(branch_history) +# br_hist = branch_history.replace("\n", "
\n") +# else: +# br_hist = err +#---- myDB = db.DBConnection() CCOMICS = myDB.action("SELECT COUNT(*) FROM comics").fetchall() CHAVES = myDB.action("SELECT COUNT(*) FROM issues WHERE Status='Downloaded' OR Status='Archived'").fetchall() @@ -2040,7 +2129,8 @@ class WebInterface(object): "prog_dir" : mylar.PROG_DIR, "cache_dir" : mylar.CACHE_DIR, "config_file" : mylar.CONFIG_FILE, -# "branch_history" : br_hist + "branch_history" : 'None', +# "branch_history" : br_hist, "enable_pre_scripts" : helpers.checked(mylar.ENABLE_PRE_SCRIPTS), "pre_scripts" : mylar.PRE_SCRIPTS, "log_dir" : mylar.LOG_DIR @@ -2355,6 +2445,10 @@ class WebInterface(object): logger.info("CHMOD File value is not a valid numeric - please correct. Defaulting to 0660") mylar.CHMOD_FILE = '0660' + if mylar.SAB_HOST.endswith('/'): + logger.info("Auto-correcting trailing slash in SABnzbd url (not required)") + mylar.SAB_HOST = mylar.SAB_HOST[:-1] + if mylar.ENABLE_META: if mylar.CMTAGGER_PATH is None or mylar.CMTAGGER_PATH == '': logger.info("ComicTagger Path not set - defaulting to Mylar Program Directory : " + mylar.PROG_DIR) diff --git a/mylar/weeklypull.py b/mylar/weeklypull.py index a8d89eb2..d1da5b04 100755 --- a/mylar/weeklypull.py +++ b/mylar/weeklypull.py @@ -56,6 +56,27 @@ def pullit(forcecheck=None): PULLURL = 'http://www.previewsworld.com/shipping/newreleases.txt' #PULLURL = 'http://www.previewsworld.com/Archive/GetFile/1/1/71/994/081512.txt' + #Prepare the Substitute name switch for pulllist to comic vine conversion + substitutes = os.path.join(mylar.DATA_DIR,"substitutes.csv") + if not os.path.exists(substitutes): + logger.debug('no substitues.csv file located - not performing substitutions on weekly pull list') + substitute_check = False + else: + substitute_check = True + #shortrep is the name to be replaced, longrep the replacement + shortrep=[] + longrep=[] + #open the file data + with open(substitutes) as f: + reader = csv.reader(f, delimiter='|') + for row in reader: + if not row.startswith('#'): + logger.debug ("Substitutes file read : "+str(row)) + shortrep.append(row[0]) + longrep.append(row[1]) + + f.close() + not_these=['PREVIEWS', 'Shipping', 'Every Wednesday', @@ -188,7 +209,7 @@ def pullit(forcecheck=None): dupefound = "no" if '#' in i: issname = i.split() - print (issname) + #print (issname) issnamec = len(issname) n = 0 while (n < issnamec): @@ -241,7 +262,7 @@ def pullit(forcecheck=None): #if it doesn't have a '#' in the line, then we know it's either #a special edition of some kind, or a non-comic issname = i.split() - print (issname) + #print (issname) issnamec = len(issname) n = 1 issue = '' @@ -301,6 +322,16 @@ def pullit(forcecheck=None): # pullist has shortforms of a series' title sometimes and causes problems if 'O/T' in comicnm: comicnm = re.sub('O/T', 'OF THE', comicnm) + + if substitute_check == True: + #Step through the list - storing an index + for repindex,repcheck in enumerate(shortrep): + if len(comicnm)>= len(shortrep): + #if the leftmost chars match the short text then replace them with the long text + if comicnm[:len(repcheck)]==repcheck: + logger.info("Switch worked on "+comicnm + " replacing " + str(repcheck) + " with " + str(longrep[repindex])) + comicnm = re.sub(repcheck, longrep[repindex], comicnm) + for excl in excludes: if excl in str(comicrm): #duplicate comic / issue detected - don't add... diff --git a/substitutes_sample.csv b/substitutes_sample.csv new file mode 100644 index 00000000..39708c56 --- /dev/null +++ b/substitutes_sample.csv @@ -0,0 +1,22 @@ +#-------------------------------------------- +# This is a substitutes file for the weekly pull list +# (Thanks to IanHub for this) +# --- +# What this does is it will abbreviations for series on the weekly pull list +# with the full series title, so that Mylar can match it up properly against ComicVine. +# This is due to the pull-list having abbreviations in some titles, and +# ComicVine not using the same abbreviations. +# ---- +# To use this you have to first either rename this file to substitutes.csv +# Make sure it's in the the same directory as the Mylar.py file (root directory). +# +# Within the csv, you can add text in the format of: +# pull-list text|replacement text +# +# ie. GFT|GRIMM FAIRY TALES PRESENTS +#-------------------------------------------- +GFT GRIMM FAIRY TALES|GRIMM FAIRY TALES PRESENTS +GFT|GRIMM FAIRY TALES PRESENTS +HELLRAISER|CLIVE BARKER'S HELLRAISER +BTVS SEASON 9|BUFFY THE VAMPIRE SLAYER SEASON NINE +SUPURBIA|GRACE RANDOLPH'S SUPURBIA diff --git a/mylar.init.d b/ubuntu-mylar.init.d similarity index 100% rename from mylar.init.d rename to ubuntu-mylar.init.d