IMP:(#559) Differences between pull-list naming and ComicVine - substitutions.csv usage, FIX:(#583) When Importing Series, if filenames had a volume label it would error out, FIX:(#597) When Importing Series, decimal in issue would cause error, FIX:(#600) Annuals file format now inluded as (thanks uspider7), IMP:(#571) centos-init.d and ubuntu-init.d added, Various other fixes....

This commit is contained in:
evilhero 2014-01-16 15:25:02 -05:00
parent b27584af81
commit aad93a2314
14 changed files with 449 additions and 66 deletions

93
centos-mylar.init.d Normal file
View File

@ -0,0 +1,93 @@
#!/bin/sh
#
### BEGIN INIT INFO
# Provides: mylar
# Required-Start: $all
# Required-Stop: $all
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: starts Mylar
# Description: starts Mylar
### END INIT INFO
# Source function library.
. /etc/init.d/functions
# Source SickBeard configuration
if [ -f /etc/sysconfig/mylar ]; then
. /etc/sysconfig/mylar
fi
prog=mylar
lockfile=/var/lock/subsys/$prog
## Edit user configuation in /etc/sysconfig/mylar to change
## the defaults
username=${MY_USER-mylar}
homedir=${MY_HOME-/opt/mylar}
datadir=${MY_DATA-/opt/mylar}
pidfile=${MY_PIDFILE-/var/run/mylar/mylar.pid}
nice=${MY_NICE-}
##
pidpath=`dirname ${pidfile}`
options=" --daemon --nolaunch --pidfile=${pidfile} --datadir=${datadir}"
# create PID directory if not exist and ensure the SickBeard user can write to it
if [ ! -d $pidpath ]; then
mkdir -p $pidpath
chown $username $pidpath
fi
if [ ! -d $datadir ]; then
mkdir -p $datadir
chown $username $datadir
fi
start() {
# Start daemon.
echo -n $"Starting $prog: "
daemon --user=${username} --pidfile=${pidfile} ${nice} python ${homedir}/Mylar.py ${options}
RETVAL=$?
echo
[ $RETVAL -eq 0 ] && touch $lockfile
return $RETVAL
}
stop() {
echo -n $"Shutting down $prog: "
killproc -p ${pidfile} python
RETVAL=$?
echo
[ $RETVAL -eq 0 ] && rm -f $lockfile
return $RETVAL
}
# See how we were called.
case "$1" in
start)
start
;;
stop)
stop
;;
status)
status $prog
;;
restart|force-reload)
stop
start
;;
try-restart|condrestart)
if status $prog > /dev/null; then
stop
start
fi
;;
reload)
exit 3
;;
*)
echo $"Usage: $0 {start|stop|status|restart|try-restart|force-reload}"
exit 2
esac

View File

@ -84,6 +84,9 @@
<fieldset>
<legend>Branch history</legend>
<label>This would be a nice place to see revision history...</label>
<!--
<div>${config['branch_history']}</div>
-->
</fieldset>
</td>
</tr>
@ -621,7 +624,7 @@
<label> File Format</label>
<input type="text" name="file_format" value="${config['file_format']}" size="43">
<%
file_options = "$Series = SeriesName\n$Year = SeriesYear\n$Issue = IssueNumber\n$VolumeY = V{SeriesYear}\n$VolumeN = V{Volume#}"
file_options = "$Series = SeriesName\n$Year = SeriesYear\n$Annual = Annual (word)\n$Issue = IssueNumber\n$VolumeY = V{SeriesYear}\n$VolumeN = V{Volume#}"
%>
<a href="#" title="${file_options}"><img src="interfaces/default/images/info32.png" height="16" alt="" /></a>
<small>Use: $Series, $Year, $Issue<br />
@ -732,13 +735,13 @@
<fieldset>
<h3><img src="interfaces/default/images/nma_logo.png" style="vertical-align: middle; margin: 3px; margin-top: -1px;" height="30" width="30"/>NotifyMyAndroid</h3>
<div class="checkbox row">
<input type="checkbox" name="nma_enabled" id="nma" value="1" ${config['nma_enabled']} /><label>Enable NotifyMyAndroid</label>
<input type="checkbox" name="nma_enabled" id="nma" value="1" ${config['nma_enabled']} /><label>Enable NotifyMyAndroid</label>
</div>
<div id="nmaoptions">
<div class="row">
<div class="row checkbox">
<input type="checkbox" name="nma_onsnatch" value="1" ${config['nma_onsnatch']} /><label>Notify on snatch?</label>
</div>
<input type="checkbox" name="nma_onsnatch" value="1" ${config['nma_onsnatch']} /><label>Notify on snatch?</label>
</div>
<label>NotifyMyAndroid API Key</label>
<input type="text" name="nma_apikey" value="${config['nma_apikey']}" size="30">
<small>Separate multiple api keys with commas</small>

View File

@ -175,7 +175,7 @@ class PostProcessor(object):
myDB = db.DBConnection()
if self.nzb_name == 'Manual Run':
print ("manual run initiated")
logger.fdebug ("manual run initiated")
#Manual postprocessing on a folder.
#use the nzb_folder to determine every file
#walk the dir,
@ -197,7 +197,7 @@ class PostProcessor(object):
"ComicVersion": cs['ComicVersion'],
"Total": cs['Total']}
watchmatch = filechecker.listFiles(self.nzb_folder,cs['ComicName'],cs['AlternateSearch'], manual=watchvals)
if watchmatch is None:
if watchmatch['comiccount'] == 0: # is None:
nm+=1
continue
else:
@ -267,10 +267,10 @@ class PostProcessor(object):
"IssueNumber": issuechk['Issue_Number'],
"ComicName": cs['ComicName']})
ccnt+=1
print manual_list
#print manual_list
wdc+=1
fn+=1
print("There are " + str(len(manual_list)) + " files found that match on your watchlist, " + str(nm) + " do not match anything and will be ignored.")
logger.fdebug("There are " + str(len(manual_list)) + " files found that match on your watchlist, " + str(nm) + " do not match anything and will be ignored.")
else:
@ -455,14 +455,11 @@ class PostProcessor(object):
myDB = db.DBConnection()
comicnzb = myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone()
issuenzb = myDB.action("SELECT * from issues WHERE issueid=? AND comicid=? AND ComicName NOT NULL", [issueid,comicid]).fetchone()
print "issueid: " + str(issueid)
print "issuenumOG: " + str(issuenumOG)
logger.fdebug('issueid: ' + str(issueid))
logger.fdebug('issuenumOG: ' + str(issuenumOG))
if issuenzb is None:
print "chk1"
issuenzb = myDB.action("SELECT * from annuals WHERE issueid=? and comicid=?", [issueid,comicid]).fetchone()
print "chk2"
annchk = "yes"
print issuenzb
#issueno = str(issuenum).split('.')[0]
#new CV API - removed all decimals...here we go AGAIN!
issuenum = issuenzb['Issue_Number']
@ -551,7 +548,6 @@ class PostProcessor(object):
self._log("issue length error - cannot determine length. Defaulting to None: " + str(prettycomiss), logger.DEBUG)
if annchk == "yes":
prettycomiss = "Annual " + str(prettycomiss)
self._log("Annual detected.")
logger.fdebug("Pretty Comic Issue is : " + str(prettycomiss))
issueyear = issuenzb['IssueDate'][:4]
@ -587,6 +583,22 @@ class PostProcessor(object):
else:
chunk_file_format = mylar.FILE_FORMAT
if annchk == "no":
chunk_f_f = re.sub('\$Annual','',chunk_file_format)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
logger.fdebug('not an annual - removing from filename paramaters')
logger.fdebug('new format: ' + str(chunk_file_format))
else:
logger.fdebug('chunk_file_format is: ' + str(chunk_file_format))
if '$Annual' not in chunk_file_format:
#if it's an annual, but $Annual isn't specified in file_format, we need to
#force it in there, by default in the format of $Annual $Issue
prettycomiss = "Annual " + str(prettycomiss)
logger.fdebug('prettycomiss: ' + str(prettycomiss))
ofilename = None
#if meta-tagging is not enabled, we need to declare the check as being fail
@ -652,7 +664,8 @@ class PostProcessor(object):
'$Publisher': publisher,
'$publisher': publisher.lower(),
'$VolumeY': 'V' + str(seriesyear),
'$VolumeN': comversion
'$VolumeN': comversion,
'$Annual': 'Annual'
}
@ -667,12 +680,12 @@ class PostProcessor(object):
else:
if pcheck == "fail":
otofilename = ml['ComicLocation']
print "otofilename:" + str(otofilename)
logger.fdebug('otofilename:' + str(otofilename))
odir, ofilename = os.path.split(otofilename)
print "ofilename: " + str(ofilename)
logger.fdebug('ofilename: ' + str(ofilename))
path, ext = os.path.splitext(ofilename)
print "path: " + str(path)
print "ext:" + str(ext)
logger.fdebug('path: ' + str(path))
logger.fdebug('ext:' + str(ext))
if ofilename is None:
logger.error(u"Aborting PostProcessing - the filename doesn't exist in the location given. Make sure that " + str(self.nzb_folder) + " exists and is the correct location.")
@ -696,6 +709,7 @@ class PostProcessor(object):
#mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
nfilename = nfilename.replace(' ', mylar.REPLACE_CHAR)
nfilename = re.sub('[\,\:\?]', '', nfilename)
nfilename = re.sub('[\/]', '-', nfilename)
self._log("New Filename: " + nfilename, logger.DEBUG)
logger.fdebug("New Filename: " + str(nfilename))
@ -714,6 +728,9 @@ class PostProcessor(object):
if ml is None:
#non-manual run moving/deleting...
logger.fdebug('self.nzb_folder: ' + self.nzb_folder)
logger.fdebug('ofilename:' + str(ofilename))
logger.fdebug('nfilename:' + str(nfilename + ext))
os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder,str(nfilename + ext)))
src = os.path.join(self.nzb_folder, str(nfilename + ext))
try:

View File

@ -144,6 +144,7 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None):
nonocount = 0
charpos = 0
detneg = "no"
leavehyphen = False
for nono in not_these:
if nono in subname:
subcnt = subname.count(nono)
@ -158,10 +159,11 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None):
logger.fdebug('possible negative issue detected.')
nonocount = nonocount + subcnt - 1
detneg = "yes"
if '-' in watchcomic and i < len(watchcomic):
elif '-' in watchcomic and i < len(watchcomic):
logger.fdebug('- appears in series title.')
leavehyphen = True
i+=1
if detneg == "no":
if detneg == "no" or leavehyphen == False:
subname = re.sub(str(nono), ' ', subname)
nonocount = nonocount + subcnt
#logger.fdebug(str(nono) + " detected " + str(subcnt) + " times.")
@ -197,7 +199,12 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None):
subname = re.sub(str(nono), ' ', subname)
nonocount = nonocount + subcnt + blspc
#subname = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\+\'\?\@]',' ', subname)
modwatchcomic = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\'\?\@]', ' ', u_watchcomic)
modwatchcomic = re.sub('[\_\#\,\/\:\;\.\!\$\%\'\?\@\-]', ' ', u_watchcomic)
#if leavehyphen == False:
# logger.fdebug('removing hyphen for comparisons')
# modwatchcomic = re.sub('-', ' ', modwatchcomic)
# subname = re.sub('-', ' ', subname)
detectand = False
detectthe = False
modwatchcomic = re.sub('\&', ' and ', modwatchcomic)
@ -513,6 +520,10 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None):
if yearmatch == "false": continue
if 'annual' in subname.lower():
subname = re.sub('annual', '', subname.lower())
subname = re.sub('\s+', ' ', subname)
#tmpitem = item[:jtd_len]
# if it's an alphanumeric with a space, rejoin, so we can remove it cleanly just below this.
substring_removal = None
@ -575,9 +586,9 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None):
else:
pass
#print ("directory found - ignoring")
logger.fdebug('you have a total of ' + str(comiccnt) + ' ' + watchcomic + ' comics')
watchmatch['comiccount'] = comiccnt
#print watchmatch
return watchmatch
def validateAndCreateDirectory(dir, create=False):

View File

@ -239,7 +239,7 @@ def decimal_issue(iss):
deciss = (int(iss_b4dec) * 1000) + issdec
return deciss, dec_except
def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=None):
def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=None, annualize=None):
import db, logger
myDB = db.DBConnection()
logger.fdebug('comicid: ' + str(comicid))
@ -262,22 +262,40 @@ def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=N
# issue = iss
# print ("converted issue#: " + str(issue))
logger.fdebug('issueid:' + str(issueid))
if issueid is None:
chkissue = myDB.action("SELECT * from issues WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone()
logger.fdebug('annualize is ' + str(annualize))
if annualize is None:
chkissue = myDB.action("SELECT * from issues WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone()
else:
chkissue = myDB.action("SELECT * from annuals WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone()
if chkissue is None:
#rechk chkissue against int value of issue #
chkissue = myDB.action("SELECT * from issues WHERE ComicID=? AND Issue_Number=?", [comicid, int(issue)]).fetchone()
chkissue = myDB.action("SELECT * from issues WHERE ComicID=? AND Int_IssueNumber=?", [comicid, issuedigits(issue)]).fetchone()
if chkissue is None:
logger.error('Invalid Issue_Number - please validate.')
return
if chkissue is None:
logger.error('Invalid Issue_Number - please validate.')
return
else:
logger.info('Int Issue_number compare found. continuing...')
issueid = chkissue['IssueID']
issueid = chkissue['IssueID']
else:
issueid = chkissue['IssueID']
#use issueid to get publisher, series, year, issue number
issuenzb = myDB.action("SELECT * from issues WHERE issueid=?", [issueid]).fetchone()
logger.fdebug('issueid is now : ' + str(issueid))
issuenzb = myDB.action("SELECT * from issues WHERE ComicID=? AND IssueID=?", [comicid,issueid]).fetchone()
if issuenzb is None:
logger.fdebug('not an issue, checking against annuals')
issuenzb = myDB.action("SELECT * from annuals WHERE ComicID=? AND IssueID=?", [comicid,issueid]).fetchone()
if issuenzb is None:
logger.fdebug('Unable to rename - cannot locate issue id within db')
return
else:
annualize = True
logger.fdebug('blah')
#comicid = issuenzb['ComicID']
issuenum = issuenzb['Issue_Number']
#issueno = str(issuenum).split('.')[0]
@ -306,6 +324,8 @@ def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=N
else:
iss = issuenum
issueno = str(iss)
logger.fdebug('iss:' + str(iss))
logger.fdebug('issueno:' + str(issueno))
# issue zero-suppression here
if mylar.ZERO_LEVEL == "0":
zeroadd = ""
@ -381,10 +401,25 @@ def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=N
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
logger.fdebug('No version # found for series, removing from filename')
print ("new format: " + str(chunk_file_format))
logger.fdebug("new format: " + str(chunk_file_format))
else:
chunk_file_format = mylar.FILE_FORMAT
if annualize is None:
chunk_f_f = re.sub('\$Annual','',chunk_file_format)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
logger.fdebug('not an annual - removing from filename paramaters')
logger.fdebug('new format: ' + str(chunk_file_format))
else:
logger.fdebug('chunk_file_format is: ' + str(chunk_file_format))
if '$Annual' not in chunk_file_format:
#if it's an annual, but $annual isn't specified in file_format, we need to
#force it in there, by default in the format of $Annual $Issue
prettycomiss = "Annual " + str(prettycomiss)
logger.fdebug('prettycomiss: ' + str(prettycomiss))
file_values = {'$Series': series,
'$Issue': prettycomiss,
'$Year': issueyear,
@ -392,7 +427,8 @@ def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=N
'$Publisher': publisher,
'$publisher': publisher.lower(),
'$VolumeY': 'V' + str(seriesyear),
'$VolumeN': comversion
'$VolumeN': comversion,
'$Annual': 'Annual'
}
extensions = ('.cbr', '.cbz')

View File

@ -296,7 +296,8 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None,c
'$series': series.lower(),
'$publisher': publisher.lower(),
'$VolumeY': 'V' + str(year),
'$VolumeN': comversion
'$VolumeN': comversion,
'$Annual': 'Annual'
}

View File

@ -138,9 +138,30 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None)
cfilename = re.sub('[\_\#\,\/\:\;\-\!\$\%\&\+\'\?\@]', ' ', comfilename)
#cfilename = re.sub('\s', '_', str(cfilename))
#versioning - remove it
subsplit = cfilename.replace('_', ' ').split()
volno = None
volyr = None
for subit in subsplit:
if subit[0].lower() == 'v':
vfull = 0
if subit[1:].isdigit():
#if in format v1, v2009 etc...
if len(subit) > 3:
# if it's greater than 3 in length, then the format is Vyyyy
vfull = 1 # add on 1 character length to account for extra space
cfilename = re.sub(subit, '', cfilename)
volno = re.sub("[^0-9]", " ", subit)
elif subit.lower()[:3] == 'vol':
#if in format vol.2013 etc
#because the '.' in Vol. gets removed, let's loop thru again after the Vol hit to remove it entirely
logger.fdebug('volume indicator detected as version #:' + str(subit))
cfilename = re.sub(subit, '', cfilename)
volyr = re.sub("[^0-9]", " ", subit)
cm_cn = 0
#we need to track the counter to make sure we are comparing the right array parts
#we need to track the counter to make sure we are comparing the right array parts
#this takes care of the brackets :)
m = re.findall('[^()]+', cfilename)
lenm = len(m)
@ -150,6 +171,7 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None)
foundonwatch = "False"
issue = 999999
while (cnt < lenm):
if m[cnt] is None: break
if m[cnt] == ' ':
@ -239,11 +261,13 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None)
splitit = []
watchcomic_split = []
logger.fdebug("filename comic and issue: " + cfilename)
logger.fdebug("filename comic and issue: " + comic_andiss)
#changed this from '' to ' '
comic_iss_b4 = re.sub('[\-\:\,]', ' ', com_NAME)
comic_iss_b4 = re.sub('[\-\:\,]', ' ', comic_andiss)
comic_iss = comic_iss_b4.replace('.',' ')
logger.fdebug("adjusted comic and issue: " + str(comic_iss))
comic_iss = re.sub('[\s+]', ' ', comic_iss).strip()
logger.fdebug("adjusted comic and issue: " + str(comic_iss))
#remove 'the' from here for proper comparisons.
if ' the ' in comic_iss.lower():
comic_iss = comic_iss[-4:]
@ -417,7 +441,13 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None)
# cmnam+=1
# logger.fdebug("comic: " + str(com_NAME))
# n+=1
if result_comyear is None: result_comyear = '0000' #no year in filename basically.
if volyr is None:
if result_comyear is None:
result_comyear = '0000' #no year in filename basically.
else:
if result_comyear is None:
result_comyear = volyr
print ("adding " + com_NAME + " to the import-queue!")
impid = com_NAME + "-" + str(result_comyear) + "-" + str(comiss)
print ("impid: " + str(impid))

View File

@ -428,10 +428,16 @@ def torrentdbsearch(seriesname,issue,comicid=None,nzbprov=None):
while (i < len(torsplit)):
#we'll rebuild the string here so that it's formatted accordingly to be passed back to the parser.
logger.fdebug('section(' + str(i) + '): ' + str(torsplit[i]))
#remove extensions
titletemp = torsplit[i]
titletemp = re.sub('cbr', '', str(titletemp))
titletemp = re.sub('cbz', '', str(titletemp))
titletemp = re.sub('none', '', str(titletemp))
if i == 0:
rebuiltline = str(torsplit[i])
rebuiltline = str(titletemp)
else:
rebuiltline = rebuiltline + ' (' + str(torsplit[i]) + ')'
rebuiltline = rebuiltline + ' (' + str(titletemp) + ')'
i+=1
logger.fdebug('rebuiltline is :' + str(rebuiltline))
@ -462,9 +468,12 @@ def torrentdbsearch(seriesname,issue,comicid=None,nzbprov=None):
logger.fdebug('matched on series title: ' + seriesname)
titleend = formatrem_torsplit[len(formatrem_seriesname):]
titleend = re.sub('\-', '', titleend) #remove the '-' which is unnecessary
titleend = re.sub('cbr', '', str(titleend)) #remove extensions
#remove extensions
titleend = re.sub('cbr', '', str(titleend))
titleend = re.sub('cbz', '', str(titleend))
titleend = re.sub('none', '', str(titleend))
logger.fdebug('titleend: ' + str(titleend))
sptitle = titleend.split()
extra = ''
# for sp in sptitle:
@ -480,14 +489,16 @@ def torrentdbsearch(seriesname,issue,comicid=None,nzbprov=None):
ctitle = tor['Title'].find('cbr')
if ctitle == 0:
ctitle = tor['Title'].find('cbz')
if ctitle == 0:
logger.fdebug('cannot determine title properly - ignoring for now.')
continue
if ctitle == 0:
ctitle = tor['Title'].find('none')
if ctitle == 0:
logger.fdebug('cannot determine title properly - ignoring for now.')
continue
cttitle = tor['Title'][:ctitle]
#print("change title to : " + str(cttitle))
# if extra == '':
tortheinfo.append({
'title': cttitle, #tor['Title'],
'title': rebuiltline, #cttitle,
'link': tor['Link'],
'pubdate': tor['Pubdate'],
'site': tor['Site'],

View File

@ -33,6 +33,7 @@ import sys
import getopt
import re
import time
import urlparse
from xml.dom.minidom import parseString
import urllib2
from datetime import datetime
@ -389,6 +390,9 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is
tmpprov = nzbprov
logger.info(u"Shhh be very quiet...I'm looking for " + ComicName + " issue: " + str(IssueNumber) + " (" + str(ComicYear) + ") using " + str(tmpprov))
#load in do not download db here for given series
#myDB = db.DBConnection()
#nodown = myDB.action('SELECT * FROM nzblog')
if mylar.PREFERRED_QUALITY == 0: filetype = ""
elif mylar.PREFERRED_QUALITY == 1: filetype = ".cbr"
@ -681,12 +685,12 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is
thisentry = entry['title']
logger.fdebug("Entry: " + thisentry)
cleantitle = re.sub('[_/.]', ' ', entry['title'])
cleantitle = re.sub('[\_\.]', ' ', entry['title'])
cleantitle = helpers.cleanName(cleantitle)
# this is new - if title contains a '&' in the title it will assume the filename has ended at that point
# which causes false positives (ie. wolverine & the x-men becomes the x-men, which matches on x-men.
# 'the' is removed for comparisons later on
if '&' in cleantitle: cleantitle = re.sub('[/&]','and', cleantitle)
if '&' in cleantitle: cleantitle = re.sub('[\&]','and', cleantitle)
nzbname = cleantitle
@ -737,16 +741,20 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is
if vers4year == "no" and vers4vol == "no":
# if the series is a v1, let's remove the requirements for year and volume label
if ComVersChk != 0:
# even if it's a v1, the nzbname might not contain a valid year format (20xx) or v3,
# and since it's already known that there is no (year) or vYEAR given
# let's push it through (and edit out the following if constraint)...
#if ComVersChk != 0:
# if there are no () in the string, try to add them if it looks like a year (19xx or 20xx)
if len(re.findall('[^()]+', cleantitle)):
logger.fdebug("detected invalid nzb filename - attempting to detect year to continue")
cleantitle = re.sub('(.*)\s+(19\d{2}|20\d{2})(.*)', '\\1 (\\2) \\3', cleantitle)
continue
else:
logger.fdebug("invalid nzb and/or cover only - skipping.")
cleantitle = "abcdefghijk 0 (1901).cbz"
continue
if len(re.findall('[^()]+', cleantitle)):
logger.fdebug("detected invalid nzb filename - attempting to detect year to continue")
cleantitle = re.sub('(.*)\s+(19\d{2}|20\d{2})(.*)', '\\1 (\\2) \\3', cleantitle)
continue
else:
logger.fdebug("invalid nzb and/or cover only - skipping.")
cleantitle = "abcdefghijk 0 (1901).cbz"
continue
#adjust for covers only by removing them entirely...
logger.fdebug("Cleantitle: " + str(cleantitle))
@ -1078,6 +1086,32 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is
#issue comparison now as well
if int(intIss) == int(comintIss):
#check if nzb is in do not download list ;)
if nzbprov == 'experimental':
#id is located after the /download/ portion
url_parts = urlparse.urlparse(entry['link'])
path_parts = url_parts[2].rpartition('/')
nzbtempid = path_parts[0].rpartition('/')
nzblen = len(nzbtempid)
nzbid = nzbtempid[nzblen-1]
elif nzbprov == 'CBT':
url_parts = urlparse.urlparse(entry['link'])
nzbtemp = url_parts[4] # get the query paramater string
nzbtemp = re.sub('torrent=', '', nzbtemp).rstrip()
nzbid = re.sub('.torrent', '', nzbtemp).rstrip()
elif nzbprov == 'KAT':
url_parts = urllib.parse.urlparse(entry['link'])
path_parts = url_parts[2].rpartition('/')
nzbtempid = pathparts[2]
nzbid = re.sub('.torrent', '', nzbtempid).rstrip()
elif nzbprov == 'nzb.su':
pass
elif nzbprov == 'dognzb':
pass
elif nzbprov == 'newznab':
#if in format of http://newznab/getnzb/<id>.nzb&i=1&r=apikey
nzbid = os.path.splitext(entry['link'])[0].rsplit('/', 1)[1]
logger.fdebug('issues match!')
logger.info(u"Found " + ComicName + " (" + str(comyear) + ") issue: " + str(IssueNumber) + " using " + str(tmpprov) )
## -- inherit issue. Comic year is non-standard. nzb year is the year

View File

@ -80,7 +80,7 @@ def solicit(month, year):
loopthis = (cnt-1)
#this loops through each 'found' solicit page
shipdate = str(month) + '-' + str(year)
shipdate = str(month_string) + '-' + str(year)
while ( loopthis >= 0 ):
upcoming += populate(resultURL[loopthis], publish[loopthis], shipdate)
loopthis -=1
@ -178,7 +178,7 @@ def populate(link,publisher,shipdate):
get_next = False
if prev_chk == True:
tempName = titlet.findNext(text=True)
if ' TPB' not in tempName and ' HC' not in tempName and 'GN-TPB' not in tempName and 'for $1' not in tempName.lower() and 'subscription variant' not in tempName.lower():
if ' TPB' not in tempName and ' HC' not in tempName and 'GN-TPB' not in tempName and 'for $1' not in tempName.lower() and 'subscription variant' not in tempName.lower() and 'poster' not in tempName.lower():
#print publisher + ' found upcoming'
if '#' in tempName:
#tempName = tempName.replace(u'.',u"'")

View File

@ -859,7 +859,46 @@ class WebInterface(object):
def upcoming(self):
myDB = db.DBConnection()
#upcoming = myDB.select("SELECT * from issues WHERE ReleaseDate > date('now') order by ReleaseDate DESC")
upcoming = myDB.select("SELECT * from upcoming WHERE IssueDate > date('now') AND IssueID is NULL order by IssueDate DESC")
upcomingdata = myDB.select("SELECT * from upcoming WHERE IssueID is NULL order by IssueDate DESC")
upcoming = []
for upc in upcomingdata:
if len(upc['IssueDate']) <= 7 :
#if it's less than or equal 7, then it's a future-pull so let's check the date and display
#tmpdate = datetime.datetime.com
tmpdatethis = upc['IssueDate']
if tmpdatethis[:2] == '20':
tmpdate = tmpdatethis #in correct format of yyyymm
else:
findst = tmpdatethis.find('-') #find the '-'
tmpdate = tmpdatethis[findst+1:] + tmpdatethis[:findst] #rebuild in format of yyyymm
timenow = datetime.datetime.now().strftime('%Y%m')
logger.fdebug('comparing pubdate of: ' + str(tmpdate) + ' to now date of: ' + str(timenow))
if int(tmpdate) >= int(timenow):
if upc['Status'] == 'Wanted':
upcoming.append({"ComicName": upc['ComicName'],
"IssueNumber": upc['IssueNumber'],
"IssueDate": upc['IssueDate'],
"ComicID": upc['ComicID'],
"IssueID": upc['IssueID'],
"Status": upc['Status'],
"DisplayComicName": upc['DisplayComicName']})
else:
#if it's greater than 7 it's a full date, and shouldn't be displayed ;)
timenow = datetime.datetime.now().strftime('%Y%m%d') #convert to yyyymmdd
tmpdate = re.sub("[^0-9]", "", upc['IssueDate']) #convert date to numerics only (should be in yyyymmdd)
logger.fdebug('comparing pubdate of: ' + str(tmpdate) + ' to now date of: ' + str(timenow))
if int(tmpdate) >= int(timenow):
if upc['Status'] == 'Wanted':
upcoming.append({"ComicName": upc['ComicName'],
"IssueNumber": upc['IssueNumber'],
"IssueDate": upc['IssueDate'],
"ComicID": upc['ComicID'],
"IssueID": upc['IssueID'],
"Status": upc['Status'],
"DisplayComicName": upc['DisplayComicName']})
issues = myDB.select("SELECT * from issues WHERE Status='Wanted'")
ann_list = []
@ -940,6 +979,8 @@ class WebInterface(object):
comicname = comic['ComicName']
extensions = ('.cbr', '.cbz')
issues = myDB.action("SELECT * FROM issues WHERE ComicID=?", [comicid]).fetchall()
if mylar.ANNUALS_ON:
issues += myDB.action("SELECT * FROM annuals WHERE ComicID=?", [comicid]).fetchall()
comfiles = []
filefind = 0
for root, dirnames, filenames in os.walk(comicdir):
@ -949,7 +990,11 @@ class WebInterface(object):
for issue in issues:
if issue['Location'] == filename:
#logger.error("matched " + str(filename) + " to DB file " + str(issue['Location']))
renameiss = helpers.rename_param(comicid, comicname, issue['Issue_Number'], filename, comicyear=None, issueid=None)
if 'annual' in issue['Location'].lower():
annualize = 'yes'
else:
annualize = None
renameiss = helpers.rename_param(comicid, comicname, issue['Issue_Number'], filename, comicyear=None, issueid=None, annualize=annualize)
nfilename = renameiss['nfilename']
srciss = os.path.join(comicdir,filename)
if mylar.LOWERCASE_FILENAMES:
@ -1901,14 +1946,58 @@ class WebInterface(object):
return serve_template(templatename="searchresults.html", title='Import Results for: "' + ComicName + '"',searchresults=sresults, type=type, imported='yes', ogcname=ogcname) #imported=comicstoIMP, ogcname=ogcname)
preSearchit.exposed = True
def pretty_git(self, br_history):
#in order to 'prettify' the history log for display, we need to break it down so it's line by line.
br_split = br_history.split("\n") #split it on each commit
for br in br_split:
br_commit_st = br.find('-') #first - will represent end of commit numeric
br_commit = br[:br_commit_st].strip()
br_time_en = br.replace('-', 'XXX', 1).find('-') #2nd - is end of time datestamp
br_time = br[br_commit_st+1:br_time_en].strip()
print 'COMMIT:' + str(br_commit)
print 'TIME:' + str(br_time)
commit_split = br.split() #split it by space to break it further down..
tag_chk = False
statline = ''
commit = []
for cs in commit_split:
if tag_chk == True:
if 'FIX:' in cs or 'IMP:' in cs:
commit.append({"commit": br_commit,
"time": br_time,
"stat": tag_status,
"line": statline})
print commit
tag_chk == False
statline = ''
else:
statline += str(cs) + ' '
else:
if 'FIX:' in cs:
tag_status = 'FIX'
tag_chk = True
print 'status: ' + str(tag_status)
elif 'IMP:' in cs:
tag_status = 'IMPROVEMENT'
tag_chk = True
print 'status: ' + str(tag_status)
pretty_git.exposed = True
#---
def config(self):
interface_dir = os.path.join(mylar.PROG_DIR, 'data/interfaces/')
interface_list = [ name for name in os.listdir(interface_dir) if os.path.isdir(os.path.join(interface_dir, name)) ]
#----
# to be implemented in the future.
# branch_history, err = mylar.versioncheck.runGit("log --oneline --pretty=format:'%h - %ar - %s' -n 4")
# br_hist = branch_history.replace("\n", "<br />\n")
# #here we pass the branch_history to the pretty_git module to break it down
# if branch_history:
# self.pretty_git(branch_history)
# br_hist = branch_history.replace("\n", "<br />\n")
# else:
# br_hist = err
#----
myDB = db.DBConnection()
CCOMICS = myDB.action("SELECT COUNT(*) FROM comics").fetchall()
CHAVES = myDB.action("SELECT COUNT(*) FROM issues WHERE Status='Downloaded' OR Status='Archived'").fetchall()
@ -2040,7 +2129,8 @@ class WebInterface(object):
"prog_dir" : mylar.PROG_DIR,
"cache_dir" : mylar.CACHE_DIR,
"config_file" : mylar.CONFIG_FILE,
# "branch_history" : br_hist
"branch_history" : 'None',
# "branch_history" : br_hist,
"enable_pre_scripts" : helpers.checked(mylar.ENABLE_PRE_SCRIPTS),
"pre_scripts" : mylar.PRE_SCRIPTS,
"log_dir" : mylar.LOG_DIR
@ -2355,6 +2445,10 @@ class WebInterface(object):
logger.info("CHMOD File value is not a valid numeric - please correct. Defaulting to 0660")
mylar.CHMOD_FILE = '0660'
if mylar.SAB_HOST.endswith('/'):
logger.info("Auto-correcting trailing slash in SABnzbd url (not required)")
mylar.SAB_HOST = mylar.SAB_HOST[:-1]
if mylar.ENABLE_META:
if mylar.CMTAGGER_PATH is None or mylar.CMTAGGER_PATH == '':
logger.info("ComicTagger Path not set - defaulting to Mylar Program Directory : " + mylar.PROG_DIR)

View File

@ -56,6 +56,27 @@ def pullit(forcecheck=None):
PULLURL = 'http://www.previewsworld.com/shipping/newreleases.txt'
#PULLURL = 'http://www.previewsworld.com/Archive/GetFile/1/1/71/994/081512.txt'
#Prepare the Substitute name switch for pulllist to comic vine conversion
substitutes = os.path.join(mylar.DATA_DIR,"substitutes.csv")
if not os.path.exists(substitutes):
logger.debug('no substitues.csv file located - not performing substitutions on weekly pull list')
substitute_check = False
else:
substitute_check = True
#shortrep is the name to be replaced, longrep the replacement
shortrep=[]
longrep=[]
#open the file data
with open(substitutes) as f:
reader = csv.reader(f, delimiter='|')
for row in reader:
if not row.startswith('#'):
logger.debug ("Substitutes file read : "+str(row))
shortrep.append(row[0])
longrep.append(row[1])
f.close()
not_these=['PREVIEWS',
'Shipping',
'Every Wednesday',
@ -188,7 +209,7 @@ def pullit(forcecheck=None):
dupefound = "no"
if '#' in i:
issname = i.split()
print (issname)
#print (issname)
issnamec = len(issname)
n = 0
while (n < issnamec):
@ -241,7 +262,7 @@ def pullit(forcecheck=None):
#if it doesn't have a '#' in the line, then we know it's either
#a special edition of some kind, or a non-comic
issname = i.split()
print (issname)
#print (issname)
issnamec = len(issname)
n = 1
issue = ''
@ -301,6 +322,16 @@ def pullit(forcecheck=None):
# pullist has shortforms of a series' title sometimes and causes problems
if 'O/T' in comicnm:
comicnm = re.sub('O/T', 'OF THE', comicnm)
if substitute_check == True:
#Step through the list - storing an index
for repindex,repcheck in enumerate(shortrep):
if len(comicnm)>= len(shortrep):
#if the leftmost chars match the short text then replace them with the long text
if comicnm[:len(repcheck)]==repcheck:
logger.info("Switch worked on "+comicnm + " replacing " + str(repcheck) + " with " + str(longrep[repindex]))
comicnm = re.sub(repcheck, longrep[repindex], comicnm)
for excl in excludes:
if excl in str(comicrm):
#duplicate comic / issue detected - don't add...

22
substitutes_sample.csv Normal file
View File

@ -0,0 +1,22 @@
#--------------------------------------------
# This is a substitutes file for the weekly pull list
# (Thanks to IanHub for this)
# ---
# What this does is it will abbreviations for series on the weekly pull list
# with the full series title, so that Mylar can match it up properly against ComicVine.
# This is due to the pull-list having abbreviations in some titles, and
# ComicVine not using the same abbreviations.
# ----
# To use this you have to first either rename this file to substitutes.csv
# Make sure it's in the the same directory as the Mylar.py file (root directory).
#
# Within the csv, you can add text in the format of:
# pull-list text|replacement text
#
# ie. GFT|GRIMM FAIRY TALES PRESENTS
#--------------------------------------------
GFT GRIMM FAIRY TALES|GRIMM FAIRY TALES PRESENTS
GFT|GRIMM FAIRY TALES PRESENTS
HELLRAISER|CLIVE BARKER'S HELLRAISER
BTVS SEASON 9|BUFFY THE VAMPIRE SLAYER SEASON NINE
SUPURBIA|GRACE RANDOLPH'S SUPURBIA
1 #--------------------------------------------
2 # This is a substitutes file for the weekly pull list
3 # (Thanks to IanHub for this)
4 # ---
5 # What this does is it will abbreviations for series on the weekly pull list
6 # with the full series title, so that Mylar can match it up properly against ComicVine.
7 # This is due to the pull-list having abbreviations in some titles, and
8 # ComicVine not using the same abbreviations.
9 # ----
10 # To use this you have to first either rename this file to substitutes.csv
11 # Make sure it's in the the same directory as the Mylar.py file (root directory).
12 #
13 # Within the csv, you can add text in the format of:
14 # pull-list text|replacement text
15 #
16 # ie. GFT|GRIMM FAIRY TALES PRESENTS
17 #--------------------------------------------
18 GFT GRIMM FAIRY TALES|GRIMM FAIRY TALES PRESENTS
19 GFT|GRIMM FAIRY TALES PRESENTS
20 HELLRAISER|CLIVE BARKER'S HELLRAISER
21 BTVS SEASON 9|BUFFY THE VAMPIRE SLAYER SEASON NINE
22 SUPURBIA|GRACE RANDOLPH'S SUPURBIA