1
0
Fork 0
mirror of https://github.com/evilhero/mylar synced 2025-03-10 05:52:48 +00:00

FIX: Post-Processing will allow for spaces/decimals regardless of SAB settings now, IMP: Post-Processing passing metadata to extra-scripts, IMP: passing nzbs to SAB is done by link only now (no more saving messy files), IMP: Removed some verbose logging from parsing on multi-volumes, FIX: Weekly error on startup (couldn't mark), IMP: Wanted comics on Weekly tab will now appear in Upcoming tab properly, and be marked wanted when data is available from CV/GCD and removed from Upcoming tab

This commit is contained in:
evilhero 2013-01-07 13:26:15 -05:00
parent 3e2ffdd71f
commit 254793fd22
4 changed files with 146 additions and 152 deletions

View file

@ -23,6 +23,9 @@ import time
import logging
import mylar
import subprocess
import urllib2
import sqlite3
from xml.dom.minidom import parseString
from mylar import logger, db, helpers, updater
@ -80,7 +83,7 @@ class PostProcessor(object):
# logger.log(message, level)
self.log += message + '\n'
def _run_extra_scripts(self, nzb_name, nzb_folder):
def _run_extra_scripts(self, nzb_name, nzb_folder, filen, folderp, seriesmetadata):
"""
Executes any extra scripts defined in the config.
@ -92,7 +95,7 @@ class PostProcessor(object):
curScriptName = mylar.EXTRA_SCRIPTS
self._log("extra script detected...enabling: " + str(curScriptName), logger.DEBUG)
# generate a safe command line string to execute the script and provide all the parameters
script_cmd = shlex.split(curScriptName) + [nzb_name, nzb_folder]
script_cmd = shlex.split(curScriptName) + [str(nzb_name), str(nzb_folder), str(filen), str(folderp), str(seriesmetadata)]
self._log("cmd to be executed: " + str(script_cmd), logger.DEBUG)
# use subprocess to run the command and capture output
@ -108,26 +111,66 @@ class PostProcessor(object):
# def PostProcess(nzb_name, nzb_folder):
def Process(self):
print ("..here.")
self._log("nzb name: " + str(self.nzb_name), logger.DEBUG)
self._log("nzb folder: " + str(self.nzb_folder), logger.DEBUG)
# log2screen = ""
# log2screen = log2screen + "Nzb Name:" + self.nzb_name + "\n"
# log2screen = log2screen + "Nzb Folder:" + self.nzb_folder + "\n"
#lookup nzb_name in nzblog table to get issueid
#lookup nzb_name in nzblog table to get issueid
#query SAB to find out if Replace Spaces enabled / not as well as Replace Decimals
#http://localhost:8080/sabnzbd/api?mode=set_config&section=misc&keyword=dirscan_speed&value=5
querysab = str(mylar.SAB_HOST) + "/api?mode=get_config&section=misc&output=xml&apikey=" + str(mylar.SAB_APIKEY)
#logger.info("querysab_string:" + str(querysab))
file = urllib2.urlopen(querysab)
data = file.read()
file.close()
dom = parseString(data)
sabreps = dom.getElementsByTagName('replace_spaces')[0].firstChild.wholeText
sabrepd = dom.getElementsByTagName('replace_dots')[0].firstChild.wholeText
#logger.fdebug("sabreps:" + str(sabreps))
myDB = db.DBConnection()
nzbiss = myDB.action("SELECT * from nzblog WHERE nzbname=?", [self.nzb_name]).fetchone()
nzbname = self.nzb_name
nzbiss = myDB.action("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone()
if nzbiss is None:
self._log("Epic failure - could not locate file to rename.", logger.DEBUG)
logger.error(u"Unable to locate downloaded file to rename. PostProcessing aborted.")
return
self._log("Failure - could not initially locate nzbfile in my database to rename.", logger.DEBUG)
#decimals need to be accounted for....
if str(sabrepd) == '0':
#Replace decimals is enabled so decimlas should already be passed through converted.
logger.info("SABnzbd setting: Replace Decimals is disabled.")
elif str(sabrepd) == '1':
#Replace decimals is enabled.
#By default SAB will pass back the value with decimals replaced with a ' ' or a '_'
logger.info("SABnzbd setting: Replace Decimals is enabled.")
self._log("I'm going to try to rejig the nzbname passed from SAB accounting for decmials to see if I can find it.")
nzbname = re.sub('[\.]', '_', str(nzbname))
#spaces need to be accounted for
if str(sabreps) == '1':
#Replace spaces is enabled so spaces should already be passed through converted.
logger.info("SABnzbd setting: Replace spaces is enabled.")
elif str(sabreps) == '0':
#Replace space is disabled.
#By default SAB will pass back the value with spaces replaced with a '+'
logger.info("SABnzbd setting: Replace spaces is disabled.")
self._log("I'm going to try to rejig the nzbname passed from SAB accouting for spaces to see if I can find it.")
nzbname = re.sub(' ', '_', str(nzbname))
#let's remove the - cause it will cause problems at this point...
nzbname = re.sub('[\-]', '_', str(nzbname))
logger.fdebug("trying again with this nzbname: " + str(nzbname))
nzbiss = myDB.action("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone()
if nzbiss is None:
logger.error(u"Unable to locate downloaded file to rename. PostProcessing aborted.")
return
else:
self._log("I corrected and found the nzb as : " + str(nzbname))
issueid = nzbiss['IssueID']
else:
issueid = nzbiss['IssueID']
#log2screen = log2screen + "IssueID: " + issueid + "\n"
#use issueid to get publisher, series, year, issue number
issuenzb = myDB.action("SELECT * from issues WHERE issueid=?", [issueid]).fetchone()
comicid = issuenzb['ComicID']
#log2screen = log2screen + "ComicID: " + comicid + "\n"
issuenum = issuenzb['Issue_Number']
#issueno = str(issuenum).split('.')[0]
@ -199,22 +242,6 @@ class PostProcessor(object):
self._log("Year: " + seriesyear, logger.DEBUG)
comlocation = comicnzb['ComicLocation']
self._log("Comic Location: " + comlocation, logger.DEBUG)
#---move to importer.py
#get output path format
# if ':' in series:
# series = series.replace(':','')
#do work to generate folder path
# values = {'$Series': series,
# '$Publisher': publisher,
# '$Year': seriesyear
# }
# comlocation = mylar.DESTINATION_DIR + "/" + helpers.replace_all(mylar.FOLDER_FORMAT, values)
#last perform space replace
# if mylar.REPLACE_SPACES:
#mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
# comlocation = comlocation.replace(' ', mylar.REPLACE_CHAR)
# log2screen = log2screen + "Final Location: " + comlocation + "\n"
#---
#rename file and move to new path
#nfilename = series + " " + issueno + " (" + seriesyear + ")"
file_values = {'$Series': series,
@ -243,8 +270,6 @@ class PostProcessor(object):
if mylar.REPLACE_SPACES:
#mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
nfilename = nfilename.replace(' ', mylar.REPLACE_CHAR)
#TODO - sort issue numbering 12.00 should be 12
#replace funky characters so it doesn't break things
nfilename = re.sub('[\,\:]', '', nfilename)
self._log("New Filename: " + nfilename, logger.DEBUG)
@ -279,7 +304,24 @@ class PostProcessor(object):
# retrieve/create the corresponding comic objects
if mylar.ENABLE_EXTRA_SCRIPTS:
self._run_extra_scripts(self.nzb_name, self.nzb_folder)
folderp = str(dst) #folder location after move/rename
nzbn = self.nzb_name #original nzb name
filen = str(nfilename + ext) #new filename
#name, comicyear, comicid , issueid, issueyear, issue, publisher
#create the dic and send it.
seriesmeta = []
seriesmetadata = {}
seriesmeta.append({
'name': series,
'comicyear': seriesyear,
'comicid': comicid,
'issueid': issueid,
'issueyear': issueyear,
'issue': issuenum,
'publisher': publisher
})
seriesmetadata['seriesmeta'] = seriesmeta
self._run_extra_scripts(nzbname, self.nzb_folder, filen, folderp, seriesmetadata )
return self.log

View file

@ -389,7 +389,7 @@ def GCDAdd(gcdcomicid):
#logger.fdebug("series_data: " + str(parsed))
#print ("parse:" + str(parsed))
subtxt3 = parsed.find("dd", {"id" : "publication_dates"})
logger.fdebug("publication_dates: " + str(subtxt3))
#logger.fdebug("publication_dates: " + str(subtxt3))
pubdate = subtxt3.findNext(text=True).rstrip()
logger.fdebug("pubdate:" + str(pubdate))
subtxt4 = parsed.find("dd", {"id" : "issues_published"})

View file

@ -55,6 +55,9 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, IssueDate, IssueI
if mylar.EXPERIMENTAL == 1:
nzbprovider.append('experimental')
nzbp+=1
newznabs = 0
if mylar.NEWZNAB == 1:
logger.fdebug("mylar.newznab:" + str(mylar.NEWZNAB))
if mylar.NEWZNAB_ENABLED:
@ -65,7 +68,6 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, IssueDate, IssueI
else:
newznab_hosts = []
logger.fdebug("initial newznab provider not enabled...checking for additional newznabs.")
newznabs = 0
logger.fdebug("mylar.EXTRA_NEWZNABS:" + str(mylar.EXTRA_NEWZNABS))
@ -558,98 +560,41 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is
#end blackhole
else:
if nzbprov != 'nzb.su':
tmppath = mylar.CACHE_DIR
if os.path.exists(tmppath):
logger.fdebug("cache directory successfully found at : " + str(tmppath))
pass
else:
#let's make the dir.
logger.fdebug("couldn't locate cache directory, attempting to create at : " + str(mylar.CACHE_DIR))
try:
os.makedirs(str(mylar.CACHE_DIR))
logger.info(u"Cache Directory successfully created at: " + str(mylar.CACHE_DIR))
except OSError.e:
if e.errno != errno.EEXIST:
raise
filenamenzb = os.path.split(linkapi)[1]
#filenzb = os.path.join(tmppath,filenamenzb)
logger.fdebug("unalterted nzb name: " + str(filenamenzb))
#let's send a clean copy to SAB because the names are random characters and/or could be stupid.
ComicName = re.sub('[\:\,]', '', ComicName)
filenzb = str(ComicName.replace(' ', '_')) + "_" + str(IssueNumber) + "_(" + str(comyear) + ")"
logger.fdebug("prettified nzb name: " + str(filenzb))
if mylar.RENAME_FILES == 1:
logger.fdebug("Rename Files enabled..")
filenzb = str(ComicName.replace(' ', '_')) + "_" + str(IssueNumber) + "_(" + str(comyear) + ")"
logger.fdebug("this should be the same as prettified nzb name:" + str(filenzb))
if mylar.REPLACE_SPACES:
logger.fdebug("Replace spaces option enabled")
logger.fdebug("replace character: " + str(mylar.REPLACE_CHAR))
repchar = mylar.REPLACE_CHAR
repurlchar = mylar.REPLACE_CHAR
else:
logger.fdebug("Replace spaces option NOT enabled")
repchar = ' '
repurlchar = "%20"
#let's make sure there's no crap in the ComicName since it's O.G.
logger.fdebug("original Name of comic: " + str(ComicName))
ComicNM = re.sub('[\:\,]', '', str(ComicName))
logger.fdebug("altered Name of comic: " + str(ComicNM))
renameit = str(ComicNM) + " " + str(IssueNumber) + " (" + str(SeriesYear) + ")" + " " + "(" + str(comyear) + ")"
logger.fdebug("altered Name with additional info: " + str(renameit))
renamethis = renameit.replace(' ', repchar)
logger.fdebug("...with replace spaces: " + str(renamethis))
renamer1 = renameit.replace(' ', repurlchar)
renamer = re.sub("\&", "%26", str(renamer1))
logger.fdebug("...adjusting for url restrictions: " + str(renamer))
filenext = str(filenzb) + ".nzb"
savefile = os.path.join(tmppath, filenext)
logger.fdebug("nzb file to be saved: " + str(savefile))
tmppath = mylar.CACHE_DIR
if os.path.exists(tmppath):
logger.fdebug("cache directory successfully found at : " + str(tmppath))
pass
else:
#let's make the dir.
logger.fdebug("couldn't locate cache directory, attempting to create at : " + str(mylar.CACHE_DIR))
try:
urllib.urlretrieve(linkapi, str(savefile))
except urllib.URLError:
logger.fdebug(u"Unable to retrieve nzb using link: " + str(linkapi))
logger.fdebug(u"Possibly unable to save nzb: " + str(savefile))
logger.error(u"Unable to retrieve nzb file.")
return
os.makedirs(str(mylar.CACHE_DIR))
logger.info(u"Cache Directory successfully created at: " + str(mylar.CACHE_DIR))
if os.path.getsize(str(savefile)) == 0:
logger.error(u"nzb size detected as zero bytes.")
continue
except OSError.e:
if e.errno != errno.EEXIST:
raise
logger.info(u"Sucessfully retrieved nzb file using " + str(nzbprov))
nzbname = str(filenzb)
logger.fdebug("link to retrieve via api:" + str(linkapi))
elif nzbprov == 'nzb.su':
logger.fdebug("NZB.SU - linkapi:" + str(linkapi))
nzbname = re.sub(" ", "_", str(entry['title']))
#when the series contains a '-' or the issue a decimal it can't convert
#if SAB's settings differ on how it handles spaces / decimals.
#let's attempt to adjust by removing the - and .
nzbname = re.sub(" ", "_", str(entry['title']))
nzbname = re.sub('[\.\-]', '_', str(nzbname))
logger.fdebug("nzbname used for post-processing:" + str(nzbname))
# let's build the send-to-SAB string now:
tmpapi = str(mylar.SAB_HOST)
logger.fdebug("send-to-SAB host string: " + str(tmpapi))
# nzb.su only works with direct links for some reason...
if nzbprov == 'nzb.su':
SABtype = "/api?mode=addurl&name="
savefileURL = str(linkapi)
else:
SABtype = "/api?mode=addlocalfile&name="
# if the savefile location has spaces in the path, could cause problems.
# if the savefile has a &, escape it otherwise will botch up send-to-SAB link
# let's adjust.
saveF = re.sub("\&", "%26", str(savefile))
savefileURL = re.sub(" ","%20", str(saveF))
# changed to just work with direct links now...
SABtype = "/api?mode=addurl&name="
fileURL = str(linkapi)
tmpapi = tmpapi + str(SABtype)
logger.fdebug("...selecting API type: " + str(tmpapi))
tmpapi = tmpapi + str(savefileURL)
logger.fdebug("...attaching nzbfile: " + str(tmpapi))
tmpapi = tmpapi + str(fileURL)
logger.fdebug("...attaching nzb provider link: " + str(tmpapi))
# determine SAB priority
if mylar.SAB_PRIORITY:
tmpapi = tmpapi + "&priority=" + str(sabpriority)
@ -674,10 +619,10 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is
logger.info(u"Successfully sent nzb file to SABnzbd")
#delete the .nzb now.
if mylar.PROG_DIR is not "/" and nzbprov != 'nzb.su':
logger.fdebug("preparing to remove temporary nzb file at: " + str(savefile))
os.remove(savefile)
logger.info(u"Removed temporary save file")
#if mylar.PROG_DIR is not "/" and nzbprov != 'nzb.su':
# logger.fdebug("preparing to remove temporary nzb file at: " + str(savefile))
# os.remove(savefile)
# logger.info(u"Removed temporary save file")
#raise an exception to break out of loop
foundc = "yes"
done = True

View file

@ -71,7 +71,7 @@ def upcoming_update(ComicID, ComicName, IssueNumber, IssueDate):
if ComicID[:1] == "G": mylar.importer.GCDimport(ComicID,pullupd)
else: mylar.importer.addComictoDB(ComicID,mismatch,pullupd)
if issuechk['Issue_Number'] == IssueNumber:
elif issuechk['Issue_Number'] == IssueNumber:
logger.fdebug("Comic series already up-to-date ... no need to refresh at this time.")
logger.fdebug("Available to be marked for download - checking..." + str(issuechk['ComicName']) + " Issue: " + str(issuechk['Issue_Number']))
logger.fdebug("...Existing status: " + str(issuechk['Status']))
@ -89,18 +89,25 @@ def upcoming_update(ComicID, ComicName, IssueNumber, IssueDate):
values = { "Status": "Skipped"}
newValue['Status'] = "Skipped"
#was in wrong place :(
if mylar.AUTOWANT_UPCOMING:
if issuechk['Status'] == "Skipped":
newValue['Status'] = "Wanted"
values = { "Status": "Wanted"}
logger.fdebug("...New status of Wanted")
elif issuechk['Status'] == "Wanted":
logger.fdebug("...Status already Wanted .. not changing.")
else:
logger.fdebug("...Already have issue - keeping existing status of : " + issuechk['Status'])
if mylar.AUTOWANT_UPCOMING:
#for issues not in db - to be added to Upcoming table.
if issuechk is None:
newValue['Status'] = "Wanted"
logger.fdebug("...Changing Status to Wanted and throwing it in the Upcoming section since it's not published yet.")
#this works for issues existing in DB...
elif issuechk['Status'] == "Skipped":
newValue['Status'] = "Wanted"
values = { "Status": "Wanted"}
logger.fdebug("...New status of Wanted")
elif issuechk['Status'] == "Wanted":
logger.fdebug("...Status already Wanted .. not changing.")
else:
logger.fdebug("...Already have issue - keeping existing status of : " + issuechk['Status'])
myDB.upsert("upcoming", newValue, controlValue)
myDB.upsert("issues", values, control)
if issuechk is None:
myDB.upsert("upcoming", newValue, controlValue)
else:
myDB.upsert("issues", values, control)
else:
logger.fdebug("Issues don't match for some reason... db issue: " + str(issuechk['Issue_Number']) + " ...weekly new issue: " + str(IssueNumber))
@ -198,7 +205,7 @@ def forceRescan(ComicID):
arcissues = myDB.select("SELECT * FROM issues WHERE ComicID=? and Status='Archived'", [ComicID])
if len(arcissues) > 0:
havefiles = len(arcissues)
print "have count adjusted to:" + str(len(arcissues))
logger.fdebug("Adjusting have total because of this many archive files:" + str(len(arcissues))
while (fn < fccnt):
haveissue = "no"
issuedupe = "no"
@ -223,28 +230,28 @@ def forceRescan(ComicID):
issyear = reiss['IssueDate'][:4]
old_status = reiss['Status']
logger.fdebug("integer_issue:" + str(int_iss) + " ... status: " + str(old_status))
#logger.fdebug("integer_issue:" + str(int_iss) + " ... status: " + str(old_status))
while (som < fcn):
#counts get buggered up when the issue is the last field in the filename - ie. '50.cbr'
logger.fdebug("checking word - " + str(fcnew[som]))
#logger.fdebug("checking word - " + str(fcnew[som]))
if ".cbr" in fcnew[som]:
fcnew[som] = fcnew[som].replace(".cbr", "")
elif ".cbz" in fcnew[som]:
fcnew[som] = fcnew[som].replace(".cbz", "")
if '.' in fcnew[som]:
logger.fdebug("decimal detected...adjusting.")
#logger.fdebug("decimal detected...adjusting.")
try:
i = float(fcnew[som])
except ValueError, TypeError:
#not numeric
fcnew[som] = fcnew[som].replace(".", "")
logger.fdebug("new word: " + str(fcnew[som]))
#logger.fdebug("new word: " + str(fcnew[som]))
else:
#numeric
pass
if fcnew[som].isdigit():
#this won't match on decimal issues - need to fix.
logger.fdebug("digit detected")
#logger.fdebug("digit detected")
if int(fcnew[som]) > 0:
# fcdigit = fcnew[som].lstrip('0')
#fcdigit = str(int(fcnew[som]))
@ -255,24 +262,24 @@ def forceRescan(ComicID):
elif "." in fcnew[som]:
#this will match on decimal issues
IssueChk = fcnew[som]
logger.fdebug("decimal detected...analyzing if issue")
#logger.fdebug("decimal detected...analyzing if issue")
isschk_find = IssueChk.find('.')
isschk_b4dec = IssueChk[:isschk_find]
isschk_decval = IssueChk[isschk_find+1:]
if isschk_b4dec.isdigit():
logger.fdebug("digit detected prior to decimal.")
#logger.fdebug("digit detected prior to decimal.")
if isschk_decval.isdigit():
#pass
logger.fdebug("digit detected after decimal.")
pass
#logger.fdebug("digit detected after decimal.")
else:
logger.fdebug("not an issue - no digit detected after decimal")
#logger.fdebug("not an issue - no digit detected after decimal")
break
else:
logger.fdebug("not an issue - no digit detected prior to decimal")
#logger.fdebug("not an issue - no digit detected prior to decimal")
break
logger.fdebug("IssueNumber: " + str(IssueChk))
logger.fdebug("..before decimal: " + str(isschk_b4dec))
logger.fdebug("...after decimal: " + str(isschk_decval))
#logger.fdebug("IssueNumber: " + str(IssueChk))
#logger.fdebug("..before decimal: " + str(isschk_b4dec))
#logger.fdebug("...after decimal: " + str(isschk_decval))
#--let's make sure we don't wipe out decimal issues ;)
if int(isschk_decval) == 0:
iss = isschk_b4dec
@ -285,18 +292,18 @@ def forceRescan(ComicID):
iss = isschk_b4dec + "." + isschk_decval.rstrip('0')
intdec = int(isschk_decval.rstrip('0')) * 10
fcdigit = (int(isschk_b4dec) * 1000) + intdec
logger.fdebug("b4dec: " + str(isschk_b4dec))
logger.fdebug("decval: " + str(isschk_decval))
logger.fdebug("intdec: " + str(intdec))
logger.fdebug("let's compare with this issue value: " + str(fcdigit))
#logger.fdebug("b4dec: " + str(isschk_b4dec))
#logger.fdebug("decval: " + str(isschk_decval))
#logger.fdebug("intdec: " + str(intdec))
#logger.fdebug("let's compare with this issue value: " + str(fcdigit))
else:
# it's a word, skip it.
fcdigit = 1000000
logger.fdebug("fcdigit: " + str(fcdigit))
logger.fdebug("int_iss: " + str(int_iss))
#logger.fdebug("fcdigit: " + str(fcdigit))
#logger.fdebug("int_iss: " + str(int_iss))
if "." in str(int_iss):
int_iss = helpers.decimal_issue(int_iss)
logger.fdebug("this is the int issue:" + str(int_iss))
#logger.fdebug("this is the int issue:" + str(int_iss))
if int(fcdigit) == int_iss:
#if issyear in fcnew[som+1]: