From 254793fd225313d6b6fb2b3d858b29d76f64fe62 Mon Sep 17 00:00:00 2001 From: evilhero Date: Mon, 7 Jan 2013 13:26:15 -0500 Subject: [PATCH] FIX: Post-Processing will allow for spaces/decimals regardless of SAB settings now, IMP: Post-Processing passing metadata to extra-scripts, IMP: passing nzbs to SAB is done by link only now (no more saving messy files), IMP: Removed some verbose logging from parsing on multi-volumes, FIX: Weekly error on startup (couldn't mark), IMP: Wanted comics on Weekly tab will now appear in Upcoming tab properly, and be marked wanted when data is available from CV/GCD and removed from Upcoming tab --- mylar/PostProcessor.py | 106 +++++++++++++++++++++++++------------ mylar/parseit.py | 2 +- mylar/search.py | 115 +++++++++++------------------------------ mylar/updater.py | 75 +++++++++++++++------------ 4 files changed, 146 insertions(+), 152 deletions(-) diff --git a/mylar/PostProcessor.py b/mylar/PostProcessor.py index 0430c771..971f522a 100755 --- a/mylar/PostProcessor.py +++ b/mylar/PostProcessor.py @@ -23,6 +23,9 @@ import time import logging import mylar import subprocess +import urllib2 +import sqlite3 +from xml.dom.minidom import parseString from mylar import logger, db, helpers, updater @@ -80,7 +83,7 @@ class PostProcessor(object): # logger.log(message, level) self.log += message + '\n' - def _run_extra_scripts(self, nzb_name, nzb_folder): + def _run_extra_scripts(self, nzb_name, nzb_folder, filen, folderp, seriesmetadata): """ Executes any extra scripts defined in the config. @@ -92,7 +95,7 @@ class PostProcessor(object): curScriptName = mylar.EXTRA_SCRIPTS self._log("extra script detected...enabling: " + str(curScriptName), logger.DEBUG) # generate a safe command line string to execute the script and provide all the parameters - script_cmd = shlex.split(curScriptName) + [nzb_name, nzb_folder] + script_cmd = shlex.split(curScriptName) + [str(nzb_name), str(nzb_folder), str(filen), str(folderp), str(seriesmetadata)] self._log("cmd to be executed: " + str(script_cmd), logger.DEBUG) # use subprocess to run the command and capture output @@ -108,26 +111,66 @@ class PostProcessor(object): # def PostProcess(nzb_name, nzb_folder): def Process(self): - print ("..here.") self._log("nzb name: " + str(self.nzb_name), logger.DEBUG) self._log("nzb folder: " + str(self.nzb_folder), logger.DEBUG) -# log2screen = "" -# log2screen = log2screen + "Nzb Name:" + self.nzb_name + "\n" -# log2screen = log2screen + "Nzb Folder:" + self.nzb_folder + "\n" - #lookup nzb_name in nzblog table to get issueid + #lookup nzb_name in nzblog table to get issueid + + #query SAB to find out if Replace Spaces enabled / not as well as Replace Decimals + #http://localhost:8080/sabnzbd/api?mode=set_config§ion=misc&keyword=dirscan_speed&value=5 + querysab = str(mylar.SAB_HOST) + "/api?mode=get_config§ion=misc&output=xml&apikey=" + str(mylar.SAB_APIKEY) + #logger.info("querysab_string:" + str(querysab)) + file = urllib2.urlopen(querysab) + data = file.read() + file.close() + dom = parseString(data) + + sabreps = dom.getElementsByTagName('replace_spaces')[0].firstChild.wholeText + sabrepd = dom.getElementsByTagName('replace_dots')[0].firstChild.wholeText + #logger.fdebug("sabreps:" + str(sabreps)) + myDB = db.DBConnection() - nzbiss = myDB.action("SELECT * from nzblog WHERE nzbname=?", [self.nzb_name]).fetchone() + + nzbname = self.nzb_name + + nzbiss = myDB.action("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone() if nzbiss is None: - self._log("Epic failure - could not locate file to rename.", logger.DEBUG) - logger.error(u"Unable to locate downloaded file to rename. PostProcessing aborted.") - return + self._log("Failure - could not initially locate nzbfile in my database to rename.", logger.DEBUG) + #decimals need to be accounted for.... + if str(sabrepd) == '0': + #Replace decimals is enabled so decimlas should already be passed through converted. + logger.info("SABnzbd setting: Replace Decimals is disabled.") + elif str(sabrepd) == '1': + #Replace decimals is enabled. + #By default SAB will pass back the value with decimals replaced with a ' ' or a '_' + logger.info("SABnzbd setting: Replace Decimals is enabled.") + self._log("I'm going to try to rejig the nzbname passed from SAB accounting for decmials to see if I can find it.") + nzbname = re.sub('[\.]', '_', str(nzbname)) + #spaces need to be accounted for + if str(sabreps) == '1': + #Replace spaces is enabled so spaces should already be passed through converted. + logger.info("SABnzbd setting: Replace spaces is enabled.") + elif str(sabreps) == '0': + #Replace space is disabled. + #By default SAB will pass back the value with spaces replaced with a '+' + logger.info("SABnzbd setting: Replace spaces is disabled.") + self._log("I'm going to try to rejig the nzbname passed from SAB accouting for spaces to see if I can find it.") + nzbname = re.sub(' ', '_', str(nzbname)) + #let's remove the - cause it will cause problems at this point... + nzbname = re.sub('[\-]', '_', str(nzbname)) + + logger.fdebug("trying again with this nzbname: " + str(nzbname)) + nzbiss = myDB.action("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone() + if nzbiss is None: + logger.error(u"Unable to locate downloaded file to rename. PostProcessing aborted.") + return + else: + self._log("I corrected and found the nzb as : " + str(nzbname)) + issueid = nzbiss['IssueID'] else: issueid = nzbiss['IssueID'] - #log2screen = log2screen + "IssueID: " + issueid + "\n" #use issueid to get publisher, series, year, issue number issuenzb = myDB.action("SELECT * from issues WHERE issueid=?", [issueid]).fetchone() comicid = issuenzb['ComicID'] - #log2screen = log2screen + "ComicID: " + comicid + "\n" issuenum = issuenzb['Issue_Number'] #issueno = str(issuenum).split('.')[0] @@ -199,22 +242,6 @@ class PostProcessor(object): self._log("Year: " + seriesyear, logger.DEBUG) comlocation = comicnzb['ComicLocation'] self._log("Comic Location: " + comlocation, logger.DEBUG) -#---move to importer.py - #get output path format -# if ':' in series: -# series = series.replace(':','') - #do work to generate folder path -# values = {'$Series': series, -# '$Publisher': publisher, -# '$Year': seriesyear -# } -# comlocation = mylar.DESTINATION_DIR + "/" + helpers.replace_all(mylar.FOLDER_FORMAT, values) - #last perform space replace -# if mylar.REPLACE_SPACES: - #mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot -# comlocation = comlocation.replace(' ', mylar.REPLACE_CHAR) -# log2screen = log2screen + "Final Location: " + comlocation + "\n" -#--- #rename file and move to new path #nfilename = series + " " + issueno + " (" + seriesyear + ")" file_values = {'$Series': series, @@ -243,8 +270,6 @@ class PostProcessor(object): if mylar.REPLACE_SPACES: #mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot nfilename = nfilename.replace(' ', mylar.REPLACE_CHAR) - #TODO - sort issue numbering 12.00 should be 12 - #replace funky characters so it doesn't break things nfilename = re.sub('[\,\:]', '', nfilename) self._log("New Filename: " + nfilename, logger.DEBUG) @@ -279,7 +304,24 @@ class PostProcessor(object): # retrieve/create the corresponding comic objects if mylar.ENABLE_EXTRA_SCRIPTS: - self._run_extra_scripts(self.nzb_name, self.nzb_folder) + folderp = str(dst) #folder location after move/rename + nzbn = self.nzb_name #original nzb name + filen = str(nfilename + ext) #new filename + #name, comicyear, comicid , issueid, issueyear, issue, publisher + #create the dic and send it. + seriesmeta = [] + seriesmetadata = {} + seriesmeta.append({ + 'name': series, + 'comicyear': seriesyear, + 'comicid': comicid, + 'issueid': issueid, + 'issueyear': issueyear, + 'issue': issuenum, + 'publisher': publisher + }) + seriesmetadata['seriesmeta'] = seriesmeta + self._run_extra_scripts(nzbname, self.nzb_folder, filen, folderp, seriesmetadata ) return self.log diff --git a/mylar/parseit.py b/mylar/parseit.py index 0c2315f8..d1aa8140 100755 --- a/mylar/parseit.py +++ b/mylar/parseit.py @@ -389,7 +389,7 @@ def GCDAdd(gcdcomicid): #logger.fdebug("series_data: " + str(parsed)) #print ("parse:" + str(parsed)) subtxt3 = parsed.find("dd", {"id" : "publication_dates"}) - logger.fdebug("publication_dates: " + str(subtxt3)) + #logger.fdebug("publication_dates: " + str(subtxt3)) pubdate = subtxt3.findNext(text=True).rstrip() logger.fdebug("pubdate:" + str(pubdate)) subtxt4 = parsed.find("dd", {"id" : "issues_published"}) diff --git a/mylar/search.py b/mylar/search.py index 6b706b1b..720afde8 100755 --- a/mylar/search.py +++ b/mylar/search.py @@ -55,6 +55,9 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, IssueDate, IssueI if mylar.EXPERIMENTAL == 1: nzbprovider.append('experimental') nzbp+=1 + + newznabs = 0 + if mylar.NEWZNAB == 1: logger.fdebug("mylar.newznab:" + str(mylar.NEWZNAB)) if mylar.NEWZNAB_ENABLED: @@ -65,7 +68,6 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, IssueDate, IssueI else: newznab_hosts = [] logger.fdebug("initial newznab provider not enabled...checking for additional newznabs.") - newznabs = 0 logger.fdebug("mylar.EXTRA_NEWZNABS:" + str(mylar.EXTRA_NEWZNABS)) @@ -558,98 +560,41 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is #end blackhole else: - if nzbprov != 'nzb.su': - tmppath = mylar.CACHE_DIR - if os.path.exists(tmppath): - logger.fdebug("cache directory successfully found at : " + str(tmppath)) - pass - else: - #let's make the dir. - logger.fdebug("couldn't locate cache directory, attempting to create at : " + str(mylar.CACHE_DIR)) - try: - os.makedirs(str(mylar.CACHE_DIR)) - logger.info(u"Cache Directory successfully created at: " + str(mylar.CACHE_DIR)) - - except OSError.e: - if e.errno != errno.EEXIST: - raise - - filenamenzb = os.path.split(linkapi)[1] - #filenzb = os.path.join(tmppath,filenamenzb) - logger.fdebug("unalterted nzb name: " + str(filenamenzb)) - #let's send a clean copy to SAB because the names are random characters and/or could be stupid. - ComicName = re.sub('[\:\,]', '', ComicName) - filenzb = str(ComicName.replace(' ', '_')) + "_" + str(IssueNumber) + "_(" + str(comyear) + ")" - logger.fdebug("prettified nzb name: " + str(filenzb)) - - if mylar.RENAME_FILES == 1: - logger.fdebug("Rename Files enabled..") - filenzb = str(ComicName.replace(' ', '_')) + "_" + str(IssueNumber) + "_(" + str(comyear) + ")" - logger.fdebug("this should be the same as prettified nzb name:" + str(filenzb)) - if mylar.REPLACE_SPACES: - logger.fdebug("Replace spaces option enabled") - logger.fdebug("replace character: " + str(mylar.REPLACE_CHAR)) - repchar = mylar.REPLACE_CHAR - repurlchar = mylar.REPLACE_CHAR - else: - logger.fdebug("Replace spaces option NOT enabled") - repchar = ' ' - repurlchar = "%20" - #let's make sure there's no crap in the ComicName since it's O.G. - logger.fdebug("original Name of comic: " + str(ComicName)) - ComicNM = re.sub('[\:\,]', '', str(ComicName)) - logger.fdebug("altered Name of comic: " + str(ComicNM)) - renameit = str(ComicNM) + " " + str(IssueNumber) + " (" + str(SeriesYear) + ")" + " " + "(" + str(comyear) + ")" - logger.fdebug("altered Name with additional info: " + str(renameit)) - renamethis = renameit.replace(' ', repchar) - logger.fdebug("...with replace spaces: " + str(renamethis)) - renamer1 = renameit.replace(' ', repurlchar) - renamer = re.sub("\&", "%26", str(renamer1)) - logger.fdebug("...adjusting for url restrictions: " + str(renamer)) - - filenext = str(filenzb) + ".nzb" - savefile = os.path.join(tmppath, filenext) - logger.fdebug("nzb file to be saved: " + str(savefile)) - + tmppath = mylar.CACHE_DIR + if os.path.exists(tmppath): + logger.fdebug("cache directory successfully found at : " + str(tmppath)) + pass + else: + #let's make the dir. + logger.fdebug("couldn't locate cache directory, attempting to create at : " + str(mylar.CACHE_DIR)) try: - urllib.urlretrieve(linkapi, str(savefile)) - except urllib.URLError: - logger.fdebug(u"Unable to retrieve nzb using link: " + str(linkapi)) - logger.fdebug(u"Possibly unable to save nzb: " + str(savefile)) - logger.error(u"Unable to retrieve nzb file.") - return + os.makedirs(str(mylar.CACHE_DIR)) + logger.info(u"Cache Directory successfully created at: " + str(mylar.CACHE_DIR)) - if os.path.getsize(str(savefile)) == 0: - logger.error(u"nzb size detected as zero bytes.") - continue + except OSError.e: + if e.errno != errno.EEXIST: + raise - logger.info(u"Sucessfully retrieved nzb file using " + str(nzbprov)) - nzbname = str(filenzb) + logger.fdebug("link to retrieve via api:" + str(linkapi)) - elif nzbprov == 'nzb.su': - logger.fdebug("NZB.SU - linkapi:" + str(linkapi)) - nzbname = re.sub(" ", "_", str(entry['title'])) + #when the series contains a '-' or the issue a decimal it can't convert + #if SAB's settings differ on how it handles spaces / decimals. + #let's attempt to adjust by removing the - and . + nzbname = re.sub(" ", "_", str(entry['title'])) + nzbname = re.sub('[\.\-]', '_', str(nzbname)) logger.fdebug("nzbname used for post-processing:" + str(nzbname)) # let's build the send-to-SAB string now: tmpapi = str(mylar.SAB_HOST) logger.fdebug("send-to-SAB host string: " + str(tmpapi)) - # nzb.su only works with direct links for some reason... - if nzbprov == 'nzb.su': - SABtype = "/api?mode=addurl&name=" - savefileURL = str(linkapi) - else: - SABtype = "/api?mode=addlocalfile&name=" - # if the savefile location has spaces in the path, could cause problems. - # if the savefile has a &, escape it otherwise will botch up send-to-SAB link - # let's adjust. - saveF = re.sub("\&", "%26", str(savefile)) - savefileURL = re.sub(" ","%20", str(saveF)) + # changed to just work with direct links now... + SABtype = "/api?mode=addurl&name=" + fileURL = str(linkapi) tmpapi = tmpapi + str(SABtype) logger.fdebug("...selecting API type: " + str(tmpapi)) - tmpapi = tmpapi + str(savefileURL) - logger.fdebug("...attaching nzbfile: " + str(tmpapi)) + tmpapi = tmpapi + str(fileURL) + logger.fdebug("...attaching nzb provider link: " + str(tmpapi)) # determine SAB priority if mylar.SAB_PRIORITY: tmpapi = tmpapi + "&priority=" + str(sabpriority) @@ -674,10 +619,10 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is logger.info(u"Successfully sent nzb file to SABnzbd") #delete the .nzb now. - if mylar.PROG_DIR is not "/" and nzbprov != 'nzb.su': - logger.fdebug("preparing to remove temporary nzb file at: " + str(savefile)) - os.remove(savefile) - logger.info(u"Removed temporary save file") + #if mylar.PROG_DIR is not "/" and nzbprov != 'nzb.su': + # logger.fdebug("preparing to remove temporary nzb file at: " + str(savefile)) + # os.remove(savefile) + # logger.info(u"Removed temporary save file") #raise an exception to break out of loop foundc = "yes" done = True diff --git a/mylar/updater.py b/mylar/updater.py index 179ee26f..328045c6 100755 --- a/mylar/updater.py +++ b/mylar/updater.py @@ -71,7 +71,7 @@ def upcoming_update(ComicID, ComicName, IssueNumber, IssueDate): if ComicID[:1] == "G": mylar.importer.GCDimport(ComicID,pullupd) else: mylar.importer.addComictoDB(ComicID,mismatch,pullupd) - if issuechk['Issue_Number'] == IssueNumber: + elif issuechk['Issue_Number'] == IssueNumber: logger.fdebug("Comic series already up-to-date ... no need to refresh at this time.") logger.fdebug("Available to be marked for download - checking..." + str(issuechk['ComicName']) + " Issue: " + str(issuechk['Issue_Number'])) logger.fdebug("...Existing status: " + str(issuechk['Status'])) @@ -89,18 +89,25 @@ def upcoming_update(ComicID, ComicName, IssueNumber, IssueDate): values = { "Status": "Skipped"} newValue['Status'] = "Skipped" #was in wrong place :( - if mylar.AUTOWANT_UPCOMING: - if issuechk['Status'] == "Skipped": - newValue['Status'] = "Wanted" - values = { "Status": "Wanted"} - logger.fdebug("...New status of Wanted") - elif issuechk['Status'] == "Wanted": - logger.fdebug("...Status already Wanted .. not changing.") - else: - logger.fdebug("...Already have issue - keeping existing status of : " + issuechk['Status']) + if mylar.AUTOWANT_UPCOMING: + #for issues not in db - to be added to Upcoming table. + if issuechk is None: + newValue['Status'] = "Wanted" + logger.fdebug("...Changing Status to Wanted and throwing it in the Upcoming section since it's not published yet.") + #this works for issues existing in DB... + elif issuechk['Status'] == "Skipped": + newValue['Status'] = "Wanted" + values = { "Status": "Wanted"} + logger.fdebug("...New status of Wanted") + elif issuechk['Status'] == "Wanted": + logger.fdebug("...Status already Wanted .. not changing.") + else: + logger.fdebug("...Already have issue - keeping existing status of : " + issuechk['Status']) - myDB.upsert("upcoming", newValue, controlValue) - myDB.upsert("issues", values, control) + if issuechk is None: + myDB.upsert("upcoming", newValue, controlValue) + else: + myDB.upsert("issues", values, control) else: logger.fdebug("Issues don't match for some reason... db issue: " + str(issuechk['Issue_Number']) + " ...weekly new issue: " + str(IssueNumber)) @@ -198,7 +205,7 @@ def forceRescan(ComicID): arcissues = myDB.select("SELECT * FROM issues WHERE ComicID=? and Status='Archived'", [ComicID]) if len(arcissues) > 0: havefiles = len(arcissues) - print "have count adjusted to:" + str(len(arcissues)) + logger.fdebug("Adjusting have total because of this many archive files:" + str(len(arcissues)) while (fn < fccnt): haveissue = "no" issuedupe = "no" @@ -223,28 +230,28 @@ def forceRescan(ComicID): issyear = reiss['IssueDate'][:4] old_status = reiss['Status'] - logger.fdebug("integer_issue:" + str(int_iss) + " ... status: " + str(old_status)) + #logger.fdebug("integer_issue:" + str(int_iss) + " ... status: " + str(old_status)) while (som < fcn): #counts get buggered up when the issue is the last field in the filename - ie. '50.cbr' - logger.fdebug("checking word - " + str(fcnew[som])) + #logger.fdebug("checking word - " + str(fcnew[som])) if ".cbr" in fcnew[som]: fcnew[som] = fcnew[som].replace(".cbr", "") elif ".cbz" in fcnew[som]: fcnew[som] = fcnew[som].replace(".cbz", "") if '.' in fcnew[som]: - logger.fdebug("decimal detected...adjusting.") + #logger.fdebug("decimal detected...adjusting.") try: i = float(fcnew[som]) except ValueError, TypeError: #not numeric fcnew[som] = fcnew[som].replace(".", "") - logger.fdebug("new word: " + str(fcnew[som])) + #logger.fdebug("new word: " + str(fcnew[som])) else: #numeric pass if fcnew[som].isdigit(): #this won't match on decimal issues - need to fix. - logger.fdebug("digit detected") + #logger.fdebug("digit detected") if int(fcnew[som]) > 0: # fcdigit = fcnew[som].lstrip('0') #fcdigit = str(int(fcnew[som])) @@ -255,24 +262,24 @@ def forceRescan(ComicID): elif "." in fcnew[som]: #this will match on decimal issues IssueChk = fcnew[som] - logger.fdebug("decimal detected...analyzing if issue") + #logger.fdebug("decimal detected...analyzing if issue") isschk_find = IssueChk.find('.') isschk_b4dec = IssueChk[:isschk_find] isschk_decval = IssueChk[isschk_find+1:] if isschk_b4dec.isdigit(): - logger.fdebug("digit detected prior to decimal.") + #logger.fdebug("digit detected prior to decimal.") if isschk_decval.isdigit(): - #pass - logger.fdebug("digit detected after decimal.") + pass + #logger.fdebug("digit detected after decimal.") else: - logger.fdebug("not an issue - no digit detected after decimal") + #logger.fdebug("not an issue - no digit detected after decimal") break else: - logger.fdebug("not an issue - no digit detected prior to decimal") + #logger.fdebug("not an issue - no digit detected prior to decimal") break - logger.fdebug("IssueNumber: " + str(IssueChk)) - logger.fdebug("..before decimal: " + str(isschk_b4dec)) - logger.fdebug("...after decimal: " + str(isschk_decval)) + #logger.fdebug("IssueNumber: " + str(IssueChk)) + #logger.fdebug("..before decimal: " + str(isschk_b4dec)) + #logger.fdebug("...after decimal: " + str(isschk_decval)) #--let's make sure we don't wipe out decimal issues ;) if int(isschk_decval) == 0: iss = isschk_b4dec @@ -285,18 +292,18 @@ def forceRescan(ComicID): iss = isschk_b4dec + "." + isschk_decval.rstrip('0') intdec = int(isschk_decval.rstrip('0')) * 10 fcdigit = (int(isschk_b4dec) * 1000) + intdec - logger.fdebug("b4dec: " + str(isschk_b4dec)) - logger.fdebug("decval: " + str(isschk_decval)) - logger.fdebug("intdec: " + str(intdec)) - logger.fdebug("let's compare with this issue value: " + str(fcdigit)) + #logger.fdebug("b4dec: " + str(isschk_b4dec)) + #logger.fdebug("decval: " + str(isschk_decval)) + #logger.fdebug("intdec: " + str(intdec)) + #logger.fdebug("let's compare with this issue value: " + str(fcdigit)) else: # it's a word, skip it. fcdigit = 1000000 - logger.fdebug("fcdigit: " + str(fcdigit)) - logger.fdebug("int_iss: " + str(int_iss)) + #logger.fdebug("fcdigit: " + str(fcdigit)) + #logger.fdebug("int_iss: " + str(int_iss)) if "." in str(int_iss): int_iss = helpers.decimal_issue(int_iss) - logger.fdebug("this is the int issue:" + str(int_iss)) + #logger.fdebug("this is the int issue:" + str(int_iss)) if int(fcdigit) == int_iss: #if issyear in fcnew[som+1]: