From 974e49d2768dbff3a8aae03c50324a6bdedf7632 Mon Sep 17 00:00:00 2001 From: evilhero Date: Tue, 22 Jan 2013 11:20:47 -0500 Subject: [PATCH 01/16] FIX: added some direct verbose logging to PostProcessing to help with troubleshooting --- mylar/PostProcessor.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/mylar/PostProcessor.py b/mylar/PostProcessor.py index 05bd2d5d..5cb2fbac 100755 --- a/mylar/PostProcessor.py +++ b/mylar/PostProcessor.py @@ -134,10 +134,11 @@ class PostProcessor(object): self._log(u"Unable to run extra_script: " + str(script_cmd)) -# def PostProcess(nzb_name, nzb_folder): def Process(self): self._log("nzb name: " + str(self.nzb_name), logger.DEBUG) self._log("nzb folder: " + str(self.nzb_folder), logger.DEBUG) + logger.fdebug("nzb name: " + str(self.nzb_name)) + logger.fdebug("nzb folder: " + str(self.nzb_folder)) #lookup nzb_name in nzblog table to get issueid #query SAB to find out if Replace Spaces enabled / not as well as Replace Decimals @@ -151,8 +152,8 @@ class PostProcessor(object): sabreps = dom.getElementsByTagName('replace_spaces')[0].firstChild.wholeText sabrepd = dom.getElementsByTagName('replace_dots')[0].firstChild.wholeText - #logger.fdebug("sabreps:" + str(sabreps)) - + logger.fdebug("SAB Replace Spaces: " + str(sabreps)) + logger.fdebug("SAB Replace Dots: " + str(sabrepd)) myDB = db.DBConnection() nzbname = self.nzb_name @@ -168,21 +169,25 @@ class PostProcessor(object): nzbname = re.sub(' ', '.', str(nzbname)) nzbname = re.sub('[\,\:]', '', str(nzbname)) + logger.fdebug("After conversions, nzbname is : " + str(nzbname)) self._log("nzbname: " + str(nzbname), logger.DEBUG) nzbiss = myDB.action("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone() if nzbiss is None: self._log("Failure - could not initially locate nzbfile in my database to rename.", logger.DEBUG) + logger.fdebug("Failure - could not locate nzbfile initially.") # if failed on spaces, change it all to decimals and try again. nzbname = re.sub('_', '.', str(nzbname)) self._log("trying again with this nzbname: " + str(nzbname), logger.DEBUG) + logger.fdebug("trying again with nzbname of : " + str(nzbname)) nzbiss = myDB.action("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone() if nzbiss is None: logger.error(u"Unable to locate downloaded file to rename. PostProcessing aborted.") return else: self._log("I corrected and found the nzb as : " + str(nzbname)) + logger.fdebug("auto-corrected and found the nzb as : " + str(nzbname)) issueid = nzbiss['IssueID'] else: issueid = nzbiss['IssueID'] @@ -200,6 +205,7 @@ class PostProcessor(object): issdec = int(iss_decval) issueno = str(iss) self._log("Issue Number: " + str(issueno), logger.DEBUG) + logger.fdebug("Issue Number: " + str(issueno)) else: if len(iss_decval) == 1: iss = iss_b4dec + "." + iss_decval @@ -209,6 +215,7 @@ class PostProcessor(object): issdec = int(iss_decval.rstrip('0')) * 10 issueno = iss_b4dec self._log("Issue Number: " + str(iss), logger.DEBUG) + logger.fdebug("Issue Number: " + str(iss)) # issue zero-suppression here if mylar.ZERO_LEVEL == "0": @@ -218,6 +225,8 @@ class PostProcessor(object): elif mylar.ZERO_LEVEL_N == "0x": zeroadd = "0" elif mylar.ZERO_LEVEL_N == "00x": zeroadd = "00" + logger.fdebug("Zero Suppression set to : " + str(mylar.ZERO_LEVEL_N)) + if str(len(issueno)) > 1: if int(issueno) < 10: self._log("issue detected less than 10", logger.DEBUG) @@ -249,17 +258,23 @@ class PostProcessor(object): prettycomiss = str(issueno) self._log("issue length error - cannot determine length. Defaulting to None: " + str(prettycomiss), logger.DEBUG) + logger.fdebug("Pretty Comic Issue is : " + str(prettycomiss)) issueyear = issuenzb['IssueDate'][:4] self._log("Issue Year: " + str(issueyear), logger.DEBUG) + logger.fdebug("Issue Year : " + str(issueyear)) comicnzb= myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone() publisher = comicnzb['ComicPublisher'] self._log("Publisher: " + publisher, logger.DEBUG) + logger.fdebug("Publisher: " + str(publisher)) series = comicnzb['ComicName'] self._log("Series: " + series, logger.DEBUG) + logger.fdebug("Series: " + str(series)) seriesyear = comicnzb['ComicYear'] self._log("Year: " + seriesyear, logger.DEBUG) + logger.fdebug("Year: " + str(seriesyear)) comlocation = comicnzb['ComicLocation'] self._log("Comic Location: " + comlocation, logger.DEBUG) + logger.fdebug("Comic Location: " + str(comlocation)) #Run Pre-script @@ -301,8 +316,12 @@ class PostProcessor(object): path, ext = os.path.splitext(ofilename) self._log("Original Filename: " + ofilename, logger.DEBUG) self._log("Original Extension: " + ext, logger.DEBUG) + logger.fdebug("Original Filname: " + str(ofilename)) + logger.fdebug("Original Extension: " + str(ext)) + if mylar.FILE_FORMAT == '': self._log("Rename Files isn't enabled...keeping original filename.", logger.DEBUG) + logger.fdebug("Rename Files isn't enabled - keeping original filename.") #check if extension is in nzb_name - will screw up otherwise if ofilename.lower().endswith(extensions): nfilename = ofilename[:-4] @@ -315,16 +334,20 @@ class PostProcessor(object): nfilename = nfilename.replace(' ', mylar.REPLACE_CHAR) nfilename = re.sub('[\,\:]', '', nfilename) self._log("New Filename: " + nfilename, logger.DEBUG) + logger.fdebug("New Filename: " + str(nfilename)) - src = self.nzb_folder + "/" + ofilename + src = os.path.join(self.nzb_folder, ofilename) if mylar.LOWERCASE_FILENAMES: dst = (comlocation + "/" + nfilename + ext).lower() else: dst = comlocation + "/" + nfilename + ext.lower() self._log("Source:" + src, logger.DEBUG) self._log("Destination:" + dst, logger.DEBUG) - os.rename(self.nzb_folder + "/" + ofilename, self.nzb_folder + "/" + nfilename + ext) - src = self.nzb_folder + "/" + nfilename + ext + logger.fdebug("Source: " + str(src)) + logger.fdebug("Destination: " + str(dst)) + + os.rename(os.path.join(self.nzb_folder, ofilename), os.path.join(self.nzb_folder,str(nfilename + ext))) + #src = os.path.join(self.nzb_folder, str(nfilename + ext)) try: shutil.move(src, dst) except (OSError, IOError): From 6ef8ab7b8d77bb534889f4cf9c2ea3d959793f7e Mon Sep 17 00:00:00 2001 From: evilhero Date: Wed, 23 Jan 2013 02:34:50 -0500 Subject: [PATCH 02/16] FIX:(#173) If a search result contained a watchcomic title, it would match incorrectly, FIX:(#172)(#163) Post-Processing would fail on finding directory - hopefully fixed now, FIX:(#167) NZBX would fail on send to SAB, IMP:(#109) Make All Skipped Wanted option now available on Comic Details page --- data/interfaces/default/artistredone.html | 3 ++- mylar/__init__.py | 2 +- mylar/search.py | 31 ++++++++++------------- mylar/webserve.py | 23 ++++++++++++++++- 4 files changed, 39 insertions(+), 20 deletions(-) diff --git a/data/interfaces/default/artistredone.html b/data/interfaces/default/artistredone.html index 13198915..eba7cb5a 100755 --- a/data/interfaces/default/artistredone.html +++ b/data/interfaces/default/artistredone.html @@ -220,11 +220,12 @@ selected issues + - + diff --git a/mylar/__init__.py b/mylar/__init__.py index efb72f85..f4bfdf8b 100755 --- a/mylar/__init__.py +++ b/mylar/__init__.py @@ -115,6 +115,7 @@ AUTOWANT_UPCOMING = True AUTOWANT_ALL = False COMIC_COVER_LOCAL = False ADD_TO_CSV = True +SKIPPED2WANTED = False SAB_HOST = None SAB_USERNAME = None @@ -530,7 +531,6 @@ def config_write(): new_config['General']['use_maxsize'] = int(USE_MAXSIZE) new_config['General']['maxsize'] = MAXSIZE new_config['General']['add_to_csv'] = int(ADD_TO_CSV) - new_config['General']['enable_extra_scripts'] = int(ENABLE_EXTRA_SCRIPTS) new_config['General']['extra_scripts'] = EXTRA_SCRIPTS new_config['General']['enable_pre_scripts'] = int(ENABLE_PRE_SCRIPTS) diff --git a/mylar/search.py b/mylar/search.py index 7af39860..df00f585 100755 --- a/mylar/search.py +++ b/mylar/search.py @@ -503,7 +503,7 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is logger.fdebug(str(n) + " Comparing: " + str(watchcomic_split[n]) + " .to. " + str(splitit[n])) if '+' in watchcomic_split[n]: watchcomic_split[n] = re.sub('+', '', str(watchcomic_split[n])) - if str(watchcomic_split[n].lower()) in str(splitit[n].lower()): + if str(watchcomic_split[n].lower()) in str(splitit[n].lower()) and len(watchcomic_split[n]) >= len(splitit[n]): logger.fdebug("word matched on : " + str(splitit[n])) scount+=1 #elif ':' in splitit[n] or '-' in splitit[n]: @@ -607,6 +607,8 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is urllib.urlretrieve(linkapi, str(mylar.BLACKHOLE_DIR) + str(filenamenzb)) logger.fdebug("filename saved to your blackhole as : " + str(filenamenzb)) logger.info(u"Successfully sent .nzb to your Blackhole directory : " + str(mylar.BLACKHOLE_DIR) + str(filenamenzb) ) + nzbname = filenamenzb[:-4] + logger.fdebug("nzb name to be used for post-processing is : " + str(nzbname)) #end blackhole else: @@ -627,6 +629,17 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is logger.fdebug("link to retrieve via api:" + str(linkapi)) + #let's change all space to decimals for simplicity + nzbname = re.sub(" ", ".", str(entry['title'])) + nzbname = re.sub('[\,\:]', '', str(nzbname)) + extensions = ('.cbr', '.cbz') + + if nzbname.lower().endswith(extensions): + fd, ext = os.path.splitext(nzbname) + logger.fdebug("Removed extension from nzb: " + ext) + nzbname = re.sub(str(ext), '', str(nzbname)) + + logger.fdebug("nzbname used for post-processing:" + str(nzbname)) #we need to change the nzbx string now to allow for the nzbname rename. if nzbprov == 'nzbx': @@ -673,22 +686,6 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is # logger.info(u"Removed temporary save file") #raise an exception to break out of loop - #let's change all space to decimals for simplicity - if mylar.BLACKHOLE: - bhole_cname = re.sub('[/:/,\/]', '', str(ComicName)) - nzbname = str(re.sub(" ", ".", str(bhole_cname))) + "." + str(IssueNumber) + ".(" + str(comyear) + ")" - else: - nzbname = re.sub(" ", ".", str(entry['title'])) - nzbname = re.sub('[\,\:]', '', str(nzbname)) - extensions = ('.cbr', '.cbz') - - if nzbname.lower().endswith(extensions): - fd, ext = os.path.splitext(nzbname) - logger.fdebug("Removed extension from nzb: " + ext) - nzbname = re.sub(str(ext), '', str(nzbname)) - - - logger.fdebug("nzbname used for post-processing:" + str(nzbname)) foundc = "yes" done = True diff --git a/mylar/webserve.py b/mylar/webserve.py index ec837306..582bbb18 100755 --- a/mylar/webserve.py +++ b/mylar/webserve.py @@ -70,12 +70,14 @@ class WebInterface(object): if comic is None: raise cherrypy.HTTPRedirect("home") usethefuzzy = comic['UseFuzzy'] + skipped2wanted = "0" if usethefuzzy is None: usethefuzzy = "0" comicConfig = { "comiclocation" : mylar.COMIC_LOCATION, "fuzzy_year0" : helpers.radio(int(usethefuzzy), 0), "fuzzy_year1" : helpers.radio(int(usethefuzzy), 1), - "fuzzy_year2" : helpers.radio(int(usethefuzzy), 2) + "fuzzy_year2" : helpers.radio(int(usethefuzzy), 2), + "skipped2wanted" : helpers.checked(skipped2wanted) } return serve_template(templatename="artistredone.html", title=comic['ComicName'], comic=comic, issues=issues, comicConfig=comicConfig) artistPage.exposed = True @@ -444,6 +446,25 @@ class WebInterface(object): return serve_template(templatename="upcoming.html", title="Upcoming", upcoming=upcoming, issues=issues) upcoming.exposed = True + def skipped2wanted(self, comicid): + # change all issues for a given ComicID that are Skipped, into Wanted. + issuestowanted = [] + issuesnumwant = [] + myDB = db.DBConnection() + skipped2 = myDB.select("SELECT * from issues WHERE ComicID=? AND Status='Skipped'", [comicid]) + for skippy in skipped2: + mvcontroldict = {"IssueID": skippy['IssueID']} + mvvalues = {"Status": "Wanted"} + #print ("Changing issue " + str(skippy['Issue_Number']) + " to Wanted.") + myDB.upsert("issues", mvvalues, mvcontroldict) + issuestowanted.append(skippy['IssueID']) + issuesnumwant.append(skippy['Issue_Number']) + if len(issuestowanted) > 0 : + logger.info("Marking issues: %s as Wanted" % issuesnumwant) + threading.Thread(target=search.searchIssueIDList, args=[issuestowanted]).start() + raise cherrypy.HTTPRedirect("artistPage?ComicID=%s" % [comicid]) + skipped2wanted.exposed = True + def searchScan(self, name): return serve_template(templatename="searchfix.html", title="Manage", name=name) searchScan.exposed = True From f30d507ab94aefa4e4898519d192b48d82aa1358 Mon Sep 17 00:00:00 2001 From: evilhero Date: Wed, 23 Jan 2013 02:50:49 -0500 Subject: [PATCH 03/16] FIX:(#169) 500 Error on startup trying to retrieve Weekly Pull-list --- mylar/weeklypull.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mylar/weeklypull.py b/mylar/weeklypull.py index a12b6c9c..3d316cf8 100755 --- a/mylar/weeklypull.py +++ b/mylar/weeklypull.py @@ -37,7 +37,7 @@ def pullit(): pull_date = myDB.action("SELECT SHIPDATE from weekly").fetchone() logger.info(u"Weekly pull list present - checking if it's up-to-date..") pulldate = pull_date['SHIPDATE'] - except sqlite3.OperationalError, msg: + except (sqlite3.OperationalError, TypeError),msg: conn=sqlite3.connect(mylar.DB_FILE) c=conn.cursor() logger.info(u"Error Retrieving weekly pull list - attempting to adjust") From 25c83bdab46fd94936b88dab89c08aff0281bbc5 Mon Sep 17 00:00:00 2001 From: evilhero Date: Wed, 23 Jan 2013 03:22:22 -0500 Subject: [PATCH 04/16] IMP:(#133) CVinfo option added in Configuration/Advanced Options --- data/interfaces/default/config.html | 10 ++++++---- mylar/__init__.py | 6 ++++-- mylar/importer.py | 10 ++++++++++ mylar/webserve.py | 4 +++- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index cd0da1b2..0c82e5bb 100755 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -354,9 +354,6 @@ - - + + From 3e5448e62a7c96b6777b435dcecf9b371c991076 Mon Sep 17 00:00:00 2001 From: evilhero Date: Wed, 23 Jan 2013 04:47:21 -0500 Subject: [PATCH 06/16] FIX: removed unnecessary html files (forgot last commit) --- data/interfaces/default/album.html | 163 --------- .../default/artist-descincluded.html | 194 ----------- data/interfaces/default/artist.html | 262 --------------- .../interfaces/default/artistredone.html.save | 318 ------------------ data/interfaces/default/editcomic.html | 137 -------- 5 files changed, 1074 deletions(-) delete mode 100755 data/interfaces/default/album.html delete mode 100755 data/interfaces/default/artist-descincluded.html delete mode 100755 data/interfaces/default/artist.html delete mode 100755 data/interfaces/default/artistredone.html.save delete mode 100755 data/interfaces/default/editcomic.html diff --git a/data/interfaces/default/album.html b/data/interfaces/default/album.html deleted file mode 100755 index 0ebc283b..00000000 --- a/data/interfaces/default/album.html +++ /dev/null @@ -1,163 +0,0 @@ -<%inherit file="base.html" /> -<%! - from mylar import db, helpers - myDB = db.DBConnection() -%> - -<%def name="headerIncludes()"> -
-
- Delete Album - %if album['Status'] == 'Skipped': - Mark Album as Wanted - %elif album['Status'] == 'Wanted': - Force Check - Mark Album as Skipped - %else: - Retry Download - Try New Version - %endif -
-
- « Back to ${album['ArtistName']} - - -<%def name="body()"> -
-
-
- -
- -

${album['AlbumTitle']}

-

${album['ArtistName']}

- <% - totalduration = myDB.action("SELECT SUM(TrackDuration) FROM tracks WHERE AlbumID=?", [album['AlbumID']]).fetchone()[0] - totaltracks = len(myDB.select("SELECT TrackTitle from tracks WHERE AlbumID=?", [album['AlbumID']])) - try: - albumduration = helpers.convert_milliseconds(totalduration) - except: - albumduration = 'n/a' - - %> -
-
-
    -
  • Tracks: ${totaltracks}
    • -
  • Duration: ${albumduration}
    • -
-
- -
-
-
Number Name Date
Post-Processing -
- -
@@ -385,6 +382,9 @@
Renaming options +
+ +
@@ -423,7 +423,7 @@
-
+
+ +
diff --git a/mylar/__init__.py b/mylar/__init__.py index f4bfdf8b..37eeb174 100755 --- a/mylar/__init__.py +++ b/mylar/__init__.py @@ -116,6 +116,7 @@ AUTOWANT_ALL = False COMIC_COVER_LOCAL = False ADD_TO_CSV = True SKIPPED2WANTED = False +CVINFO = False SAB_HOST = None SAB_USERNAME = None @@ -216,7 +217,7 @@ def initialize(): NZBSU, NZBSU_APIKEY, DOGNZB, DOGNZB_APIKEY, NZBX,\ NEWZNAB, NEWZNAB_HOST, NEWZNAB_APIKEY, NEWZNAB_ENABLED, EXTRA_NEWZNABS,\ RAW, RAW_PROVIDER, RAW_USERNAME, RAW_PASSWORD, RAW_GROUPS, EXPERIMENTAL, \ - PREFERRED_QUALITY, MOVE_FILES, RENAME_FILES, LOWERCASE_FILENAMES, USE_MINSIZE, MINSIZE, USE_MAXSIZE, MAXSIZE, CORRECT_METADATA, FOLDER_FORMAT, FILE_FORMAT, REPLACE_CHAR, REPLACE_SPACES, ADD_TO_CSV, \ + PREFERRED_QUALITY, MOVE_FILES, RENAME_FILES, LOWERCASE_FILENAMES, USE_MINSIZE, MINSIZE, USE_MAXSIZE, MAXSIZE, CORRECT_METADATA, FOLDER_FORMAT, FILE_FORMAT, REPLACE_CHAR, REPLACE_SPACES, ADD_TO_CSV, CVINFO, \ COMIC_LOCATION, QUAL_ALTVERS, QUAL_SCANNER, QUAL_TYPE, QUAL_QUALITY, ENABLE_EXTRA_SCRIPTS, EXTRA_SCRIPTS, ENABLE_PRE_SCRIPTS, PRE_SCRIPTS if __INITIALIZED__: @@ -284,7 +285,7 @@ def initialize(): USE_MAXSIZE = bool(check_setting_int(CFG, 'General', 'use_maxsize', 0)) MAXSIZE = check_setting_str(CFG, 'General', 'maxsize', '') ADD_TO_CSV = bool(check_setting_int(CFG, 'General', 'add_to_csv', 1)) - + CVINFO = bool(check_setting_int(CFG, 'General', 'cvinfo', 0)) ENABLE_EXTRA_SCRIPTS = bool(check_setting_int(CFG, 'General', 'enable_extra_scripts', 0)) EXTRA_SCRIPTS = check_setting_str(CFG, 'General', 'extra_scripts', '') @@ -531,6 +532,7 @@ def config_write(): new_config['General']['use_maxsize'] = int(USE_MAXSIZE) new_config['General']['maxsize'] = MAXSIZE new_config['General']['add_to_csv'] = int(ADD_TO_CSV) + new_config['General']['cvinfo'] = int(CVINFO) new_config['General']['enable_extra_scripts'] = int(ENABLE_EXTRA_SCRIPTS) new_config['General']['extra_scripts'] = EXTRA_SCRIPTS new_config['General']['enable_pre_scripts'] = int(ENABLE_PRE_SCRIPTS) diff --git a/mylar/importer.py b/mylar/importer.py index 4814075c..511c1eca 100755 --- a/mylar/importer.py +++ b/mylar/importer.py @@ -363,6 +363,11 @@ def addComictoDB(comicid,mismatch=None,pullupd=None): } myDB.upsert("comics", newValueStat, controlValueStat) + + if mylar.CVINFO: + if not os.path.exists(comlocation + "/cvinfo"): + with open(comlocation + "/cvinfo","w") as text_file: + text_file.write("http://www.comicvine.com/" + str(comic['ComicName']).replace(" ", "-") + "/49-" + str(comicid)) logger.info(u"Updating complete for: " + comic['ComicName']) @@ -657,6 +662,11 @@ def GCDimport(gcomicid, pullupd=None): myDB.upsert("comics", newValueStat, controlValueStat) + if mylar.CVINFO: + if not os.path.exists(comlocation + "/cvinfo"): + with open(comlocation + "/cvinfo","w") as text_file: + text_file.write("http://www.comicvine.com/" + str(comic['ComicName']).replace(" ", "-") + "/49-" + str(comicid)) + logger.info(u"Updating complete for: " + ComicName) if pullupd is None: diff --git a/mylar/webserve.py b/mylar/webserve.py index 582bbb18..23b72ba9 100755 --- a/mylar/webserve.py +++ b/mylar/webserve.py @@ -620,6 +620,7 @@ class WebInterface(object): "zero_level" : helpers.checked(mylar.ZERO_LEVEL), "zero_level_n" : mylar.ZERO_LEVEL_N, "add_to_csv" : helpers.checked(mylar.ADD_TO_CSV), + "cvinfo" : helpers.checked(mylar.CVINFO), "lowercase_filenames" : helpers.checked(mylar.LOWERCASE_FILENAMES), "enable_extra_scripts" : helpers.checked(mylar.ENABLE_EXTRA_SCRIPTS), "extra_scripts" : mylar.EXTRA_SCRIPTS, @@ -718,7 +719,7 @@ class WebInterface(object): sab_host=None, sab_username=None, sab_apikey=None, sab_password=None, sab_category=None, sab_priority=None, log_dir=None, blackhole=0, blackhole_dir=None, usenet_retention=None, nzbsu=0, nzbsu_apikey=None, dognzb=0, dognzb_apikey=None, nzbx=0, newznab=0, newznab_host=None, newznab_apikey=None, newznab_enabled=0, raw=0, raw_provider=None, raw_username=None, raw_password=None, raw_groups=None, experimental=0, - preferred_quality=0, move_files=0, rename_files=0, add_to_csv=1, lowercase_filenames=0, folder_format=None, file_format=None, enable_extra_scripts=0, extra_scripts=None, enable_pre_scripts=0, pre_scripts=None, + preferred_quality=0, move_files=0, rename_files=0, add_to_csv=1, cvinfo=0, lowercase_filenames=0, folder_format=None, file_format=None, enable_extra_scripts=0, extra_scripts=None, enable_pre_scripts=0, pre_scripts=None, destination_dir=None, replace_spaces=0, replace_char=None, use_minsize=0, minsize=None, use_maxsize=0, maxsize=None, autowant_all=0, autowant_upcoming=0, comic_cover_local=0, zero_level=0, zero_level_n=None, interface=None, **kwargs): mylar.HTTP_HOST = http_host mylar.HTTP_PORT = http_port @@ -762,6 +763,7 @@ class WebInterface(object): mylar.ZERO_LEVEL = zero_level mylar.ZERO_LEVEL_N = zero_level_n mylar.ADD_TO_CSV = add_to_csv + mylar.CVINFO = cvinfo mylar.LOWERCASE_FILENAMES = lowercase_filenames mylar.USE_MINSIZE = use_minsize mylar.MINSIZE = minsize From 22f492b781b7e6feb8a2904f65d48a2f3110e3d2 Mon Sep 17 00:00:00 2001 From: evilhero Date: Wed, 23 Jan 2013 04:43:21 -0500 Subject: [PATCH 05/16] IMP: Filtered data and colours now working on main page and comic detail page (meaning you can now display 10,25,50,100 titles / issues at a time per page), IMP: Removed unnecessary html files --- data/interfaces/default/artistredone.html | 53 ++++++------- data/interfaces/default/extras.html | 15 ---- data/interfaces/default/index.html | 86 ++++++++++++---------- data/interfaces/default/searchresults.html | 6 +- 4 files changed, 74 insertions(+), 86 deletions(-) delete mode 100755 data/interfaces/default/extras.html diff --git a/data/interfaces/default/artistredone.html b/data/interfaces/default/artistredone.html index eba7cb5a..9999ac08 100755 --- a/data/interfaces/default/artistredone.html +++ b/data/interfaces/default/artistredone.html @@ -298,45 +298,36 @@ } }; - function initThisPage() - { + function initThisPage(){ $(function() { $( "#tabs" ).tabs(); }); initActions(); - $('#issue_table').dataTable({ - "bDestroy": true, - "aoColumns": [ - null, - null, - null, - null, - null - - ], - "aoColumnDefs": [ - { 'bSortable': false, 'aTargets': [ 0,1 ] } - ], - "oLanguage": { - "sLengthMenu":"Show _MENU_ issues per page", - "sEmptyTable": "No issue information available", - "sInfo":"Showing _TOTAL_ issues", - "sInfoEmpty":"Showing 0 to 0 of 0 issues", - "sInfoFiltered":"(filtered from _MAX_ total issues)", - "sSearch": ""}, - "bPaginate": false, - "aaSorting": [[1, 'desc'],[3,'desc']] - - }); - resetFilters("issues"); - setTimeout(function(){ - initFancybox(); - },1500) + $('#issue_table').dataTable( + { + "bDestroy": true, + "aoColumnDefs": [ + { 'bSortable': false, 'aTargets': [ 0, 2 ] } + ], + "oLanguage": { + "sLengthMenu":"Show _MENU_ issues per page", + "sEmptyTable": "No issue information available", + "sInfo":"Showing _TOTAL_ issues", + "sInfoEmpty":"Showing 0 to 0 of 0 issues", + "sInfoFiltered":"(filtered from _MAX_ total issues)", + "sSearch": ""}, + "iDisplayLength": 25, + "sPaginationType": "full_numbers", + "aaSorting": [[1, 'desc'],[3,'desc']] + }); + resetFilters("issues"); + setTimeout(function(){ + initFancybox(); + },1500) } - $(document).ready(function() { initThisPage(); }); diff --git a/data/interfaces/default/extras.html b/data/interfaces/default/extras.html deleted file mode 100755 index 2c25d357..00000000 --- a/data/interfaces/default/extras.html +++ /dev/null @@ -1,15 +0,0 @@ -<%inherit file="base.html" /> -<%def name="body()"> -
-

extraArtists You Might Like

-
-
-
- -
-
- diff --git a/data/interfaces/default/index.html b/data/interfaces/default/index.html index 5435d9ee..be939ab1 100755 --- a/data/interfaces/default/index.html +++ b/data/interfaces/default/index.html @@ -1,9 +1,11 @@ <%inherit file="base.html"/> <%! - from mylar import helpers + from mylar import helpers, db %> <%def name="body()"> +
+ @@ -18,10 +20,28 @@ %for comic in comics: <% - totaltracks = comic['Total'] - havetracks = comic['Have'] - if not havetracks: - havetracks = 0 + myDB = db.DBConnection() + issue = myDB.select("SELECT * FROM issues WHERE ComicID=?", [comic['ComicID']]) + wantedc = myDB.action("SELECT COUNT(*) as count FROM issues WHERE ComicID=? AND Status='Wanted'", [comic['ComicID']]).fetchone() + archedc = myDB.action("SELECT COUNT(*) as count FROM issues WHERE ComicID=? AND Status='Archived'", [comic['ComicID']]).fetchone() + totaltracks = comic['Total'] + havetracks = comic['Have'] + wants = wantedc[0] + arcs = archedc[0] + if not havetracks: + havetracks = 0 + if not wants: + wants = 0 + if not arcs: + arcs = 0 + try: + wantpercent = (wants*100.0)/totaltracks + if wantpercent > 100: + wantpercent = 100 + except (ZeroDivisionError, TypeError): + wantpercent = 0 + wants = '?' + try: percent = (havetracks*100.0)/totaltracks if percent > 100: @@ -34,7 +54,7 @@ if comic['Status'] == 'Paused': grade = 'X' elif comic['Status'] == 'Loading': - grade = 'L' + grade = 'L' elif comic['Status'] == 'Error': grade = 'Z' else: @@ -47,11 +67,11 @@ - - + %endfor
${comic['ComicYear']} ${comic['Status']} # ${comic['LatestIssue']} (${comic['LatestDate']})
${havetracks}/${totaltracks}
${havetracks}/${totaltracks}
+
<%def name="headIncludes()"> @@ -62,40 +82,32 @@ diff --git a/data/interfaces/default/searchresults.html b/data/interfaces/default/searchresults.html index 87d07699..6627ddaa 100755 --- a/data/interfaces/default/searchresults.html +++ b/data/interfaces/default/searchresults.html @@ -20,7 +20,7 @@ %if searchresults: %for result in searchresults: <% - if result['comicyear'] == '2012': + if result['comicyear'] == '2013': grade = 'A' else: grade = 'Z' @@ -28,8 +28,8 @@
${result['name']}${result['publisher']}${result['comicyear']}${result['publisher']}${result['comicyear']} ${result['issues']} Add this Comic
- - - - - - - - - - - - %for track in tracks: - <% - if track['Location']: - grade = 'A' - location = track['Location'] - else: - grade = 'X' - location = '' - - if track['BitRate']: - bitrate = str(track['BitRate']/1000) + ' kbps' - else: - bitrate = '' - - try: - trackduration = helpers.convert_milliseconds(track['TrackDuration']) - except: - trackduration = 'n/a' - - if not track['Format']: - format = 'Unknown' - else: - format = track['Format'] - %> - - - - - - - - - %endfor - <% - unmatched = myDB.select('SELECT * from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']]) - %> - %if unmatched: - %for track in unmatched: - <% - duration = helpers.convert_seconds(float(track['TrackLength'])) - %> - - - - - - - - - %endfor - %endif - -
#Track TitleDurationLocal FileBit RateFormat
${track['TrackNumber']}${track['TrackTitle']}${trackduration}${location}${bitrate}${format}
${track['TrackNumber']}${track['TrackTitle']}${duration}${track['Location']}${int(track['BitRate'])/1000} kbps${track['Format']}
- - - - -<%def name="headIncludes()"> - - - -<%def name="javascriptIncludes()"> - - - diff --git a/data/interfaces/default/artist-descincluded.html b/data/interfaces/default/artist-descincluded.html deleted file mode 100755 index 4d03b279..00000000 --- a/data/interfaces/default/artist-descincluded.html +++ /dev/null @@ -1,194 +0,0 @@ -<%inherit file="base.html"/> -<%! - from mylar import db - import mylar -%> - -<%def name="headerIncludes()"> -
-
- Edit Comic - Refresh Comic - Delete Comic - %if comic['Status'] == 'Paused': - Resume Comic - %else: - Pause Comic - %endif -
-
- « Back to overview - - -<%def name="body()"> -
-
- -
-

- %if comic['Status'] == 'Loading': - loading - %endif - ${comic['ComicName']} (${comic['ComicYear']}) - %if comic['Status'] == 'Loading': -

(Comic information for this comic is currently being loaded)

- %endif - -
- - Publisher: ${comic['ComicPublisher']}
- Comics in Series: ${comic['Total']} issues (${comic['ComicPublished']})
- %if comic['ComicPublisher'] == 'DC Comics': - DC - %elif comic['ComicPublisher'] == 'Marvel': - Marvel - %elif comic['ComicPublisher'] == 'Image': - Image - %endif - - Status: ${comic['Status']} - %if comic['Status'] == 'Active': - - %endif -
- Comic Directory Location:
   ${comic['ComicLocation']} -

- Quality Control: - - - - - - -
Version: v5
Scanner: GreenGiant (Fallback ON)
Type: NOADS (Fallback ON)over-rides default settings*
Pixel Quality: None (Fallback OFF)
-
-
-
- -
Mark selected issues as - - -
- - - - - - - - - - - - - %for issue in issues: - <% - if issue['Status'] == 'Skipped': - grade = 'Z' - elif issue['Status'] == 'Wanted': - grade = 'X' - elif issue['Status'] == 'Snatched': - grade = 'C' - else: - grade = 'A' - %> - - - - - - - - - %endfor - -
NumberNameDateTypeStatus
${issue['Issue_Number']}${issue['IssueName']}${issue['IssueDate']}${issue['Type']}${issue['Status']} - %if issue['Status'] == 'Skipped': - [want] - %elif (issue['Status'] == 'Wanted'): - [skip] - %else: - [retry][new] - %endif -
-
- - -<%def name="headIncludes()"> - - %if comic['Status'] == 'Loading': - - %endif - - -<%def name="javascriptIncludes()"> - - - - diff --git a/data/interfaces/default/artist.html b/data/interfaces/default/artist.html deleted file mode 100755 index 1d4706e4..00000000 --- a/data/interfaces/default/artist.html +++ /dev/null @@ -1,262 +0,0 @@ -<%inherit file="base.html"/> -<%! - from mylar import db - import mylar -%> - -<%def name="headerIncludes()"> -
-
- Edit Comic - Refresh Comic - Delete Comic - %if comic['Status'] == 'Paused': - Resume Comic - %else: - Pause Comic - %endif -
-
- « Back to overview - - -<%def name="body()"> -
- - -
- - - - - - -
-
-
- -
-
- 		-
-

- %if comic['Status'] == 'Loading': - loading - %endif -
- ${comic['ComicName']} (${comic['ComicYear']}) - %if comic['Status'] == 'Loading': -

(Comic information for this comic is currently being loaded)

- %endif -
-

-
- -
-
- -
- %if comic['ComicPublisher'] == 'DC Comics': - DC - %elif comic['ComicPublisher'] == 'Marvel': - Marvel - %endif -
- -
-
- - -
-
-
-
- -
- - - - -
-
- Quality Control -
- -
-
- -
-
- -
-
- -
-
-
-
- -
- - -
- -
Mark selected issues as - - -
- - - - - - - - - - - - - %for issue in issues: - <% - if issue['Status'] == 'Skipped': - grade = 'Z' - elif issue['Status'] == 'Wanted': - grade = 'X' - elif issue['Status'] == 'Snatched': - grade = 'C' - else: - grade = 'A' - %> - - - - - - - - - %endfor - -
NumberNameDateTypeStatus
${issue['Issue_Number']}${issue['IssueName']}${issue['IssueDate']}${issue['Type']}${issue['Status']} - %if issue['Status'] == 'Skipped': - [want] - %elif (issue['Status'] == 'Wanted'): - [skip] - %else: - [retry][new] - %endif -
-
- - -<%def name="headIncludes()"> - - %if comic['Status'] == 'Loading': - - %endif - - -<%def name="javascriptIncludes()"> - - - - diff --git a/data/interfaces/default/artistredone.html.save b/data/interfaces/default/artistredone.html.save deleted file mode 100755 index a9668e2e..00000000 --- a/data/interfaces/default/artistredone.html.save +++ /dev/null @@ -1,318 +0,0 @@ -<%inherit file="base.html"/> -<%! - from mylar import db - import mylar -%> - -<%def name="headerIncludes()"> -
-
- Refresh Comic - Delete Comic - %if comic['Status'] == 'Paused': - Resume Comic - %else: - Pause Comic - %endif - Shut Down - Restart - - -
-
- -<%def name="body()"> - <% - totaltracks = comic['Total'] - havetracks = comic['Have'] - if not havetracks: - havetracks = 0 - try: - percent = (havetracks*100.0)/totaltracks - if percent > 100: - percent = 100 - except (ZeroDivisionError, TypeError): - percent = 0 - totaltracks = '?' - %> - - -
-

- %if comic['Status'] == 'Loading': - loading - %endif -
- ${comic['ComicName']} (${comic['ComicYear']}) - %if comic['Status'] == 'Loading': -

(Comic information for this comic is currently being loaded)

- %endif -
-

- -
-
- -
- -
- - - - - - + %endfor
-
-
- -
-
- 		- %if comic['ComicPublisher'] == 'DC Comics': - DC - %elif comic['ComicPublisher'] == 'Marvel': - Marvel - %if comic['ComicPublisher'] == 'DC Comics': - DC - - - - - -
-
-
- -
-
- 		- %if comic['ComicPublisher'] == 'DC Comics': - DC - %elif comic['ComicPublisher'] == 'Marvel': - Marvel - %endif -
-
- -
-
- -
-
- -
-
- -
-
-
- - -
- - - - - - -
-
-
- -
-
- 		-
-
- - - the directory where all the comics are for this comic -
-
- - - if the comic is v5 or whatever, enter 'v5' here -
-
- - - list preference of scanner -
-
- - - c2c / noads -
-
- - - resolution of scan (ie.1440px) -
- - -
-
-
- - - - -
Mark selected issues as - - -
- - - - - - - - - - - - - %for issue in issues: - <% - if issue['Status'] == 'Skipped': - grade = 'Z' - elif issue['Status'] == 'Wanted': - grade = 'X' - elif issue['Status'] == 'Snatched': - grade = 'C' - else: - grade = 'A' - %> - - - - - - - - - %endfor - -
NumberNameDateTypeStatus
${issue['Issue_Number']}${issue['IssueName']}${issue['IssueDate']}${issue['Type']}${issue['Status']} - %if issue['Status'] == 'Skipped': - [want] - %elif (issue['Status'] == 'Wanted'): - [skip] - %else: - [retry][new] - %endif -
- - - - -<%def name="headIncludes()"> - - %if comic['Status'] == 'Loading': - - %endif - - - -<%def name="javascriptIncludes()"> - - - - diff --git a/data/interfaces/default/editcomic.html b/data/interfaces/default/editcomic.html deleted file mode 100755 index 36567d70..00000000 --- a/data/interfaces/default/editcomic.html +++ /dev/null @@ -1,137 +0,0 @@ -<%inherit file="base.html"/> -<%! - import mylar -%> - -<%def name="headerIncludes()"> -
- -
- -<%def name="body()"> - - - -
-

settings${comic['ComicName']} - Settings

-
-
- -
- -
- - - - - - - -
-
- Basic -
- - - the directory where all the comics are for this comic -
-
- 		-
-
- -
-
-
-
- -
- - - - - -
-
- Quality -
- - - if the comic is v5 or whatever, enter 'v5' here -
-
- - -
-
- - -
-
- - -
-
-
- -
-

Web Interface changes require a restart to take effect

-
- -
- - -<%def name="javascriptIncludes()"> - - From 6bea1d781951d1a74c7b1e21175d438d5af27fd4 Mon Sep 17 00:00:00 2001 From: evilhero Date: Wed, 23 Jan 2013 04:50:33 -0500 Subject: [PATCH 07/16] FIX: Home screen will remember number of results/page now --- data/interfaces/default/index.html | 1 + 1 file changed, 1 insertion(+) diff --git a/data/interfaces/default/index.html b/data/interfaces/default/index.html index be939ab1..6df722b2 100755 --- a/data/interfaces/default/index.html +++ b/data/interfaces/default/index.html @@ -96,6 +96,7 @@ "sInfoEmpty":"Showing 0 to 0 of 0 results", "sInfoFiltered":"(filtered from _MAX_ total results)", "sSearch" : ""}, + "bStateSave": true, "iDisplayLength": 15, "sPaginationType": "full_numbers", "aaSorting": [] From eecf06b4ef685cef0226312ab489d44e66e0b290 Mon Sep 17 00:00:00 2001 From: evilhero Date: Wed, 23 Jan 2013 05:07:27 -0500 Subject: [PATCH 08/16] FIX: Post-Processing should now fully work once again. --- mylar/PostProcessor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mylar/PostProcessor.py b/mylar/PostProcessor.py index 5cb2fbac..834faf56 100755 --- a/mylar/PostProcessor.py +++ b/mylar/PostProcessor.py @@ -346,8 +346,8 @@ class PostProcessor(object): logger.fdebug("Source: " + str(src)) logger.fdebug("Destination: " + str(dst)) - os.rename(os.path.join(self.nzb_folder, ofilename), os.path.join(self.nzb_folder,str(nfilename + ext))) - #src = os.path.join(self.nzb_folder, str(nfilename + ext)) + os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder,str(nfilename + ext))) + src = os.path.join(self.nzb_folder, str(nfilename + ext)) try: shutil.move(src, dst) except (OSError, IOError): From 16c69e7bf54ae799a64aa4996db4563887461833 Mon Sep 17 00:00:00 2001 From: evilhero Date: Thu, 24 Jan 2013 04:42:47 -0500 Subject: [PATCH 09/16] FIX:(#170) crashing on Checking diretory for existing issues, FIX:(#179) won't try and search for 'None', TMP IMP: added some console logging to parse for (#170) --- mylar/parseit.py | 32 +++++++++++++++++++------------- mylar/search.py | 2 +- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/mylar/parseit.py b/mylar/parseit.py index 904f0270..0cdeada8 100755 --- a/mylar/parseit.py +++ b/mylar/parseit.py @@ -184,7 +184,7 @@ def GCDdetails(comseries, resultURL, vari_loop, ComicID, TotalIssues, issvariati parsed = soup.find("div", {"id" : "series_data"}) subtxt3 = parsed.find("dd", {"id" : "publication_dates"}) resultPublished = subtxt3.findNext(text=True).rstrip() - #print ("pubdate:" + str(resultPublished)) + print ("pubdate:" + str(resultPublished)) coverst = soup.find("div", {"id" : "series_cover"}) if coverst < 0: gcdcover = "None" @@ -192,21 +192,21 @@ def GCDdetails(comseries, resultURL, vari_loop, ComicID, TotalIssues, issvariati subcoverst = coverst('img',src=True)[0] gcdcover = subcoverst['src'] - #print ("resultURL:" + str(resultURL)) - #print ("comicID:" + str(ComicID)) + print ("resultURL:" + str(resultURL)) + print ("comicID:" + str(ComicID)) input2 = 'http://www.comics.org' + str(resultURL) + 'details/' - resp = urllib2.urlopen ( input2 ) - soup = BeautifulSoup ( resp ) + resp = urllib2.urlopen(input2) + soup = BeautifulSoup(resp) #for newer comics, on-sale date has complete date... #for older comics, pub.date is to be used type = soup.find(text=' On-sale date ') if type: - #print ("on-sale date detected....adjusting") + print ("on-sale date detected....adjusting") datetype = "on-sale" else: - #print ("pub date defaulting") + print ("pub date defaulting") datetype = "pub" cnt1 = len(soup.findAll("tr", {"class" : "row_even_False"})) @@ -214,7 +214,7 @@ def GCDdetails(comseries, resultURL, vari_loop, ComicID, TotalIssues, issvariati cnt = int(cnt1 + cnt2) - #print (str(cnt) + " Issues in Total (this may be wrong due to alternate prints, etc") + print (str(cnt) + " Issues in Total (this may be wrong due to alternate prints, etc") n_odd = -1 n_even = -1 @@ -236,14 +236,19 @@ def GCDdetails(comseries, resultURL, vari_loop, ComicID, TotalIssues, issvariati fid = parsed('a',href=True)[0] resultGID = fid['href'] resultID = resultGID[7:-1] - #print ( "ID: " + str(resultID) ) + print ( "ID: " + str(resultID) ) if ',' in ParseIssue: ParseIssue = re.sub("\,", "", ParseIssue) + #print ("ParseIssue before : " + str(ParseIssue)) + if 'Vol' in ParseIssue or '[' in ParseIssue: + ParseIssue = re.sub("[^0-9]", "", ParseIssue) isslen = ParseIssue.find(' ') #if 'isslen' exists, it means that it's an alternative cover. #however, if ONLY alternate covers exist of an issue it won't work. #let's use the FIRST record, and ignore all other covers for the given issue. isschk = ParseIssue[:isslen] + print ("Parse is now: " + str(isschk)) + #check if decimal or '1/2' exists or not, and store decimal results halfchk = "no" if '.' in isschk: @@ -292,23 +297,24 @@ def GCDdetails(comseries, resultURL, vari_loop, ComicID, TotalIssues, issvariati #logger.fdebug("adding issue to db : " + str(ParseIssue)) # in order to get the compare right, let's decimialize the string to '.00'. gcdinfo['ComicIssue'] = ParseIssue - #print "Issue: " + str(ParseIssue) + print "Issue: " + str(ParseIssue) #^^ will retrieve issue #if datetype == "on-sale": subtxt1 = parsed('td')[2] ParseDate = subtxt1.findNext(text=True) pdlen = len(ParseDate) - #print "sale-date..ParseDate:" + str(ParseDate) - #print ("Parsed Date length: " + str(pdlen)) + print "sale-date..ParseDate:" + str(ParseDate) + print ("Parsed Date length: " + str(pdlen)) if len(ParseDate) < 7: subtxt3 = parsed('td')[0] ParseDate = subtxt3.findNext(text=True) - #print "pub-date..ParseDate:" + str(ParseDate) + print "pub-date..ParseDate:" + str(ParseDate) if ParseDate == ' ': #default to empty so doesn't error out. ParseDate = "0000-00-00" #ParseDate = ParseDate.replace('?','') ParseDate = ParseDate.replace(' ','') + print "Parse date: " + str(ParseDate) gcdinfo['ComicDate'] = ParseDate #^^ will retrieve date # if not any(d.get('GCDIssue', None) == str(gcdinfo['ComicIssue']) for d in gcdchoice): diff --git a/mylar/search.py b/mylar/search.py index df00f585..1ef543fe 100755 --- a/mylar/search.py +++ b/mylar/search.py @@ -725,7 +725,7 @@ def searchforissue(issueid=None, new=False): new = True for result in results: - comic = myDB.action('SELECT * from comics WHERE ComicID=?', [result['ComicID']]).fetchone() + comic = myDB.action("SELECT * from comics WHERE ComicID=? AND ComicName != 'None'", [result['ComicID']]).fetchone() foundNZB = "none" SeriesYear = comic['ComicYear'] AlternateSearch = comic['AlternateSearch'] From a831608d377aeea98febe750c9a152be7cb615c1 Mon Sep 17 00:00:00 2001 From: evilhero Date: Thu, 24 Jan 2013 12:01:27 -0500 Subject: [PATCH 10/16] FIX: (#181) 2000 AD exception added to csv for mulit-volume support, FIX: ComicYear was parsed wrong when using multi-volume occassionally, IMP: Turned off console printing for parsing --- exceptions.csv | 3 +++ mylar/importer.py | 2 ++ mylar/parseit.py | 36 ++++++++++++++++++------------------ mylar/webserve.py | 20 +++++++++++++++++--- 4 files changed, 40 insertions(+), 21 deletions(-) diff --git a/exceptions.csv b/exceptions.csv index 480f7c98..f3ecd4af 100644 --- a/exceptions.csv +++ b/exceptions.csv @@ -14,6 +14,8 @@ 2,2127,none,1570/7794/11288 #The Boys 1,18033,none,19531/25058 +#2000 A.D. +4,19752,none,11289/11295/11294/11292/11293 #-------- #-----Mismatched Names------ 99,3092,2605,none @@ -27,3 +29,4 @@ 99,42821,60934,none 99,42947,61242,none 99,42322,60917,none +99,53920,69871,none diff --git a/mylar/importer.py b/mylar/importer.py index 511c1eca..042c0c33 100755 --- a/mylar/importer.py +++ b/mylar/importer.py @@ -343,6 +343,8 @@ def addComictoDB(comicid,mismatch=None,pullupd=None): #print ("Existing status : " + str(iss_exists['Status'])) newValueDict['Status'] = iss_exists['Status'] + #logger.fdebug("newValueDict:" + str(newValueDict)) + myDB.upsert("issues", newValueDict, controlValueDict) n+=1 diff --git a/mylar/parseit.py b/mylar/parseit.py index 0cdeada8..7899d919 100755 --- a/mylar/parseit.py +++ b/mylar/parseit.py @@ -184,7 +184,7 @@ def GCDdetails(comseries, resultURL, vari_loop, ComicID, TotalIssues, issvariati parsed = soup.find("div", {"id" : "series_data"}) subtxt3 = parsed.find("dd", {"id" : "publication_dates"}) resultPublished = subtxt3.findNext(text=True).rstrip() - print ("pubdate:" + str(resultPublished)) + #print ("pubdate:" + str(resultPublished)) coverst = soup.find("div", {"id" : "series_cover"}) if coverst < 0: gcdcover = "None" @@ -192,8 +192,8 @@ def GCDdetails(comseries, resultURL, vari_loop, ComicID, TotalIssues, issvariati subcoverst = coverst('img',src=True)[0] gcdcover = subcoverst['src'] - print ("resultURL:" + str(resultURL)) - print ("comicID:" + str(ComicID)) + #print ("resultURL:" + str(resultURL)) + #print ("comicID:" + str(ComicID)) input2 = 'http://www.comics.org' + str(resultURL) + 'details/' resp = urllib2.urlopen(input2) soup = BeautifulSoup(resp) @@ -203,10 +203,10 @@ def GCDdetails(comseries, resultURL, vari_loop, ComicID, TotalIssues, issvariati type = soup.find(text=' On-sale date ') if type: - print ("on-sale date detected....adjusting") + #print ("on-sale date detected....adjusting") datetype = "on-sale" else: - print ("pub date defaulting") + #print ("pub date defaulting") datetype = "pub" cnt1 = len(soup.findAll("tr", {"class" : "row_even_False"})) @@ -214,7 +214,7 @@ def GCDdetails(comseries, resultURL, vari_loop, ComicID, TotalIssues, issvariati cnt = int(cnt1 + cnt2) - print (str(cnt) + " Issues in Total (this may be wrong due to alternate prints, etc") + #print (str(cnt) + " Issues in Total (this may be wrong due to alternate prints, etc") n_odd = -1 n_even = -1 @@ -236,7 +236,7 @@ def GCDdetails(comseries, resultURL, vari_loop, ComicID, TotalIssues, issvariati fid = parsed('a',href=True)[0] resultGID = fid['href'] resultID = resultGID[7:-1] - print ( "ID: " + str(resultID) ) + #print ( "ID: " + str(resultID) ) if ',' in ParseIssue: ParseIssue = re.sub("\,", "", ParseIssue) #print ("ParseIssue before : " + str(ParseIssue)) @@ -247,7 +247,7 @@ def GCDdetails(comseries, resultURL, vari_loop, ComicID, TotalIssues, issvariati #however, if ONLY alternate covers exist of an issue it won't work. #let's use the FIRST record, and ignore all other covers for the given issue. isschk = ParseIssue[:isslen] - print ("Parse is now: " + str(isschk)) + #print ("Parse is now: " + str(isschk)) #check if decimal or '1/2' exists or not, and store decimal results halfchk = "no" @@ -297,24 +297,24 @@ def GCDdetails(comseries, resultURL, vari_loop, ComicID, TotalIssues, issvariati #logger.fdebug("adding issue to db : " + str(ParseIssue)) # in order to get the compare right, let's decimialize the string to '.00'. gcdinfo['ComicIssue'] = ParseIssue - print "Issue: " + str(ParseIssue) + #print "Issue: " + str(ParseIssue) #^^ will retrieve issue #if datetype == "on-sale": subtxt1 = parsed('td')[2] ParseDate = subtxt1.findNext(text=True) pdlen = len(ParseDate) - print "sale-date..ParseDate:" + str(ParseDate) - print ("Parsed Date length: " + str(pdlen)) + #print "sale-date..ParseDate:" + str(ParseDate) + #print ("Parsed Date length: " + str(pdlen)) if len(ParseDate) < 7: subtxt3 = parsed('td')[0] ParseDate = subtxt3.findNext(text=True) - print "pub-date..ParseDate:" + str(ParseDate) + #print "pub-date..ParseDate:" + str(ParseDate) if ParseDate == ' ': #default to empty so doesn't error out. ParseDate = "0000-00-00" #ParseDate = ParseDate.replace('?','') ParseDate = ParseDate.replace(' ','') - print "Parse date: " + str(ParseDate) + #print "Parse date: " + str(ParseDate) gcdinfo['ComicDate'] = ParseDate #^^ will retrieve date # if not any(d.get('GCDIssue', None) == str(gcdinfo['ComicIssue']) for d in gcdchoice): @@ -461,11 +461,11 @@ def ComChk(ComicName, ComicYear, ComicPublisher, Total, ComicID): comicis = Total comicid = ComicID comicpub = ComicPublisher - print ( "comicname: " + str(comicnm) ) - print ( "comicyear: " + str(comicyr) ) - print ( "comichave: " + str(comicis) ) - print ( "comicpub: " + str(comicpub) ) - print ( "comicid: " + str(comicid) ) + #print ( "comicname: " + str(comicnm) ) + #print ( "comicyear: " + str(comicyr) ) + #print ( "comichave: " + str(comicis) ) + #print ( "comicpub: " + str(comicpub) ) + #print ( "comicid: " + str(comicid) ) # do 3 runs at the comics.org search to get the best results comicrun = [] # &pub_name=DC diff --git a/mylar/webserve.py b/mylar/webserve.py index 23b72ba9..6ab5a831 100755 --- a/mylar/webserve.py +++ b/mylar/webserve.py @@ -179,7 +179,7 @@ class WebInterface(object): def from_Exceptions(self, comicid, gcdid, comicname=None, comicyear=None, comicissues=None, comicpublisher=None): mismatch = "yes" - print ("gcdid:" + str(gcdid)) + #print ("gcdid:" + str(gcdid)) #write it to the custom_exceptions.csv and reload it so that importer will pick it up and do it's thing :) #custom_exceptions in this format... #99, (comicid), (gcdid), none @@ -209,6 +209,20 @@ class WebInterface(object): gcomicid = "G" + str(comicid) comicyear_len = comicyear.find(' ', 2) comyear = comicyear[comicyear_len+1:comicyear_len+5] + if comyear.isdigit(): + logger.fdebug("Series year set to : " + str(comyear)) + else: + logger.fdebug("Invalid Series year detected - trying to adjust from " + str(comyear)) + #comicyear_len above will trap wrong year if it's 10 October 2010 - etc ( 2000 AD)... + find_comicyear = comicyear.split() + for i in find_comicyear: + if len(i) == 4: + logger.fdebug("Series year detected as : " + str(i)) + comyear = str(i) + continue + + logger.fdebug("Series year set to: " + str(comyear)) + controlValueDict = { 'ComicID': gcomicid } newValueDict = {'ComicName': comicname, 'ComicYear': comyear, @@ -398,7 +412,7 @@ class WebInterface(object): #raise cherrypy.HTTPRedirect("home") else: return self.manualpull() - return serve_template(templatename="weeklypull.html", title="Weekly Pull", weeklyresults=weeklyresults, pulldate=pulldate['SHIPDATE'],pullfilter=False) + return serve_template(templatename="weeklypull.html", title="Weekly Pull", weeklyresults=weeklyresults, pulldate=pulldate['SHIPDATE']) pullist.exposed = True def filterpull(self): @@ -407,7 +421,7 @@ class WebInterface(object): pulldate = myDB.action("SELECT * from weekly").fetchone() if pulldate is None: raise cherrypy.HTTPRedirect("home") - return serve_template(templatename="weeklypull.html", title="Weekly Pull", weeklyresults=weeklyresults, pulldate=pulldate['SHIPDATE'], pullfilter=True) + return serve_template(templatename="weeklypull.html", title="Weekly Pull", weeklyresults=weeklyresults, pulldate=pulldate['SHIPDATE']) filterpull.exposed = True def manualpull(self): From 4d1b61846b5e2d8021a237edf715d211d5b7f884 Mon Sep 17 00:00:00 2001 From: evilhero Date: Thu, 24 Jan 2013 13:31:03 -0500 Subject: [PATCH 11/16] FIX: comics that had alot of issues would take forever to initially add (improved speed performance). --- mylar/parseit.py | 7 +++++-- mylar/webserve.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mylar/parseit.py b/mylar/parseit.py index 7899d919..2971039e 100755 --- a/mylar/parseit.py +++ b/mylar/parseit.py @@ -23,7 +23,7 @@ import datetime from decimal import Decimal from HTMLParser import HTMLParseError -def GCDScraper(ComicName, ComicYear, Total, ComicID): +def GCDScraper(ComicName, ComicYear, Total, ComicID, quickmatch=None): NOWyr = datetime.date.today().year if datetime.date.today().month == 12: NOWyr = NOWyr + 1 @@ -148,8 +148,11 @@ def GCDScraper(ComicName, ComicYear, Total, ComicID): if 'and' in ComicName.lower(): ComicName = ComicName.replace('and', '&') return GCDScraper(ComicName, ComicYear, Total, ComicID) - return 'No Match' + if not quickmatch: return 'No Match' #vari_loop = 0 + if quickmatch == "yes": + if resultURL is None: return 'No Match' + else: return 'Match' return GCDdetails(comseries=None, resultURL=resultURL, vari_loop=0, ComicID=ComicID, TotalIssues=TotalIssues, issvariation=issvariation, resultPublished=resultPublished) diff --git a/mylar/webserve.py b/mylar/webserve.py index 6ab5a831..92dda2ec 100755 --- a/mylar/webserve.py +++ b/mylar/webserve.py @@ -115,7 +115,7 @@ class WebInterface(object): #here we test for exception matches (ie. comics spanning more than one volume, known mismatches, etc). CV_EXcomicid = myDB.action("SELECT * from exceptions WHERE ComicID=?", [comicid]).fetchone() if CV_EXcomicid is None: # pass # - gcdinfo=parseit.GCDScraper(comicname, comicyear, comicissues, comicid) + gcdinfo=parseit.GCDScraper(comicname, comicyear, comicissues, comicid, quickmatch="yes") if gcdinfo == "No Match": #when it no matches, the image will always be blank...let's fix it. cvdata = mylar.cv.getComic(comicid,'comic') From 8c70872c111dcd7a32a1f99db238455f94315d15 Mon Sep 17 00:00:00 2001 From: evilhero Date: Sun, 27 Jan 2013 05:07:08 -0500 Subject: [PATCH 12/16] FIX:(#170) Possible Fix due to page-encoding - added some error traps, FIX:(#184) unclosed table row cell, possible fix, FIX:(#77) A+X would not match on results, IMP: Delete all None and Error-d out (ComicID displaying) from DB on startup - 'if Present in watchd...'(#170) --- data/interfaces/default/index.html | 3 ++- mylar/__init__.py | 17 ++++++++++++++--- mylar/parseit.py | 24 +++++++++++++++++++++--- mylar/search.py | 4 +++- 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/data/interfaces/default/index.html b/data/interfaces/default/index.html index 6df722b2..ba13b994 100755 --- a/data/interfaces/default/index.html +++ b/data/interfaces/default/index.html @@ -68,6 +68,7 @@
${comic['Status']} # ${comic['LatestIssue']} (${comic['LatestDate']})
${havetracks}/${totaltracks}
@@ -97,7 +98,7 @@ "sInfoFiltered":"(filtered from _MAX_ total results)", "sSearch" : ""}, "bStateSave": true, - "iDisplayLength": 15, + "iDisplayLength": 25, "sPaginationType": "full_numbers", "aaSorting": [] }); diff --git a/mylar/__init__.py b/mylar/__init__.py index 37eeb174..4b3671fa 100755 --- a/mylar/__init__.py +++ b/mylar/__init__.py @@ -84,6 +84,9 @@ DESTINATION_DIR = None USENET_RETENTION = None ADD_COMICS = False +COMIC_DIR = None +LIBRARYSCAN = False +IMP_MOVE = False SEARCH_INTERVAL = 360 NZB_STARTUP_SEARCH = False @@ -213,7 +216,7 @@ def initialize(): HTTP_PORT, HTTP_HOST, HTTP_USERNAME, HTTP_PASSWORD, HTTP_ROOT, LAUNCH_BROWSER, GIT_PATH, \ CURRENT_VERSION, LATEST_VERSION, CHECK_GITHUB, CHECK_GITHUB_ON_STARTUP, CHECK_GITHUB_INTERVAL, MUSIC_DIR, DESTINATION_DIR, \ DOWNLOAD_DIR, USENET_RETENTION, SEARCH_INTERVAL, NZB_STARTUP_SEARCH, INTERFACE, AUTOWANT_ALL, AUTOWANT_UPCOMING, ZERO_LEVEL, ZERO_LEVEL_N, COMIC_COVER_LOCAL, \ - LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, SAB_PRIORITY, BLACKHOLE, BLACKHOLE_DIR, \ + LIBRARYSCAN, LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, SAB_PRIORITY, BLACKHOLE, BLACKHOLE_DIR, ADD_COMICS, COMIC_DIR, IMP_MOVE, \ NZBSU, NZBSU_APIKEY, DOGNZB, DOGNZB_APIKEY, NZBX,\ NEWZNAB, NEWZNAB_HOST, NEWZNAB_APIKEY, NEWZNAB_ENABLED, EXTRA_NEWZNABS,\ RAW, RAW_PROVIDER, RAW_USERNAME, RAW_PASSWORD, RAW_GROUPS, EXPERIMENTAL, \ @@ -261,7 +264,11 @@ def initialize(): SEARCH_INTERVAL = check_setting_int(CFG, 'General', 'search_interval', 360) NZB_STARTUP_SEARCH = bool(check_setting_int(CFG, 'General', 'nzb_startup_search', 0)) + LIBRARYSCAN = bool(check_setting_int(CFG, 'General', 'libraryscan', 1)) LIBRARYSCAN_INTERVAL = check_setting_int(CFG, 'General', 'libraryscan_interval', 300) + ADD_COMICS = bool(check_setting_int(CFG, 'General', 'add_comics', 0)) + COMIC_DIR = check_setting_str(CFG, 'General', 'comic_dir', '') + IMP_MOVE = bool(check_setting_int(CFG, 'General', 'imp_move', 0)) DOWNLOAD_SCAN_INTERVAL = check_setting_int(CFG, 'General', 'download_scan_interval', 5) INTERFACE = check_setting_str(CFG, 'General', 'interface', 'default') AUTOWANT_ALL = bool(check_setting_int(CFG, 'General', 'autowant_all', 0)) @@ -508,7 +515,11 @@ def config_write(): new_config['General']['search_interval'] = SEARCH_INTERVAL new_config['General']['nzb_startup_search'] = int(NZB_STARTUP_SEARCH) + new_config['General']['libraryscan'] = int(LIBRARYSCAN) new_config['General']['libraryscan_interval'] = LIBRARYSCAN_INTERVAL + new_config['General']['add_comics'] = int(ADD_COMICS) + new_config['General']['comic_dir'] = COMIC_DIR + new_config['General']['imp_move'] = int(IMP_MOVE) new_config['General']['download_scan_interval'] = DOWNLOAD_SCAN_INTERVAL new_config['General']['interface'] = INTERFACE new_config['General']['autowant_all'] = int(AUTOWANT_ALL) @@ -674,8 +685,8 @@ def dbcheck(): except sqlite3.OperationalError: c.execute('ALTER TABLE comics ADD COLUMN UseFuzzy TEXT') - #let's delete errant comics that are stranded (ie. None) - c.execute("DELETE from COMICS WHERE ComicName='None'") + #let's delete errant comics that are stranded (ie. Comicname = Comic ID: ) + c.execute("DELETE from COMICS WHERE ComicName='None' OR ComicName LIKE 'Comic ID%'") logger.info(u"Ensuring DB integrity - Removing all Erroneous Comics (ie. named None)") conn.commit() diff --git a/mylar/parseit.py b/mylar/parseit.py index 2971039e..961973ce 100755 --- a/mylar/parseit.py +++ b/mylar/parseit.py @@ -14,12 +14,13 @@ # along with Mylar. If not, see . -from bs4 import BeautifulSoup +from bs4 import BeautifulSoup, UnicodeDammit import urllib2 import re import helpers import logger import datetime +import sys from decimal import Decimal from HTMLParser import HTMLParseError @@ -182,8 +183,17 @@ def GCDdetails(comseries, resultURL, vari_loop, ComicID, TotalIssues, issvariati # let's pull down the publication date as it'll be blank otherwise inputMIS = 'http://www.comics.org' + str(resultURL) resp = urllib2.urlopen ( inputMIS ) - soup = BeautifulSoup ( resp ) - +# soup = BeautifulSoup ( resp ) + try: + soup = BeautifulSoup(urllib2.urlopen(inputMIS)) + except UnicodeDecodeError: + logger.info("I've detected your system is using: " + sys.stdout.encoding) + logger.info("unable to parse properly due to utf-8 problem, ignoring wrong symbols") + try: + soup = BeautifulSoup(urllib2.urlopen(inputMIS)).decode('utf-8', 'ignore') + except UnicodeDecodeError: + logger.info("not working...aborting. Tell Evilhero.") + return parsed = soup.find("div", {"id" : "series_data"}) subtxt3 = parsed.find("dd", {"id" : "publication_dates"}) resultPublished = subtxt3.findNext(text=True).rstrip() @@ -583,3 +593,11 @@ def ComChk(ComicName, ComicYear, ComicPublisher, Total, ComicID): comchoice['comchkchoice'] = comchkchoice return comchoice, totalcount +def decode_html(html_string): + converted = UnicodeDammit(html_string, isHTML=True) + if not converted.unicode: + raise UnicodeDecodeError( + "Failed to detect encoding, tried [%s]", + ', '.join(converted.triedEncodings)) + # print converted.originalEncoding + return converted.unicode diff --git a/mylar/search.py b/mylar/search.py index 1ef543fe..0c8941b9 100755 --- a/mylar/search.py +++ b/mylar/search.py @@ -362,6 +362,7 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is logger.fdebug("Entry: " + str(thisentry)) cleantitle = re.sub('[_/.]', ' ', str(entry['title'])) cleantitle = helpers.cleanName(str(cleantitle)) + nzbname = cleantitle logger.fdebug("Cleantitle: " + str(cleantitle)) @@ -482,7 +483,8 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is #else: splitst = len(splitit) - 1 # make sure that things like - in watchcomic are accounted for when comparing to nzb. - watchcomic_split = re.sub('[\-\:\,\.]', ' ', findcomic[findloop]).split(None) + watchcomic_split = helpers.cleanName(str(findcomic[findloop])) + watchcomic_split = re.sub('[\-\:\,\.]', ' ', watchcomic_split).split(None) logger.fdebug(str(splitit) + " nzb series word count: " + str(splitst)) logger.fdebug(str(watchcomic_split) + " watchlist word count: " + str(len(watchcomic_split))) From e8efcfac794c4c951086079c128fd2a694735520 Mon Sep 17 00:00:00 2001 From: evilhero Date: Sun, 27 Jan 2013 05:08:32 -0500 Subject: [PATCH 13/16] FIX: Removed erroneous Issue Details link from Comic Details pages --- data/interfaces/default/artistredone.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/interfaces/default/artistredone.html b/data/interfaces/default/artistredone.html index 9999ac08..ba1ca9d7 100755 --- a/data/interfaces/default/artistredone.html +++ b/data/interfaces/default/artistredone.html @@ -247,7 +247,7 @@ ${issue['Issue_Number']} - ${issue['IssueName']} + ${issue['IssueName']} ${issue['IssueDate']} ${issue['Status']} %if issue['Status'] == 'Skipped': From 7e0ef1774c21e47f90d1909013dcd4142459cb1d Mon Sep 17 00:00:00 2001 From: evilhero Date: Mon, 28 Jan 2013 15:31:43 -0500 Subject: [PATCH 14/16] FIX:(#186)(#187) Weekly pull list would either error out on start-up, or wouldn't display any data, IMP: Removed 'want' from Weeklypull as wasn't working, IMP: setting up the base for some mass importing. --- data/css/style.css | 1 - data/interfaces/default/base.html | 4 + data/interfaces/default/css/style.css | 32 ++ data/interfaces/default/idirectory.html | 85 ++++ data/interfaces/default/importresults.html | 77 ++++ data/interfaces/default/searchfix.html | 1 - data/interfaces/default/weeklypull.html | 139 ++----- mylar/__init__.py | 7 +- mylar/helpers.py | 129 ++++++ mylar/librarysync.py | 444 +++++++++++++++++++++ mylar/search.py | 20 + mylar/webserve.py | 82 +++- mylar/weeklypull.py | 7 +- 13 files changed, 916 insertions(+), 112 deletions(-) create mode 100755 data/interfaces/default/idirectory.html create mode 100644 data/interfaces/default/importresults.html create mode 100644 mylar/librarysync.py diff --git a/data/css/style.css b/data/css/style.css index adf2bdeb..6c80315c 100755 --- a/data/css/style.css +++ b/data/css/style.css @@ -246,7 +246,6 @@ table#searchresults_table td#comicyear { vertical-align: middle; text-align: lef table#searchresults_table td#issues { vertical-align: middle; text-align: center; min-width: 50px; } div.progress-container { border: 1px solid #ccc; width: 100px; height: 14px; margin: 2px 5px 2px 0; padding: 1px; float: left; background: white; } -div.progress-container > div { background-color: #a3e532; height: 14px; } .havetracks { font-size: 13px; margin-left: 36px; padding-bottom: 3px; vertical-align: middle; } footer { margin: 20px auto 20px auto; } diff --git a/data/interfaces/default/base.html b/data/interfaces/default/base.html index bc8b2ced..5149f6f6 100755 --- a/data/interfaces/default/base.html +++ b/data/interfaces/default/base.html @@ -86,6 +86,10 @@ (${version.MYLAR_VERSION}) %endif
+
+ + +
Back to top
diff --git a/data/interfaces/default/css/style.css b/data/interfaces/default/css/style.css index a8ea6038..511587ed 100755 --- a/data/interfaces/default/css/style.css +++ b/data/interfaces/default/css/style.css @@ -879,6 +879,38 @@ div#artistheader h2 a { font-weight: bold; font-family: "Trebuchet MS", Helvetica, Arial, sans-serif; } +#weekly_pull th#publisher { + min-width: 150px; + text-align: left; +} +#weekly_pull th#comicname { + min-width: 250px; + text-align: left; +} +#weekly_pull th#comicnumber, +#weekly_pull th#status, +#weekly_pull th#series { + min-width: 50px; + text-align: center; +} +#weekly_pull td#comicname { + min-width: 275px; + text-align: left; + vertical-align: middle; + font-size: 12px; +} +#weekly_pull td#status, +#weekly_pull td#series, +#weekly_pull td#comicnumber { + min-width: 50px; + text-align: left; + vertical-align: middle; +} +#weekly_pull td#publisher { + min-width: 150px; + text-align: left; + vertical-align: middle; +} #manage_comic th#name { min-width: 275px; text-align: left; diff --git a/data/interfaces/default/idirectory.html b/data/interfaces/default/idirectory.html new file mode 100755 index 00000000..be14e9fd --- /dev/null +++ b/data/interfaces/default/idirectory.html @@ -0,0 +1,85 @@ +<%inherit file="base.html" /> +<%! + import mylar + from mylar.helpers import checked +%> +<%def name="headerIncludes()"> +
+ +
+ + +<%def name="body()"> +
+

Manage

+
+
+ +
+ +
+ Scan Comic Library +

Where do you keep your comics?

+

You can put in any directory, and it will scan for comic files in that folder + (including all subdirectories).
For example: '/Users/name/Comics'

+

+ It may take a while depending on how many files you have. You can navigate away from the page
+ as soon as you click 'Save changes' +

+
+

THIS IS CURRENTLY DISABLED UNTIL WORKING..

+
+ + %if mylar.COMIC_DIR: + + %else: + + %endif +
+
+ +
+
+ +
+
+ + Leaving this unchecked will not move anything, but will mark the issues as Archived +
+
+
+ + Rename files to configuration settings +
+
+
+ + +
+ +
+ +
+ +<%def name="javascriptIncludes()"> + + diff --git a/data/interfaces/default/importresults.html b/data/interfaces/default/importresults.html new file mode 100644 index 00000000..352a59fc --- /dev/null +++ b/data/interfaces/default/importresults.html @@ -0,0 +1,77 @@ +<%inherit file="base.html" /> <%! + import mylar + from mylar.helpers import checked +%> +<%def name="headerIncludes()"> +
+ +
+ + +<%def name="body()"> +
+

Borg Importing Results

+
+
+ +
+ + + + + + +
+
+
+ 		+
+ + + + + + + + + + + + %if results: + %for result in results: + + + + + + + + %endfor + %else: + + + + %endif + + + +
Comic NameYearStatusImport DateOptions
${result['ComicName']}${result['ComicYear']}${result['Status']}${result['ImportDate']}Add Series
There are no results to display
+
+
+ +<%def name="javascriptIncludes()"> + + + diff --git a/data/interfaces/default/searchfix.html b/data/interfaces/default/searchfix.html index bda98a09..db6aafa5 100755 --- a/data/interfaces/default/searchfix.html +++ b/data/interfaces/default/searchfix.html @@ -97,7 +97,6 @@ function initThisPage() { jQuery( "#tabs" ).tabs(); initActions(); - initConfigCheckbox("#addtocsv"); }; $(document).ready(function() { initThisPage(); diff --git a/data/interfaces/default/weeklypull.html b/data/interfaces/default/weeklypull.html index b29013a8..2a21f683 100755 --- a/data/interfaces/default/weeklypull.html +++ b/data/interfaces/default/weeklypull.html @@ -8,11 +8,6 @@
Refresh Pull-list - %if pullfilter is False: - Filter Non-Comics - %else: - Show All Comics - %endif
« Back to overview @@ -23,30 +18,15 @@

Weekly Pull list for : ${pulldate}

-
- -
Mark selected issues as - - -
+
+ - - %if pullfilter is False: - - %endif - @@ -64,60 +44,21 @@ %if pullfilter is True: %if str(weekly['ISSUE']).isdigit() > 0: - - - - - + + + - %elif (weekly['STATUS'] == 'Wanted'): - [skip] - %else: - [retry][new] + add series %endif %endif - %elif pullfilter is False: - - - - %if str(weekly['ISSUE']).isdigit() > 0: - - %else: - - %endif - %if str(weekly['ISSUE']).isdigit() > 0: - - %else: - - %endif - %if str(weekly['ISSUE']).isdigit() > 0: - - %elif (weekly['STATUS'] == 'Wanted'): - [skip] - %else: - [retry][new] - %endif - %else: - %endfor
Publisher COMIC NumberTypeStatusSeries
${weekly['PUBLISHER']}${weekly['COMIC']} - ${weekly['ISSUE']}${weekly['STATUS']} + ${weekly['PUBLISHER']}${weekly['COMIC']}${weekly['ISSUE']}${weekly['STATUS']} %if weekly['STATUS'] == 'Skipped': - [want] - add series${weekly['PUBLISHER']}${weekly['COMIC']}${weekly['ISSUE']} ${weekly['ISSUE']}${weekly['STATUS']} - %if weekly['STATUS'] == 'Skipped': - [want] - add series${weekly['STATUS']} - %if weekly['STATUS'] == 'Skipped': - [want] - %elif (weekly['STATUS'] == 'Wanted'): - [skip] - %endif - %endif %endif
- +
<%def name="headIncludes()"> @@ -129,42 +70,34 @@ + $(document).ready(function() { + initThisPage(); + }); + diff --git a/mylar/__init__.py b/mylar/__init__.py index 4b3671fa..db86e426 100755 --- a/mylar/__init__.py +++ b/mylar/__init__.py @@ -87,6 +87,7 @@ ADD_COMICS = False COMIC_DIR = None LIBRARYSCAN = False IMP_MOVE = False +IMP_RENAME = False SEARCH_INTERVAL = 360 NZB_STARTUP_SEARCH = False @@ -216,7 +217,7 @@ def initialize(): HTTP_PORT, HTTP_HOST, HTTP_USERNAME, HTTP_PASSWORD, HTTP_ROOT, LAUNCH_BROWSER, GIT_PATH, \ CURRENT_VERSION, LATEST_VERSION, CHECK_GITHUB, CHECK_GITHUB_ON_STARTUP, CHECK_GITHUB_INTERVAL, MUSIC_DIR, DESTINATION_DIR, \ DOWNLOAD_DIR, USENET_RETENTION, SEARCH_INTERVAL, NZB_STARTUP_SEARCH, INTERFACE, AUTOWANT_ALL, AUTOWANT_UPCOMING, ZERO_LEVEL, ZERO_LEVEL_N, COMIC_COVER_LOCAL, \ - LIBRARYSCAN, LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, SAB_PRIORITY, BLACKHOLE, BLACKHOLE_DIR, ADD_COMICS, COMIC_DIR, IMP_MOVE, \ + LIBRARYSCAN, LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, SAB_PRIORITY, BLACKHOLE, BLACKHOLE_DIR, ADD_COMICS, COMIC_DIR, IMP_MOVE, IMP_RENAME, \ NZBSU, NZBSU_APIKEY, DOGNZB, DOGNZB_APIKEY, NZBX,\ NEWZNAB, NEWZNAB_HOST, NEWZNAB_APIKEY, NEWZNAB_ENABLED, EXTRA_NEWZNABS,\ RAW, RAW_PROVIDER, RAW_USERNAME, RAW_PASSWORD, RAW_GROUPS, EXPERIMENTAL, \ @@ -269,6 +270,7 @@ def initialize(): ADD_COMICS = bool(check_setting_int(CFG, 'General', 'add_comics', 0)) COMIC_DIR = check_setting_str(CFG, 'General', 'comic_dir', '') IMP_MOVE = bool(check_setting_int(CFG, 'General', 'imp_move', 0)) + IMP_RENAME = bool(check_setting_int(CFG, 'General', 'imp_rename', 0)) DOWNLOAD_SCAN_INTERVAL = check_setting_int(CFG, 'General', 'download_scan_interval', 5) INTERFACE = check_setting_str(CFG, 'General', 'interface', 'default') AUTOWANT_ALL = bool(check_setting_int(CFG, 'General', 'autowant_all', 0)) @@ -520,6 +522,7 @@ def config_write(): new_config['General']['add_comics'] = int(ADD_COMICS) new_config['General']['comic_dir'] = COMIC_DIR new_config['General']['imp_move'] = int(IMP_MOVE) + new_config['General']['imp_rename'] = int(IMP_RENAME) new_config['General']['download_scan_interval'] = DOWNLOAD_SCAN_INTERVAL new_config['General']['interface'] = INTERFACE new_config['General']['autowant_all'] = int(AUTOWANT_ALL) @@ -639,7 +642,7 @@ def dbcheck(): c.execute('CREATE TABLE IF NOT EXISTS nzblog (IssueID TEXT, NZBName TEXT)') c.execute('CREATE TABLE IF NOT EXISTS weekly (SHIPDATE text, PUBLISHER text, ISSUE text, COMIC VARCHAR(150), EXTRA text, STATUS text)') # c.execute('CREATE TABLE IF NOT EXISTS sablog (nzo_id TEXT, ComicName TEXT, ComicYEAR TEXT, ComicIssue TEXT, name TEXT, nzo_complete TEXT)') - + c.execute('CREATE TABLE IF NOT EXISTS importresults (ComicName TEXT, ComicYear TEXT, Status TEXT, ImportDate TEXT)') conn.commit c.close #new diff --git a/mylar/helpers.py b/mylar/helpers.py index 6443fad4..b0ca8ac1 100755 --- a/mylar/helpers.py +++ b/mylar/helpers.py @@ -183,3 +183,132 @@ def decimal_issue(iss): issdec = int(iss_decval.rstrip('0')) * 10 deciss = (int(iss_b4dec) * 1000) + issdec return deciss + +def rename_param(comicid, comicname, comicyear, issue, issueid=None): + myDB = db.DBConnection() + if issueid is None: + chkissue = myDB.action("SELECT * from issues WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone() + if chkissue is None: + logger.error("Invalid Issue_Number - please validate.") + return + else: + issueid = chkissue['IssueID'] + + #use issueid to get publisher, series, year, issue number + issuenzb = myDB.action("SELECT * from issues WHERE issueid=?", [issueid]).fetchone() + #comicid = issuenzb['ComicID'] + issuenum = issuenzb['Issue_Number'] + #issueno = str(issuenum).split('.')[0] + + iss_find = issuenum.find('.') + iss_b4dec = issuenum[:iss_find] + iss_decval = issuenum[iss_find+1:] + if int(iss_decval) == 0: + iss = iss_b4dec + issdec = int(iss_decval) + issueno = str(iss) + logger.fdebug("Issue Number: " + str(issueno)) + else: + if len(iss_decval) == 1: + iss = iss_b4dec + "." + iss_decval + issdec = int(iss_decval) * 10 + else: + iss = iss_b4dec + "." + iss_decval.rstrip('0') + issdec = int(iss_decval.rstrip('0')) * 10 + issueno = iss_b4dec + logger.fdebug("Issue Number: " + str(iss)) + + # issue zero-suppression here + if mylar.ZERO_LEVEL == "0": + zeroadd = "" + else: + if mylar.ZERO_LEVEL_N == "none": zeroadd = "" + elif mylar.ZERO_LEVEL_N == "0x": zeroadd = "0" + elif mylar.ZERO_LEVEL_N == "00x": zeroadd = "00" + + logger.fdebug("Zero Suppression set to : " + str(mylar.ZERO_LEVEL_N)) + + if str(len(issueno)) > 1: + if int(issueno) < 10: + logger.fdebug("issue detected less than 10") + if int(iss_decval) > 0: + issueno = str(iss) + prettycomiss = str(zeroadd) + str(iss) + else: + prettycomiss = str(zeroadd) + str(int(issueno)) + logger.fdebug("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss)) + elif int(issueno) >= 10 and int(issueno) < 100: + logger.fdebug("issue detected greater than 10, but less than 100") + if mylar.ZERO_LEVEL_N == "none": + zeroadd = "" + else: + zeroadd = "0" + if int(iss_decval) > 0: + issueno = str(iss) + prettycomiss = str(zeroadd) + str(iss) + else: + prettycomiss = str(zeroadd) + str(int(issueno)) + logger.fdebug("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ".Issue will be set as : " + str(prettycomiss)) + else: + logger.fdebug("issue detected greater than 100") + if int(iss_decval) > 0: + issueno = str(iss) + prettycomiss = str(issueno) + logger.fdebug("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss)) + else: + prettycomiss = str(issueno) + logger.fdebug("issue length error - cannot determine length. Defaulting to None: " + str(prettycomiss)) + + logger.fdebug("Pretty Comic Issue is : " + str(prettycomiss)) + issueyear = issuenzb['IssueDate'][:4] + logger.fdebug("Issue Year : " + str(issueyear)) + comicnzb= myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone() + publisher = comicnzb['ComicPublisher'] + logger.fdebug("Publisher: " + str(publisher)) + series = comicnzb['ComicName'] + logger.fdebug("Series: " + str(series)) + seriesyear = comicnzb['ComicYear'] + logger.fdebug("Year: " + str(seriesyear)) + comlocation = comicnzb['ComicLocation'] + logger.fdebug("Comic Location: " + str(comlocation)) + + file_values = {'$Series': series, + '$Issue': prettycomiss, + '$Year': issueyear, + '$series': series.lower(), + '$Publisher': publisher, + '$publisher': publisher.lower(), + '$Volume': seriesyear + } + + extensions = ('.cbr', '.cbz') + + if mylar.FILE_FORMAT == '': + self._log("Rename Files isn't enabled...keeping original filename.", logger.DEBUG) + logger.fdebug("Rename Files isn't enabled - keeping original filename.") + #check if extension is in nzb_name - will screw up otherwise + if ofilename.lower().endswith(extensions): + nfilename = ofilename[:-4] + else: + nfilename = ofilename + else: + nfilename = helpers.replace_all(mylar.FILE_FORMAT, file_values) + if mylar.REPLACE_SPACES: + #mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot + nfilename = nfilename.replace(' ', mylar.REPLACE_CHAR) + nfilename = re.sub('[\,\:]', '', nfilename) + logger.fdebug("New Filename: " + str(nfilename)) + + if mylar.LOWERCASE_FILENAMES: + dst = (comlocation + "/" + nfilename + ext).lower() + else: + dst = comlocation + "/" + nfilename + ext.lower() + logger.fdebug("Source: " + str(src)) + logger.fdebug("Destination: " + str(dst)) + + rename_this = { "destination_dir" : dst, + "nfilename" : nfilename, + "issueid" : issueid, + "comicid" : comicid } + + return rename_this diff --git a/mylar/librarysync.py b/mylar/librarysync.py new file mode 100644 index 00000000..e0aa767b --- /dev/null +++ b/mylar/librarysync.py @@ -0,0 +1,444 @@ +# This file is part of Mylar. +# +# Mylar is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Mylar is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Mylar. If not, see . + +from __future__ import with_statement + +import os +import glob +import re +import shutil + +import mylar +from mylar import db, logger, helpers, importer, updater + +# You can scan a single directory and append it to the current library by specifying append=True +def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None): + + if cron and not mylar.LIBRARYSCAN: + return + + if not dir: + dir = mylar.COMIC_DIR + + # If we're appending a dir, it's coming from the post processor which is + # already bytestring + if not append: + dir = dir.encode(mylar.SYS_ENCODING) + + if not os.path.isdir(dir): + logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(mylar.SYS_ENCODING, 'replace')) + return + + + logger.info('Scanning comic directory: %s' % dir.decode(mylar.SYS_ENCODING, 'replace')) + + basedir = dir + + watchmatch = {} + comic_list = [] + comiccnt = 0 + extensions = ('cbr','cbz') + for r,d,f in os.walk(dir): + #for directory in d[:]: + # if directory.startswith("."): + # d.remove(directory) + for files in f: + if any(files.lower().endswith('.' + x.lower()) for x in extensions): + comic = files + comicpath = os.path.join(r, files) + comicsize = os.path.getsize(comicpath) + print "Comic: " + comic + print "Comic Path: " + comicpath + print "Comic Size: " + str(comicsize) + + # We need the unicode path to use for logging, inserting into database + unicode_comic_path = comicpath.decode(mylar.SYS_ENCODING, 'replace') + + comiccnt+=1 + comic_dict = { 'ComicFilename': comic, + 'ComicLocation': comicpath, + 'ComicSize': comicsize, + 'Unicode_ComicLocation': unicode_comic_path } + comic_list.append(comic_dict) + + logger.info("I've found a total of " + str(comiccnt) + " comics....analyzing now") + + myDB = db.DBConnection() + + #let's load in the watchlist to see if we have any matches. + logger.info("loading in the watchlist to see if a series is being watched already...") + watchlist = myDB.action("SELECT * from comics") + ComicName = [] + ComicYear = [] + ComicPublisher = [] + ComicTotal = [] + ComicID = [] + ComicLocation = [] + + AltName = [] + watchcnt = 0 + + watch_kchoice = [] + watchchoice = {} + import_by_comicids = [] + import_comicids = {} + + for watch in watchlist: + # let's clean up the name, just in case for comparison purposes... + watchcomic = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', ' ', str(watch['ComicName'])) + #watchcomic = re.sub('\s+', ' ', str(watchcomic)).strip() + alt_chk = "no" # alt-checker flag (default to no) + + # account for alternate names as well + if watch['AlternateSearch'] is not None and watch['AlternateSearch'] is not 'None': + altcomic = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', ' ', str(watch['AlternateSearch'])) + #altcomic = re.sub('\s+', ' ', str(altcomic)).strip() + AltName.append(altcomic) + alt_chk = "yes" # alt-checker flag + + ComicName.append(watchcomic) + ComicYear.append(watch['ComicYear']) + ComicPublisher.append(watch['ComicPublisher']) + ComicTotal.append(watch['Total']) + ComicID.append(watch['ComicID']) + ComicLocation.append(watch['ComicLocation']) + watchcnt+=1 + + logger.info("Successfully loaded " + str(watchcnt) + " series from your watchlist.") + + ripperlist=['digital-', + 'empire', + 'dcp'] + + watchfound = 0 + + for i in comic_list: + #print i['ComicFilename'] + + comfilename = i['ComicFilename'] + comlocation = i['ComicLocation'] + #let's clean up the filename for matching purposes + + cfilename = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', ' ', str(comfilename)) + #cfilename = re.sub('\s+', ' ', str(cfilename)).strip() + + cm_cn = 0 + + #we need to track the counter to make sure we are comparing the right array parts + #this takes care of the brackets :) + m = re.findall('[^()]+', cfilename) + lenm = len(m) + print ("there are " + str(lenm) + " words.") + cnt = 0 + yearmatch = "false" + foundonwatch = "False" + + while (cnt < lenm): + if m[cnt] is None: break + if m[cnt] == ' ': + pass + else: + logger.fdebug(str(cnt) + ". Bracket Word: " + str(m[cnt])) + if cnt == 0: + comic_andiss = m[cnt] + logger.fdebug("Comic: " + str(comic_andiss)) + if m[cnt][:-2] == '19' or m[cnt][:-2] == '20': + logger.fdebug("year detected: " + str(m[cnt])) + result_comyear = m[cnt] + yearmatch = "true" + # if str(comyear) in result_comyear: + # logger.fdebug(str(comyear) + " - right years match baby!") + # yearmatch = "true" + # else: + # logger.fdebug(str(comyear) + " - not right - years do not match") + # yearmatch = "false" + #let's do this hear and save a few extra loops ;) + if 'digital' in m[cnt] and len(m[cnt]) == 7: + logger.fdebug("digital edition detected") + pass + if ' of ' in m[cnt]: + logger.fdebug("mini-series detected : " + str(m[cnt])) + result_of = m[cnt] + if 'cover' in m[cnt]: + logger.fdebug("covers detected: " + str(m[cnt])) + result_comcovers = m[cnt] + for ripper in ripperlist: + if ripper in m[cnt]: + logger.fdebug("Scanner detected: " + str(m[cnt])) + result_comscanner = m[cnt] + cnt+=1 + + if yearmatch == "false": + logger.fdebug("failed to match...skipping.") + break + splitit = [] + watchcomic_split = [] + logger.fdebug("filename comic and issue: " + str(cfilename)) + #changed this from '' to ' ' + comic_iss_b4 = re.sub('[\-\:\,]', ' ', str(comic_andiss)) + comic_iss = comic_iss_b4.replace('.',' ') + logger.fdebug("adjusted comic and issue: " + str(comic_iss)) + splitit = comic_iss.split(None) + logger.fdebug("adjusting from: " + str(comic_iss_b4) + " to: " + str(comic_iss)) + #bmm = re.findall('v\d', comic_iss) + #if len(bmm) > 0: splitst = len(splitit) - 2 + #else: splitst = len(splitit) - 1 + #----- + #here we cycle through the Watchlist looking for a match. + while (cm_cn < watchcnt): + #setup the watchlist + comname = ComicName[cm_cn] + print ("watch_comic:" + str(comname)) + comyear = ComicYear[cm_cn] + compub = ComicPublisher[cm_cn] + comtotal = ComicTotal[cm_cn] + comicid = ComicID[cm_cn] + watch_location = ComicLocation[cm_cn] + + if splitit[(len(splitit)-1)].isdigit(): + #compares - if the last digit and second last digit are #'s seperated by spaces assume decimal + comic_iss = splitit[(len(splitit)-1)] + splitst = len(splitit) - 1 + if splitit[(len(splitit)-2)].isdigit(): + # for series that have a digit at the end, it screws up the logistics. + i = 1 + chg_comic = splitit[0] + while (i < (len(splitit)-1)): + chg_comic = chg_comic + " " + splitit[i] + i+=1 + logger.fdebug("chg_comic:" + str(chg_comic)) + if chg_comic.upper() == comname.upper(): + logger.fdebug("series contains numerics...adjusting..") + else: + changeup = "." + splitit[(len(splitit)-1)] + logger.fdebug("changeup to decimal: " + str(changeup)) + comic_iss = splitit[(len(splitit)-2)] + "." + comic_iss + splitst = len(splitit) - 2 + else: + # if the nzb name doesn't follow the series-issue-year format even closely..ignore nzb + logger.fdebug("invalid naming format of filename detected - cannot properly determine issue") + continue + + # make sure that things like - in watchcomic are accounted for when comparing to nzb. + watchcomic_split = helpers.cleanName(str(comname)) + watchcomic_split = re.sub('[\-\:\,\.]', ' ', watchcomic_split).split(None) + + logger.fdebug(str(splitit) + " file series word count: " + str(splitst)) + logger.fdebug(str(watchcomic_split) + " watchlist word count: " + str(len(watchcomic_split))) + if (splitst) != len(watchcomic_split): + logger.fdebug("incorrect comic lengths...not a match") + if str(splitit[0]).lower() == "the": + logger.fdebug("THE word detected...attempting to adjust pattern matching") + splitit[0] = splitit[4:] + else: + logger.fdebug("length match..proceeding") + n = 0 + scount = 0 + logger.fdebug("search-length: " + str(splitst)) + logger.fdebug("Watchlist-length: " + str(len(watchcomic_split))) + while ( n <= (splitst)-1 ): + logger.fdebug("splitit: " + str(splitit[n])) + if n < (splitst) and n < len(watchcomic_split): + logger.fdebug(str(n) + " Comparing: " + str(watchcomic_split[n]) + " .to. " + str(splitit[n])) + if '+' in watchcomic_split[n]: + watchcomic_split[n] = re.sub('+', '', str(watchcomic_split[n])) + if str(watchcomic_split[n].lower()) in str(splitit[n].lower()) and len(watchcomic_split[n]) >= len(splitit[n]): + logger.fdebug("word matched on : " + str(splitit[n])) + scount+=1 + #elif ':' in splitit[n] or '-' in splitit[n]: + # splitrep = splitit[n].replace('-', '') + # print ("non-character keyword...skipped on " + splitit[n]) + elif str(splitit[n].lower()).startswith('v'): + logger.fdebug("possible versioning..checking") + #we hit a versioning # - account for it + if splitit[n][1:].isdigit(): + comicversion = str(splitit[n]) + logger.fdebug("version found: " + str(comicversion)) + else: + logger.fdebug("Comic / Issue section") + if splitit[n].isdigit(): + logger.fdebug("issue detected") + #comiss = splitit[n] + comicNAMER = n - 1 + com_NAME = splitit[0] + cmnam = 1 + while (cmnam <= comicNAMER): + com_NAME = str(com_NAME) + " " + str(splitit[cmnam]) + cmnam+=1 + logger.fdebug("comic: " + str(com_NAME)) + else: + logger.fdebug("non-match for: "+ str(splitit[n])) + pass + n+=1 + #set the match threshold to 80% (for now) + # if it's less than 80% consider it a non-match and discard. + #splitit has to splitit-1 because last position is issue. + wordcnt = int(scount) + logger.fdebug("scount:" + str(wordcnt)) + totalcnt = int(splitst) + logger.fdebug("splitit-len:" + str(totalcnt)) + spercent = (wordcnt/totalcnt) * 100 + logger.fdebug("we got " + str(spercent) + " percent.") + if int(spercent) >= 80: + logger.fdebug("it's a go captain... - we matched " + str(spercent) + "%!") + logger.fdebug("this should be a match!") + if '.' in comic_iss: + comisschk_find = comic_iss.find('.') + comisschk_b4dec = comic_iss[:comisschk_find] + comisschk_decval = comic_iss[comisschk_find+1:] + logger.fdebug("Found IssueNumber: " + str(comic_iss)) + logger.fdebug("..before decimal: " + str(comisschk_b4dec)) + logger.fdebug("...after decimal: " + str(comisschk_decval)) + #--let's make sure we don't wipe out decimal issues ;) + if int(comisschk_decval) == 0: + ciss = comisschk_b4dec + cintdec = int(comisschk_decval) + else: + if len(comisschk_decval) == 1: + ciss = comisschk_b4dec + "." + comisschk_decval + cintdec = int(comisschk_decval) * 10 + else: + ciss = comisschk_b4dec + "." + comisschk_decval.rstrip('0') + cintdec = int(comisschk_decval.rstrip('0')) * 10 + comintIss = (int(comisschk_b4dec) * 1000) + cintdec + else: + comintIss = int(comic_iss) * 1000 + logger.fdebug("issue we found for is : " + str(comic_iss)) + + #issue comparison now as well + logger.info(u"Found " + str(comname) + " (" + str(comyear) + ") issue: " + str(comic_iss)) + watchfound+=1 +# updater.forceRescan(ComicID=comicid) +# if not any(d.get('ComicID', None) == str(comicid) for d in watch_kchoice): + watch_kchoice.append({ + "ComicID": str(comicid), + "ComicName": str(comname), + "ComicYear": str(comyear), + "ComicIssue": str(int(comic_iss)), + "ComicLocation": str(watch_location), + "OriginalLocation" : str(comlocation), + "OriginalFilename" : str(comfilename) + }) + foundonwatch = "True" + break + elif int(spercent) < 80: + logger.fdebug("failure - we only got " + str(spercent) + "% right!") + cm_cn+=1 + + if foundonwatch == "False": + #---if it's not a match - send it to the importer. + n = 0 + csplit = comic_andiss.split(None) + while ( n <= (len(csplit)-1) ): + if csplit[n].isdigit(): + logger.fdebug("issue detected") + #comiss = splitit[n] + comicNAMER = n - 1 + com_NAME = csplit[0] + cmnam = 1 + while (cmnam <= comicNAMER): + com_NAME = str(com_NAME) + " " + str(csplit[cmnam]) + cmnam+=1 + logger.fdebug("comic: " + str(com_NAME)) + n+=1 + + print ("adding " + str(com_NAME) + " to the import-queue!") + import_by_comicids.append({ + "comicname" : com_NAME, + "comicyear" : result_comyear, + "comfilename" : comfilename, + "comlocation" : comlocation + }) + + if len(watch_kchoice) > 0: + watchchoice['watchlist'] = watch_kchoice + print ("watchchoice: " + str(watchchoice)) + + logger.info("I have found " + str(watchfound) + " out of " + str(comiccnt) + " comics for series that are being watched.") + wat = 0 + comicids = [] + + if watchfound > 0: + if mylar.IMP_MOVE: + logger.info("You checked off Move Files...so that's what I'm going to do") + #check to see if Move Files is enabled. + #if not being moved, set the archive bit. + print("Moving files into appropriate directory") + while (wat < watchfound): + watch_the_list = watchchoice['watchlist'][wat] + watch_comlocation = watch_the_list['ComicLocation'] + watch_comicid = watch_the_list['ComicID'] + watch_comicname = watch_the_list['ComicName'] + watch_comicyear = watch_the_list['ComicYear'] + watch_comiciss = watch_the_list['ComicIssue'] + print ("ComicLocation: " + str(watch_comlocation)) + orig_comlocation = watch_the_list['OriginalLocation'] + orig_filename = watch_the_list['OriginalFilename'] + print ("Orig. Location: " + str(orig_comlocation)) + print ("Orig. Filename: " + str(orig_filename)) + #before moving check to see if Rename to Mylar structure is enabled. + if mylar.IMP_RENAME: + print("Renaming files according to configuration details : " + str(mylar.FILE_FORMAT)) + renameit = helpers.rename_param(watch_comicid, watch_comicname, watch_comicyear, watch_comiciss) + nfilename = renameit['nfilename'] + + dst_path = os.path.join(watch_comlocation,nfilename) + if str(watch_comicid) not in comicids: + comicids.append(watch_comicid) + else: + print("Renaming files not enabled, keeping original filename(s)") + dst_path = os.path.join(watch_comlocation,orig_filename) + + #os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder,str(nfilename + ext))) + #src = os.path.join(, str(nfilename + ext)) + print ("I'm going to move " + str(orig_comlocation) + " to .." + str(dst_path)) + try: + shutil.move(orig_comlocation, dst_path) + except (OSError, IOError): + logger.info("Failed to move directory - check directories and manually re-run.") + wat+=1 + else: + # if move files isn't enabled, let's set all found comics to Archive status :) + while (wat < watchfound): + watch_the_list = watchchoice['watchlist'][wat] + watch_comicid = watch_the_list['ComicID'] + watch_issue = watch_the_list['ComicIssue'] + print ("ComicID: " + str(watch_comicid)) + print ("Issue#: " + str(watch_issue)) + issuechk = myDB.action("SELECT * from issues where ComicID=? AND INT_IssueNumber=?", [watch_comicid, watch_issue]).fetchone() + if issuechk is None: + print ("no matching issues for this comic#") + else: + print("...Existing status: " + str(issuechk['Status'])) + control = {"IssueID": issuechk['IssueID']} + values = { "Status": "Archived"} + print ("...changing status of " + str(issuechk['Issue_Number']) + " to Archived ") + myDB.upsert("issues", values, control) + if str(watch_comicid) not in comicids: + comicids.append(watch_comicid) + wat+=1 + if comicids is None: pass + else: + c_upd = len(comicids) + c = 0 + while (c < c_upd ): + print ("Rescanning.. " + str(c)) + updater.forceRescan(c) + if not len(import_by_comicids): + return "Completed" + if len(import_by_comicids) > 0: + import_comicids['comic_info'] = import_by_comicids + print ("import comicids: " + str(import_by_comicids)) + return import_comicids diff --git a/mylar/search.py b/mylar/search.py index 0c8941b9..f2c8a98d 100755 --- a/mylar/search.py +++ b/mylar/search.py @@ -648,6 +648,26 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is nzbxlink_st = linkapi.find("*|*") linkapi = linkapi[:(nzbxlink_st + 3)] + str(nzbname) logger.fdebug("new linkapi (this should =nzbname) :" + str(linkapi)) + +# #test nzb.get +# if mylar.NZBGET: +# from xmlrpclib import ServerProxy +# if mylar.NZBGET_HOST[:4] = 'http': +# tmpapi = "http://" +# nzbget_host = mylar.NZBGET_HOST[7] +# elif mylar.NZBGET_HOST[:5] = 'https': +# tmpapi = "https://" +# nzbget_host = mylar.NZBGET_HOST[8] +# tmpapi = tmpapi + str(mylar.NZBGET_USERNAME) + ":" + str(mylar.NZBGET_PASSWORD) +# tmpapi = tmpapi + "@" + nzbget_host + ":" + str(mylar.NZBGET_PORT) + "/xmlrpc" +# server = ServerProxy(tmpapi) +# send_to_nzbget = server.appendurl(nzbname, mylar.NZBGET_CATEGORY, mylar.NZBGET_PRIORITY, True, str(linkapi)) +# if send_to_nzbget is True: +# logger.info("Successfully sent nzb to NZBGet!") +# else: +# logger.info("Unable to send nzb to NZBGet - check your configs.") +# #end nzb.get test + # let's build the send-to-SAB string now: tmpapi = str(mylar.SAB_HOST) logger.fdebug("send-to-SAB host string: " + str(tmpapi)) diff --git a/mylar/webserve.py b/mylar/webserve.py index 92dda2ec..2bd5648a 100755 --- a/mylar/webserve.py +++ b/mylar/webserve.py @@ -28,10 +28,11 @@ import time import threading import csv import platform +import Queue import mylar -from mylar import logger, db, importer, mb, search, filechecker, helpers, updater, parseit, weeklypull, PostProcessor, version +from mylar import logger, db, importer, mb, search, filechecker, helpers, updater, parseit, weeklypull, PostProcessor, version, librarysync #from mylar.helpers import checked, radio, today import lib.simplejson as simplejson @@ -97,6 +98,7 @@ class WebInterface(object): searchresults = mb.findComic(name, mode, issue=None) elif type == 'comic' and mode == 'want': searchresults = mb.findComic(name, mode, issue) + searchresults = sorted(searchresults, key=itemgetter('comicyear','issues'), reverse=True) #print ("Results: " + str(searchresults)) return serve_template(templatename="searchresults.html", title='Search Results for: "' + name + '"', searchresults=searchresults, type=type) @@ -403,16 +405,26 @@ class WebInterface(object): def pullist(self): myDB = db.DBConnection() + weeklyresults = [] popit = myDB.select("SELECT * FROM sqlite_master WHERE name='weekly' and type='table'") if popit: - weeklyresults = myDB.select("SELECT * from weekly") + w_results = myDB.select("SELECT PUBLISHER, ISSUE, COMIC, STATUS from weekly") + for weekly in w_results: + if weekly['ISSUE'].isdigit(): + weeklyresults.append({ + "PUBLISHER" : weekly['PUBLISHER'], + "ISSUE" : weekly['ISSUE'], + "COMIC" : weekly['COMIC'], + "STATUS" : weekly['STATUS'] + }) + weeklyresults = sorted(weeklyresults, key=itemgetter('PUBLISHER','COMIC'), reverse=False) pulldate = myDB.action("SELECT * from weekly").fetchone() if pulldate is None: return self.manualpull() #raise cherrypy.HTTPRedirect("home") else: return self.manualpull() - return serve_template(templatename="weeklypull.html", title="Weekly Pull", weeklyresults=weeklyresults, pulldate=pulldate['SHIPDATE']) + return serve_template(templatename="weeklypull.html", title="Weekly Pull", weeklyresults=weeklyresults, pulldate=pulldate['SHIPDATE'], pullfilter=True) pullist.exposed = True def filterpull(self): @@ -421,7 +433,7 @@ class WebInterface(object): pulldate = myDB.action("SELECT * from weekly").fetchone() if pulldate is None: raise cherrypy.HTTPRedirect("home") - return serve_template(templatename="weeklypull.html", title="Weekly Pull", weeklyresults=weeklyresults, pulldate=pulldate['SHIPDATE']) + return serve_template(templatename="weeklypull.html", title="Weekly Pull", weeklyresults=weeklyresults, pulldate=pulldate['SHIPDATE'], pullfilter=True) filterpull.exposed = True def manualpull(self): @@ -479,6 +491,10 @@ class WebInterface(object): raise cherrypy.HTTPRedirect("artistPage?ComicID=%s" % [comicid]) skipped2wanted.exposed = True + def ManualRename(self): + print ("hello") + ManualRename.exposed = True + def searchScan(self, name): return serve_template(templatename="searchfix.html", title="Manage", name=name) searchScan.exposed = True @@ -573,6 +589,64 @@ class WebInterface(object): raise cherrypy.HTTPRedirect("history") clearhistory.exposed = True + #for testing. + def idirectory(self): + return serve_template(templatename="idirectory.html", title="Import a Directory") + idirectory.exposed = True + + def comicScan(self, path, scan=0, redirect=None, autoadd=0, libraryscan=0, imp_move=0, imp_rename=0): + mylar.LIBRARYSCAN = libraryscan + mylar.ADD_COMICS = autoadd + mylar.COMIC_DIR = path + mylar.IMP_MOVE = imp_move + mylar.IMP_RENAME = imp_rename + mylar.config_write() + if scan: + try: + soma = librarysync.libraryScan() + except Exception, e: + logger.error('Unable to complete the scan: %s' % e) + if soma == "Completed": + print ("sucessfully completed import.") + else: + logger.info(u"Starting mass importing...") + #this is what it should do... + #store soma (the list of comic_details from importing) into sql table so import can be whenever + #display webpage showing results + #allow user to select comic to add (one at a time) + #call addComic off of the webpage to initiate the add. + #return to result page to finish or continue adding. + #.... + #threading.Thread(target=self.searchit).start() + #threadthis = threadit.ThreadUrl() + #result = threadthis.main(soma) + myDB = db.DBConnection() + sl = 0 + while (sl < len(soma)): + soma_sl = soma['comic_info'][sl] + print ("cname: " + soma_sl['comicname']) + + controlValue = {"ComicName": soma_sl['comicname']} + newValue = {"ComicYear": soma_sl['comicyear'], + "Status": "Not Imported", + "ImportDate": helpers.today()} + myDB.upsert("importresults", newValue, controlValue) + sl+=1 + + self.importResults() + + if redirect: + raise cherrypy.HTTPRedirect(redirect) + else: + raise cherrypy.HTTPRedirect("home") + comicScan.exposed = True + + def importResults(self): + myDB = db.DBConnection() + results = myDB.select("SELECT * FROM importresults") + return serve_template(templatename="importresults.html", title="Import Results", results=results) + importResults.exposed = True + #--- def config(self): interface_dir = os.path.join(mylar.PROG_DIR, 'data/interfaces/') diff --git a/mylar/weeklypull.py b/mylar/weeklypull.py index 3d316cf8..ec3428bf 100755 --- a/mylar/weeklypull.py +++ b/mylar/weeklypull.py @@ -397,9 +397,10 @@ def pullitcheck(comic1off_name=None,comic1off_id=None): lines[cnt] = str(lines[cnt]).upper() #llen[cnt] = str(llen[cnt]) logger.fdebug("looking for : " + str(lines[cnt])) - sqlsearch = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', ' ', str(lines[cnt])) + sqlsearch = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\'\?\@]', ' ', str(lines[cnt])) sqlsearch = re.sub(r'\s', '%', sqlsearch) if 'THE' in sqlsearch: sqlsearch = re.sub('THE', '', sqlsearch) + if '+' in sqlsearch: sqlsearch = re.sub('\+', '%PLUS%', sqlsearch) logger.fdebug("searchsql: " + str(sqlsearch)) weekly = myDB.select('SELECT PUBLISHER, ISSUE, COMIC, EXTRA, SHIPDATE FROM weekly WHERE COMIC LIKE (?)', [sqlsearch]) #cur.execute('SELECT PUBLISHER, ISSUE, COMIC, EXTRA, SHIPDATE FROM weekly WHERE COMIC LIKE (?)', [lines[cnt]]) @@ -435,6 +436,10 @@ def pullitcheck(comic1off_name=None,comic1off_id=None): logger.fdebug("ComicNM: " + str(comicnm)) if 'THE' in str(watchcomic): modcomicnm = re.sub('THE', '', comicnm) + #thnx to A+X for this... + if '+' in str(watchcomic): + if 'plus' in str(comicnm).lower(): + modcomicnm = re.sub('plus', '+', comicnm) if str(comicnm) == str(watchcomic).upper() or str(modcomicnm) == str(watchcomic).upper(): logger.fdebug("matched on:" + str(comicnm) + "..." + str(watchcomic).upper()) #pass From 1072a51f0cc3e8c83dd07faffbf528499308f853 Mon Sep 17 00:00:00 2001 From: evilhero Date: Mon, 28 Jan 2013 17:35:04 -0500 Subject: [PATCH 15/16] FIX:(#189) Weekly Pull list not matching on some series with symbols in the name --- mylar/weeklypull.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mylar/weeklypull.py b/mylar/weeklypull.py index ec3428bf..acd985e4 100755 --- a/mylar/weeklypull.py +++ b/mylar/weeklypull.py @@ -422,12 +422,12 @@ def pullitcheck(comic1off_name=None,comic1off_id=None): comicnm = week['COMIC'] #here's the tricky part, ie. BATMAN will match on #every batman comic, not exact -# logger.fdebug("comparing" + str(comicnm) + "..to.." + str(unlines[cnt]).upper()) - logger.fdebug("comparing" + str(sqlsearch) + "..to.." + str(unlines[cnt]).upper()) + logger.fdebug("comparing" + str(comicnm) + "..to.." + str(unlines[cnt]).upper()) + #logger.fdebug("comparing" + str(sqlsearch) + "..to.." + str(unlines[cnt]).upper()) #-NEW- # strip out all special characters and compare - watchcomic = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', '', str(sqlsearch)) + watchcomic = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', '', str(unlines[cnt])) comicnm = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', '', str(comicnm)) watchcomic = re.sub(r'\s', '', watchcomic) comicnm = re.sub(r'\s', '', comicnm) From 4eb71abefab9cddaa2086701d5374b33d133f537 Mon Sep 17 00:00:00 2001 From: evilhero Date: Tue, 29 Jan 2013 04:02:23 -0500 Subject: [PATCH 16/16] FIX:(#141) Attempted fix to accomodate the '//' issues with windows paths, FIX: Custom_Exceptions error on Ultron EC adding in a title with a ',' - will add comicname to csv now --- data/interfaces/default/searchfix.html | 2 +- mylar/PostProcessor.py | 4 ++-- mylar/webserve.py | 13 ++++++++++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/data/interfaces/default/searchfix.html b/data/interfaces/default/searchfix.html index db6aafa5..f78f5d6a 100755 --- a/data/interfaces/default/searchfix.html +++ b/data/interfaces/default/searchfix.html @@ -82,7 +82,7 @@
- +