diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index e4d02996..7dfe5a89 100755 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -22,15 +22,43 @@
-
- - + +
+ + + + + +
+
+ Configuration Options +
+
+

+ MYLAR PROGRAM OPTIONS +
+
+
+
+ +
+
+ 		+
+ Branch history + +
+
+
+ +
@@ -99,7 +127,7 @@
-
+
@@ -163,7 +191,7 @@
@@ -126,13 +154,13 @@ -
+
- - - - - + + + + +
-
+
@@ -183,7 +211,7 @@
DOGNZB
- +
@@ -192,6 +220,15 @@
+ +
+ NZBX +
+ +
+
+ +
EXPERIMENTAL
@@ -264,16 +301,16 @@
-
+
Quality -
- - - +
+ + +
@@ -299,7 +336,7 @@
-
+
%else: - + %endif %endfor diff --git a/data/interfaces/default/weeklypull.html b/data/interfaces/default/weeklypull.html index 7c27da2a..b29013a8 100755 --- a/data/interfaces/default/weeklypull.html +++ b/data/interfaces/default/weeklypull.html @@ -53,13 +53,13 @@ %for weekly in weeklyresults: <% if weekly['STATUS'] == 'Skipped': - grade = 'X' + grade = 'Z' elif weekly['STATUS'] == 'Wanted': - grade = 'A' + grade = 'X' elif weekly['STATUS'] == 'Snatched': grade = 'C' else: - grade = 'Z' + grade = 'A' %> %if pullfilter is True: diff --git a/mylar/__init__.py b/mylar/__init__.py index 296847f7..000064ce 100755 --- a/mylar/__init__.py +++ b/mylar/__init__.py @@ -122,6 +122,8 @@ NZBSU_APIKEY = None DOGNZB = False DOGNZB_APIKEY = None +NZBX = False + NEWZNAB = False NEWZNAB_HOST = None NEWZNAB_APIKEY = None @@ -200,7 +202,7 @@ def initialize(): CURRENT_VERSION, LATEST_VERSION, CHECK_GITHUB, CHECK_GITHUB_ON_STARTUP, CHECK_GITHUB_INTERVAL, MUSIC_DIR, DESTINATION_DIR, \ DOWNLOAD_DIR, USENET_RETENTION, SEARCH_INTERVAL, INTERFACE, AUTOWANT_ALL, AUTOWANT_UPCOMING, ZERO_LEVEL, ZERO_LEVEL_N, COMIC_COVER_LOCAL, \ LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, SAB_PRIORITY, BLACKHOLE, BLACKHOLE_DIR, \ - NZBSU, NZBSU_APIKEY, DOGNZB, DOGNZB_APIKEY, \ + NZBSU, NZBSU_APIKEY, DOGNZB, DOGNZB_APIKEY, NZBX,\ NEWZNAB, NEWZNAB_HOST, NEWZNAB_APIKEY, NEWZNAB_ENABLED, EXTRA_NEWZNABS,\ RAW, RAW_PROVIDER, RAW_USERNAME, RAW_PASSWORD, RAW_GROUPS, EXPERIMENTAL, \ PREFERRED_QUALITY, MOVE_FILES, RENAME_FILES, CORRECT_METADATA, FOLDER_FORMAT, FILE_FORMAT, REPLACE_CHAR, REPLACE_SPACES, \ @@ -278,6 +280,8 @@ def initialize(): DOGNZB = bool(check_setting_int(CFG, 'DOGnzb', 'dognzb', 0)) DOGNZB_APIKEY = check_setting_str(CFG, 'DOGnzb', 'dognzb_apikey', '') + NZBX = bool(check_setting_int(CFG, 'nzbx', 'nzbx', 0)) + RAW = bool(check_setting_int(CFG, 'Raw', 'raw', 0)) RAW_PROVIDER = check_setting_str(CFG, 'Raw', 'raw_provider', '') RAW_USERNAME = check_setting_str(CFG, 'Raw', 'raw_username', '') @@ -506,6 +510,9 @@ def config_write(): new_config['DOGnzb']['dognzb'] = int(DOGNZB) new_config['DOGnzb']['dognzb_apikey'] = DOGNZB_APIKEY + new_config['nzbx'] = {} + new_config['nzbx']['nzbx'] = int(NZBX) + new_config['Experimental'] = {} new_config['Experimental']['experimental'] = int(EXPERIMENTAL) @@ -542,6 +549,7 @@ def start(): #from mylar import updater, searcher, librarysync, postprocessor from mylar import updater, search, weeklypull + SCHED.add_interval_job(updater.dbUpdate, hours=48) SCHED.add_interval_job(search.searchforissue, minutes=SEARCH_INTERVAL) #SCHED.add_interval_job(librarysync.libraryScan, minutes=LIBRARYSCAN_INTERVAL) @@ -551,7 +559,9 @@ def start(): threading.Thread(target=weeklypull.pullit).start() #now the scheduler (check every 24 hours) SCHED.add_interval_job(weeklypull.pullit, hours=24) - + + #let's do a run at the Wanted issues here (on startup). + threading.Thread(target=search.searchforissue).start() if CHECK_GITHUB: SCHED.add_interval_job(versioncheck.checkGithub, minutes=CHECK_GITHUB_INTERVAL) @@ -575,50 +585,13 @@ def dbcheck(): c.execute('CREATE TABLE IF NOT EXISTS weekly (SHIPDATE text, PUBLISHER text, ISSUE text, COMIC VARCHAR(150), EXTRA text, STATUS text)') # c.execute('CREATE TABLE IF NOT EXISTS sablog (nzo_id TEXT, ComicName TEXT, ComicYEAR TEXT, ComicIssue TEXT, name TEXT, nzo_complete TEXT)') + conn.commit + c.close #new - c.execute('DROP TABLE IF EXISTS exceptions') - c.execute('CREATE TABLE IF NOT EXISTS exceptions (variloop TEXT, ComicID TEXT, NewComicID TEXT, GComicID TEXT)') - - # for Mylar-based Exception Updates.... - i = 0 - EXCEPTIONS = [] - EXCEPTIONS.append('exceptions.csv') - EXCEPTIONS.append('custom_exceptions.csv') - - while (i <= 1): - #EXCEPTIONS_FILE = os.path.join(DATA_DIR, 'exceptions.csv') - EXCEPTIONS_FILE = os.path.join(DATA_DIR, EXCEPTIONS[i]) - - if not os.path.exists(EXCEPTIONS_FILE): - try: - csvfile = open(str(EXCEPTIONS_FILE), "rb") - except (OSError,IOError): - if i == 1: - logger.error("No Custom Exceptions found. Using base exceptions only.") - else: - logger.error("Could not locate " + str(EXCEPTIONS[i]) + " file. Make sure it's in datadir: " + DATA_DIR) - break - else: - csvfile = open(str(EXCEPTIONS_FILE), "rb") - if i == 0: - logger.info(u"Populating Base Exception listings into Mylar....") - elif i == 1: - logger.info(u"Populating Custom Exception listings into Mylar....") - - creader = csv.reader(csvfile, delimiter=',') - - for row in creader: - try: - c.execute("INSERT INTO exceptions VALUES (?,?,?,?);", row) - except Exception, e: - #print ("Error - invald arguments...-skipping") - pass - csvfile.close() - i+=1 - - #c.executemany("INSERT INTO exceptions VALUES (?, ?);", to_db) + csv_load() + #add in the late players to the game.... try: c.execute('SELECT LastUpdated from comics') @@ -652,10 +625,64 @@ def dbcheck(): except sqlite3.OperationalError: c.execute('ALTER TABLE issues ADD COLUMN ComicSize TEXT') + #let's delete errant comics that are stranded (ie. None) + c.execute("DELETE from COMICS WHERE ComicName='None'") + logger.info(u"Ensuring DB integrity - Removing all Erroneous Comics (ie. named None)") + conn.commit() c.close() - + +def csv_load(): + # for redudant module calls..include this. + conn=sqlite3.connect(DB_FILE) + c=conn.cursor() + + c.execute('DROP TABLE IF EXISTS exceptions') + + c.execute('CREATE TABLE IF NOT EXISTS exceptions (variloop TEXT, ComicID TEXT, NewComicID TEXT, GComicID TEXT)') + + # for Mylar-based Exception Updates.... + i = 0 + EXCEPTIONS = [] + EXCEPTIONS.append('exceptions.csv') + EXCEPTIONS.append('custom_exceptions.csv') + + while (i <= 1): + #EXCEPTIONS_FILE = os.path.join(DATA_DIR, 'exceptions.csv') + EXCEPTIONS_FILE = os.path.join(DATA_DIR, EXCEPTIONS[i]) + + if not os.path.exists(EXCEPTIONS_FILE): + try: + csvfile = open(str(EXCEPTIONS_FILE), "rb") + except (OSError,IOError): + if i == 1: + logger.error("No Custom Exceptions found. Using base exceptions only.") + else: + logger.error("Could not locate " + str(EXCEPTIONS[i]) + " file. Make sure it's in datadir: " + DATA_DIR) + break + else: + csvfile = open(str(EXCEPTIONS_FILE), "rb") + if i == 0: + logger.info(u"Populating Base Exception listings into Mylar....") + elif i == 1: + logger.info(u"Populating Custom Exception listings into Mylar....") + + creader = csv.reader(csvfile, delimiter=',') + + for row in creader: + try: + c.execute("INSERT INTO exceptions VALUES (?,?,?,?);", row) + except Exception, e: + #print ("Error - invald arguments...-skipping") + pass + pass + csvfile.close() + i+=1 + + conn.commit() + c.close() + def shutdown(restart=False, update=False): cherrypy.engine.exit() diff --git a/mylar/mb.py b/mylar/mb.py index 199b5435..22b95868 100755 --- a/mylar/mb.py +++ b/mylar/mb.py @@ -91,7 +91,8 @@ def findComic(name, mode, issue): pubnames = publishers[0].getElementsByTagName('name') if len(pubnames) >0: xmlpub = pubnames[0].firstChild.wholeText - if (result.getElementsByTagName('name')[0].childNodes[0].nodeValue) is None: +# if (result.getElementsByTagName('name')[0].childNodes[0].nodeValue) is None: + if (result.getElementsByTagName('image')[0].childNodes[0].nodeValue) is None: xmlimage = result.getElementsByTagName('super_url')[0].firstChild.wholeText else: xmlimage = "cache/blankcover.jpg" diff --git a/mylar/parseit.py b/mylar/parseit.py index d1aa8140..172de522 100755 --- a/mylar/parseit.py +++ b/mylar/parseit.py @@ -14,13 +14,13 @@ # along with Mylar. If not, see . -from bs4 import BeautifulSoup -import urllib2 -import re -import helpers -import logger -import datetime -from decimal import Decimal +from bs4 import BeautifulSoup +import urllib2 +import re +import helpers +import logger +import datetime +from decimal import Decimal from HTMLParser import HTMLParseError def GCDScraper(ComicName, ComicYear, Total, ComicID): @@ -32,10 +32,10 @@ def GCDScraper(ComicName, ComicYear, Total, ComicID): comicyr = ComicYear comicis = Total comicid = ComicID - #print ( "comicname: " + str(comicnm) ) - #print ( "comicyear: " + str(comicyr) ) - #print ( "comichave: " + str(comicis) ) - #print ( "comicid: " + str(comicid) ) + print ( "comicname: " + str(comicnm) ) + print ( "comicyear: " + str(comicyr) ) + print ( "comichave: " + str(comicis) ) + print ( "comicid: " + str(comicid) ) comicnm = re.sub(' ', '+', comicnm) input = 'http://www.comics.org/search/advanced/process/?target=series&method=icontains&logic=False&order2=date&order3=&start_date=' + str(comicyr) + '-01-01&end_date=' + str(NOWyr) + '-12-31&series=' + str(comicnm) + '&is_indexed=None' response = urllib2.urlopen ( input ) @@ -413,3 +413,160 @@ def GCDAdd(gcdcomicid): }) series['serieschoice'] = serieschoice return series + + +def ComChk(ComicName, ComicYear, ComicPublisher, Total, ComicID): + comchkchoice = [] + comchoice = {} + + NOWyr = datetime.date.today().year + if datetime.date.today().month == 12: + NOWyr = NOWyr + 1 + logger.fdebug("We're in December, incremented search Year to increase search results: " + str(NOWyr)) + comicnm = ComicName + comicyr = ComicYear + comicis = Total + comicid = ComicID + comicpub = ComicPublisher + print ( "comicname: " + str(comicnm) ) + print ( "comicyear: " + str(comicyr) ) + print ( "comichave: " + str(comicis) ) + print ( "comicpub: " + str(comicpub) ) + print ( "comicid: " + str(comicid) ) + # do 3 runs at the comics.org search to get the best results + comicrun = [] + # &pub_name=DC + # have to remove the spaces from Publisher or else will not work (ie. DC Comics vs DC will not match) + # take the 1st word ;) + #comicpub = comicpub.split()[0] + # if it's not one of the BIG publisher's it might fail - so let's increase the odds. + pubbiggies = [ 'DC', + 'Marvel', + 'Image', + 'IDW' ] + uhuh = "no" + for pb in pubbiggies: + if pb in comicpub: + #keep publisher in url if a biggie. + uhuh = "yes" + print (" publisher match : " + str(comicpub)) + conv_pub = comicpub.split()[0] + print (" converted publisher to : " + str(conv_pub)) + #1st run setup - leave it all as it is. + comicrun.append(comicnm) + cruncnt = 0 + #2nd run setup - remove the last character and do a broad search (keep year or else will blow up) + if len(str(comicnm).split()) > 2: + comicrun.append(' '.join(comicnm.split(' ')[:-1])) + cruncnt+=1 + # to increase the likely hood of matches and to get a broader scope... + # lets remove extra characters + if re.sub('[\.\,\:]', '', comicnm) != comicnm: + comicrun.append(re.sub('[\.\,\:]', '', comicnm)) + cruncnt+=1 + totalcount = 0 + cr = 0 + print ("cruncnt is " + str(cruncnt)) + while (cr <= cruncnt): + print ("cr is " + str(cr)) + comicnm = comicrun[cr] + #leaving spaces in will screw up the search...let's take care of it + comicnm = re.sub(' ', '+', comicnm) + print ("comicnm: " + str(comicnm)) + #input = 'http://www.comics.org/series/name/' + str(comicnm) + '/sort/alpha' + if uhuh == "yes": + publink = "&pub_name=" + str(conv_pub) + if uhuh == "no": + publink = "&pub_name=" +# input = 'http://www.comics.org/search/advanced/process/?target=series&method=icontains&logic=False&order2=date&order3=&start_date=' + str(comicyr) + '-01-01&end_date=' + str(NOWyr) + '-12-31&series=' + str(comicnm) + str(publink) + '&is_indexed=None' + input = 'http://www.comics.org/search/advanced/process/?target=series&method=icontains&logic=False&keywords=&order1=series&order2=date&order3=&start_date=' + str(comicyr) + '-01-01&end_date=' + str(NOWyr) + '-12-31' + '&title=&feature=&job_number=&pages=&script=&pencils=&inks=&colors=&letters=&story_editing=&genre=&characters=&synopsis=&reprint_notes=&story_reprinted=None¬es=' + str(publink) + '&pub_notes=&brand=&brand_notes=&indicia_publisher=&is_surrogate=None&ind_pub_notes=&series=' + str(comicnm) + '&series_year_began=&series_notes=&tracking_notes=&issue_count=&is_comics=None&format=&color=&dimensions=&paper_stock=&binding=&publishing_format=&issues=&volume=&issue_title=&variant_name=&issue_date=&indicia_frequency=&price=&issue_pages=&issue_editing=&isbn=&barcode=&issue_notes=&issue_reprinted=None&is_indexed=None' + print ("input: " + str(input)) + response = urllib2.urlopen ( input ) + soup = BeautifulSoup ( response) + cnt1 = len(soup.findAll("tr", {"class" : "listing_even"})) + cnt2 = len(soup.findAll("tr", {"class" : "listing_odd"})) + + try: + cntit = soup.find("div", {"class" : "item_data"}) +# catchit = pubst('a')[0] + + except (IndexError, TypeError): + cntit = soup.findAll("div", {"class" : "left"})[1] +# catchit = pubst.find("a") + + truecnt = cntit.findNext(text=True) + cnt = int(cnt1 + cnt2) + print ("truecnt: " + str(truecnt)) + print ("cnt1: " + str(cnt1)) + print ("cnt2: " + str(cnt2)) + print (str(cnt) + " results") + + resultName = [] + resultID = [] + resultYear = [] + resultIssues = [] + resultPublisher = [] + resultURL = None + n_odd = -1 + n_even = -1 + n = 0 + while ( n < cnt ): + if n%2==0: + n_even+=1 + resultp = soup.findAll("tr", {"class" : "listing_even"})[n_even] + else: + n_odd+=1 + resultp = soup.findAll("tr", {"class" : "listing_odd"})[n_odd] + rtp = resultp('a')[1] + resultName.append(helpers.cleanName(rtp.findNext(text=True))) + print ( "Comic Name: " + str(resultName[n]) ) + + pub = resultp('a')[0] + resultPublisher.append(pub.findNext(text=True)) + print ( "Publisher: " + str(resultPublisher[n]) ) + + fip = resultp('a',href=True)[1] + resultID.append(fip['href']) +# print ( "ID: " + str(resultID[n]) ) + + subtxt3 = resultp('td')[3] + resultYear.append(subtxt3.findNext(text=True)) + resultYear[n] = resultYear[n].replace(' ','') + subtxt4 = resultp('td')[4] + resultIssues.append(helpers.cleanName(subtxt4.findNext(text=True))) + resiss = resultIssues[n].find('issue') + resiss = int(resiss) + resultIssues[n] = resultIssues[n].replace('','')[:resiss] + resultIssues[n] = resultIssues[n].replace(' ','') +# print ( "Year: " + str(resultYear[n]) ) +# print ( "Issues: " + str(resultIssues[n]) ) + print ("comchkchoice: " + str(comchkchoice)) +# if (cr == 0 and n == 0) or (comchkchoice is None): +# print ("initial add.") +# comchkchoice.append({ +# "ComicID": str(comicid), +# "ComicName": str(resultName[n]), +# "GCDID": str(resultID[n]), +# "ComicYear" : str(resultYear[n]), +# "ComicPublisher" : str(resultPublisher[n]), +# "ComicIssues" : str(resultIssues[n]) +# }) + if not any(d.get('GCDID', None) == str(resultID[n]) for d in comchkchoice): + print ( str(resultID[n]) + " not in DB...adding.") + comchkchoice.append({ + "ComicID": str(comicid), + "ComicName": str(resultName[n]), + "GCDID": str(resultID[n]).split('/')[2], + "ComicYear" : str(resultYear[n]), + "ComicPublisher" : str(resultPublisher[n]), + "ComicURL" : "http://www.comics.org" + str(resultID[n]), + "ComicIssues" : str(resultIssues[n]) + }) + else: + print ( str(resultID[n]) + " already in DB...skipping" ) + n+=1 + cr+=1 + totalcount= totalcount + cnt + comchoice['comchkchoice'] = comchkchoice + return comchoice, totalcount + diff --git a/mylar/search.py b/mylar/search.py index 8bbcc4cb..12a038c0 100755 --- a/mylar/search.py +++ b/mylar/search.py @@ -16,14 +16,13 @@ from __future__ import division import mylar -from mylar import logger, db, updater, helpers, parseit, findcomicfeed +from mylar import logger, db, updater, helpers, parseit, findcomicfeed, prov_nzbx nzbsu_APIkey = mylar.NZBSU_APIKEY dognzb_APIkey = mylar.DOGNZB_APIKEY LOG = mylar.LOG_DIR -import pickle import lib.feedparser as feedparser import urllib import os, errno @@ -50,6 +49,9 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, IssueDate, IssueI if mylar.DOGNZB == 1: nzbprovider.append('dognzb') nzbp+=1 + if mylar.NZBX == 1: + nzbprovider.append('nzbx') + nzbp+=1 # -------- # Xperimental if mylar.EXPERIMENTAL == 1: @@ -122,7 +124,6 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, IssueDate, IssueI findit = NZB_SEARCH(AlternateSearch, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, IssDateFix, IssueID, newznab_host) if findit == 'yes': break - nzbpr-=1 elif nzbprovider[nzbpr] == 'experimental': @@ -141,6 +142,22 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, IssueDate, IssueI nzbpr-=1 + elif nzbprovider[nzbpr] == 'nzbx': + # this is for nzbx.co + nzbprov = 'nzbx' + findit = NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, IssDateFix, IssueID) + if findit == 'yes': + logger.fdebug("findit = found!") + break + else: + if AlternateSearch is not None: + logger.info(u"Alternate Search pattern detected...re-adjusting to : " + str(AlternateSearch) + " " + str(ComicYear)) + findit = NZB_SEARCH(AlternateSearch, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, IssDateFix, IssueID) + if findit == 'yes': + break + + nzbpr-=1 + elif nzbprovider[nzbpr] == 'nzb.su': # this is for nzb.su nzbprov = 'nzb.su' @@ -183,11 +200,13 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, IssueDate, IssueI return findit def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, IssDateFix, IssueID, newznab_host=None): - logger.info(u"Shhh be very quiet...I'm looking for " + ComicName + " issue: " + str(IssueNumber) + "(" + str(ComicYear) + ") using " + str(nzbprov)) + if nzbprov == 'nzb.su': apikey = mylar.NZBSU_APIKEY elif nzbprov == 'dognzb': apikey = mylar.DOGNZB_APIKEY + elif nzbprov == 'nzbx': + apikey = 'none' elif nzbprov == 'experimental': apikey = 'none' elif nzbprov == 'newznab': @@ -195,6 +214,9 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is apikey = newznab_host[1] logger.fdebug("using Newznab host of : " + str(host_newznab)) + logger.info(u"Shhh be very quiet...I'm looking for " + ComicName + " issue: " + str(IssueNumber) + "(" + str(ComicYear) + ") using " + str(nzbprov)) + + if mylar.PREFERRED_QUALITY == 0: filetype = "" elif mylar.PREFERRED_QUALITY == 1: filetype = ".cbr" elif mylar.PREFERRED_QUALITY == 2: filetype = ".cbz" @@ -311,12 +333,17 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is elif nzbprov == 'newznab': findurl = str(host_newznab) + "api?t=search&q=" + str(comsearch[findloop]) + "&apikey=" + str(apikey) + "&o=xml&cat=7030" logger.fdebug("search-url: " + str(findurl)) - bb = feedparser.parse(findurl) + elif nzbprov == 'nzbx': + bb = prov_nzbx.searchit(comsearch[findloop]) + logger.fdebug("nzbx.co!") + if nzbprov != 'nzbx': + bb = feedparser.parse(findurl) elif nzbprov == 'experimental': #bb = parseit.MysterBinScrape(comsearch[findloop], comyear) bb = findcomicfeed.Startit(cm, isssearch[findloop], comyear) # since the regexs in findcomicfeed do the 3 loops, lets force the exit after cmloopit == 1 + done = False foundc = "no" log2file = "" @@ -552,11 +579,13 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is if mylar.BLACKHOLE: logger.fdebug("using blackhole directory at : " + str(mylar.BLACKHOLE_DIR)) if os.path.exists(mylar.BLACKHOLE_DIR): - filenamenzb = str(ComicName) + " " + str(IssueNumber) + " (" + str(comyear) + ").nzb" + #pretty this biatch up. + Bl_ComicName = re.sub('[/:/,\/]', '', str(ComicName)) + filenamenzb = str(Bl_ComicName) + " " + str(IssueNumber) + " (" + str(comyear) + ").nzb" urllib.urlretrieve(linkapi, str(mylar.BLACKHOLE_DIR) + str(filenamenzb)) logger.fdebug("filename saved to your blackhole as : " + str(filenamenzb)) logger.info(u"Successfully sent .nzb to your Blackhole directory : " + str(mylar.BLACKHOLE_DIR) + str(filenamenzb) ) - nzbname = str(ComicName) + " " + str(IssueNumber) + " (" + str(comyear) + ")" + nzbname = str(Bl_ComicName) + " " + str(IssueNumber) + " (" + str(comyear) + ")" #end blackhole else: @@ -676,7 +705,7 @@ def searchforissue(issueid=None, new=False): else: ComicYear = str(result['IssueDate'])[:4] - if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL or mylar.NEWZNAB) and (mylar.SAB_HOST): + if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL or mylar.NEWZNAB or mylar.NZBX) and (mylar.SAB_HOST): foundNZB = search_init(result['ComicName'], result['Issue_Number'], str(ComicYear), comic['ComicYear'], IssueDate, result['IssueID'], AlternateSearch) if foundNZB == "yes": #print ("found!") @@ -697,7 +726,7 @@ def searchforissue(issueid=None, new=False): IssueYear = str(result['IssueDate'])[:4] foundNZB = "none" - if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL or mylar.NEWZNAB) and (mylar.SAB_HOST): + if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL or mylar.NEWZNAB or mylar.NZBX) and (mylar.SAB_HOST): foundNZB = search_init(result['ComicName'], result['Issue_Number'], str(IssueYear), comic['ComicYear'], IssueDate, result['IssueID'], AlternateSearch) if foundNZB == "yes": #print ("found!") @@ -719,7 +748,7 @@ def searchIssueIDList(issuelist): ComicYear = comic['ComicYear'] else: ComicYear = str(issue['IssueDate'])[:4] - if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL or mylar.NEWZNAB) and (mylar.SAB_HOST): + if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL or mylar.NEWZNAB or mylar.NZBX) and (mylar.SAB_HOST): foundNZB = search_init(comic['ComicName'], issue['Issue_Number'], str(ComicYear), comic['ComicYear'], issue['IssueDate'], issue['IssueID'], AlternateSearch) if foundNZB == "yes": #print ("found!") diff --git a/mylar/updater.py b/mylar/updater.py index 3e906675..525b2e35 100755 --- a/mylar/updater.py +++ b/mylar/updater.py @@ -149,7 +149,8 @@ def no_searchresults(ComicID): # something other than 'Loaded' myDB = db.DBConnection() controlValue = { "ComicID": ComicID} - newValue = {"Status": "Error"} + newValue = {"Status": "Error", + "ComicName": "None"} myDB.upsert("comics", newValue, controlValue) def nzblog(IssueID, NZBName): @@ -232,7 +233,7 @@ def forceRescan(ComicID): break temploc = tmpfc['ComicFilename'].replace('_', ' ') temploc = re.sub('[\#\']', '', temploc) - logger.fdebug("temploc: " + str(temploc)) + #logger.fdebug("temploc: " + str(temploc)) if 'annual' not in temploc: fcnew = shlex.split(str(temploc)) fcn = len(fcnew) @@ -247,14 +248,14 @@ def forceRescan(ComicID): issyear = reiss['IssueDate'][:4] old_status = reiss['Status'] - logger.fdebug("integer_issue:" + str(int_iss) + " ... status: " + str(old_status)) + #logger.fdebug("integer_issue:" + str(int_iss) + " ... status: " + str(old_status)) #if comic in format of "SomeSeries 5(c2c)(2013).cbr" whatever...it'll die. #can't distinguish the 5(c2c) to tell it's the issue #... while (som < fcn): #counts get buggered up when the issue is the last field in the filename - ie. '50.cbr' - logger.fdebug("checking word - " + str(fcnew[som])) + #logger.fdebug("checking word - " + str(fcnew[som])) if ".cbr" in fcnew[som]: fcnew[som] = fcnew[som].replace(".cbr", "") elif ".cbz" in fcnew[som]: @@ -265,7 +266,7 @@ def forceRescan(ComicID): if fcnew[som] != " ": fcnew[som] = get_issue[0] if '.' in fcnew[som]: - logger.fdebug("decimal detected...adjusting.") + #logger.fdebug("decimal detected...adjusting.") try: i = float(fcnew[som]) except ValueError, TypeError: @@ -277,7 +278,7 @@ def forceRescan(ComicID): pass if fcnew[som].isdigit(): #this won't match on decimal issues - need to fix. - logger.fdebug("digit detected") + #logger.fdebug("digit detected") if int(fcnew[som]) > 0: # fcdigit = fcnew[som].lstrip('0') #fcdigit = str(int(fcnew[som])) @@ -288,7 +289,7 @@ def forceRescan(ComicID): elif "." in fcnew[som]: #this will match on decimal issues IssueChk = fcnew[som] - logger.fdebug("decimal detected...analyzing if issue") + #logger.fdebug("decimal detected...analyzing if issue") isschk_find = IssueChk.find('.') isschk_b4dec = IssueChk[:isschk_find] isschk_decval = IssueChk[isschk_find+1:] @@ -325,11 +326,11 @@ def forceRescan(ComicID): else: # it's a word, skip it. fcdigit = 1000000 - logger.fdebug("fcdigit: " + str(fcdigit)) - logger.fdebug("int_iss: " + str(int_iss)) + #logger.fdebug("fcdigit: " + str(fcdigit)) + #logger.fdebug("int_iss: " + str(int_iss)) if "." in str(int_iss): int_iss = helpers.decimal_issue(int_iss) - logger.fdebug("this is the int issue:" + str(int_iss)) + #logger.fdebug("this is the int issue:" + str(int_iss)) if int(fcdigit) == int_iss: #if issyear in fcnew[som+1]: diff --git a/mylar/webserve.py b/mylar/webserve.py index b3d705fd..438d0d74 100755 --- a/mylar/webserve.py +++ b/mylar/webserve.py @@ -24,10 +24,12 @@ from mako import exceptions import time import threading +import csv +import platform import mylar -from mylar import logger, db, importer, mb, search, filechecker, helpers, updater, parseit, weeklypull, PostProcessor +from mylar import logger, db, importer, mb, search, filechecker, helpers, updater, parseit, weeklypull, PostProcessor, version #from mylar.helpers import checked, radio, today import lib.simplejson as simplejson @@ -63,9 +65,6 @@ class WebInterface(object): myDB = db.DBConnection() comic = myDB.action('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone() issues = myDB.select('SELECT * from issues WHERE ComicID=? order by Int_IssueNumber DESC', [ComicID]) - #print (pickle.loads(comic['AlternateSearch'])) - #AlternateSearch = [] - #AlternateSearch.append(pickle.loads (comic['AlternateSearch'])) if comic is None: raise cherrypy.HTTPRedirect("home") comicConfig = { @@ -94,13 +93,46 @@ class WebInterface(object): return serve_template(templatename="searchresults.html", title='Search Results for: "' + name + '"', searchresults=searchresults, type=type) searchit.exposed = True - def addComic(self, comicid, comicname=None, comicyear=None, comicissues=None): + def addComic(self, comicid, comicname=None, comicyear=None, comicimage=None, comicissues=None, comicpublisher=None): myDB = db.DBConnection() sresults = [] + cresults = [] mismatch = "no" + print ("comicid: " + str(comicid)) + print ("comicname: " + str(comicname)) + print ("comicyear: " + str(comicyear)) + print ("comicissues: " + str(comicissues)) + print ("comicimage: " + str(comicimage)) #here we test for exception matches (ie. comics spanning more than one volume, known mismatches, etc). CV_EXcomicid = myDB.action("SELECT * from exceptions WHERE ComicID=?", [comicid]).fetchone() - if CV_EXcomicid is None: pass + if CV_EXcomicid is None: # pass # + gcdinfo=parseit.GCDScraper(comicname, comicyear, comicissues, comicid) + if gcdinfo == "No Match": + updater.no_searchresults(comicid) + nomatch = "true" + logger.info(u"I couldn't find an exact match for " + str(comicname) + " (" + str(comicyear) + ") - gathering data for Error-Checking screen (this could take a minute)..." ) + i = 0 + loopie, cnt = parseit.ComChk(comicname, comicyear, comicpublisher, comicissues, comicid) + print ("total count : " + str(cnt)) + while (i < cnt): + try: + stoopie = loopie['comchkchoice'][i] + except (IndexError, TypeError): + break + cresults.append({ + 'ComicID' : stoopie['ComicID'], + 'ComicName' : stoopie['ComicName'], + 'ComicYear' : stoopie['ComicYear'], + 'ComicIssues' : stoopie['ComicIssues'], + 'ComicURL' : stoopie['ComicURL'], + 'ComicPublisher' : stoopie['ComicPublisher'], + 'GCDID' : stoopie['GCDID'] + }) + i+=1 + return serve_template(templatename="searchfix.html", title="Error Check", comicname=comicname, comicid=comicid, comicyear=comicyear, comicimage=comicimage, comicissues=comicissues,cresults=cresults) + else: + nomatch = "false" + logger.info(u"Quick match success..continuing.") else: if CV_EXcomicid['variloop'] == '99': logger.info(u"mismatched name...autocorrecting to correct GID and auto-adding.") @@ -133,6 +165,24 @@ class WebInterface(object): raise cherrypy.HTTPRedirect("artistPage?ComicID=%s" % comicid) addComic.exposed = True + def from_Exceptions(self, comicid, gcdid, comicname=None, comicyear=None, comicissues=None, comicpublisher=None): + mismatch = "yes" + print ("gcdid:" + str(gcdid)) + #write it to the custom_exceptions.csv and reload it so that importer will pick it up and do it's thing :) + #custom_exceptions in this format... + #99, (comicid), (gcdid), none + logger.info("saving new information into custom_exceptions.csv...") + except_info = "none #" + str(comicname) + "-(" + str(comicyear) + ")" + with open('custom_exceptions.csv', 'a') as f: + f.write('%s,%s,%s,%s\n' % ("99", str(comicid), str(gcdid), str(except_info)) ) + + logger.info("re-loading csv file so it's all nice and current.") + mylar.csv_load() + + threading.Thread(target=importer.addComictoDB, args=[comicid,mismatch]).start() + raise cherrypy.HTTPRedirect("artistPage?ComicID=%s" % comicid) + from_Exceptions.exposed = True + def GCDaddComic(self, comicid, comicname=None, comicyear=None, comicissues=None, comiccover=None, comicpublisher=None): #since we already know most of the info, let's add it to the db so we can reference it later. myDB = db.DBConnection() @@ -475,6 +525,8 @@ class WebInterface(object): interface_dir = os.path.join(mylar.PROG_DIR, 'data/interfaces/') interface_list = [ name for name in os.listdir(interface_dir) if os.path.isdir(os.path.join(interface_dir, name)) ] + branch_history, err = mylar.versioncheck.runGit("log --oneline --pretty=format:'%h - %ar - %s' -n 4") + config = { "http_host" : mylar.HTTP_HOST, "http_user" : mylar.HTTP_USERNAME, @@ -502,6 +554,7 @@ class WebInterface(object): "nzbsu_api" : mylar.NZBSU_APIKEY, "use_dognzb" : helpers.checked(mylar.DOGNZB), "dognzb_api" : mylar.DOGNZB_APIKEY, + "use_nzbx" : helpers.checked(mylar.NZBX), "use_experimental" : helpers.checked(mylar.EXPERIMENTAL), "use_newznab" : helpers.checked(mylar.NEWZNAB), "newznab_host" : mylar.NEWZNAB_HOST, @@ -527,10 +580,31 @@ class WebInterface(object): "zero_level_n" : mylar.ZERO_LEVEL_N, "enable_extra_scripts" : helpers.checked(mylar.ENABLE_EXTRA_SCRIPTS), "extra_scripts" : mylar.EXTRA_SCRIPTS, - "log_dir" : mylar.LOG_DIR + "log_dir" : mylar.LOG_DIR, + "branch" : version.MYLAR_VERSION, + "br_type" : mylar.INSTALL_TYPE, + "br_version" : mylar.versioncheck.getVersion(), + "py_version" : platform.python_version(), + "data_dir" : mylar.DATA_DIR, + "prog_dir" : mylar.PROG_DIR, + "cache_dir" : mylar.CACHE_DIR, + "config_file" : mylar.CONFIG_FILE, + "branch_history" : re.sub('[\n]', '
', branch_history) } return serve_template(templatename="config.html", title="Settings", config=config) config.exposed = True + + def error_change(self, comicid, errorgcd): + if errorgcd[:5].isdigit(): + print ("GCD-ID detected : + str(errorgcd)[:5]") + print ("I'm assuming you know what you're doing - going to force-match.") + self.from_Exceptions(comicid=comicid,gcdid=errorgcd) + else: + print ("Assuming rewording of Comic - adjusting to : " + str(errorgcd)) + self.addComic(errorgcd) + + error_change.exposed = True + def comic_config(self, com_location, alt_search, ComicID): myDB = db.DBConnection() @@ -587,7 +661,7 @@ class WebInterface(object): def configUpdate(self, http_host='0.0.0.0', http_username=None, http_port=8090, http_password=None, launch_browser=0, logverbose=0, download_scan_interval=None, nzb_search_interval=None, libraryscan_interval=None, sab_host=None, sab_username=None, sab_apikey=None, sab_password=None, sab_category=None, sab_priority=0, log_dir=None, blackhole=0, blackhole_dir=None, - usenet_retention=None, nzbsu=0, nzbsu_apikey=None, dognzb=0, dognzb_apikey=None, newznab=0, newznab_host=None, newznab_apikey=None, newznab_enabled=0, + usenet_retention=None, nzbsu=0, nzbsu_apikey=None, dognzb=0, dognzb_apikey=None, nzbx=0, newznab=0, newznab_host=None, newznab_apikey=None, newznab_enabled=0, raw=0, raw_provider=None, raw_username=None, raw_password=None, raw_groups=None, experimental=0, preferred_quality=0, move_files=0, rename_files=0, folder_format=None, file_format=None, enable_extra_scripts=0, extra_scripts=None, destination_dir=None, replace_spaces=0, replace_char=None, autowant_all=0, autowant_upcoming=0, comic_cover_local=0, zero_level=0, zero_level_n=None, interface=None, **kwargs): @@ -613,6 +687,7 @@ class WebInterface(object): mylar.NZBSU_APIKEY = nzbsu_apikey mylar.DOGNZB = dognzb mylar.DOGNZB_APIKEY = dognzb_apikey + mylar.NZBX = nzbx mylar.RAW = raw mylar.RAW_PROVIDER = raw_provider mylar.RAW_USERNAME = raw_username diff --git a/mylar/weeklypull.py b/mylar/weeklypull.py index a78f4655..4ce638d6 100755 --- a/mylar/weeklypull.py +++ b/mylar/weeklypull.py @@ -356,7 +356,8 @@ def pullitcheck(comic1off_name=None,comic1off_id=None): cur.execute("SELECT ComicID, ComicName, ComicYear, ComicPublisher from comics") while True: watchd = cur.fetchone() - if watchd == None: + #print ("watchd: " + str(watchd)) + if watchd is None: break a_list.append(watchd[1]) b_list.append(watchd[2]) @@ -387,6 +388,7 @@ def pullitcheck(comic1off_name=None,comic1off_id=None): logger.fdebug("looking for : " + str(lines[cnt])) sqlsearch = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', ' ', str(lines[cnt])) sqlsearch = re.sub(r'\s', '%', sqlsearch) + if 'THE' in sqlsearch: sqlsearch = re.sub('THE', '', sqlsearch) logger.fdebug("searchsql: " + str(sqlsearch)) weekly = myDB.select('SELECT PUBLISHER, ISSUE, COMIC, EXTRA, SHIPDATE FROM weekly WHERE COMIC LIKE (?)', [sqlsearch]) #cur.execute('SELECT PUBLISHER, ISSUE, COMIC, EXTRA, SHIPDATE FROM weekly WHERE COMIC LIKE (?)', [lines[cnt]]) @@ -419,13 +421,17 @@ def pullitcheck(comic1off_name=None,comic1off_id=None): comicnm = re.sub(r'\s', '', comicnm) logger.fdebug("Revised_Watch: " + str(watchcomic)) logger.fdebug("ComicNM: " + str(comicnm)) - if str(comicnm) == str(watchcomic).upper(): + if 'THE' in str(watchcomic): + modcomicnm = re.sub('THE', '', comicnm) + if str(comicnm) == str(watchcomic).upper() or str(modcomicnm) == str(watchcomic).upper(): logger.fdebug("matched on:" + str(comicnm) + "..." + str(watchcomic).upper()) #pass elif ("ANNUAL" in week['EXTRA']): pass #print ( row[3] + " matched on ANNUAL") else: + if 'THE' in str(comicnm): + modcomicnm = re.sub('THE', '', comicnm) #print ( row[2] + " not an EXACT match...") break #if "WOLVERINE AND X-MEN" in str(comicnm):
@@ -366,15 +403,10 @@
Miscellaneous -
- -
- -
-
- -
- +
+ + +
diff --git a/data/interfaces/default/searchfix.html b/data/interfaces/default/searchfix.html new file mode 100755 index 00000000..ac53eb6a --- /dev/null +++ b/data/interfaces/default/searchfix.html @@ -0,0 +1,108 @@ +<%inherit file="base.html" /> +<%! + import mylar + from mylar.helpers import checked +%> +<%def name="headerIncludes()"> +
+ +
+ + +<%def name="body()"> +
+

Search Question

+
+
+ +
+ + + + + +
+
+
+ +
+
+ +
+
+ 		+
+ Error-Checking... +

I can't add the requsted comic.

+

I've figured out that the Comic that you've selected to watch isn't listed + correctly on the other databases I need to query. This is most likely due to + an incorrect spelling, but sometimes it could because the year is wrong, or even + the issues are incorrect. So you need to help me out on this one - please select + below which series from the list is the one that you want to add.

+
+
This is what you've given me to look for:

+ ${comicname} (${comicyear})
+ ${comicissues} Issues
+
+
+
+ + + + + + + + + + + + %if cresults: + %for result in cresults: + + + + + + + + + %endfor + %else: + + + + %endif + + + +
PublisherComic NameYearIssues
${result['ComicPublisher']}${result['ComicName']}${result['ComicYear']}${result['ComicIssues']}Add Series
There are no results to display
+
+ + + +

+
+ +
+
+
+ +<%def name="javascriptIncludes()"> + + diff --git a/data/interfaces/default/searchresults.html b/data/interfaces/default/searchresults.html index 4faf2c70..5a169d12 100755 --- a/data/interfaces/default/searchresults.html +++ b/data/interfaces/default/searchresults.html @@ -34,7 +34,7 @@ %if type == 'album':
Add this album Add this Comic Add this Comic