mirror of https://github.com/evilhero/mylar
FIX: user-agent added for nzb-providers, IMP: Removed some unnecessary logging from weeklypull
This commit is contained in:
evilhero
parent
49dcba8161
commit
5817ff18f6
|
|
@ -30,7 +30,6 @@ from lib.configobj import ConfigObj
|
|||
import cherrypy
|
||||
|
||||
from mylar import versioncheck, logger, version
|
||||
from mylar.common import *
|
||||
|
||||
FULL_PATH = None
|
||||
PROG_DIR = None
|
||||
|
|
@ -77,6 +76,7 @@ INSTALL_TYPE = None
|
|||
CURRENT_VERSION = None
|
||||
LATEST_VERSION = None
|
||||
COMMITS_BEHIND = None
|
||||
USER_AGENT = None
|
||||
|
||||
CHECK_GITHUB = False
|
||||
CHECK_GITHUB_ON_STARTUP = False
|
||||
|
|
@ -221,7 +221,7 @@ def initialize():
|
|||
|
||||
global __INITIALIZED__, FULL_PATH, PROG_DIR, VERBOSE, DAEMON, DATA_DIR, CONFIG_FILE, CFG, CONFIG_VERSION, LOG_DIR, CACHE_DIR, LOGVERBOSE, \
|
||||
HTTP_PORT, HTTP_HOST, HTTP_USERNAME, HTTP_PASSWORD, HTTP_ROOT, LAUNCH_BROWSER, GIT_PATH, \
|
||||
CURRENT_VERSION, LATEST_VERSION, CHECK_GITHUB, CHECK_GITHUB_ON_STARTUP, CHECK_GITHUB_INTERVAL, MUSIC_DIR, DESTINATION_DIR, \
|
||||
CURRENT_VERSION, LATEST_VERSION, CHECK_GITHUB, CHECK_GITHUB_ON_STARTUP, CHECK_GITHUB_INTERVAL, USER_AGENT, MUSIC_DIR, DESTINATION_DIR, \
|
||||
DOWNLOAD_DIR, USENET_RETENTION, SEARCH_INTERVAL, NZB_STARTUP_SEARCH, INTERFACE, AUTOWANT_ALL, AUTOWANT_UPCOMING, ZERO_LEVEL, ZERO_LEVEL_N, COMIC_COVER_LOCAL, \
|
||||
LIBRARYSCAN, LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, SAB_PRIORITY, SAB_DIRECTORY, BLACKHOLE, BLACKHOLE_DIR, ADD_COMICS, COMIC_DIR, IMP_MOVE, IMP_RENAME, IMP_METADATA, \
|
||||
NZBSU, NZBSU_APIKEY, DOGNZB, DOGNZB_APIKEY, NZBX,\
|
||||
|
|
@ -429,6 +429,15 @@ def initialize():
|
|||
# Get the currently installed version - returns None, 'win32' or the git hash
|
||||
# Also sets INSTALL_TYPE variable to 'win', 'git' or 'source'
|
||||
CURRENT_VERSION = versioncheck.getVersion()
|
||||
hash = CURRENT_VERSION[:7]
|
||||
print ("hash is set to : " + str(hash))
|
||||
|
||||
if version.MYLAR_VERSION == 'master':
|
||||
vers = 'M'
|
||||
else:
|
||||
vers = 'D'
|
||||
|
||||
USER_AGENT = 'Mylar/'+str(hash)+'('+vers+') +http://www.github.com/evilhero/mylar/'
|
||||
|
||||
# Check for new versions
|
||||
if CHECK_GITHUB_ON_STARTUP:
|
||||
|
|
@ -707,6 +716,12 @@ def dbcheck():
|
|||
c.execute('SELECT WatchMatch from importresults')
|
||||
except sqlite3.OperationalError:
|
||||
c.execute('ALTER TABLE importresults ADD COLUMN WatchMatch TEXT')
|
||||
|
||||
try:
|
||||
c.execute('SELECT inCacheDIR from issues')
|
||||
except sqlite3.OperationalError:
|
||||
c.execute('ALTER TABLE issues ADD COLUMN inCacheDIR TEXT')
|
||||
|
||||
# -- not implemented just yet ;)
|
||||
|
||||
# for metadata...
|
||||
|
|
|
|||
|
|
@ -1,34 +0,0 @@
|
|||
# This file is part of Mylar.
|
||||
#
|
||||
# Mylar is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# Mylar is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with Mylar. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Created on Aug 1, 2011
|
||||
|
||||
@author: Michael
|
||||
'''
|
||||
import platform, operator, os, re
|
||||
|
||||
from mylar import version
|
||||
|
||||
#Identify Our Application
|
||||
USER_AGENT = 'Mylar/-'+version.MYLAR_VERSION+' ('+platform.system()+' '+platform.release()+')'
|
||||
|
||||
### Notification Types
|
||||
NOTIFY_SNATCH = 1
|
||||
NOTIFY_DOWNLOAD = 2
|
||||
|
||||
notifyStrings = {}
|
||||
notifyStrings[NOTIFY_SNATCH] = "Started Download"
|
||||
notifyStrings[NOTIFY_DOWNLOAD] = "Download Finished"
|
||||
|
|
@ -14,14 +14,17 @@ def searchit(cm):
|
|||
searchURL = 'https://nzbx.co/api/search?cat=7030&q=' + str(cm)
|
||||
|
||||
logger.fdebug(u'Parsing results from <a href="%s">nzbx.co</a>' % searchURL)
|
||||
|
||||
request = urllib2.Request(searchURL)
|
||||
request.add_header(mylar.USER_AGENT)
|
||||
opener = urllib2.build_opener()
|
||||
|
||||
try:
|
||||
data = urllib2.urlopen(searchURL, timeout=20).read()
|
||||
except urllib2.URLError, e:
|
||||
logger.fdebug('Error fetching data from nzbx.co: %s' % str(e))
|
||||
data = opener.open(request).read()
|
||||
except Exception, e:
|
||||
logger.warn('Error fetching data from nzbx.co : %s' % str(e))
|
||||
data = False
|
||||
return "no results"
|
||||
|
||||
|
||||
if data:
|
||||
|
||||
d = json.loads(data)
|
||||
|
|
|
|||
|
|
@ -327,9 +327,10 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is
|
|||
comsearch[findloop] = comsrc + "%200" + isssearch[findloop] + "%20" + str(filetype)
|
||||
elif cmloopit == 1:
|
||||
comsearch[findloop] = comsrc + "%20" + isssearch[findloop] + "%20" + str(filetype)
|
||||
logger.fdebug("comsearch: " + str(comsearch))
|
||||
logger.fdebug("cmloopit: " + str(cmloopit))
|
||||
logger.fdebug("done: " + str(done))
|
||||
#logger.fdebug("comsearch: " + str(comsearch))
|
||||
#logger.fdebug("cmloopit: " + str(cmloopit))
|
||||
#logger.fdebug("done: " + str(done))
|
||||
|
||||
if nzbprov != 'experimental':
|
||||
if nzbprov == 'dognzb':
|
||||
findurl = "http://dognzb.cr/api?t=search&apikey=" + str(apikey) + "&q=" + str(comsearch[findloop]) + "&o=xml&cat=7030"
|
||||
|
|
@ -342,9 +343,24 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is
|
|||
logger.fdebug("search-url: " + str(findurl))
|
||||
elif nzbprov == 'nzbx':
|
||||
bb = prov_nzbx.searchit(comsearch[findloop])
|
||||
logger.fdebug("nzbx.co!")
|
||||
if nzbprov != 'nzbx':
|
||||
bb = feedparser.parse(findurl)
|
||||
# Add a user-agent
|
||||
print ("user-agent:" + str(mylar.USER_AGENT))
|
||||
request = urllib2.Request(findurl)
|
||||
request.add_header('User Agent', str(mylar.USER_AGENT))
|
||||
opener = urllib2.build_opener()
|
||||
|
||||
try:
|
||||
data = opener.open(request).read()
|
||||
except Exception, e:
|
||||
logger.warn('Error fetching data from %s: %s' % (nzbprov, e))
|
||||
data = False
|
||||
|
||||
if data:
|
||||
bb = feedparser.parse(data)
|
||||
else:
|
||||
bb = "no results"
|
||||
|
||||
elif nzbprov == 'experimental':
|
||||
#bb = parseit.MysterBinScrape(comsearch[findloop], comyear)
|
||||
bb = findcomicfeed.Startit(cm, isssearch[findloop], comyear)
|
||||
|
|
|
|||
|
|
@ -246,7 +246,7 @@ def pullit():
|
|||
try:
|
||||
comicnm = comicnm + " " + issname[n]
|
||||
except IndexError:
|
||||
print ("went too far looking at this comic...adjusting.")
|
||||
#print ("went too far looking at this comic...adjusting.")
|
||||
comicnm = comicnm
|
||||
break
|
||||
n+=1
|
||||
|
|
@ -404,12 +404,12 @@ def pullitcheck(comic1off_name=None,comic1off_id=None):
|
|||
while (cnt > -1):
|
||||
lines[cnt] = str(lines[cnt]).upper()
|
||||
#llen[cnt] = str(llen[cnt])
|
||||
logger.fdebug("looking for : " + str(lines[cnt]))
|
||||
#logger.fdebug("looking for : " + str(lines[cnt]))
|
||||
sqlsearch = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\'\?\@]', ' ', str(lines[cnt]))
|
||||
sqlsearch = re.sub(r'\s', '%', sqlsearch)
|
||||
if 'THE' in sqlsearch: sqlsearch = re.sub('THE', '', sqlsearch)
|
||||
if '+' in sqlsearch: sqlsearch = re.sub('\+', '%PLUS%', sqlsearch)
|
||||
logger.fdebug("searchsql: " + str(sqlsearch))
|
||||
#logger.fdebug("searchsql: " + str(sqlsearch))
|
||||
weekly = myDB.select('SELECT PUBLISHER, ISSUE, COMIC, EXTRA, SHIPDATE FROM weekly WHERE COMIC LIKE (?)', [sqlsearch])
|
||||
#cur.execute('SELECT PUBLISHER, ISSUE, COMIC, EXTRA, SHIPDATE FROM weekly WHERE COMIC LIKE (?)', [lines[cnt]])
|
||||
for week in weekly:
|
||||
|
|
@ -417,21 +417,20 @@ def pullitcheck(comic1off_name=None,comic1off_id=None):
|
|||
break
|
||||
for nono in not_t:
|
||||
if nono in week['PUBLISHER']:
|
||||
logger.fdebug("nono present")
|
||||
#logger.fdebug("nono present")
|
||||
break
|
||||
if nono in week['ISSUE']:
|
||||
logger.fdebug("graphic novel/tradeback detected..ignoring.")
|
||||
#logger.fdebug("graphic novel/tradeback detected..ignoring.")
|
||||
break
|
||||
for nothere in not_c:
|
||||
if nothere in week['EXTRA']:
|
||||
logger.fdebug("nothere present")
|
||||
#logger.fdebug("nothere present")
|
||||
break
|
||||
else:
|
||||
comicnm = week['COMIC']
|
||||
#here's the tricky part, ie. BATMAN will match on
|
||||
#every batman comic, not exact
|
||||
logger.fdebug("comparing" + str(comicnm) + "..to.." + str(unlines[cnt]).upper())
|
||||
#logger.fdebug("comparing" + str(sqlsearch) + "..to.." + str(unlines[cnt]).upper())
|
||||
#logger.fdebug("comparing" + str(comicnm) + "..to.." + str(unlines[cnt]).upper())
|
||||
|
||||
#-NEW-
|
||||
# strip out all special characters and compare
|
||||
|
|
@ -439,8 +438,8 @@ def pullitcheck(comic1off_name=None,comic1off_id=None):
|
|||
comicnm = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', '', str(comicnm))
|
||||
watchcomic = re.sub(r'\s', '', watchcomic)
|
||||
comicnm = re.sub(r'\s', '', comicnm)
|
||||
logger.fdebug("Revised_Watch: " + str(watchcomic))
|
||||
logger.fdebug("ComicNM: " + str(comicnm))
|
||||
#logger.fdebug("Revised_Watch: " + str(watchcomic))
|
||||
#logger.fdebug("ComicNM: " + str(comicnm))
|
||||
if 'THE' in str(watchcomic).upper():
|
||||
modwatchcomic = re.sub('THE', '', watchcomic.upper())
|
||||
modcomicnm = re.sub('THE', '', comicnm)
|
||||
|
|
@ -452,8 +451,8 @@ def pullitcheck(comic1off_name=None,comic1off_id=None):
|
|||
if 'plus' in str(comicnm).lower():
|
||||
modcomicnm = re.sub('plus', '+', comicnm)
|
||||
if str(comicnm) == str(watchcomic).upper() or str(modcomicnm) == str(modwatchcomic).upper():
|
||||
logger.fdebug("matched on:" + str(comicnm) + "..." + str(watchcomic).upper())
|
||||
#pass
|
||||
#logger.fdebug("matched on:" + str(comicnm) + "..." + str(watchcomic).upper())
|
||||
pass
|
||||
elif ("ANNUAL" in week['EXTRA']):
|
||||
pass
|
||||
#print ( row[3] + " matched on ANNUAL")
|
||||
|
|
|
|||
Loading…
Reference in New Issue