mirror of https://github.com/evilhero/mylar
IMP: Added maintenance mode to command line option. Allows for exporting/importing of either json file/db files, as well as updating Mylar without having to load the GUI which sometimes causes problems, FIX: Removed extra Mark as Skipped option from appearing on comic details page, FIX: Added series cover fallback for alternate issue images if issue images were not populated yet, FIX: Removed some unneeded logging lines during weekly pull check
This commit is contained in:
evilhero
parent
3ffec19628
commit
e060f37c8e
100
Mylar.py
100
Mylar.py
|
|
@ -25,7 +25,7 @@ sys.path.insert(1, os.path.join(os.path.dirname(__file__), 'lib'))
|
|||
|
||||
import mylar
|
||||
|
||||
from mylar import webstart, logger, filechecker, versioncheck
|
||||
from mylar import webstart, logger, filechecker, versioncheck, maintenance
|
||||
|
||||
import argparse
|
||||
|
||||
|
|
@ -64,7 +64,10 @@ def main():
|
|||
|
||||
# Set up and gather command line arguments
|
||||
parser = argparse.ArgumentParser(description='Automated Comic Book Downloader')
|
||||
subparsers = parser.add_subparsers(title='Subcommands', dest='maintenance')
|
||||
parser_maintenance = subparsers.add_parser('maintenance', help='Enter maintenance mode (no GUI). Additional commands are available (maintenance --help)')
|
||||
|
||||
#main parser
|
||||
parser.add_argument('-v', '--verbose', action='store_true', help='Increase console logging verbosity')
|
||||
parser.add_argument('-q', '--quiet', action='store_true', help='Turn off console logging')
|
||||
parser.add_argument('-d', '--daemon', action='store_true', help='Run as a daemon')
|
||||
|
|
@ -76,10 +79,23 @@ def main():
|
|||
parser.add_argument('--nolaunch', action='store_true', help='Prevent browser from launching on startup')
|
||||
parser.add_argument('--pidfile', help='Create a pid file (only relevant when running as a daemon)')
|
||||
parser.add_argument('--safe', action='store_true', help='redirect the startup page to point to the Manage Comics screen on startup')
|
||||
#parser.add_argument('-u', '--update', action='store_true', help='force mylar to perform an update as if in GUI')
|
||||
parser_maintenance.add_argument('-xj', '--exportjson', action='store', help='Export existing mylar.db to json file')
|
||||
parser_maintenance.add_argument('-id', '--importdatabase', action='store', help='Import a mylar.db into current db')
|
||||
parser_maintenance.add_argument('-ij', '--importjson', action='store', help='Import a specified json file containing just {"ComicID": "XXXXX"} into current db')
|
||||
parser_maintenance.add_argument('-st', '--importstatus', action='store_true', help='Provide current maintenance status')
|
||||
parser_maintenance.add_argument('-u', '--update', action='store_true', help='force mylar to perform an update as if in GUI')
|
||||
#parser_maintenance.add_argument('-it', '--importtext', action='store', help='Import a specified text file into current db')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.maintenance:
|
||||
if all([args.exportjson is None, args.importdatabase is None, args.importjson is None, args.importstatus is False, args.update is False]):
|
||||
print 'Expecting subcommand with the maintenance positional argumeent'
|
||||
sys.exit()
|
||||
mylar.MAINTENANCE = True
|
||||
else:
|
||||
mylar.MAINTENANCE = False
|
||||
|
||||
if args.verbose:
|
||||
mylar.VERBOSE = True
|
||||
if args.quiet:
|
||||
|
|
@ -88,15 +104,6 @@ def main():
|
|||
# Do an intial setup of the logger.
|
||||
logger.initLogger(console=not mylar.QUIET, log_dir=False, init=True, verbose=mylar.VERBOSE)
|
||||
|
||||
#if args.update:
|
||||
# print('Attempting to update Mylar so things can work again...')
|
||||
# try:
|
||||
# versioncheck.update()
|
||||
# except Exception, e:
|
||||
# sys.exit('Mylar failed to update.')
|
||||
# else:
|
||||
# mylar.shutdown(restart=True)
|
||||
|
||||
if args.daemon:
|
||||
if sys.platform == 'win32':
|
||||
print "Daemonize not supported under Windows, starting normally"
|
||||
|
|
@ -140,18 +147,12 @@ def main():
|
|||
else:
|
||||
mylar.NOWEEKLY = False
|
||||
|
||||
# Try to create the DATA_DIR if it doesn't exist
|
||||
#if not os.path.exists(mylar.DATA_DIR):
|
||||
# try:
|
||||
# os.makedirs(mylar.DATA_DIR)
|
||||
# except OSError:
|
||||
# raise SystemExit('Could not create data directory: ' + mylar.DATA_DIR + '. Exiting....')
|
||||
if mylar.MAINTENANCE is False:
|
||||
filechecker.validateAndCreateDirectory(mylar.DATA_DIR, True)
|
||||
|
||||
filechecker.validateAndCreateDirectory(mylar.DATA_DIR, True)
|
||||
|
||||
# Make sure the DATA_DIR is writeable
|
||||
if not os.access(mylar.DATA_DIR, os.W_OK):
|
||||
raise SystemExit('Cannot write to the data directory: ' + mylar.DATA_DIR + '. Exiting...')
|
||||
# Make sure the DATA_DIR is writeable
|
||||
if not os.access(mylar.DATA_DIR, os.W_OK):
|
||||
raise SystemExit('Cannot write to the data directory: ' + mylar.DATA_DIR + '. Exiting...')
|
||||
|
||||
# Put the database in the DATA_DIR
|
||||
mylar.DB_FILE = os.path.join(mylar.DATA_DIR, 'mylar.db')
|
||||
|
|
@ -197,23 +198,70 @@ def main():
|
|||
|
||||
i += 1
|
||||
|
||||
#from configobj import ConfigObj
|
||||
#mylar.CFG = ConfigObj(mylar.CONFIG_FILE, encoding='utf-8')
|
||||
|
||||
# Read config and start logging
|
||||
try:
|
||||
if mylar.MAINTENANCE is False:
|
||||
logger.info('Initializing startup sequence....')
|
||||
|
||||
try:
|
||||
mylar.initialize(mylar.CONFIG_FILE)
|
||||
except Exception as e:
|
||||
print e
|
||||
raise SystemExit('FATAL ERROR')
|
||||
|
||||
|
||||
# Rename the main thread
|
||||
threading.currentThread().name = "MAIN"
|
||||
|
||||
if mylar.DAEMON:
|
||||
mylar.daemonize()
|
||||
|
||||
if mylar.MAINTENANCE is True and any([args.exportjson, args.importjson, args.update is True, args.importstatus is True]):
|
||||
loggermode = '[MAINTENANCE-MODE]'
|
||||
if args.importstatus: #mylar.MAINTENANCE is True:
|
||||
cs = maintenance.Maintenance('status')
|
||||
cstat = cs.check_status()
|
||||
else:
|
||||
logger.info('%s Initializing maintenance mode' % loggermode)
|
||||
|
||||
if args.update is True:
|
||||
logger.info('%s Attempting to update Mylar so things can work again...' % loggermode)
|
||||
try:
|
||||
mylar.shutdown(restart=True, update=True, maintenance=True)
|
||||
except Exception as e:
|
||||
sys.exit('%s Mylar failed to update: %s' % (loggermode, e))
|
||||
|
||||
elif args.importdatabase:
|
||||
#for attempted db import.
|
||||
maintenance_path = args.importdatabase
|
||||
logger.info('%s db path accepted as %s' % (loggermode, maintenance_path))
|
||||
di = maintenance.Maintenance('database-import', file=maintenance_path)
|
||||
d = di.database_import()
|
||||
elif args.importjson:
|
||||
#for attempted file re-import (json format)
|
||||
maintenance_path = args.importjson
|
||||
logger.info('%s file indicated as being in json format - path accepted as %s' % (loggermode, maintenance_path))
|
||||
ij = maintenance.Maintenance('json-import', file=maintenance_path)
|
||||
j = ij.json_import()
|
||||
#elif args.importtext:
|
||||
# #for attempted file re-import (list format)
|
||||
# maintenance_path = args.importtext
|
||||
# logger.info('%s file indicated as being in list format - path accepted as %s' % (loggermode, maintenance_path))
|
||||
# it = maintenance.Maintenance('list-import', file=maintenance_path)
|
||||
# t = it.list_import()
|
||||
elif args.exportjson:
|
||||
#for export of db comicid's in json format
|
||||
maintenance_path = args.exportjson
|
||||
logger.info('%s file indicated as being written to json format - destination accepted as %s' % (loggermode, maintenance_path))
|
||||
ej = maintenance.Maintenance('json-export', output=maintenance_path)
|
||||
j = ej.json_export()
|
||||
else:
|
||||
logger.info('%s Not a valid command: %s' % (loggermode, maintenance_info))
|
||||
sys.exit()
|
||||
logger.info('%s Exiting Maintenance mode' % (loggermode))
|
||||
|
||||
#possible option to restart automatically after maintenance has completed...
|
||||
sys.exit()
|
||||
|
||||
# Force the http port if neccessary
|
||||
if args.port:
|
||||
http_port = args.port
|
||||
|
|
|
|||
|
|
@ -387,6 +387,7 @@
|
|||
%endif
|
||||
%endif
|
||||
<a href="#" title="Add to Reading List" onclick="doAjaxCall('addtoreadlist?IssueID=${issue['IssueID']}',$(this),'table')" data-success="${comic['ComicName']} #${issue['Issue_Number']} added to Reading List"><img src="interfaces/default/images/glasses-icon.png" height="25" width="25" class="highqual" /></a>
|
||||
<a href="#" title="Mark issue as Skipped" onclick="doAjaxCall('unqueueissue?IssueID=${issue['IssueID']}&ComicID=${issue['ComicID']}',$(this),'table')" data-success="'${issue['Issue_Number']}' has been marked as skipped"><img src="interfaces/default/images/skipped_icon.png" height="25" width="25" class="highqual" /></a>
|
||||
%else:
|
||||
<a href="#" title="Retry the same download again" onclick="doAjaxCall('queueit?ComicID=${issue['ComicID']}&IssueID=${issue['IssueID']}&ComicIssue=${issue['Issue_Number']}&mode=want', $(this),'table')" data-success="Retrying the same version of '${issue['ComicName']}' '${issue['Issue_Number']}'"><img src="interfaces/default/images/retry_icon.png" height="25" width="25" class="highqual" /></a>
|
||||
<a href="#" title="Mark issue as Skipped" onclick="doAjaxCall('unqueueissue?IssueID=${issue['IssueID']}&ComicID=${issue['ComicID']}',$(this),'table',true);" data-success="'${issue['Issue_Number']}' has been marked as skipped"><img src="interfaces/default/images/skipped_icon.png" height="25" width="25" class="highqual" /></a>
|
||||
|
|
@ -549,7 +550,6 @@
|
|||
-->
|
||||
<a href="#" title="Add to Reading List" onclick="doAjaxCall('addtoreadlist?IssueID=${annual['IssueID']}',$(this),'table')" data-success="${aninfo['annualComicName']} #${annual['Issue_Number']} added to Reading List"><img src="interfaces/default/images/glasses-icon.png" height="25" width="25" class="highqual" /></a>
|
||||
|
||||
<a href="#" onclick="doAjaxCall('retryit?ComicName=${annual['ComicName'] |u}&ComicID=${annual['ComicID']}&IssueID=${annual['IssueID']}&IssueNumber=${annual['Issue_Number']}&ComicYear=${annual['IssueDate']}&ReleaseComicID=${annual['ReleaseComicID']}', $(this),'table')" data-success="Retrying the same version of '${annual['ComicName']}' '${annual['Issue_Number']}'" title="Retry the same download again"><img src="interfaces/default/images/retry_icon.png" height="25" width="25" class="highqual" /></a>
|
||||
<a href="#" title="Mark annual as Skipped" onclick="doAjaxCall('unqueueissue?IssueID=${annual['IssueID']}&ComicID=${annual['ComicID']}&ReleaseComicID=${annual['ReleaseComicID']}',$(this),'table')" data-success="'${annual['Issue_Number']}' has been marked as skipped"><img src="interfaces/default/images/skipped_icon.png" height="25" width="25" class="highqual" /></a>
|
||||
|
||||
%else:
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ import mylar.config
|
|||
PROG_DIR = None
|
||||
DATA_DIR = None
|
||||
FULL_PATH = None
|
||||
MAINTENANCE = False
|
||||
LOG_DIR = None
|
||||
LOGTYPE = 'log'
|
||||
ARGS = None
|
||||
|
|
@ -151,7 +152,7 @@ SCHED = BackgroundScheduler({
|
|||
def initialize(config_file):
|
||||
with INIT_LOCK:
|
||||
|
||||
global CONFIG, _INITIALIZED, QUIET, CONFIG_FILE, OS_DETECT, CURRENT_VERSION, LATEST_VERSION, COMMITS_BEHIND, INSTALL_TYPE, IMPORTLOCK, PULLBYFILE, INKDROPS_32P, \
|
||||
global CONFIG, _INITIALIZED, QUIET, CONFIG_FILE, OS_DETECT, MAINTENANCE, CURRENT_VERSION, LATEST_VERSION, COMMITS_BEHIND, INSTALL_TYPE, IMPORTLOCK, PULLBYFILE, INKDROPS_32P, \
|
||||
DONATEBUTTON, CURRENT_WEEKNUMBER, CURRENT_YEAR, UMASK, USER_AGENT, SNATCHED_QUEUE, NZB_QUEUE, PULLNEW, COMICSORT, WANTED_TAB_OFF, CV_HEADERS, \
|
||||
IMPORTBUTTON, IMPORT_FILES, IMPORT_TOTALFILES, IMPORT_CID_COUNT, IMPORT_PARSED_COUNT, IMPORT_FAILURE_COUNT, CHECKENABLED, CVURL, DEMURL, WWTURL, \
|
||||
USE_SABNZBD, USE_NZBGET, USE_BLACKHOLE, USE_RTORRENT, USE_UTORRENT, USE_QBITTORRENT, USE_DELUGE, USE_TRANSMISSION, USE_WATCHDIR, SAB_PARAMS, \
|
||||
|
|
@ -173,38 +174,11 @@ def initialize(config_file):
|
|||
# Start the logger, silence console logging if we need to
|
||||
logger.initLogger(console=not QUIET, log_dir=CONFIG.LOG_DIR, verbose=VERBOSE) #logger.mylar_log.initLogger(verbose=VERBOSE)
|
||||
|
||||
#try to get the local IP using socket. Get this on every startup so it's at least current for existing session.
|
||||
import socket
|
||||
try:
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
s.connect(('8.8.8.8', 80))
|
||||
LOCAL_IP = s.getsockname()[0]
|
||||
s.close()
|
||||
logger.info('Successfully discovered local IP and locking it in as : ' + str(LOCAL_IP))
|
||||
except:
|
||||
logger.warn('Unable to determine local IP - this might cause problems when downloading (maybe use host_return in the config.ini)')
|
||||
LOCAL_IP = CONFIG.HTTP_HOST
|
||||
|
||||
|
||||
# verbatim back the logger being used since it's now started.
|
||||
if LOGTYPE == 'clog':
|
||||
logprog = 'Concurrent Rotational Log Handler'
|
||||
else:
|
||||
logprog = 'Rotational Log Handler (default)'
|
||||
|
||||
logger.fdebug('Logger set to use : ' + logprog)
|
||||
if LOGTYPE == 'log' and OS_DETECT == 'Windows':
|
||||
logger.fdebug('ConcurrentLogHandler package not installed. Using builtin log handler for Rotational logs (default)')
|
||||
logger.fdebug('[Windows Users] If you are experiencing log file locking and want this auto-enabled, you need to install Python Extensions for Windows ( http://sourceforge.net/projects/pywin32/ )')
|
||||
|
||||
logger.info('Config GIT Branch: %s' % CONFIG.GIT_BRANCH)
|
||||
|
||||
# Get the currently installed version - returns None, 'win32' or the git hash
|
||||
# Also sets INSTALL_TYPE variable to 'win', 'git' or 'source'
|
||||
CURRENT_VERSION, CONFIG.GIT_BRANCH = versioncheck.getVersion()
|
||||
|
||||
#versioncheck.getVersion()
|
||||
#config_write()
|
||||
|
||||
if CURRENT_VERSION is not None:
|
||||
hash = CURRENT_VERSION[:7]
|
||||
else:
|
||||
|
|
@ -213,10 +187,73 @@ def initialize(config_file):
|
|||
if CONFIG.GIT_BRANCH == 'master':
|
||||
vers = 'M'
|
||||
elif CONFIG.GIT_BRANCH == 'development':
|
||||
vers = 'D'
|
||||
vers = 'D'
|
||||
else:
|
||||
vers = 'NONE'
|
||||
|
||||
if MAINTENANCE is False:
|
||||
#try to get the local IP using socket. Get this on every startup so it's at least current for existing session.
|
||||
import socket
|
||||
try:
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
s.connect(('8.8.8.8', 80))
|
||||
LOCAL_IP = s.getsockname()[0]
|
||||
s.close()
|
||||
logger.info('Successfully discovered local IP and locking it in as : ' + str(LOCAL_IP))
|
||||
except:
|
||||
logger.warn('Unable to determine local IP - this might cause problems when downloading (maybe use host_return in the config.ini)')
|
||||
LOCAL_IP = CONFIG.HTTP_HOST
|
||||
|
||||
|
||||
# verbatim back the logger being used since it's now started.
|
||||
if LOGTYPE == 'clog':
|
||||
logprog = 'Concurrent Rotational Log Handler'
|
||||
else:
|
||||
logprog = 'Rotational Log Handler (default)'
|
||||
|
||||
logger.fdebug('Logger set to use : ' + logprog)
|
||||
if LOGTYPE == 'log' and OS_DETECT == 'Windows':
|
||||
logger.fdebug('ConcurrentLogHandler package not installed. Using builtin log handler for Rotational logs (default)')
|
||||
logger.fdebug('[Windows Users] If you are experiencing log file locking and want this auto-enabled, you need to install Python Extensions for Windows ( http://sourceforge.net/projects/pywin32/ )')
|
||||
|
||||
logger.info('Config GIT Branch: %s' % CONFIG.GIT_BRANCH)
|
||||
|
||||
# Initialize the database
|
||||
logger.info('Checking to see if the database has all tables....')
|
||||
try:
|
||||
dbcheck()
|
||||
except Exception, e:
|
||||
logger.error('Cannot connect to the database: %s' % e)
|
||||
|
||||
# Check for new versions (autoupdate)
|
||||
if CONFIG.CHECK_GITHUB_ON_STARTUP:
|
||||
try:
|
||||
LATEST_VERSION = versioncheck.checkGithub()
|
||||
except:
|
||||
LATEST_VERSION = CURRENT_VERSION
|
||||
else:
|
||||
LATEST_VERSION = CURRENT_VERSION
|
||||
#
|
||||
if CONFIG.AUTO_UPDATE:
|
||||
if CURRENT_VERSION != LATEST_VERSION and INSTALL_TYPE != 'win' and COMMITS_BEHIND > 0:
|
||||
logger.info('Auto-updating has been enabled. Attempting to auto-update.')
|
||||
# SIGNAL = 'update'
|
||||
|
||||
#check for syno_fix here
|
||||
if CONFIG.SYNO_FIX:
|
||||
parsepath = os.path.join(DATA_DIR, 'bs4', 'builder', '_lxml.py')
|
||||
if os.path.isfile(parsepath):
|
||||
print ("found bs4...renaming appropriate file.")
|
||||
src = os.path.join(parsepath)
|
||||
dst = os.path.join(DATA_DIR, 'bs4', 'builder', 'lxml.py')
|
||||
try:
|
||||
shutil.move(src, dst)
|
||||
except (OSError, IOError):
|
||||
logger.error('Unable to rename file...shutdown Mylar and go to ' + src.encode('utf-8') + ' and rename the _lxml.py file to lxml.py')
|
||||
logger.error('NOT doing this will result in errors when adding / refreshing a series')
|
||||
else:
|
||||
logger.info('Synology Parsing Fix already implemented. No changes required at this time.')
|
||||
|
||||
USER_AGENT = 'Mylar/' +str(hash) +'(' +vers +') +http://www.github.com/evilhero/mylar/'
|
||||
|
||||
CV_HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1'}
|
||||
|
|
@ -226,42 +263,6 @@ def initialize(config_file):
|
|||
CURRENT_WEEKNUMBER = todaydate.strftime("%U")
|
||||
CURRENT_YEAR = todaydate.strftime("%Y")
|
||||
|
||||
# Initialize the database
|
||||
logger.info('Checking to see if the database has all tables....')
|
||||
try:
|
||||
dbcheck()
|
||||
except Exception, e:
|
||||
logger.error('Cannot connect to the database: %s' % e)
|
||||
|
||||
# Check for new versions (autoupdate)
|
||||
if CONFIG.CHECK_GITHUB_ON_STARTUP:
|
||||
try:
|
||||
LATEST_VERSION = versioncheck.checkGithub()
|
||||
except:
|
||||
LATEST_VERSION = CURRENT_VERSION
|
||||
else:
|
||||
LATEST_VERSION = CURRENT_VERSION
|
||||
#
|
||||
if CONFIG.AUTO_UPDATE:
|
||||
if CURRENT_VERSION != LATEST_VERSION and INSTALL_TYPE != 'win' and COMMITS_BEHIND > 0:
|
||||
logger.info('Auto-updating has been enabled. Attempting to auto-update.')
|
||||
# SIGNAL = 'update'
|
||||
|
||||
#check for syno_fix here
|
||||
if CONFIG.SYNO_FIX:
|
||||
parsepath = os.path.join(DATA_DIR, 'bs4', 'builder', '_lxml.py')
|
||||
if os.path.isfile(parsepath):
|
||||
print ("found bs4...renaming appropriate file.")
|
||||
src = os.path.join(parsepath)
|
||||
dst = os.path.join(DATA_DIR, 'bs4', 'builder', 'lxml.py')
|
||||
try:
|
||||
shutil.move(src, dst)
|
||||
except (OSError, IOError):
|
||||
logger.error('Unable to rename file...shutdown Mylar and go to ' + src.encode('utf-8') + ' and rename the _lxml.py file to lxml.py')
|
||||
logger.error('NOT doing this will result in errors when adding / refreshing a series')
|
||||
else:
|
||||
logger.info('Synology Parsing Fix already implemented. No changes required at this time.')
|
||||
|
||||
#set the default URL for ComicVine API here.
|
||||
CVURL = 'https://comicvine.gamespot.com/api/'
|
||||
|
||||
|
|
@ -272,10 +273,10 @@ def initialize(config_file):
|
|||
if CONFIG.LOCMOVE:
|
||||
helpers.updateComicLocation()
|
||||
|
||||
|
||||
#Ordering comics here
|
||||
logger.info('Remapping the sorting to allow for new additions.')
|
||||
COMICSORT = helpers.ComicSort(sequence='startup')
|
||||
if mylar.MAINTENANCE is False:
|
||||
logger.info('Remapping the sorting to allow for new additions.')
|
||||
COMICSORT = helpers.ComicSort(sequence='startup')
|
||||
|
||||
# Store the original umask
|
||||
UMASK = os.umask(0)
|
||||
|
|
@ -1164,10 +1165,11 @@ def halt():
|
|||
os._exit(0)
|
||||
_INITIALIZED = False
|
||||
|
||||
def shutdown(restart=False, update=False):
|
||||
def shutdown(restart=False, update=False, maintenance=False):
|
||||
|
||||
cherrypy.engine.exit()
|
||||
halt()
|
||||
if maintenance is False:
|
||||
cherrypy.engine.exit()
|
||||
halt()
|
||||
|
||||
if not restart and not update:
|
||||
logger.info('Mylar is shutting down...')
|
||||
|
|
@ -1185,9 +1187,12 @@ def shutdown(restart=False, update=False):
|
|||
if restart:
|
||||
logger.info('Mylar is restarting...')
|
||||
popen_list = [sys.executable, FULL_PATH]
|
||||
popen_list += ARGS
|
||||
# if '--nolaunch' not in popen_list:
|
||||
# popen_list += ['--nolaunch']
|
||||
if 'maintenance' not in ARGS:
|
||||
popen_list += ARGS
|
||||
else:
|
||||
for x in ARGS:
|
||||
if all([x != 'maintenance', x != '-u']):
|
||||
popen_list += x
|
||||
logger.info('Restarting Mylar with ' + str(popen_list))
|
||||
subprocess.Popen(popen_list, cwd=os.getcwd())
|
||||
|
||||
|
|
|
|||
|
|
@ -557,11 +557,15 @@ def Getissue(issueid, dom, type):
|
|||
else:
|
||||
try:
|
||||
image = dom.getElementsByTagName('super_url')[0].firstChild.wholeText
|
||||
image_alt = dom.getElementsByTagName('small_url')[0].firstChild.wholeText
|
||||
except:
|
||||
image = None
|
||||
try:
|
||||
image_alt = dom.getElementsByTagName('small_url')[0].firstChild.wholeText
|
||||
except:
|
||||
image_alt = None
|
||||
|
||||
return image
|
||||
return {'image': image,
|
||||
'image_alt': image_alt}
|
||||
|
||||
def GetSeriesYears(dom):
|
||||
#used by the 'add a story arc' option to individually populate the Series Year for each series within the given arc.
|
||||
|
|
|
|||
|
|
@ -29,7 +29,6 @@ import imghdr
|
|||
import sqlite3
|
||||
import cherrypy
|
||||
import requests
|
||||
import gzip
|
||||
|
||||
import mylar
|
||||
from mylar import logger, helpers, db, mb, cv, parseit, filechecker, search, updater, moveit, comicbookdb
|
||||
|
|
@ -341,7 +340,7 @@ def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=No
|
|||
logger.warn('Unable to complete Refreshing / Adding issue data - this WILL create future problems if not addressed.')
|
||||
return {'status': 'incomplete'}
|
||||
|
||||
if calledfrom is None:
|
||||
if any([calledfrom is None, calledfrom == 'maintenance']):
|
||||
issue_collection(issuedata, nostatus='False')
|
||||
#need to update annuals at this point too....
|
||||
if anndata:
|
||||
|
|
@ -349,12 +348,18 @@ def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=No
|
|||
|
||||
#let's download the image...
|
||||
if mylar.CONFIG.ALTERNATE_LATEST_SERIES_COVERS is True:
|
||||
imagetopull = myDB.selectone('SELECT issueid from issues where ComicID=? AND Int_IssueNumber=?', [comicid, helpers.issuedigits(importantdates['LatestIssue'])]).fetchone()
|
||||
imageurl = mylar.cv.getComic(comicid, 'image', issueid=imagetopull['IssueID'])
|
||||
covercheck = helpers.getImage(comicid, imageurl)
|
||||
ls = helpers.issuedigits(importantdates['LatestIssue'])
|
||||
imagetopull = myDB.selectone('SELECT issueid from issues where ComicID=? AND Int_IssueNumber=?', [comicid, ls]).fetchone()
|
||||
imageurl = mylar.cv.getComic(comicid, 'image', issueid=imagetopull[0])
|
||||
covercheck = helpers.getImage(comicid, imageurl['image'])
|
||||
if covercheck == 'retry':
|
||||
logger.info('Attempting to retrieve alternate comic image for the series.')
|
||||
covercheck = helpers.getImage(comicid, comic['ComicImageALT'])
|
||||
logger.fdebug('Attempting to retrieve a different comic image for this particular issue.')
|
||||
if imageurl['image_alt'] is not None:
|
||||
covercheck = helpers.getImage(comicid, imageurl['image_alt'])
|
||||
else:
|
||||
if not os.path.isfile(os.path.join(mylar.CACHE_DIR, str(comicid) + '.jpg')):
|
||||
logger.fdebug('Failed to retrieve issue image, possibly because not available. Reverting back to series image.')
|
||||
covercheck = helpers.getImage(comicid, comic['ComicImage'])
|
||||
PRComicImage = os.path.join('cache', str(comicid) + ".jpg")
|
||||
ComicImage = helpers.replacetheslash(PRComicImage)
|
||||
|
||||
|
|
@ -442,6 +447,7 @@ def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=No
|
|||
updater.newpullcheck(ComicName=cn_pull, ComicID=comicid, issue=latestiss)
|
||||
|
||||
#here we grab issues that have been marked as wanted above...
|
||||
if calledfrom != 'maintenance':
|
||||
results = []
|
||||
issresults = myDB.select("SELECT * FROM issues where ComicID=? AND Status='Wanted'", [comicid])
|
||||
if issresults:
|
||||
|
|
@ -501,6 +507,11 @@ def addComictoDB(comicid, mismatch=None, pullupd=None, imported=None, ogcname=No
|
|||
if calledfrom == 'addbyid':
|
||||
logger.info('Sucessfully added %s (%s) to the watchlist by directly using the ComicVine ID' % (comic['ComicName'], SeriesYear))
|
||||
return {'status': 'complete'}
|
||||
elif calledfrom == 'maintenance':
|
||||
logger.info('Sucessfully added %s (%s) to the watchlist' % (comic['ComicName'], SeriesYear))
|
||||
return {'status': 'complete',
|
||||
'comicname': comic['ComicName'],
|
||||
'year': SeriesYear}
|
||||
else:
|
||||
logger.info('Sucessfully added %s (%s) to the watchlist' % (comic['ComicName'], SeriesYear))
|
||||
return {'status': 'complete'}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,169 @@
|
|||
# This file is part of Mylar.
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Mylar is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# Mylar is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with Mylar. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import sqlite3
|
||||
import json
|
||||
|
||||
import mylar
|
||||
from mylar import logger, importer
|
||||
|
||||
class Maintenance(object):
|
||||
|
||||
def __init__(self, mode, file=None, output=None):
|
||||
self.mode = mode
|
||||
self.maintenance_db = os.path.join(mylar.DATA_DIR, '.mylar_maintenance.db')
|
||||
if self.mode == 'database-import':
|
||||
self.dbfile = file
|
||||
else:
|
||||
self.dbfile = mylar.DB_FILE
|
||||
self.file = file
|
||||
self.outputfile = output
|
||||
self.comiclist = []
|
||||
self.maintenance_success = []
|
||||
self.maintenance_fail = []
|
||||
|
||||
def sql_attachmylar(self):
|
||||
self.connectmylar = sqlite3.connect(self.dbfile)
|
||||
self.dbmylar = self.connectmylar.cursor()
|
||||
|
||||
def sql_closemylar(self):
|
||||
self.connectmylar.commit()
|
||||
self.dbmylar.close()
|
||||
|
||||
def sql_attach(self):
|
||||
self.conn = sqlite3.connect(self.maintenance_db)
|
||||
self.db = self.conn.cursor()
|
||||
self.db.execute('CREATE TABLE IF NOT EXISTS maintenance (id TEXT, mode TEXT, status TEXT, progress TEXT, total TEXT, current TEXT, last_comicid TEXT, last_series TEXT, last_seriesyear TEXT)')
|
||||
|
||||
def sql_close(self):
|
||||
self.conn.commit()
|
||||
self.db.close()
|
||||
|
||||
def database_import(self):
|
||||
self.sql_attachmylar()
|
||||
|
||||
comicidlist = self.dbmylar.execute('SELECT * FROM comics')
|
||||
for i in comicidlist:
|
||||
self.comiclist.append(i['ComicID'])
|
||||
|
||||
self.sql_closemylar()
|
||||
|
||||
self.importIT()
|
||||
|
||||
def json_import(self):
|
||||
self.comiclist = json.load(open(self.file))
|
||||
logger.info('[MAINTENANCE-MODE][JSON-IMPORT] Found %s series within json listing. Preparing to mass import to existing db.' % (len(self.comiclist)))
|
||||
self.importIT()
|
||||
|
||||
def json_export(self):
|
||||
self.sql_attachmylar()
|
||||
|
||||
for i in self.dbmylar.execute('SELECT ComicID FROM comics'):
|
||||
self.comiclist.append({'ComicID': i[0]})
|
||||
|
||||
self.sql_closemylar()
|
||||
|
||||
with open(self.outputfile, 'wb') as outfile:
|
||||
json.dump(self.comiclist, outfile)
|
||||
|
||||
logger.info('[MAINTENANCE-MODE][%s] Successfully exported %s ComicID\'s to json file: %s' % (self.mode.upper(), len(self.comiclist), self.outputfile))
|
||||
|
||||
def check_status(self):
|
||||
try:
|
||||
found = False
|
||||
self.sql_attach()
|
||||
checkm = self.db.execute('SELECT * FROM maintenance')
|
||||
for cm in checkm:
|
||||
found = True
|
||||
if 'import' in cm[1]:
|
||||
logger.info('[MAINTENANCE-MODE][STATUS] Current Progress: %s [%s / %s]' % (cm[2].upper(), cm[3], cm[4]))
|
||||
if cm[2] == 'running':
|
||||
try:
|
||||
logger.info('[MAINTENANCE-MODE][STATUS] Current Import: %s' % (cm[5]))
|
||||
if cm[6] is not None:
|
||||
logger.info('[MAINTENANCE-MODE][STATUS] Last Successful Import: %s [%s]' % (cm[7], cm[6]))
|
||||
except:
|
||||
pass
|
||||
elif cm[2] == 'completed':
|
||||
logger.info('[MAINTENANCE-MODE][STATUS] Last Successful Import: %s [%s]' % (cm[7], cm[6]))
|
||||
else:
|
||||
logger.info('[MAINTENANCE-MODE][STATUS] Current Progress: %s [mode: %s]' % (cm[2].upper(), cm[1]))
|
||||
if found is False:
|
||||
raise Error
|
||||
except Exception as e:
|
||||
logger.info('[MAINTENANCE-MODE][STATUS] Nothing is currently running')
|
||||
|
||||
self.sql_close()
|
||||
|
||||
def importIT(self):
|
||||
#set startup...
|
||||
if len(self.comiclist) > 0:
|
||||
self.sql_attach()
|
||||
query = "DELETE FROM maintenance"
|
||||
self.db.execute(query)
|
||||
query = "INSERT INTO maintenance (id, mode, total, status) VALUES (%s,'%s',%s,'%s')" % ('1', self.mode, len(self.comiclist), "running")
|
||||
self.db.execute(query)
|
||||
self.sql_close()
|
||||
logger.info('[MAINTENANCE-MODE][%s] Found %s series in previous db. Preparing to migrate into existing db.' % (self.mode.upper(), len(self.comiclist)))
|
||||
count = 1
|
||||
for x in self.comiclist:
|
||||
logger.info('[MAINTENANCE-MODE][%s] [%s/%s] now attempting to add %s to watchlist...' % (self.mode.upper(), count, len(self.comiclist), x['ComicID']))
|
||||
try:
|
||||
self.sql_attach()
|
||||
self.db.execute("UPDATE maintenance SET progress=?, total=?, current=? WHERE id='1'", (count, len(self.comiclist), re.sub('4050-', '', x['ComicID'].strip())))
|
||||
self.sql_close()
|
||||
except Exception as e:
|
||||
logger.warn('[ERROR] %s' % e)
|
||||
maintenance_info = importer.addComictoDB(re.sub('4050-', '', x['ComicID']).strip(), calledfrom='maintenance')
|
||||
try:
|
||||
logger.info('MAINTENANCE: %s' % maintenance_info)
|
||||
if maintenance_info['status'] == 'complete':
|
||||
logger.fdebug('[MAINTENANCE-MODE][%s] Successfully added %s [%s] to watchlist.' % (self.mode.upper(), maintenance_info['comicname'], maintenance_info['year']))
|
||||
else:
|
||||
logger.fdebug('[MAINTENANCE-MODE][%s] Unable to add %s [%s] to watchlist.' % (self.mode.upper(), maintenance_info['comicname'], maintenance_info['year']))
|
||||
raise IOError
|
||||
self.maintenance_success.append(x)
|
||||
|
||||
try:
|
||||
self.sql_attach()
|
||||
self.db.execute("UPDATE maintenance SET progress=?, last_comicid=?, last_series=?, last_seriesyear=? WHERE id='1'", (count, re.sub('4050-', '', x['ComicID'].strip()), maintenance_info['comicname'], maintenance_info['year']))
|
||||
self.sql_close()
|
||||
except Exception as e:
|
||||
logger.warn('[ERROR] %s' % e)
|
||||
|
||||
|
||||
except IOError as e:
|
||||
logger.warn('[MAINTENANCE-MODE][%s] Unable to add series to watchlist: %s' % (self.mode.upper(), e))
|
||||
self.maintenance_fail.append(x)
|
||||
|
||||
count+=1
|
||||
else:
|
||||
logger.warn('[MAINTENANCE-MODE][%s] Unable to locate any series in db. This is probably a FATAL error and an unrecoverable db.' % self.mode.upper())
|
||||
return
|
||||
|
||||
logger.info('[MAINTENANCE-MODE][%s] Successfully imported %s series into existing db.' % (self.mode.upper(), len(self.maintenance_success)))
|
||||
if len(self.maintenance_fail) > 0:
|
||||
logger.info('[MAINTENANCE-MODE][%s] Failed to import %s series into existing db: %s' % (self.mode.upper(), len(self.maintenance_success), self.maintenance_fail))
|
||||
try:
|
||||
self.sql_attach()
|
||||
self.db.execute("UPDATE maintenance SET status=? WHERE id='1'", ["completed"])
|
||||
self.sql_close()
|
||||
except Exception as e:
|
||||
logger.warn('[ERROR] %s' % e)
|
||||
|
||||
|
|
@ -1068,7 +1068,7 @@ def new_pullcheck(weeknumber, pullyear, comic1off_name=None, comic1off_id=None,
|
|||
updater.latest_update(ComicID=comicid, LatestIssue=week['issue'], LatestDate=ComicDate)
|
||||
# here we add to upcoming table...
|
||||
statusupdate = updater.upcoming_update(ComicID=comicid, ComicName=comicname, IssueNumber=week['issue'], IssueDate=ComicDate, forcecheck=forcecheck, weekinfo={'weeknumber':weeknumber,'year':pullyear})
|
||||
logger.info('statusupdate: ' + str(statusupdate))
|
||||
logger.fdebug('statusupdate: ' + str(statusupdate))
|
||||
|
||||
# here we update status of weekly table...
|
||||
try:
|
||||
|
|
|
|||
Loading…
Reference in New Issue