IMP: DDL Queue management now added (Manage tab / Manage DDL Queue) - restart / resume queue jobs, see active download in progress, history of ddl items with status and completion dates.

This commit is contained in:
evilhero 2019-03-29 11:17:43 -04:00
parent 5805ed3877
commit bf1e479b2a
4 changed files with 525 additions and 37 deletions

View File

@ -1690,6 +1690,62 @@ div#artistheader h2 a {
min-width: 95px;
vertical-align: middle;
}
#queue_table th#qcomicid {
max-width: 10px;
text-align: center;
}
#queue_table th#qseries {
max-width: 475px;
text-align: center;
}
#queue_table th#qsize {
max-width: 30px;
text-align: center;
}
#queue_table th#qprogress {
max-width: 25px;
text-align: center;
}
#queue_table th#qstatus {
max-width: 50px;
text-align: center;
}
#queue_table th#qdate {
max-width: 90px;
text-align: center;
}
#queue_table th#qoptions {
min-width: 160px;
text-align: center;
}
#queue_table td#qcomicid {
max-width: 10px;
text-align: left;
}
#queue_table td#qseries {
max-width: 475px;
text-align: left;
}
#queue_table td#qsize {
max-width: 30px;
text-align: center;
}
#queue_table td#qprogress {
max-width: 25px;
text-align: center;
}
#queue_table td#qstatus {
max-width: 50px;
text-align: center;
}
#queue_table td#qdate {
min-width: 90px;
text-align: center;
}
#queue_table td#qoptions {
min-width: 160px;
text-align: center;
}
DIV.progress-container
{
position: relative;

View File

@ -9,6 +9,7 @@
<a id="menu_link_edit" href="manageComics">Manage Comics</a>
<a id="menu_link_edit" href="manageIssues?status=Wanted">Manage Issues</a>
<a id="menu_link_edit" href="manageFailed">Manage Failed Links</a>
<a id="menu_link_edit" href="queueManage">Manage DDL Queue</a>
</div>
</div>
</%def>

View File

@ -0,0 +1,246 @@
<%inherit file="base.html"/>
<%
import mylar
%>
<%def name="headerIncludes()">
<div id="subhead_container">
<div id="subhead_menu">
<a id="menu_link_refresh" href="#" title="Restart stalled queue" onclick="doAjaxCall('ddl_requeue?mode=restart_queue',$(this),'table')" data-success="Restarted Queue">Restart Queue</a>
</div>
</div>
</%def>
<%def name="body()">
<div id="paddingheader">
<h1 class="clearfix">QUEUE MANAGEMENT</h1></br>
</div>
<div style="text-align:center;">
<h2><center><bold>ACTIVE</bold></center></h2>
<div id="active_message" align="center" style="display:none;">
<div id="amessage" align="center"></div>
<div id="queuebuttons"><div id="queue_menu" style="display:none;"></br>
<a id="arestartddl" name="restartddl" href="#" title="Restart stalled download" data-success="Restarted Download">Restart Download</a>
<a id="aresumeddl" name="resumeddl" href="#" title="Resume download" data-success="Resumed Download">Resume Download</a>
<a id="aabortddl" name="abortddl" href="#" title="Abort download" data-success="Aborted Download">Abort Download</a>
</div></div>
</div>
<div id="active_queue" style="display:none;">
<table width="100%" cellpadding="5" cellspacing="5">
<tbody>
<div style="display: flex; justify-content: flex-end">
<div id="queuebuttons"><div id="queue_menu">
<a id="qrestartddl" name="restartddl" href="#" title="Restart stalled download" data-success="Restarted Download">Restart Download</a></br>
<a id="qresumeddl" name="resumeddl" href="#" title="Resume download" data-success="Resumed Download">Resume Download</a></br>
<a id="qabortddl" name="abortddl" href="#" title="Abort download" data-success="Aborted Download">Abort Download</a>
</div></div>
</div>
<tr><td id="series" align="center" style="text-align:center"></td></tr>
<tr><td id="filename" align="center" style="text-align:center"></td></tr>
<tr><td id="size" align="center" style="text-align:center"></td></tr>
<tr><td align="center" style="text-align:center">
<div style="display:table;position:relative;margin:auto;top:0px;"><span id="progress_percent"></span><div class="progress-container complete"><div id="prog_width"><span class="progressbar-front-text" style="margin:auto;top:-3px;" id="progress" name="progress" value="0%"></span></div></div></div>
</td></tr>
<tr><td id="status" align="center" style="text-align:center"></td></tr>
</tbody>
</table>
</div>
</div>
</br></br>
<h2><center><bold>HISTORY</bold></center></h2>
%if type(resultlist) == str:
<center>${resultlist}</center></br>
%else:
<div class="table_wrapper">
<table class="display" id="queue_table">
<thead>
<tr>
<th id="qcomicid">ComicID</th>
<th id="qseries">Series</th>
<th id="qsize">Size</th>
<th id="qprogress">%</th>
<th id="qstatus">Status</th>
<th id="qdate">Updated</th>
<th id="qoptions">Options</th>
</tr>
</thead>
<tbody>
</tbody>
</table>
</div>
%endif
</%def>
<%def name="headIncludes()">
<link rel="stylesheet" href="interfaces/default/css/data_table.css">
</%def>
<%def name="javascriptIncludes()">
<script src="js/libs/jquery.dataTables.min.js"></script>
<script src="js/libs/full_numbers_no_ellipses.js"></script>
<script>
var ImportTimer = setInterval(activecheck, 5000);
function activecheck() {
$.get('check_ActiveDDL',
function(data){
if (data.error != undefined) {
alert(data.error);
return;
};
var obj = JSON.parse(data);
var percent = obj['percent'];
var status = obj['status'];
var aid = obj['a_id'];
if (status == 'Downloading') {
acm = document.getElementById("active_message");
acm.style.display = "none";
acq = document.getElementById("active_queue");
acq.style.display = "unset";
document.getElementById("prog_width").style.width=percent;
$("#progress").html(percent);
document.getElementById("series").innerHTML = obj['a_series'];
document.getElementById("filename").innerHTML = obj['a_filename'];
document.getElementById("size").innerHTML = obj['a_size'];
document.getElementById("status").innerHTML = status;
qmm = document.getElementById("queue_menu");
qmm.style.display = "inline-block";
$("#qrestartddl").attr('onClick', "ajaxcallit('restart', "+aid+")");
$("#qresumeddl").attr('onClick', "ajaxcallit('resume', "+aid+")");
$("#qabortddl").attr('onClick', "ajaxcallit('abort', "+aid+")");
} else if ( status.indexOf("File does not exist") > -1){
acm = document.getElementById("active_message");
acm.style.display = "inline-block";
acq = document.getElementById("active_queue");
acq.style.display = "none";
document.getElementById("amessage").innerHTML = status;
qmm = document.getElementById("queue_menu");
qmm.style.display = "inline-block";
$("#arestartddl").attr('onClick', "ajaxcallit('restart', "+aid+")");
$("#aresumeddl").attr('onClick', "ajaxcallit('resume', "+aid+")");
$("#aabortddl").attr('onClick', "ajaxcallit('abort', "+aid+")");
} else {
acm = document.getElementById("active_message");
acm.style.display = "inline-block";
acq = document.getElementById("active_queue");
acq.style.display = "none";
qmm = document.getElementById("queue_menu");
qmm.style.display = "none";
document.getElementById("amessage").innerHTML = status;
}
if (percent == '100%') {
clearInterval(ImportTimer);
$('#queue_table').DataTable().ajax.reload(null, false);
ImportTimer = setInterval(activecheck, 5000);
}
});
};
function ajaxcallit(mode, queueid) {
alert(queueid);
$.get("ddl_requeue",
{ mode: mode, id: queueid },
function(data){
if (data.error != undefined) {
alert(data.error);
return;
}
$('#ajaxMsg').html("<div class='msg'><span class='ui-icon ui-icon-check'></span>Successfully restarted DDL Queue</div>");
$('#queue_table').DataTable().ajax.reload(null, false);
});
};
function initThisPage() {
initActions();
$("#queuebuttons #queue_menu #arestartddl, #qrestartddl").button({ icons: { primary: "ui-icon-refresh" } });
$("#queuebuttons #queue_menu #aresumeddl, #qresumeddl").button({ icons: { primary: "ui-icon-pencil" } });
$("#queuebuttons #queue_menu #aabortddl, #qabortddl").button({ icons: { primary: "ui-icon-pencil" } });
if ( $.fn.dataTable.isDataTable( '#queue_table' ) ) {
$('#queue_table').DataTable().ajax.reload(null, false);
} else {
$('#queue_table').DataTable( {
"processing": true,
"serverSide": true,
"ajaxSource": 'queueManageIt',
"paginationType": "full_numbers",
"sorting": [[0, 'desc']],
"displayLength": 15,
"stateSave": false,
"columnDefs": [
{
"sortable": false,
"targets": [ 0 ],
"visible": false,
},
{
"sortable": true,
"targets": [ 1 ],
"visible": true,
"data": "Series",
"render":function (data,type,full) {
return '<span title="' + full[1] + '"></span><a href="comicDetails?ComicID=' + full[0] + '">' + full[1] + '</a>';
}
},
{
"sortable": false,
"targets": [ 6 ],
"visible": true,
"render":function (data,type,full) {
val = full[4]
var restartline = "('ddl_requeue?mode=restart&id="+full[6]+"',$(this));"
var resumeline = "('ddl_requeue?mode=resume&id="+full[6]+"',$(this));"
if (val == 'Completed' || val == 'Failed' || val == 'Downloading'){
return '<span title="Restart"></span><a href="#" onclick="doAjaxCall'+restartline+'">Restart</a>';
} else if (val == 'Incomplete') {
return '<span title="Restart"></span><a href="#" onclick="doAjaxCall'+restartline+'">Restart</a><span title="Resume"></span><a href="#" onclick="doAjaxCall'+resumeline+'"><span class="ui-icon ui-icon-plus"></span>Resume</a>';
} else if (val == 'Queued') {
return '<span title="Start"></span><a href="#" onclick="doAjaxCall'+restartline+'">Start</a>';
}
}
},
],
"language": {
"search":"Filter:",
"lengthMenu":"Show _MENU_ items per page",
"emptyTable": "No information available",
"info":"Showing _START_ to _END_ of _TOTAL_ items",
"infoEmpty":"Showing 0 to 0 of 0 lines",
"infoFiltered":"(filtered from _MAX_ total items)"
},
"rowCallback": function (nRow, aData, iDisplayIndex, iDisplayIndexFull) {
if (aData[4] === "Completed") {
$('td', nRow).closest('tr').addClass("gradeA");
} else if (aData[4] === "Queued") {
$('td', nRow).closest('tr').addClass("gradeL");
} else if (aData[4] === "Incomplete" || aData[4] == "Failed") {
$('td', nRow).closest('tr').addClass("gradeX");
}
nRow.children[0].id = 'qcomicid';
nRow.children[1].id = 'qseries';
nRow.children[2].id = 'qsize';
nRow.children[3].id = 'qprogress';
nRow.children[4].id = 'qstatus';
nRow.children[5].id = 'qdate';
return nRow;
},
"drawCallback": function (o) {
// Jump to top of page
$('html,body').scrollTop(0);
},
"serverData": function ( sSource, aoData, fnCallback ) {
/* Add some extra data to the sender */
$.getJSON(sSource, aoData, function (json) {
fnCallback(json)
});
},
"fnInitComplete": function(oSettings, json)
{
},
});
};
activecheck();
};
$(document).ready(function() {
initThisPage();
});
</script>
</%def>

View File

@ -2189,45 +2189,67 @@ class WebInterface(object):
annualDelete.exposed = True
def ddl_requeue(self, id, mode):
def ddl_requeue(self, mode, id=None):
myDB = db.DBConnection()
item = myDB.selectone("SELECT * FROM DDL_INFO WHERE ID=?", [id]).fetchone()
if item is not None:
if mode == 'resume':
if item['status'] != 'Completed':
filesize = os.stat(os.path.join(mylar.CONFIG.DDL_LOCATION, item['filename'])).st_size
mylar.DDL_QUEUE.put({'link': item['link'],
'mainlink': item['mainlink'],
'series': item['series'],
'year': item['year'],
'size': item['size'],
'comicid': item['comicid'],
'issueid': item['issueid'],
'id': item['id'],
'resume': filesize})
if id is None:
items = myDB.select("SELECT * FROM ddl_info WHERE status = 'Queued' ORDER BY updated_date DESC")
else:
oneitem = myDB.selectone("SELECT * FROM DDL_INFO WHERE ID=?", [id]).fetchone()
items = [oneitem]
itemlist = [x for x in items]
if itemlist is not None:
for item in itemlist:
if all([mylar.CONFIG.DDL_AUTORESUME is True, mode == 'resume', item['status'] != 'Completed']):
try:
filesize = os.stat(os.path.join(mylar.CONFIG.DDL_LOCATION, item['filename'])).st_size
except:
filesize = 0
resume = filesize
elif mode == 'abort':
myDB.action('DELETE FROM ddl_info where ID=?', [id])
continue
else:
resume = None
mylar.DDL_QUEUE.put({'link': item['link'],
'mainlink': item['mainlink'],
'series': item['series'],
'year': item['year'],
'size': item['size'],
'comicid': item['comicid'],
'issueid': item['issueid'],
'id': item['id'],
'resume': resume})
if mode == 'restart_queue':
logger.info('[DDL-RESTART-QUEUE] DDL Queue successfully restarted. Put %s items back into the queue for downloading..' % len(itemlist))
elif mode == 'restart':
logger.info('[DDL-REQUEUE] Successfully restarted %s [%s] for downloading..' % (oneitem['series'], oneitem['size']))
elif mode == 'requeue':
logger.info('[DDL-REQUEUE] Successfully requeued %s [%s] for downloading..' % (oneitem['series'], oneitem['size']))
elif mode == 'abort':
logger.info('[DDL-ABORT] Successfully aborted downloading of %s [%s]..' % (oneitem['series'], oneitem['size']))
ddl_requeue.exposed = True
def queueManage(self): # **args):
myDB = db.DBConnection()
activelist = 'There are currently no items currently downloading via Direct Download (DDL).'
active = myDB.selectone("SELECT * FROM DDL_INFO WHERE STATUS = 'Downloading'").fetchone()
if active is not None:
activelist ={'series': active['series'],
'year': active['year'],
'size': active['size'],
'filename': active['filename'],
'status': active['status'],
'id': active['id']}
resultlist = 'There are currently no items waiting in the Direct Download (DDL) Queue for processing.'
s_info = myDB.select("SELECT a.ComicName, a.ComicVersion, a.ComicID, a.ComicYear, b.Issue_Number, b.IssueID, c.size, c.status, c.id, c.updated_date FROM comics as a INNER JOIN issues as b ON a.ComicID = b.ComicID INNER JOIN ddl_info as c ON b.IssueID = c.IssueID WHERE c.status != 'Downloading'")
s_info = myDB.select("SELECT a.ComicName, a.ComicVersion, a.ComicID, a.ComicYear, b.Issue_Number, b.IssueID, c.size, c.status, c.id, c.updated_date, c.issues, c.year FROM comics as a INNER JOIN issues as b ON a.ComicID = b.ComicID INNER JOIN ddl_info as c ON b.IssueID = c.IssueID") # WHERE c.status != 'Downloading'")
o_info = myDB.select("Select a.ComicName, b.Issue_Number, a.IssueID, a.ComicID, c.size, c.status, c.id, c.updated_date, c.issues, c.year from oneoffhistory a join snatched b on a.issueid=b.issueid join ddl_info c on b.issueid=c.issueid where b.provider = 'ddl'")
if s_info:
resultlist = []
for si in s_info:
issue = si['Issue_Number']
if issue is not None:
issue = '#%s' % issue
if si['issues'] is None:
issue = si['Issue_Number']
year = si['ComicYear']
if issue is not None:
issue = '#%s' % issue
else:
year = si['year']
issue = '#%s' % si['issues']
if si['status'] == 'Completed':
si_status = '100%'
else:
@ -2236,18 +2258,161 @@ class WebInterface(object):
'issue': issue,
'id': si['id'],
'volume': si['ComicVersion'],
'year': si['ComicYear'],
'year': year,
'size': si['size'].strip(),
'comicid': si['ComicID'],
'issueid': si['IssueID'],
'status': si['status'],
'updated_date': si['updated_date'],
'progress': si_status})
if o_info:
if type(resultlist) is str:
resultlist = []
logger.info('resultlist: %s' % resultlist)
return serve_template(templatename="queue_management.html", title="Queue Management", activelist=activelist, resultlist=resultlist)
for oi in o_info:
if oi['issues'] is None:
issue = oi['Issue_Number']
year = oi['year']
if issue is not None:
issue = '#%s' % issue
else:
year = oi['year']
issue = '#%s' % oi['issues']
if oi['status'] == 'Completed':
oi_status = '100%'
else:
oi_status = ''
resultlist.append({'series': oi['ComicName'],
'issue': issue,
'id': oi['id'],
'volume': None,
'year': year,
'size': oi['size'].strip(),
'comicid': oi['ComicID'],
'issueid': oi['IssueID'],
'status': oi['status'],
'updated_date': oi['updated_date'],
'progress': oi_status})
return serve_template(templatename="queue_management.html", title="Queue Management", resultlist=resultlist) #activelist=activelist, resultlist=resultlist)
queueManage.exposed = True
def queueManageIt(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=0, sSortDir_0="desc", sSearch="", **kwargs):
iDisplayStart = int(iDisplayStart)
iDisplayLength = int(iDisplayLength)
filtered = []
myDB = db.DBConnection()
resultlist = 'There are currently no items waiting in the Direct Download (DDL) Queue for processing.'
s_info = myDB.select("SELECT a.ComicName, a.ComicVersion, a.ComicID, a.ComicYear, b.Issue_Number, b.IssueID, c.size, c.status, c.id, c.updated_date, c.issues, c.year FROM comics as a INNER JOIN issues as b ON a.ComicID = b.ComicID INNER JOIN ddl_info as c ON b.IssueID = c.IssueID") # WHERE c.status != 'Downloading'")
o_info = myDB.select("Select a.ComicName, b.Issue_Number, a.IssueID, a.ComicID, c.size, c.status, c.id, c.updated_date, c.issues, c.year from oneoffhistory a join snatched b on a.issueid=b.issueid join ddl_info c on b.issueid=c.issueid where b.provider = 'ddl'")
if s_info:
resultlist = []
for si in s_info:
if si['issues'] is None:
issue = si['Issue_Number']
year = si['ComicYear']
if issue is not None:
issue = '#%s' % issue
else:
year = si['year']
issue = '#%s' % si['issues']
if si['status'] == 'Completed':
si_status = '100%'
else:
si_status = ''
if issue is not None:
if si['ComicVersion'] is not None:
series = '%s %s %s (%s)' % (si['ComicName'], si['ComicVersion'], issue, year)
else:
series = '%s %s (%s)' % (si['ComicName'], issue, year)
else:
if si['ComicVersion'] is not None:
series = '%s %s (%s)' % (si['ComicName'], si['ComicVersion'], year)
else:
series = '%s (%s)' % (si['ComicName'], year)
resultlist.append({'series': series, #i['ComicName'],
'issue': issue,
'queueid': si['id'],
'volume': si['ComicVersion'],
'year': year,
'size': si['size'].strip(),
'comicid': si['ComicID'],
'issueid': si['IssueID'],
'status': si['status'],
'updated_date': si['updated_date'],
'progress': si_status})
if o_info:
if type(resultlist) is str:
resultlist = []
for oi in o_info:
if oi['issues'] is None:
issue = oi['Issue_Number']
year = oi['year']
if issue is not None:
issue = '#%s' % issue
else:
year = oi['year']
issue = '#%s' % oi['issues']
if oi['status'] == 'Completed':
oi_status = '100%'
else:
oi_status = ''
if issue is not None:
series = '%s %s (%s)' % (oi['ComicName'], issue, year)
else:
series = '%s (%s)' % (oi['ComicName'], year)
resultlist.append({'series': series,
'issue': issue,
'queueid': oi['id'],
'volume': None,
'year': year,
'size': oi['size'].strip(),
'comicid': oi['ComicID'],
'issueid': oi['IssueID'],
'status': oi['status'],
'updated_date': oi['updated_date'],
'progress': oi_status})
if sSearch == "" or sSearch == None:
filtered = resultlist[::]
else:
filtered = [row for row in resultlist if any([sSearch.lower() in row['series'].lower(), sSearch.lower() in row['status'].lower()])]
sortcolumn = 'series'
if iSortCol_0 == '1':
sortcolumn = 'series'
elif iSortCol_0 == '2':
sortcolumn = 'size'
elif iSortCol_0 == '3':
sortcolumn = 'progress'
elif iSortCol_0 == '4':
sortcolumn = 'status'
elif iSortCol_0 == '5':
sortcolumn = 'updated_date'
filtered.sort(key=lambda x: x[sortcolumn], reverse=sSortDir_0 == "desc")
rows = filtered[iDisplayStart:(iDisplayStart + iDisplayLength)]
rows = [[row['comicid'], row['series'], row['size'], row['progress'], row['status'], row['updated_date'], row['queueid']] for row in rows]
#rows = [{'comicid': row['comicid'], 'series': row['series'], 'size': row['size'], 'progress': row['progress'], 'status': row['status'], 'updated_date': row['updated_date']} for row in rows]
#logger.info('rows: %s' % rows)
return json.dumps({
'iTotalDisplayRecords': len(filtered),
'iTotalRecords': len(resultlist),
'aaData': rows,
})
queueManageIt.exposed = True
def previewRename(self, **args): #comicid=None, comicidlist=None):
file_format = mylar.CONFIG.FILE_FORMAT
@ -4068,7 +4233,7 @@ class WebInterface(object):
mylar.CONFIG.IMP_METADATA = bool(imp_metadata)
mylar.CONFIG.IMP_PATHS = bool(imp_paths)
mylar.CONFIG.configure(update=True, startup=False)
mylar.CONFIG.configure(update=True)
# Write the config
logger.info('Now updating config...')
mylar.CONFIG.writeconfig()
@ -5253,7 +5418,7 @@ class WebInterface(object):
mylar.CONFIG.process_kwargs(kwargs)
#this makes sure things are set to the default values if they're not appropriately set.
mylar.CONFIG.configure(update=True)
mylar.CONFIG.configure(update=True, startup=False)
# Write the config
logger.info('Now saving config...')
@ -5894,11 +6059,31 @@ class WebInterface(object):
myDB = db.DBConnection()
active = myDB.selectone("SELECT * FROM DDL_INFO WHERE STATUS = 'Downloading'").fetchone()
if active is None:
return "There are no active downloads currently being attended to"
return json.dumps({'status': 'There are no active downloads currently being attended to',
'percent': 0,
'a_series': None,
'a_year': None,
'a_filename': None,
'a_size': None,
'a_id': None})
else:
filesize = os.stat(os.path.join(mylar.CONFIG.DDL_LOCATION, active['filename'])).st_size
cmath = int(float(filesize*100)/int(int(active['remote_filesize'])*100) * 100)
return "%s%s" % (cmath, '%')
filelocation = os.path.join(mylar.CONFIG.DDL_LOCATION, active['filename'])
#logger.fdebug('checking file existance: %s' % filelocation)
if os.path.exists(filelocation) is True:
filesize = os.stat(filelocation).st_size
cmath = int(float(filesize*100)/int(int(active['remote_filesize'])*100) * 100)
#logger.fdebug('ACTIVE DDL: %s %s [%s]' % (active['filename'], cmath, 'Downloading'))
return json.dumps({'status': 'Downloading',
'percent': "%s%s" % (cmath, '%'),
'a_series': active['series'],
'a_year': active['year'],
'a_filename': active['filename'],
'a_size': active['size'],
'a_id': active['id']})
else:
# myDB.upsert('ddl_info', {'status': 'Incomplete'}, {'id': active['id']})
return json.dumps({'a_id': active['id'], 'status': 'File does not exist in %s.</br> This probably needs to be restarted (use the option in the GUI)' % filelocation, 'percent': 0})
check_ActiveDDL.exposed = True
def create_readlist(self, list=None, weeknumber=None, year=None):