mirror of
https://github.com/evilhero/mylar
synced 2024-12-23 00:02:38 +00:00
21eee17344
torrents will now properly hide torrent information, IMP: Specified daemon port for deluge as an on-screen tip for more detail, IMP: Added 100,200,ALL as viewable watchlist views, FIX: When viewing pullist and annual integration enabled, if annual was present would incorrectly link to invalid annual series instead of the actual series itself, IMP: Added more detail error messages to metatagging errors and better handling of stranded files during cleanup, IMP: Improved some handling for weekly pull-list one-off's and refactored the nzb/oneoff post-processing into a seperate function for future callables, Moved all the main url locations for public torrent sites to the init module so that it can be cascaded down for use in other modules instead as a global, IMP: Added a 'deep_search_32p' variable in the config.ini for specific usage with 32p, where if there is more than one result will dig deeper into each result to try and figure out if there are series matches, as opposed to the default where it will only use ref32p table if available or just the first hit in a multiple series search results and ignore the remainder, FIX:Fixed some unknown characters appearing in the pullist due to unicode-related conversion problems, FIX: fixed some special cases of file parsing errors due to Volume label being named different than expected, FIX: Added a 3s pause between experimental searches to try and not hit their frequency limitation, IMP: Weekly Pullist One-off's will now show status of Snatched/Downloaded as required, FIX: Fixed some deluge parameter problems when using auto-snatch torrent script/option, IMP: Changed the downlocation in the auto-snatch option to an env variable instead of being passed to avoid unicode-related problems, FIX: Fixed some magnet-related issues for torrents when using a watchdir + TPSE, FIX: Added more verbose error message for rtorrent connection issues, FIX: Could not connect to rtorrent client if no username/password were provided, IMP: Set the db updater to run every 5 minutes on the watchlist, automatically refreshing the oldest updated series each time that is more than 5 hours old (force db update from the activity/job schedulers page will run the db updater against the entire watchlist in sequence), IMP: Attempt to handle long paths in windows (ie. > 256c) by prepending the unicode windows api character to the import a directory path (windows only), IMP: When manual metatagging a series, will update the series after all the metatagging has been completed as opposed to after each issue, IMP: Will now display available inkdrops on Config/Search Providers tab when using 32P (future will utilize/indicate inkdrop threshold when downloading)
149 lines
5 KiB
Python
149 lines
5 KiB
Python
# Copyright 2009 Brian Quinlan. All Rights Reserved.
|
|
# Licensed to PSF under a Contributor Agreement.
|
|
|
|
"""Implements ThreadPoolExecutor."""
|
|
|
|
import atexit
|
|
from concurrent.futures import _base
|
|
import Queue as queue
|
|
import threading
|
|
import weakref
|
|
import sys
|
|
|
|
try:
|
|
from multiprocessing import cpu_count
|
|
except ImportError:
|
|
# some platforms don't have multiprocessing
|
|
def cpu_count():
|
|
return None
|
|
|
|
__author__ = 'Brian Quinlan (brian@sweetapp.com)'
|
|
|
|
# Workers are created as daemon threads. This is done to allow the interpreter
|
|
# to exit when there are still idle threads in a ThreadPoolExecutor's thread
|
|
# pool (i.e. shutdown() was not called). However, allowing workers to die with
|
|
# the interpreter has two undesirable properties:
|
|
# - The workers would still be running during interpretor shutdown,
|
|
# meaning that they would fail in unpredictable ways.
|
|
# - The workers could be killed while evaluating a work item, which could
|
|
# be bad if the callable being evaluated has external side-effects e.g.
|
|
# writing to a file.
|
|
#
|
|
# To work around this problem, an exit handler is installed which tells the
|
|
# workers to exit when their work queues are empty and then waits until the
|
|
# threads finish.
|
|
|
|
_threads_queues = weakref.WeakKeyDictionary()
|
|
_shutdown = False
|
|
|
|
def _python_exit():
|
|
global _shutdown
|
|
_shutdown = True
|
|
items = list(_threads_queues.items()) if _threads_queues else ()
|
|
for t, q in items:
|
|
q.put(None)
|
|
for t, q in items:
|
|
t.join(sys.maxint)
|
|
|
|
atexit.register(_python_exit)
|
|
|
|
class _WorkItem(object):
|
|
def __init__(self, future, fn, args, kwargs):
|
|
self.future = future
|
|
self.fn = fn
|
|
self.args = args
|
|
self.kwargs = kwargs
|
|
|
|
def run(self):
|
|
if not self.future.set_running_or_notify_cancel():
|
|
return
|
|
|
|
try:
|
|
result = self.fn(*self.args, **self.kwargs)
|
|
except:
|
|
e, tb = sys.exc_info()[1:]
|
|
self.future.set_exception_info(e, tb)
|
|
else:
|
|
self.future.set_result(result)
|
|
|
|
def _worker(executor_reference, work_queue):
|
|
try:
|
|
while True:
|
|
work_item = work_queue.get(block=True)
|
|
if work_item is not None:
|
|
work_item.run()
|
|
# Delete references to object. See issue16284
|
|
del work_item
|
|
continue
|
|
executor = executor_reference()
|
|
# Exit if:
|
|
# - The interpreter is shutting down OR
|
|
# - The executor that owns the worker has been collected OR
|
|
# - The executor that owns the worker has been shutdown.
|
|
if _shutdown or executor is None or executor._shutdown:
|
|
# Notice other workers
|
|
work_queue.put(None)
|
|
return
|
|
del executor
|
|
except:
|
|
_base.LOGGER.critical('Exception in worker', exc_info=True)
|
|
|
|
|
|
class ThreadPoolExecutor(_base.Executor):
|
|
def __init__(self, max_workers=None):
|
|
"""Initializes a new ThreadPoolExecutor instance.
|
|
|
|
Args:
|
|
max_workers: The maximum number of threads that can be used to
|
|
execute the given calls.
|
|
"""
|
|
if max_workers is None:
|
|
# Use this number because ThreadPoolExecutor is often
|
|
# used to overlap I/O instead of CPU work.
|
|
max_workers = (cpu_count() or 1) * 5
|
|
if max_workers <= 0:
|
|
raise ValueError("max_workers must be greater than 0")
|
|
|
|
self._max_workers = max_workers
|
|
self._work_queue = queue.Queue()
|
|
self._threads = set()
|
|
self._shutdown = False
|
|
self._shutdown_lock = threading.Lock()
|
|
|
|
def submit(self, fn, *args, **kwargs):
|
|
with self._shutdown_lock:
|
|
if self._shutdown:
|
|
raise RuntimeError('cannot schedule new futures after shutdown')
|
|
|
|
f = _base.Future()
|
|
w = _WorkItem(f, fn, args, kwargs)
|
|
|
|
self._work_queue.put(w)
|
|
self._adjust_thread_count()
|
|
return f
|
|
submit.__doc__ = _base.Executor.submit.__doc__
|
|
|
|
def _adjust_thread_count(self):
|
|
# When the executor gets lost, the weakref callback will wake up
|
|
# the worker threads.
|
|
def weakref_cb(_, q=self._work_queue):
|
|
q.put(None)
|
|
# TODO(bquinlan): Should avoid creating new threads if there are more
|
|
# idle threads than items in the work queue.
|
|
if len(self._threads) < self._max_workers:
|
|
t = threading.Thread(target=_worker,
|
|
args=(weakref.ref(self, weakref_cb),
|
|
self._work_queue))
|
|
t.daemon = True
|
|
t.start()
|
|
self._threads.add(t)
|
|
_threads_queues[t] = self._work_queue
|
|
|
|
def shutdown(self, wait=True):
|
|
with self._shutdown_lock:
|
|
self._shutdown = True
|
|
self._work_queue.put(None)
|
|
if wait:
|
|
for t in self._threads:
|
|
t.join(sys.maxint)
|
|
shutdown.__doc__ = _base.Executor.shutdown.__doc__
|