mirror
/
mylar
mirror of https://github.com/evilhero/mylar
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'development'

This commit is contained in:
evilhero 2019-12-29 20:14:14 -05:00
parent 5ffe8fbd30 81fa69c246
commit d1ff6079a0
19 changed files with 725 additions and 340 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
data/interfaces/default/config.html
View File

@ -818,12 +818,14 @@
<input id="enable_torrent_search" type="checkbox" onclick="initConfigCheckbox($(this));" name="enable_torrent_search" value=1 ${config['enable_torrent_search']} /><legend>Torrents</legned> <input id="enable_torrent_search" type="checkbox" onclick="initConfigCheckbox($(this));" name="enable_torrent_search" value=1 ${config['enable_torrent_search']} /><legend>Torrents</legned>
</div> </div>
<div class="config"> <div class="config">
<!--
<div class="row checkbox left clearfix"> <div class="row checkbox left clearfix">
<input id="enable_public" title="Use Public Torrents" type="checkbox" name="enable_public" value=1 ${config['enable_public']} /><label>Enable Public Torrent Search</label> <input id="enable_public" title="Use Public Torrents" type="checkbox" name="enable_public" value=1 ${config['enable_public']} /><label>Enable Public Torrent Search</label>
<div align="left"> <div align="left">
<small class="heading"><span style="float: left; margin-left: .3em; margin-top: 4px;" class="ui-icon ui-icon-info"></span>Search: WWT / RSS: WWT</small> <small class="heading"><span style="float: left; margin-left: .3em; margin-top: 4px;" class="ui-icon ui-icon-info"></span>Search: None / RSS: None</small>
</div> </div>
</div> </div>
-->
<div class="row checkbox left clearfix"> <div class="row checkbox left clearfix">
<input type="checkbox" id="enable_32p" onclick="initConfigCheckbox($(this));" name="enable_32p" value=1 ${config['enable_32p']} /><label>Enable 32P</label> <input type="checkbox" id="enable_32p" onclick="initConfigCheckbox($(this));" name="enable_32p" value=1 ${config['enable_32p']} /><label>Enable 32P</label>
<div align="left"> <div align="left">
@ -1212,7 +1214,11 @@
%> %>
<a href="#" title="${folder_options}"><img src="interfaces/default/images/info32.png" height="16" alt="" /></a> <a href="#" title="${folder_options}"><img src="interfaces/default/images/info32.png" height="16" alt="" /></a>
<small>Use: $Publisher, $Series, $Year<br /> <small>Use: $Publisher, $Series, $Year<br />
E.g.: $Publisher/$Series ($Year) = DC Comics/Action Comics (2011)</small> %if 'windows' in mylar.OS_DETECT.lower():
E.g.: $Publisher\$Series ($Year) = DC Comics\Action Comics (2011)</small>
%else:
E.g.: $Publisher/$Series ($Year) = DC Comics/Action Comics (2011)</small>
%endif
</div> </div>
<div class="row"> <div class="row">
<label> File Format</label> <label> File Format</label>

4
data/js/alphabetSearch.js
View File

@ -55,7 +55,7 @@ $.fn.dataTable.ext.search.push( function ( context, searchData ) {
if ( context.alphabetSearch.match('nonalpha') && !(searchData[1].charAt(0).match(/^[a-zA-Z]/)) ) { if ( context.alphabetSearch.match('nonalpha') && !(searchData[1].charAt(0).match(/^[a-zA-Z]/)) ) {
return true; return true;
} }
if ( searchData[1].charAt(0) === context.alphabetSearch ) { if ( searchData[1].charAt(0).toUpperCase() === context.alphabetSearch ) {
return true; return true;
} }
@ -70,7 +70,7 @@ function bin ( data ) {
bins['nonalpha'] = 0; bins['nonalpha'] = 0;
for ( var i=0, ien=data.length ; i<ien ; i++ ) { for ( var i=0, ien=data.length ; i<ien ; i++ ) {
letter = data[i].charAt(13).toUpperCase(); letter = data[i].charAt(13).toUpperCase();
if ( !letter.match(/^[A-Z]/) ) { if ( !letter.match(/^[a-zA-Z]/) ) {
bins['nonalpha']++; bins['nonalpha']++;
} }
else if ( bins[letter] ) { else if ( bins[letter] ) {

9
init-scripts/systemd/mylar.initd
View File

@ -1,4 +1,13 @@
#!/usr/bin/env #!/usr/bin/env
### BEGIN INIT INFO
# Provides: mylar
# Required-Start: $all
# Required-Stop:
# Default-Start: 2 3 4 5
# Default-Stop:
# Short-Description: Mylar
### END INIT INFO
# Script name # Script name
NAME=mylar NAME=mylar

2
lib/deluge_client/__init__.py
View File

@ -1 +1 @@
from .client import DelugeRPCClient from .client import DelugeRPCClient, FailedToReconnectException

252
lib/deluge_client/client.py
View File

@ -2,6 +2,7 @@ import logging
import socket import socket
import ssl import ssl
import struct import struct
import warnings
import zlib import zlib
from .rencode import dumps, loads from .rencode import dumps, loads
@ -10,102 +11,265 @@ RPC_RESPONSE = 1
RPC_ERROR = 2 RPC_ERROR = 2
RPC_EVENT = 3 RPC_EVENT = 3
#MESSAGE_HEADER_SIZE = 5 MESSAGE_HEADER_SIZE = 5
READ_SIZE = 10 READ_SIZE = 10
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class ConnectionLostException(Exception):
class DelugeClientException(Exception):
"""Base exception for all deluge client exceptions"""
class ConnectionLostException(DelugeClientException):
pass pass
class CallTimeoutException(Exception):
class CallTimeoutException(DelugeClientException):
pass pass
class InvalidHeaderException(DelugeClientException):
pass
class FailedToReconnectException(DelugeClientException):
pass
class RemoteException(DelugeClientException):
pass
class DelugeRPCClient(object): class DelugeRPCClient(object):
timeout = 20 timeout = 20
def __init__(self, host, port, username, password): def __init__(self, host, port, username, password, decode_utf8=False, automatic_reconnect=True):
self.host = host self.host = host
self.port = port self.port = port
self.username = username self.username = username
self.password = password self.password = password
self.deluge_version = None
# This is only applicable if deluge_version is 2
self.deluge_protocol_version = None
self.decode_utf8 = decode_utf8
if not self.decode_utf8:
warnings.warn('Using `decode_utf8=False` is deprecated, please set it to True.'
'The argument will be removed in a future release where it will be always True', DeprecationWarning)
self.automatic_reconnect = automatic_reconnect
self.request_id = 1 self.request_id = 1
self.connected = False self.connected = False
self._create_socket() self._create_socket()
def _create_socket(self, ssl_version=None): def _create_socket(self, ssl_version=None):
if ssl_version is not None: if ssl_version is not None:
self._socket = ssl.wrap_socket(socket.socket(socket.AF_INET, socket.SOCK_STREAM), ssl_version=ssl_version) self._socket = ssl.wrap_socket(socket.socket(socket.AF_INET, socket.SOCK_STREAM), ssl_version=ssl_version)
else: else:
self._socket = ssl.wrap_socket(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) self._socket = ssl.wrap_socket(socket.socket(socket.AF_INET, socket.SOCK_STREAM))
self._socket.settimeout(self.timeout) self._socket.settimeout(self.timeout)
def connect(self): def connect(self):
""" """
Connects to the Deluge instance Connects to the Deluge instance
""" """
self._connect()
logger.debug('Connected to Deluge, detecting daemon version')
self._detect_deluge_version()
logger.debug('Daemon version {} detected, logging in'.format(self.deluge_version))
if self.deluge_version == 2:
result = self.call('daemon.login', self.username, self.password, client_version='deluge-client')
else:
result = self.call('daemon.login', self.username, self.password)
logger.debug('Logged in with value %r' % result)
self.connected = True
def _connect(self):
logger.info('Connecting to %s:%s' % (self.host, self.port)) logger.info('Connecting to %s:%s' % (self.host, self.port))
try: try:
self._socket.connect((self.host, self.port)) self._socket.connect((self.host, self.port))
except ssl.SSLError as e: except ssl.SSLError as e:
if e.reason != 'UNSUPPORTED_PROTOCOL' or not hasattr(ssl, 'PROTOCOL_SSLv3'): # Note: have not verified that we actually get errno 258 for this error
if (hasattr(ssl, 'PROTOCOL_SSLv3') and
(getattr(e, 'reason', None) == 'UNSUPPORTED_PROTOCOL' or e.errno == 258)):
logger.warning('Was unable to ssl handshake, trying to force SSLv3 (insecure)')
self._create_socket(ssl_version=ssl.PROTOCOL_SSLv3)
self._socket.connect((self.host, self.port))
else:
raise raise
logger.warning('Was unable to ssl handshake, trying to force SSLv3 (insecure)')
self._create_socket(ssl_version=ssl.PROTOCOL_SSLv3)
self._socket.connect((self.host, self.port))
logger.debug('Connected to Deluge, logging in')
result = self.call('daemon.login', self.username, self.password)
logger.debug('Logged in with value %r' % result)
self.connected = True
def disconnect(self): def disconnect(self):
""" """
Disconnect from deluge Disconnect from deluge
""" """
if self.connected: if self.connected:
self._socket.close() self._socket.close()
self._socket = None
def call(self, method, *args, **kwargs): self.connected = False
"""
Calls an RPC function def _detect_deluge_version(self):
""" if self.deluge_version is not None:
return
self._send_call(1, None, 'daemon.info')
self._send_call(2, None, 'daemon.info')
self._send_call(2, 1, 'daemon.info')
result = self._socket.recv(1)
if result[:1] == b'D':
# This is a protocol deluge 2.0 was using before release
self.deluge_version = 2
self.deluge_protocol_version = None
# If we need the specific version of deluge 2, this is it.
daemon_version = self._receive_response(2, None, partial_data=result)
elif ord(result[:1]) == 1:
self.deluge_version = 2
self.deluge_protocol_version = 1
# If we need the specific version of deluge 2, this is it.
daemon_version = self._receive_response(2, 1, partial_data=result)
else:
self.deluge_version = 1
# Deluge 1 doesn't recover well from the bad request. Re-connect the socket.
self._socket.close()
self._create_socket()
self._connect()
def _send_call(self, deluge_version, protocol_version, method, *args, **kwargs):
self.request_id += 1 self.request_id += 1
logger.debug('Calling reqid %s method %r with args:%r kwargs:%r' % (self.request_id, method, args, kwargs)) if method == 'daemon.login':
debug_args = list(args)
if len(debug_args) >= 2:
debug_args[1] = '<password hidden>'
logger.debug('Calling reqid %s method %r with args:%r kwargs:%r' % (self.request_id, method, debug_args, kwargs))
else:
logger.debug('Calling reqid %s method %r with args:%r kwargs:%r' % (self.request_id, method, args, kwargs))
req = ((self.request_id, method, args, kwargs), ) req = ((self.request_id, method, args, kwargs), )
req = zlib.compress(dumps(req)) req = zlib.compress(dumps(req))
#self._socket.send('D' + struct.pack("!i", len(req))) # seems to be for the future ! if deluge_version == 2:
if protocol_version is None:
# This was a protocol for deluge 2 before they introduced protocol version numbers
self._socket.send(b'D' + struct.pack("!i", len(req)))
elif protocol_version == 1:
self._socket.send(struct.pack('!BI', protocol_version, len(req)))
else:
raise Exception('Deluge protocol version {} is not (yet) supported.'.format(protocol_version))
self._socket.send(req) self._socket.send(req)
data = b'' def _receive_response(self, deluge_version, protocol_version, partial_data=b''):
expected_bytes = None
data = partial_data
while True: while True:
try: try:
d = self._socket.recv(READ_SIZE) d = self._socket.recv(READ_SIZE)
except ssl.SSLError: except ssl.SSLError:
raise CallTimeoutException() raise CallTimeoutException()
data += d data += d
try: if deluge_version == 2:
data = zlib.decompress(data) if expected_bytes is None:
except zlib.error: if len(data) < 5:
if not d: continue
raise ConnectionLostException()
continue header = data[:MESSAGE_HEADER_SIZE]
break data = data[MESSAGE_HEADER_SIZE:]
data = list(loads(data)) if protocol_version is None:
if header[0] != b'D'[0]:
raise InvalidHeaderException('Expected D as first byte in reply')
elif ord(header[:1]) != protocol_version:
raise InvalidHeaderException(
'Expected protocol version ({}) as first byte in reply'.format(protocol_version)
)
if protocol_version is None:
expected_bytes = struct.unpack('!i', header[1:])[0]
else:
expected_bytes = struct.unpack('!I', header[1:])[0]
if len(data) >= expected_bytes:
data = zlib.decompress(data)
break
else:
try:
data = zlib.decompress(data)
except zlib.error:
if not d:
raise ConnectionLostException()
continue
break
data = list(loads(data, decode_utf8=self.decode_utf8))
msg_type = data.pop(0) msg_type = data.pop(0)
request_id = data.pop(0) request_id = data.pop(0)
if msg_type == RPC_ERROR: if msg_type == RPC_ERROR:
exception_type, exception_msg, traceback = data[0] if self.deluge_version == 2:
exception = type(str(exception_type), (Exception, ), {}) exception_type, exception_msg, _, traceback = data
exception_msg = '%s\n\n%s' % (exception_msg, traceback) # On deluge 2, exception arguments are sent as tuple
if self.decode_utf8:
exception_msg = ', '.join(exception_msg)
else:
exception_msg = b', '.join(exception_msg)
else:
exception_type, exception_msg, traceback = data[0]
if self.decode_utf8:
exception = type(str(exception_type), (RemoteException, ), {})
exception_msg = '%s\n%s' % (exception_msg,
traceback)
else:
exception = type(str(exception_type.decode('utf-8', 'ignore')), (RemoteException, ), {})
exception_msg = '%s\n%s' % (exception_msg.decode('utf-8', 'ignore'),
traceback.decode('utf-8', 'ignore'))
raise exception(exception_msg) raise exception(exception_msg)
elif msg_type == RPC_RESPONSE: elif msg_type == RPC_RESPONSE:
retval = data[0] retval = data[0]
return retval return retval
def reconnect(self):
"""
Reconnect
"""
self.disconnect()
self._create_socket()
self.connect()
def call(self, method, *args, **kwargs):
"""
Calls an RPC function
"""
tried_reconnect = False
for _ in range(2):
try:
self._send_call(self.deluge_version, self.deluge_protocol_version, method, *args, **kwargs)
return self._receive_response(self.deluge_version, self.deluge_protocol_version)
except (socket.error, ConnectionLostException, CallTimeoutException):
if self.automatic_reconnect:
if tried_reconnect:
raise FailedToReconnectException()
else:
try:
self.reconnect()
except (socket.error, ConnectionLostException, CallTimeoutException):
raise FailedToReconnectException()
tried_reconnect = True
else:
raise
def __getattr__(self, item):
return RPCCaller(self.call, item)
class RPCCaller(object):
def __init__(self, caller, method=''):
self.caller = caller
self.method = method
def __getattr__(self, item):
return RPCCaller(self.caller, self.method+'.'+item)
def __call__(self, *args, **kwargs):
return self.caller(self.method, *args, **kwargs)

310
lib/deluge_client/rencode.py
View File

@ -1,27 +1,3 @@
"""
rencode -- Web safe object pickling/unpickling.
Public domain, Connelly Barnes 2006-2007.
The rencode module is a modified version of bencode from the
BitTorrent project. For complex, heterogeneous data structures with
many small elements, r-encodings take up significantly less space than
b-encodings:
>>> len(rencode.dumps({'a':0, 'b':[1,2], 'c':99}))
13
>>> len(bencode.bencode({'a':0, 'b':[1,2], 'c':99}))
26
The rencode format is not standardized, and may change with different
rencode module versions, so you should check that you are using the
same rencode version throughout your project.
"""
__version__ = '1.0.2'
__all__ = ['dumps', 'loads']
# Original bencode module by Petru Paler, et al. # Original bencode module by Petru Paler, et al.
# #
# Modifications by Connelly Barnes: # Modifications by Connelly Barnes:
@ -62,23 +38,50 @@ __all__ = ['dumps', 'loads']
# (The rencode module is licensed under the above license as well). # (The rencode module is licensed under the above license as well).
# #
import sys """
rencode -- Web safe object pickling/unpickling.
Public domain, Connelly Barnes 2006-2007.
The rencode module is a modified version of bencode from the
BitTorrent project. For complex, heterogeneous data structures with
many small elements, r-encodings take up significantly less space than
b-encodings:
>>> len(rencode.dumps({'a':0, 'b':[1,2], 'c':99}))
13
>>> len(bencode.bencode({'a':0, 'b':[1,2], 'c':99}))
26
The rencode format is not standardized, and may change with different
rencode module versions, so you should check that you are using the
same rencode version throughout your project.
"""
py3 = False
if sys.version_info >= (3, 0):
py3 = True
long = int
unicode = str
def int2byte(c):
if py3:
return bytes([c])
else:
return chr(c)
import struct import struct
import sys
from threading import Lock from threading import Lock
try:
from future_builtins import zip
except ImportError:
# Ignore on Py3.
pass
__version__ = ('Python', 1, 0, 4)
__all__ = ['dumps', 'loads']
py3 = sys.version_info[0] >= 3
if py3:
long = int # pylint: disable=redefined-builtin
unicode = str # pylint: disable=redefined-builtin
def int2byte(c):
return bytes([c])
else:
def int2byte(c):
return chr(c)
# Default number of bits for serialized floats, either 32 or 64 (also a parameter for dumps()). # Default number of bits for serialized floats, either 32 or 64 (also a parameter for dumps()).
DEFAULT_FLOAT_BITS = 32 DEFAULT_FLOAT_BITS = 32
@ -87,19 +90,19 @@ MAX_INT_LENGTH = 64
# The bencode 'typecodes' such as i, d, etc have been extended and # The bencode 'typecodes' such as i, d, etc have been extended and
# relocated on the base-256 character set. # relocated on the base-256 character set.
CHR_LIST = int2byte(59) CHR_LIST = int2byte(59)
CHR_DICT = int2byte(60) CHR_DICT = int2byte(60)
CHR_INT = int2byte(61) CHR_INT = int2byte(61)
CHR_INT1 = int2byte(62) CHR_INT1 = int2byte(62)
CHR_INT2 = int2byte(63) CHR_INT2 = int2byte(63)
CHR_INT4 = int2byte(64) CHR_INT4 = int2byte(64)
CHR_INT8 = int2byte(65) CHR_INT8 = int2byte(65)
CHR_FLOAT32 = int2byte(66) CHR_FLOAT32 = int2byte(66)
CHR_FLOAT64 = int2byte(44) CHR_FLOAT64 = int2byte(44)
CHR_TRUE = int2byte(67) CHR_TRUE = int2byte(67)
CHR_FALSE = int2byte(68) CHR_FALSE = int2byte(68)
CHR_NONE = int2byte(69) CHR_NONE = int2byte(69)
CHR_TERM = int2byte(127) CHR_TERM = int2byte(127)
# Positive integers with value embedded in typecode. # Positive integers with value embedded in typecode.
INT_POS_FIXED_START = 0 INT_POS_FIXED_START = 0
@ -118,12 +121,13 @@ STR_FIXED_START = 128
STR_FIXED_COUNT = 64 STR_FIXED_COUNT = 64
# Lists with length embedded in typecode. # Lists with length embedded in typecode.
LIST_FIXED_START = STR_FIXED_START+STR_FIXED_COUNT LIST_FIXED_START = STR_FIXED_START + STR_FIXED_COUNT
LIST_FIXED_COUNT = 64 LIST_FIXED_COUNT = 64
# Whether strings should be decoded when loading # Whether strings should be decoded when loading
_decode_utf8 = False _decode_utf8 = False
def decode_int(x, f): def decode_int(x, f):
f += 1 f += 1
newf = x.index(CHR_TERM, f) newf = x.index(CHR_TERM, f)
@ -133,39 +137,46 @@ def decode_int(x, f):
n = int(x[f:newf]) n = int(x[f:newf])
except (OverflowError, ValueError): except (OverflowError, ValueError):
n = long(x[f:newf]) n = long(x[f:newf])
if x[f:f+1] == '-': if x[f:f + 1] == '-':
if x[f + 1:f + 2] == '0': if x[f + 1:f + 2] == '0':
raise ValueError raise ValueError
elif x[f:f+1] == '0' and newf != f+1: elif x[f:f + 1] == '0' and newf != f + 1:
raise ValueError raise ValueError
return (n, newf+1) return (n, newf + 1)
def decode_intb(x, f): def decode_intb(x, f):
f += 1 f += 1
return (struct.unpack('!b', x[f:f+1])[0], f+1) return (struct.unpack('!b', x[f:f + 1])[0], f + 1)
def decode_inth(x, f): def decode_inth(x, f):
f += 1 f += 1
return (struct.unpack('!h', x[f:f+2])[0], f+2) return (struct.unpack('!h', x[f:f + 2])[0], f + 2)
def decode_intl(x, f): def decode_intl(x, f):
f += 1 f += 1
return (struct.unpack('!l', x[f:f+4])[0], f+4) return (struct.unpack('!l', x[f:f + 4])[0], f + 4)
def decode_intq(x, f): def decode_intq(x, f):
f += 1 f += 1
return (struct.unpack('!q', x[f:f+8])[0], f+8) return (struct.unpack('!q', x[f:f + 8])[0], f + 8)
def decode_float32(x, f): def decode_float32(x, f):
f += 1 f += 1
n = struct.unpack('!f', x[f:f+4])[0] n = struct.unpack('!f', x[f:f + 4])[0]
return (n, f+4) return (n, f + 4)
def decode_float64(x, f): def decode_float64(x, f):
f += 1 f += 1
n = struct.unpack('!d', x[f:f+8])[0] n = struct.unpack('!d', x[f:f + 8])[0]
return (n, f+8) return (n, f + 8)
def decode_string(x, f): def decode_string(x, f):
colon = x.index(b':', f) colon = x.index(b':', f)
@ -173,36 +184,42 @@ def decode_string(x, f):
n = int(x[f:colon]) n = int(x[f:colon])
except (OverflowError, ValueError): except (OverflowError, ValueError):
n = long(x[f:colon]) n = long(x[f:colon])
if x[f] == '0' and colon != f+1: if x[f] == '0' and colon != f + 1:
raise ValueError raise ValueError
colon += 1 colon += 1
s = x[colon:colon+n] s = x[colon:colon + n]
if _decode_utf8: if _decode_utf8:
s = s.decode('utf8') s = s.decode('utf8')
return (s, colon+n) return (s, colon + n)
def decode_list(x, f): def decode_list(x, f):
r, f = [], f+1 r, f = [], f + 1
while x[f:f+1] != CHR_TERM: while x[f:f + 1] != CHR_TERM:
v, f = decode_func[x[f:f+1]](x, f) v, f = decode_func[x[f:f + 1]](x, f)
r.append(v) r.append(v)
return (tuple(r), f + 1) return (tuple(r), f + 1)
def decode_dict(x, f): def decode_dict(x, f):
r, f = {}, f+1 r, f = {}, f + 1
while x[f:f+1] != CHR_TERM: while x[f:f + 1] != CHR_TERM:
k, f = decode_func[x[f:f+1]](x, f) k, f = decode_func[x[f:f + 1]](x, f)
r[k], f = decode_func[x[f:f+1]](x, f) r[k], f = decode_func[x[f:f + 1]](x, f)
return (r, f + 1) return (r, f + 1)
def decode_true(x, f): def decode_true(x, f):
return (True, f+1) return (True, f + 1)
def decode_false(x, f): def decode_false(x, f):
return (False, f+1) return (False, f + 1)
def decode_none(x, f): def decode_none(x, f):
return (None, f+1) return (None, f + 1)
decode_func = {} decode_func = {}
decode_func[b'0'] = decode_string decode_func[b'0'] = decode_string
@ -215,72 +232,81 @@ decode_func[b'6'] = decode_string
decode_func[b'7'] = decode_string decode_func[b'7'] = decode_string
decode_func[b'8'] = decode_string decode_func[b'8'] = decode_string
decode_func[b'9'] = decode_string decode_func[b'9'] = decode_string
decode_func[CHR_LIST ] = decode_list decode_func[CHR_LIST] = decode_list
decode_func[CHR_DICT ] = decode_dict decode_func[CHR_DICT] = decode_dict
decode_func[CHR_INT ] = decode_int decode_func[CHR_INT] = decode_int
decode_func[CHR_INT1 ] = decode_intb decode_func[CHR_INT1] = decode_intb
decode_func[CHR_INT2 ] = decode_inth decode_func[CHR_INT2] = decode_inth
decode_func[CHR_INT4 ] = decode_intl decode_func[CHR_INT4] = decode_intl
decode_func[CHR_INT8 ] = decode_intq decode_func[CHR_INT8] = decode_intq
decode_func[CHR_FLOAT32] = decode_float32 decode_func[CHR_FLOAT32] = decode_float32
decode_func[CHR_FLOAT64] = decode_float64 decode_func[CHR_FLOAT64] = decode_float64
decode_func[CHR_TRUE ] = decode_true decode_func[CHR_TRUE] = decode_true
decode_func[CHR_FALSE ] = decode_false decode_func[CHR_FALSE] = decode_false
decode_func[CHR_NONE ] = decode_none decode_func[CHR_NONE] = decode_none
def make_fixed_length_string_decoders(): def make_fixed_length_string_decoders():
def make_decoder(slen): def make_decoder(slen):
def f(x, f): def f(x, f):
s = x[f+1:f+1+slen] s = x[f + 1:f + 1 + slen]
if _decode_utf8: if _decode_utf8:
s = s.decode("utf8") s = s.decode('utf8')
return (s, f+1+slen) return (s, f + 1 + slen)
return f return f
for i in range(STR_FIXED_COUNT): for i in range(STR_FIXED_COUNT):
decode_func[int2byte(STR_FIXED_START+i)] = make_decoder(i) decode_func[int2byte(STR_FIXED_START + i)] = make_decoder(i)
make_fixed_length_string_decoders() make_fixed_length_string_decoders()
def make_fixed_length_list_decoders(): def make_fixed_length_list_decoders():
def make_decoder(slen): def make_decoder(slen):
def f(x, f): def f(x, f):
r, f = [], f+1 r, f = [], f + 1
for i in range(slen): for _ in range(slen):
v, f = decode_func[x[f:f+1]](x, f) v, f = decode_func[x[f:f + 1]](x, f)
r.append(v) r.append(v)
return (tuple(r), f) return (tuple(r), f)
return f return f
for i in range(LIST_FIXED_COUNT): for i in range(LIST_FIXED_COUNT):
decode_func[int2byte(LIST_FIXED_START+i)] = make_decoder(i) decode_func[int2byte(LIST_FIXED_START + i)] = make_decoder(i)
make_fixed_length_list_decoders() make_fixed_length_list_decoders()
def make_fixed_length_int_decoders(): def make_fixed_length_int_decoders():
def make_decoder(j): def make_decoder(j):
def f(x, f): def f(x, f):
return (j, f+1) return (j, f + 1)
return f return f
for i in range(INT_POS_FIXED_COUNT): for i in range(INT_POS_FIXED_COUNT):
decode_func[int2byte(INT_POS_FIXED_START+i)] = make_decoder(i) decode_func[int2byte(INT_POS_FIXED_START + i)] = make_decoder(i)
for i in range(INT_NEG_FIXED_COUNT): for i in range(INT_NEG_FIXED_COUNT):
decode_func[int2byte(INT_NEG_FIXED_START+i)] = make_decoder(-1-i) decode_func[int2byte(INT_NEG_FIXED_START + i)] = make_decoder(-1 - i)
make_fixed_length_int_decoders() make_fixed_length_int_decoders()
def make_fixed_length_dict_decoders(): def make_fixed_length_dict_decoders():
def make_decoder(slen): def make_decoder(slen):
def f(x, f): def f(x, f):
r, f = {}, f+1 r, f = {}, f + 1
for j in range(slen): for _ in range(slen):
k, f = decode_func[x[f:f+1]](x, f) k, f = decode_func[x[f:f + 1]](x, f)
r[k], f = decode_func[x[f:f+1]](x, f) r[k], f = decode_func[x[f:f + 1]](x, f)
return (r, f) return (r, f)
return f return f
for i in range(DICT_FIXED_COUNT): for i in range(DICT_FIXED_COUNT):
decode_func[int2byte(DICT_FIXED_START+i)] = make_decoder(i) decode_func[int2byte(DICT_FIXED_START + i)] = make_decoder(i)
make_fixed_length_dict_decoders() make_fixed_length_dict_decoders()
def loads(x, decode_utf8=False): def loads(x, decode_utf8=False):
global _decode_utf8 global _decode_utf8
_decode_utf8 = decode_utf8 _decode_utf8 = decode_utf8
@ -292,11 +318,12 @@ def loads(x, decode_utf8=False):
raise ValueError raise ValueError
return r return r
def encode_int(x, r): def encode_int(x, r):
if 0 <= x < INT_POS_FIXED_COUNT: if 0 <= x < INT_POS_FIXED_COUNT:
r.append(int2byte(INT_POS_FIXED_START+x)) r.append(int2byte(INT_POS_FIXED_START + x))
elif -INT_NEG_FIXED_COUNT <= x < 0: elif -INT_NEG_FIXED_COUNT <= x < 0:
r.append(int2byte(INT_NEG_FIXED_START-1-x)) r.append(int2byte(INT_NEG_FIXED_START - 1 - x))
elif -128 <= x < 128: elif -128 <= x < 128:
r.extend((CHR_INT1, struct.pack('!b', x))) r.extend((CHR_INT1, struct.pack('!b', x)))
elif -32768 <= x < 32768: elif -32768 <= x < 32768:
@ -308,35 +335,42 @@ def encode_int(x, r):
else: else:
s = str(x) s = str(x)
if py3: if py3:
s = bytes(s, "ascii") s = bytes(s, 'ascii')
if len(s) >= MAX_INT_LENGTH: if len(s) >= MAX_INT_LENGTH:
raise ValueError('overflow') raise ValueError('overflow')
r.extend((CHR_INT, s, CHR_TERM)) r.extend((CHR_INT, s, CHR_TERM))
def encode_float32(x, r): def encode_float32(x, r):
r.extend((CHR_FLOAT32, struct.pack('!f', x))) r.extend((CHR_FLOAT32, struct.pack('!f', x)))
def encode_float64(x, r): def encode_float64(x, r):
r.extend((CHR_FLOAT64, struct.pack('!d', x))) r.extend((CHR_FLOAT64, struct.pack('!d', x)))
def encode_bool(x, r): def encode_bool(x, r):
r.append({False: CHR_FALSE, True: CHR_TRUE}[bool(x)]) r.append({False: CHR_FALSE, True: CHR_TRUE}[bool(x)])
def encode_none(x, r): def encode_none(x, r):
r.append(CHR_NONE) r.append(CHR_NONE)
def encode_string(x, r): def encode_string(x, r):
if len(x) < STR_FIXED_COUNT: if len(x) < STR_FIXED_COUNT:
r.extend((int2byte(STR_FIXED_START + len(x)), x)) r.extend((int2byte(STR_FIXED_START + len(x)), x))
else: else:
s = str(len(x)) s = str(len(x))
if py3: if py3:
s = bytes(s, "ascii") s = bytes(s, 'ascii')
r.extend((s, b':', x)) r.extend((s, b':', x))
def encode_unicode(x, r): def encode_unicode(x, r):
encode_string(x.encode("utf8"), r) encode_string(x.encode('utf8'), r)
def encode_list(x, r): def encode_list(x, r):
if len(x) < LIST_FIXED_COUNT: if len(x) < LIST_FIXED_COUNT:
@ -349,7 +383,8 @@ def encode_list(x, r):
encode_func[type(i)](i, r) encode_func[type(i)](i, r)
r.append(CHR_TERM) r.append(CHR_TERM)
def encode_dict(x,r):
def encode_dict(x, r):
if len(x) < DICT_FIXED_COUNT: if len(x) < DICT_FIXED_COUNT:
r.append(int2byte(DICT_FIXED_START + len(x))) r.append(int2byte(DICT_FIXED_START + len(x)))
for k, v in x.items(): for k, v in x.items():
@ -362,6 +397,7 @@ def encode_dict(x,r):
encode_func[type(v)](v, r) encode_func[type(v)](v, r)
r.append(CHR_TERM) r.append(CHR_TERM)
encode_func = {} encode_func = {}
encode_func[int] = encode_int encode_func[int] = encode_int
encode_func[long] = encode_int encode_func[long] = encode_int
@ -375,14 +411,14 @@ encode_func[bool] = encode_bool
lock = Lock() lock = Lock()
def dumps(x, float_bits=DEFAULT_FLOAT_BITS): def dumps(x, float_bits=DEFAULT_FLOAT_BITS):
""" """
Dump data structure to str. Dump data structure to str.
Here float_bits is either 32 or 64. Here float_bits is either 32 or 64.
""" """
lock.acquire() with lock:
try:
if float_bits == 32: if float_bits == 32:
encode_func[float] = encode_float32 encode_func[float] = encode_float32
elif float_bits == 64: elif float_bits == 64:
@ -391,39 +427,41 @@ def dumps(x, float_bits=DEFAULT_FLOAT_BITS):
raise ValueError('Float bits (%d) is not 32 or 64' % float_bits) raise ValueError('Float bits (%d) is not 32 or 64' % float_bits)
r = [] r = []
encode_func[type(x)](x, r) encode_func[type(x)](x, r)
finally:
lock.release()
return b''.join(r) return b''.join(r)
def test(): def test():
f1 = struct.unpack('!f', struct.pack('!f', 25.5))[0] f1 = struct.unpack('!f', struct.pack('!f', 25.5))[0]
f2 = struct.unpack('!f', struct.pack('!f', 29.3))[0] f2 = struct.unpack('!f', struct.pack('!f', 29.3))[0]
f3 = struct.unpack('!f', struct.pack('!f', -0.6))[0] f3 = struct.unpack('!f', struct.pack('!f', -0.6))[0]
L = (({b'a':15, b'bb':f1, b'ccc':f2, b'':(f3,(),False,True,b'')},(b'a',10**20),tuple(range(-100000,100000)),b'b'*31,b'b'*62,b'b'*64,2**30,2**33,2**62,2**64,2**30,2**33,2**62,2**64,False,False, True, -1, 2, 0),) ld = (({b'a': 15, b'bb': f1, b'ccc': f2, b'': (f3, (), False, True, b'')}, (b'a', 10**20),
assert loads(dumps(L)) == L tuple(range(-100000, 100000)), b'b' * 31, b'b' * 62, b'b' * 64, 2**30, 2**33, 2**62,
d = dict(zip(range(-100000,100000),range(-100000,100000))) 2**64, 2**30, 2**33, 2**62, 2**64, False, False, True, -1, 2, 0),)
d.update({b'a':20, 20:40, 40:41, f1:f2, f2:f3, f3:False, False:True, True:False}) assert loads(dumps(ld)) == ld
L = (d, {}, {5:6}, {7:7,True:8}, {9:10, 22:39, 49:50, 44: b''}) d = dict(zip(range(-100000, 100000), range(-100000, 100000)))
assert loads(dumps(L)) == L d.update({b'a': 20, 20: 40, 40: 41, f1: f2, f2: f3, f3: False, False: True, True: False})
L = (b'', b'a'*10, b'a'*100, b'a'*1000, b'a'*10000, b'a'*100000, b'a'*1000000, b'a'*10000000) ld = (d, {}, {5: 6}, {7: 7, True: 8}, {9: 10, 22: 39, 49: 50, 44: b''})
assert loads(dumps(L)) == L assert loads(dumps(ld)) == ld
L = tuple([dict(zip(range(n),range(n))) for n in range(100)]) + (b'b',) ld = (b'', b'a' * 10, b'a' * 100, b'a' * 1000, b'a' * 10000, b'a' * 100000, b'a' * 1000000, b'a' * 10000000)
assert loads(dumps(L)) == L assert loads(dumps(ld)) == ld
L = tuple([dict(zip(range(n),range(-n,0))) for n in range(100)]) + (b'b',) ld = tuple([dict(zip(range(n), range(n))) for n in range(100)]) + (b'b',)
assert loads(dumps(L)) == L assert loads(dumps(ld)) == ld
L = tuple([tuple(range(n)) for n in range(100)]) + (b'b',) ld = tuple([dict(zip(range(n), range(-n, 0))) for n in range(100)]) + (b'b',)
assert loads(dumps(L)) == L assert loads(dumps(ld)) == ld
L = tuple([b'a'*n for n in range(1000)]) + (b'b',) ld = tuple([tuple(range(n)) for n in range(100)]) + (b'b',)
assert loads(dumps(L)) == L assert loads(dumps(ld)) == ld
L = tuple([b'a'*n for n in range(1000)]) + (None,True,None) ld = tuple([b'a' * n for n in range(1000)]) + (b'b',)
assert loads(dumps(L)) == L assert loads(dumps(ld)) == ld
assert loads(dumps(None)) == None ld = tuple([b'a' * n for n in range(1000)]) + (None, True, None)
assert loads(dumps({None:None})) == {None:None} assert loads(dumps(ld)) == ld
assert 1e-10<abs(loads(dumps(1.1))-1.1)<1e-6 assert loads(dumps(None)) is None
assert 1e-10<abs(loads(dumps(1.1,32))-1.1)<1e-6 assert loads(dumps({None: None})) == {None: None}
assert abs(loads(dumps(1.1,64))-1.1)<1e-12 assert 1e-10 < abs(loads(dumps(1.1)) - 1.1) < 1e-6
assert loads(dumps("Hello World!!"), decode_utf8=True) assert 1e-10 < abs(loads(dumps(1.1, 32)) - 1.1) < 1e-6
assert abs(loads(dumps(1.1, 64)) - 1.1) < 1e-12
assert loads(dumps('Hello World!!'), decode_utf8=True)
try: try:
import psyco import psyco
psyco.bind(dumps) psyco.bind(dumps)
@ -433,4 +471,4 @@ except ImportError:
if __name__ == '__main__': if __name__ == '__main__':
test() test()

96
lib/deluge_client/tests.py
View File

@ -1,41 +1,65 @@
import os import os
import sys
from unittest import TestCase import pytest
from .client import DelugeRPCClient from .client import DelugeRPCClient, RemoteException
class TestDelugeClient(TestCase):
def setUp(self): if sys.version_info > (3,):
long = int
@pytest.fixture
def client(request):
if sys.platform.startswith('win'):
auth_path = os.path.join(os.getenv('APPDATA'), 'deluge', 'auth')
else:
auth_path = os.path.expanduser("~/.config/deluge/auth") auth_path = os.path.expanduser("~/.config/deluge/auth")
with open(auth_path, 'rb') as f: with open(auth_path, 'rb') as f:
filedata = f.read().decode("utf-8").split('\n')[0].split(':') filedata = f.read().decode("utf-8").split('\n')[0].split(':')
self.username, self.password = filedata[:2] username, password = filedata[:2]
self.ip = '127.0.0.1' ip = '127.0.0.1'
self.port = 58846 port = 58846
self.client = DelugeRPCClient(self.ip, self.port, self.username, self.password) kwargs = {'decode_utf8': True}
if hasattr(request, 'param'):
def tearDown(self): kwargs.update(request.param)
try: client = DelugeRPCClient(ip, port, username, password, **kwargs)
self.client.disconnect() client.connect()
except:
pass yield client
def test_connect(self): try:
self.client.connect() client.disconnect()
except:
def test_call_method(self): pass
self.client.connect()
self.assertIsInstance(self.client.call('core.get_free_space'), int)
def test_connect(client):
def test_call_method_arguments(self): assert client.connected
self.client.connect()
self.assertIsInstance(self.client.call('core.get_free_space', '/'), int)
def test_call_method(client):
def test_call_method_exception(self): assert isinstance(client.call('core.get_free_space'), (int, long))
self.client.connect()
try:
self.client.call('core.get_free_space', '1', '2') def test_call_method_arguments(client):
except Exception as e: assert isinstance(client.call('core.get_free_space', '/'), (int, long))
self.assertEqual('deluge_client.client', e.__module__)
@pytest.mark.parametrize('client',
[{'decode_utf8': True}, {'decode_utf8': False}],
ids=['decode_utf8_on', 'decode_utf8_off'],
indirect=True)
def test_call_method_exception(client):
with pytest.raises(RemoteException) as ex_info:
client.call('core.get_free_space', '1', '2')
assert ('takes at most 2 arguments' in str(ex_info.value) or
'takes from 1 to 2 positional arguments' in str(ex_info.value)) # deluge 2.0
def test_attr_caller(client):
assert isinstance(client.core.get_free_space(), (int, long))
assert isinstance(client.core.get_free_space('/'), (int, long))

20
mylar/Failed.py
View File

@ -170,15 +170,21 @@ class FailedProcessor(object):
sandwich = issueid sandwich = issueid
elif 'G' in issueid or '-' in issueid: elif 'G' in issueid or '-' in issueid:
sandwich = 1 sandwich = 1
if helpers.is_number(sandwich): try:
if sandwich < 900000: if helpers.is_number(sandwich):
if sandwich < 900000:
# if sandwich is less than 900000 it's a normal watchlist download. Bypass. # if sandwich is less than 900000 it's a normal watchlist download. Bypass.
pass pass
else: else:
logger.info('Failed download handling for story-arcs and one-off\'s are not supported yet. Be patient!') logger.info('Failed download handling for story-arcs and one-off\'s are not supported yet. Be patient!')
self._log(' Unable to locate downloaded file to rename. PostProcessing aborted.') self._log(' Unable to locate downloaded file to rename. PostProcessing aborted.')
self.valreturn.append({"self.log": self.log,
"mode": 'stop'})
return self.queue.put(self.valreturn)
except NameError:
logger.info('sandwich was not defined. Post-processing aborted...')
self.valreturn.append({"self.log": self.log, self.valreturn.append({"self.log": self.log,
"mode": 'stop'}) "mode": 'stop'})
return self.queue.put(self.valreturn) return self.queue.put(self.valreturn)

4
mylar/PostProcessor.py
View File

@ -644,6 +644,10 @@ class PostProcessor(object):
temploc = None temploc = None
datematch = "False" datematch = "False"
if temploc is None and all([cs['WatchValues']['Type'] != 'TPB', cs['WatchValues']['Type'] != 'One-Shot']):
logger.info('this should have an issue number to match to this particular series: %s' % cs['ComicID'])
continue
if temploc is not None and (any(['annual' in temploc.lower(), 'special' in temploc.lower()]) and mylar.CONFIG.ANNUALS_ON is True): if temploc is not None and (any(['annual' in temploc.lower(), 'special' in temploc.lower()]) and mylar.CONFIG.ANNUALS_ON is True):
biannchk = re.sub('-', '', temploc.lower()).strip() biannchk = re.sub('-', '', temploc.lower()).strip()
if 'biannual' in biannchk: if 'biannual' in biannchk:

3
mylar/config.py
View File

@ -841,6 +841,9 @@ class Config(object):
self.ALT_PULL = 2 self.ALT_PULL = 2
config.set('Weekly', 'alt_pull', str(self.ALT_PULL)) config.set('Weekly', 'alt_pull', str(self.ALT_PULL))
#force off public torrents usage as currently broken.
self.ENABLE_PUBLIC = False
try: try:
if not any([self.SAB_HOST is None, self.SAB_HOST == '', 'http://' in self.SAB_HOST[:7], 'https://' in self.SAB_HOST[:8]]): if not any([self.SAB_HOST is None, self.SAB_HOST == '', 'http://' in self.SAB_HOST[:7], 'https://' in self.SAB_HOST[:8]]):
self.SAB_HOST = 'http://' + self.SAB_HOST self.SAB_HOST = 'http://' + self.SAB_HOST

6
mylar/cv.py
View File

@ -344,7 +344,7 @@ def GetComicInfo(comicid, dom, safechk=None):
comic['Type'] = 'Print' comic['Type'] = 'Print'
if comic_desc != 'None' and comic['Type'] == 'None': if comic_desc != 'None' and comic['Type'] == 'None':
if 'print' in comic_desc[:60].lower() and all(['print edition can be found' not in comic_desc.lower(), 'reprints' not in comic_desc.lower()]): if 'print' in comic_desc[:60].lower() and all(['for the printed edition' not in comic_desc.lower(), 'print edition can be found' not in comic_desc.lower(), 'reprints' not in comic_desc.lower()]):
comic['Type'] = 'Print' comic['Type'] = 'Print'
elif 'digital' in comic_desc[:60].lower() and 'digital edition can be found' not in comic_desc.lower(): elif 'digital' in comic_desc[:60].lower() and 'digital edition can be found' not in comic_desc.lower():
comic['Type'] = 'Digital' comic['Type'] = 'Digital'
@ -352,10 +352,10 @@ def GetComicInfo(comicid, dom, safechk=None):
comic['Type'] = 'TPB' comic['Type'] = 'TPB'
elif 'hardcover' in comic_desc[:60].lower() and 'hardcover can be found' not in comic_desc.lower(): elif 'hardcover' in comic_desc[:60].lower() and 'hardcover can be found' not in comic_desc.lower():
comic['Type'] = 'HC' comic['Type'] = 'HC'
elif any(['one-shot' in comic_desc[:60].lower(), 'one shot' in comic_desc[:60].lower()]) and any(['can be found' not in comic_desc.lower(), 'following the' not in comic_desc.lower()]): elif any(['one-shot' in comic_desc[:60].lower(), 'one shot' in comic_desc[:60].lower()]) and any(['can be found' not in comic_desc.lower(), 'following the' not in comic_desc.lower(), 'after the' not in comic_desc.lower()]):
i = 0 i = 0
comic['Type'] = 'One-Shot' comic['Type'] = 'One-Shot'
avoidwords = ['preceding', 'after the special', 'following the'] avoidwords = ['preceding', 'after the', 'following the']
while i < 2: while i < 2:
if i == 0: if i == 0:
cbd = 'one-shot' cbd = 'one-shot'

229
mylar/filechecker.py
View File

@ -101,7 +101,7 @@ class FileChecker(object):
self.failed_files = [] self.failed_files = []
self.dynamic_handlers = ['/','-',':',';','\'',',','&','?','!','+','(',')','\u2014','\u2013'] self.dynamic_handlers = ['/','-',':',';','\'',',','&','?','!','+','(',')','\u2014','\u2013']
self.dynamic_replacements = ['and','the'] self.dynamic_replacements = ['and','the']
self.rippers = ['-empire','-empire-hd','minutemen-','-dcp'] self.rippers = ['-empire','-empire-hd','minutemen-','-dcp','Glorith-HD']
#pre-generate the AS_Alternates now #pre-generate the AS_Alternates now
AS_Alternates = self.altcheck() AS_Alternates = self.altcheck()
@ -143,7 +143,7 @@ class FileChecker(object):
if filename.startswith('.'): if filename.startswith('.'):
continue continue
logger.debug('[FILENAME]: ' + filename) logger.debug('[FILENAME]: %s' % filename)
runresults = self.parseit(self.dir, filename, filedir) runresults = self.parseit(self.dir, filename, filedir)
if runresults: if runresults:
try: try:
@ -208,10 +208,12 @@ class FileChecker(object):
watchmatch['comiccount'] = comiccnt watchmatch['comiccount'] = comiccnt
if len(comiclist) > 0: if len(comiclist) > 0:
watchmatch['comiclist'] = comiclist watchmatch['comiclist'] = comiclist
else:
watchmatch['comiclist'] = []
if len(self.failed_files) > 0: if len(self.failed_files) > 0:
logger.info('FAILED FILES: %s' % self.failed_files) logger.info('FAILED FILES: %s' % self.failed_files)
return watchmatch return watchmatch
def parseit(self, path, filename, subpath=None): def parseit(self, path, filename, subpath=None):
@ -241,7 +243,7 @@ class FileChecker(object):
if '/' == path_list[0] or '\\' == path_list[0]: if '/' == path_list[0] or '\\' == path_list[0]:
#need to remove any leading slashes so the os join can properly join the components #need to remove any leading slashes so the os join can properly join the components
path_list = path_list[1:] path_list = path_list[1:]
logger.fdebug('[SUB-PATH] subpath set to : ' + path_list) logger.fdebug('[SUB-PATH] subpath set to : %s' % path_list)
#parse out the extension for type #parse out the extension for type
@ -261,14 +263,31 @@ class FileChecker(object):
if self.sarc and mylar.CONFIG.READ2FILENAME: if self.sarc and mylar.CONFIG.READ2FILENAME:
removest = modfilename.find('-') # the - gets removed above so we test for the first blank space... removest = modfilename.find('-') # the - gets removed above so we test for the first blank space...
if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE: if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
logger.fdebug('[SARC] Checking filename for Reading Order sequence - Reading Sequence Order found #: ' + str(modfilename[:removest])) logger.fdebug('[SARC] Checking filename for Reading Order sequence - Reading Sequence Order found #: %s' % modfilename[:removest])
if modfilename[:removest].isdigit() and removest <= 3: if modfilename[:removest].isdigit() and removest <= 3:
reading_order = {'reading_sequence': str(modfilename[:removest]), reading_order = {'reading_sequence': str(modfilename[:removest]),
'filename': filename[removest+1:]} 'filename': filename[removest+1:]}
modfilename = modfilename[removest+1:] modfilename = modfilename[removest+1:]
if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE: if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
logger.fdebug('[SARC] Removed Reading Order sequence from subname. Now set to : ' + modfilename) logger.fdebug('[SARC] Removed Reading Order sequence from subname. Now set to : %s' % modfilename)
#make sure all the brackets are properly spaced apart
m = re.findall('[^()]+', modfilename)
cnt = 1
#2019-12-24----fixed to accomodate naming convention like Amazing Mary Jane (2019) 002.cbr, and to account for brackets properly
try:
while cnt < len(m):
#logger.fdebug('[m=%s] modfilename.find: %s' % (m[cnt], modfilename[modfilename.find('('+m[cnt]+')')+len(m[cnt])+2]))
#logger.fdebug('mod_1: %s' % modfilename.find('('+m[cnt]+')'))
if modfilename[modfilename.find('('+m[cnt]+')')-1] != ' ' and modfilename.find('('+m[cnt]+')') != -1:
#logger.fdebug('before_space: %s' % modfilename[modfilename.find('('+m[cnt]+')')-1])
#logger.fdebug('after_space: %s' % modfilename[modfilename.find('('+m[cnt]+')')+len(m[cnt])+2])
modfilename = '%s%s%s' % (modfilename[:modfilename.find('('+m[cnt]+')')], ' ', modfilename[modfilename.find('('+m[cnt]+')'):])
cnt+=1
except Exception as e:
#logger.warn('[ERROR] %s' % e)
pass
#---end 2019-12-24
#grab the scanner tags here. #grab the scanner tags here.
scangroup = None scangroup = None
@ -277,6 +296,23 @@ class FileChecker(object):
#it's always possible that this could grab something else since tags aren't unique. Try and figure it out. #it's always possible that this could grab something else since tags aren't unique. Try and figure it out.
if len(rippers) > 0: if len(rippers) > 0:
m = re.findall('[^()]+', modfilename) m = re.findall('[^()]+', modfilename)
#--2019-11-30 needed for Glorith naming conventions when it's an nzb name with all formatting removed.
if len(m) == 1:
spf30 = re.compile(ur"[^.]+", re.UNICODE)
#logger.fdebug('spf30: %s' % spf30)
split_file30 = spf30.findall(modfilename)
#logger.fdebug('split_file30: %s' % split_file30)
if len(split_file30) > 3 and 'Glorith-HD' in modfilename:
scangroup = 'Glorith-HD'
sp_pos = 0
for x in split_file30:
if sp_pos+1 > len(split_file30):
break
if x[-1] == ',' and self.checkthedate(split_file30[sp_pos+1]):
modfilename = re.sub(x, x[:-1], modfilename, count=1)
break
sp_pos+=1
#-- end 2019-11-30
cnt = 1 cnt = 1
for rp in rippers: for rp in rippers:
while cnt < len(m): while cnt < len(m):
@ -284,7 +320,7 @@ class FileChecker(object):
pass pass
elif rp.lower() in m[cnt].lower(): elif rp.lower() in m[cnt].lower():
scangroup = re.sub('[\(\)]', '', m[cnt]).strip() scangroup = re.sub('[\(\)]', '', m[cnt]).strip()
logger.fdebug('Scanner group tag discovered: ' + scangroup) logger.fdebug('Scanner group tag discovered: %s' % scangroup)
modfilename = modfilename.replace(m[cnt],'').strip() modfilename = modfilename.replace(m[cnt],'').strip()
break break
cnt +=1 cnt +=1
@ -321,11 +357,13 @@ class FileChecker(object):
sf3 = re.compile(ur"[^,\s_]+", re.UNICODE) sf3 = re.compile(ur"[^,\s_]+", re.UNICODE)
split_file3 = sf3.findall(modfilename) split_file3 = sf3.findall(modfilename)
if len(split_file3) == 1: #--2019-11-30
if len(split_file3) == 1 or all([len(split_file3) == 2, scangroup == 'Glorith-HD']):
#--end 2019-11-30
logger.fdebug('Improperly formatted filename - there is no seperation using appropriate characters between wording.') logger.fdebug('Improperly formatted filename - there is no seperation using appropriate characters between wording.')
sf3 = re.compile(ur"[^,\s_\.]+", re.UNICODE) sf3 = re.compile(ur"[^,\s_\.]+", re.UNICODE)
split_file3 = sf3.findall(modfilename) split_file3 = sf3.findall(modfilename)
logger.fdebug('NEW split_file3: ' + str(split_file3)) logger.fdebug('NEW split_file3: %s' % split_file3)
ret_sf2 = ' '.join(split_file3) ret_sf2 = ' '.join(split_file3)
@ -343,8 +381,9 @@ class FileChecker(object):
ret_sf1 = re.sub('\&', 'f11', ret_sf1).strip() ret_sf1 = re.sub('\&', 'f11', ret_sf1).strip()
ret_sf1 = re.sub('\'', 'g11', ret_sf1).strip() ret_sf1 = re.sub('\'', 'g11', ret_sf1).strip()
#split_file = re.findall('(?imu)\([\w\s-]+\)|[-+]?\d*\.\d+|\d+|[\w-]+|#?\d\.\d+|#(?<![\w\d])XCV(?![\w\d])+|\)', ret_sf1, re.UNICODE) #split_file = re.findall('(?imu)\([\w\s-]+\)|[-+]?\d*\.\d+|\d+[\s]COVERS+|\d{4}-\d{2}-\d{2}|\d+[(th|nd|rd|st)]+|\d+|[\w-]+|#?\d\.\d+|#[\.-]\w+|#[\d*\.\d+|\w+\d+]+|#(?<![\w\d])XCV(?![\w\d])+|#[\w+]|\)', ret_sf1, re.UNICODE)
split_file = re.findall('(?imu)\([\w\s-]+\)|[-+]?\d*\.\d+|\d+[\s]COVERS+|\d{4}-\d{2}-\d{2}|\d+[(th|nd|rd|st)]+|\d+|[\w-]+|#?\d\.\d+|#[\.-]\w+|#[\d*\.\d+|\w+\d+]+|#(?<![\w\d])XCV(?![\w\d])+|#[\w+]|\)', ret_sf1, re.UNICODE) split_file = re.findall('(?imu)\([\w\s-]+\)|[-+]?\d*\.\d+|\d+[\s]COVERS+|\d{4}-\d{2}-\d{2}|\d+[(th|nd|rd|st)]+|[\(^\)+]|\d+|[\w-]+|#?\d\.\d+|#[\.-]\w+|#[\d*\.\d+|\w+\d+]+|#(?<![\w\d])XCV(?![\w\d])+|#[\w+]|\)', ret_sf1, re.UNICODE)
#10-20-2018 ---START -- attempt to detect '01 (of 7.3)' #10-20-2018 ---START -- attempt to detect '01 (of 7.3)'
#10-20-2018 -- attempt to detect '36p ctc' as one element #10-20-2018 -- attempt to detect '36p ctc' as one element
spf = [] spf = []
@ -369,7 +408,7 @@ class FileChecker(object):
except Exception as e: except Exception as e:
spf.append(x) spf.append(x)
elif x == ')': elif x == ')' or x == '(':
pass pass
elif x == 'p' or x == 'ctc': elif x == 'p' or x == 'ctc':
try: try:
@ -426,10 +465,10 @@ class FileChecker(object):
dtcheck = re.sub('[\(\)\,]', '', sf).strip() dtcheck = re.sub('[\(\)\,]', '', sf).strip()
#if there's more than one date, assume the right-most date is the actual issue date. #if there's more than one date, assume the right-most date is the actual issue date.
if any(['19' in dtcheck, '20' in dtcheck]) and not any([dtcheck.lower().startswith('v19'), dtcheck.lower().startswith('v20')]) and len(dtcheck) >=4: if any(['19' in dtcheck, '20' in dtcheck]) and not any([dtcheck.lower().startswith('v19'), dtcheck.lower().startswith('v20')]) and len(dtcheck) >=4:
logger.fdebug('checking date : ' + str(dtcheck)) logger.fdebug('checking date : %s' % dtcheck)
checkdate_response = self.checkthedate(dtcheck) checkdate_response = self.checkthedate(dtcheck)
if checkdate_response: if checkdate_response:
logger.fdebug('date: ' + str(checkdate_response)) logger.fdebug('date: %s' % checkdate_response)
datecheck.append({'date': dtcheck, datecheck.append({'date': dtcheck,
'position': split_file.index(sf), 'position': split_file.index(sf),
'mod_position': self.char_file_position(modfilename, sf, lastmod_position)}) 'mod_position': self.char_file_position(modfilename, sf, lastmod_position)})
@ -437,10 +476,10 @@ class FileChecker(object):
#this handles the exceptions list in the match for alpha-numerics #this handles the exceptions list in the match for alpha-numerics
test_exception = ''.join([i for i in sf if not i.isdigit()]) test_exception = ''.join([i for i in sf if not i.isdigit()])
if any([x for x in exceptions if x.lower() == test_exception.lower()]): if any([x for x in exceptions if x.lower() == test_exception.lower()]):
logger.fdebug('Exception match: ' + test_exception) logger.fdebug('Exception match: %s' % test_exception)
if lastissue_label is not None: if lastissue_label is not None:
if lastissue_position == (split_file.index(sf) -1): if lastissue_position == (split_file.index(sf) -1):
logger.fdebug('alphanumeric issue number detected as : ' + str(lastissue_label) + ' ' + str(sf)) logger.fdebug('alphanumeric issue number detected as : %s %s' % (lastissue_label,sf))
for x in possible_issuenumbers: for x in possible_issuenumbers:
possible_issuenumbers = [] possible_issuenumbers = []
if int(x['position']) != int(lastissue_position): if int(x['position']) != int(lastissue_position):
@ -449,7 +488,7 @@ class FileChecker(object):
'mod_position': x['mod_position'], 'mod_position': x['mod_position'],
'validcountchk': x['validcountchk']}) 'validcountchk': x['validcountchk']})
possible_issuenumbers.append({'number': str(lastissue_label) + ' ' + str(sf), possible_issuenumbers.append({'number': '%s %s' % (lastissue_label, sf),
'position': lastissue_position, 'position': lastissue_position,
'mod_position': self.char_file_position(modfilename, sf, lastmod_position), 'mod_position': self.char_file_position(modfilename, sf, lastmod_position),
'validcountchk': validcountchk}) 'validcountchk': validcountchk})
@ -458,7 +497,7 @@ class FileChecker(object):
#test_exception is the alpha-numeric #test_exception is the alpha-numeric
logger.fdebug('Possible alpha numeric issue (or non-numeric only). Testing my theory.') logger.fdebug('Possible alpha numeric issue (or non-numeric only). Testing my theory.')
test_sf = re.sub(test_exception.lower(), '', sf.lower()).strip() test_sf = re.sub(test_exception.lower(), '', sf.lower()).strip()
logger.fdebug('[' + test_exception + '] Removing possible alpha issue leaves: ' + test_sf + ' (Should be a numeric)') logger.fdebug('[%s] Removing possible alpha issue leaves: %s (Should be a numeric)' % (test_exception, test_sf))
if test_sf.isdigit(): if test_sf.isdigit():
possible_issuenumbers.append({'number': sf, possible_issuenumbers.append({'number': sf,
'position': split_file.index(sf), 'position': split_file.index(sf),
@ -477,7 +516,7 @@ class FileChecker(object):
for x in list(wrds): for x in list(wrds):
if x != '': if x != '':
tmpissue_number = re.sub('XCV', x, split_file[split_file.index(sf)]) tmpissue_number = re.sub('XCV', x, split_file[split_file.index(sf)])
logger.info('[SPECIAL-CHARACTER ISSUE] Possible issue # : ' + tmpissue_number) logger.info('[SPECIAL-CHARACTER ISSUE] Possible issue # : %s' % tmpissue_number)
possible_issuenumbers.append({'number': sf, possible_issuenumbers.append({'number': sf,
'position': split_file.index(sf), 'position': split_file.index(sf),
'mod_position': self.char_file_position(modfilename, sf, lastmod_position), 'mod_position': self.char_file_position(modfilename, sf, lastmod_position),
@ -501,10 +540,10 @@ class FileChecker(object):
if count: if count:
# count = count.lstrip("0") # count = count.lstrip("0")
logger.fdebug('Mini-Series Count detected. Maximum issue # set to : ' + count.lstrip('0')) logger.fdebug('Mini-Series Count detected. Maximum issue # set to : %s' % count.lstrip('0'))
# if the count was detected, then it's in a '(of 4)' or whatever pattern # if the count was detected, then it's in a '(of 4)' or whatever pattern
# 95% of the time the digit immediately preceding the '(of 4)' is the actual issue # # 95% of the time the digit immediately preceding the '(of 4)' is the actual issue #
logger.fdebug('Issue Number SHOULD BE: ' + str(lastissue_label)) logger.fdebug('Issue Number SHOULD BE: %s' % lastissue_label)
validcountchk = True validcountchk = True
match2 = re.search('(\d+[\s])covers', sf, re.IGNORECASE) match2 = re.search('(\d+[\s])covers', sf, re.IGNORECASE)
@ -516,9 +555,9 @@ class FileChecker(object):
if all([lastissue_position == (split_file.index(sf) -1), lastissue_label is not None, '#' not in sf, sf != 'p']): if all([lastissue_position == (split_file.index(sf) -1), lastissue_label is not None, '#' not in sf, sf != 'p']):
#find it in the original file to see if there's a decimal between. #find it in the original file to see if there's a decimal between.
findst = lastissue_mod_position+1 findst = lastissue_mod_position+1
if findst > len(modfilename): if findst >= len(modfilename):
findst = len(modfilename) -1 findst = len(modfilename) -1
if modfilename[findst] != '.' or modfilename[findst] != '#': #findst != '.' and findst != '#': if modfilename[findst] != '.' or modfilename[findst] != '#': #findst != '.' and findst != '#':
if sf.isdigit(): if sf.isdigit():
seper_num = False seper_num = False
@ -549,7 +588,7 @@ class FileChecker(object):
#logger.fdebug('diff: ' + str(bb) + '[' + modfilename[bb] + ']') #logger.fdebug('diff: ' + str(bb) + '[' + modfilename[bb] + ']')
if modfilename[bb] == '.': if modfilename[bb] == '.':
#logger.fdebug('decimal detected.') #logger.fdebug('decimal detected.')
logger.fdebug('[DECiMAL-DETECTION] Issue being stored for validation as : ' + modfilename[findst:cf+len(sf)]) logger.fdebug('[DECiMAL-DETECTION] Issue being stored for validation as : %s' % modfilename[findst:cf+len(sf)])
for x in possible_issuenumbers: for x in possible_issuenumbers:
possible_issuenumbers = [] possible_issuenumbers = []
#logger.fdebug('compare: ' + str(x['position']) + ' .. ' + str(lastissue_position)) #logger.fdebug('compare: ' + str(x['position']) + ' .. ' + str(lastissue_position))
@ -583,12 +622,14 @@ class FileChecker(object):
lastissue_mod_position = file_length lastissue_mod_position = file_length
elif '#' in sf: elif '#' in sf:
logger.fdebug('Iissue number found: ' + sf) logger.fdebug('Issue number found: %s' % sf)
#pound sign will almost always indicate an issue #, so just assume it's as such. #pound sign will almost always indicate an issue #, so just assume it's as such.
locateiss_st = modfilename.find('#') locateiss_st = modfilename.find('#')
locateiss_end = modfilename.find(' ', locateiss_st) locateiss_end = modfilename.find(' ', locateiss_st)
if locateiss_end == -1: if locateiss_end == -1:
locateiss_end = len(modfilename) locateiss_end = len(modfilename)
if modfilename[locateiss_end-1] == ')':
locateiss_end = locateiss_end -1
possible_issuenumbers.append({'number': modfilename[locateiss_st:locateiss_end], possible_issuenumbers.append({'number': modfilename[locateiss_st:locateiss_end],
'position': split_file.index(sf), #locateiss_st}) 'position': split_file.index(sf), #locateiss_st})
'mod_position': self.char_file_position(modfilename, sf, lastmod_position), 'mod_position': self.char_file_position(modfilename, sf, lastmod_position),
@ -597,11 +638,14 @@ class FileChecker(object):
#now we try to find the series title &/or volume lablel. #now we try to find the series title &/or volume lablel.
if any( [sf.lower().startswith('v'), sf.lower().startswith('vol'), volumeprior == True, 'volume' in sf.lower(), 'vol' in sf.lower(), 'part' in sf.lower()] ) and sf.lower() not in {'one','two','three','four','five','six'}: if any( [sf.lower().startswith('v'), sf.lower().startswith('vol'), volumeprior == True, 'volume' in sf.lower(), 'vol' in sf.lower(), 'part' in sf.lower()] ) and sf.lower() not in {'one','two','three','four','five','six'}:
if any([ split_file[split_file.index(sf)].isdigit(), split_file[split_file.index(sf)][3:].isdigit(), split_file[split_file.index(sf)][1:].isdigit() ]): if any([ split_file[split_file.index(sf)].isdigit(), split_file[split_file.index(sf)][3:].isdigit(), split_file[split_file.index(sf)][1:].isdigit() ]):
volume = re.sub("[^0-9]", "", sf) if all(identifier in sf for identifier in ['.', 'v']):
volume = sf.split('.')[0]
else:
volume = re.sub("[^0-9]", "", sf)
if volumeprior: if volumeprior:
try: try:
volume_found['position'] = split_file.index(volumeprior_label, current_pos -1) #if this passes, then we're ok, otherwise will try exception volume_found['position'] = split_file.index(volumeprior_label, current_pos -1) #if this passes, then we're ok, otherwise will try exception
logger.fdebug('volume_found: ' + str(volume_found['position'])) logger.fdebug('volume_found: %s' % volume_found['position'])
#remove volume numeric from split_file #remove volume numeric from split_file
split_file.pop(volume_found['position']) split_file.pop(volume_found['position'])
split_file.pop(split_file.index(sf, current_pos-1)) split_file.pop(split_file.index(sf, current_pos-1))
@ -662,13 +706,13 @@ class FileChecker(object):
lastissue_position = split_file.index(sf, current_pos) lastissue_position = split_file.index(sf, current_pos)
lastissue_label = sf lastissue_label = sf
lastissue_mod_position = file_length lastissue_mod_position = file_length
#logger.fdebug('possible issue found: ' + str(sf) #logger.fdebug('possible issue found: %s' % sf)
else: else:
try: try:
x = float(sf) x = float(sf)
#validity check #validity check
if x < 0: if x < 0:
logger.fdebug('I have encountered a negative issue #: ' + str(sf)) logger.fdebug('I have encountered a negative issue #: %s' % sf)
possible_issuenumbers.append({'number': sf, possible_issuenumbers.append({'number': sf,
'position': split_file.index(sf, lastissue_position), #modfilename.find(sf)}) 'position': split_file.index(sf, lastissue_position), #modfilename.find(sf)})
'mod_position': self.char_file_position(modfilename, sf, lastmod_position), 'mod_position': self.char_file_position(modfilename, sf, lastmod_position),
@ -678,7 +722,7 @@ class FileChecker(object):
lastissue_label = sf lastissue_label = sf
lastissue_mod_position = file_length lastissue_mod_position = file_length
elif x > 0: elif x > 0:
logger.fdebug('I have encountered a decimal issue #: ' + str(sf)) logger.fdebug('I have encountered a decimal issue #: %s' % sf)
possible_issuenumbers.append({'number': sf, possible_issuenumbers.append({'number': sf,
'position': split_file.index(sf, lastissue_position), #modfilename.find(sf)}) 'position': split_file.index(sf, lastissue_position), #modfilename.find(sf)})
'mod_position': self.char_file_position(modfilename, sf, lastmod_position), 'mod_position': self.char_file_position(modfilename, sf, lastmod_position),
@ -756,13 +800,13 @@ class FileChecker(object):
issue_year = None issue_year = None
possible_years = [] possible_years = []
yearmodposition = None yearmodposition = None
logger.fdebug('datecheck: ' + str(datecheck)) logger.fdebug('datecheck: %s' % datecheck)
if len(datecheck) > 0: if len(datecheck) > 0:
for dc in sorted(datecheck, key=operator.itemgetter('position'), reverse=True): for dc in sorted(datecheck, key=operator.itemgetter('position'), reverse=True):
a = self.checkthedate(dc['date']) a = self.checkthedate(dc['date'])
ab = str(a) ab = str(a)
sctd = self.checkthedate(str(dt.datetime.now().year)) sctd = self.checkthedate(str(dt.datetime.now().year))
logger.fdebug('sctd: ' + str(sctd)) logger.fdebug('sctd: %s' % sctd)
# + 1 sctd so that we can allow for issue dates that cross over into the following year when it's nearer to the end of said year. # + 1 sctd so that we can allow for issue dates that cross over into the following year when it's nearer to the end of said year.
if int(ab) > int(sctd) + 1: if int(ab) > int(sctd) + 1:
logger.fdebug('year is in the future, ignoring and assuming part of series title.') logger.fdebug('year is in the future, ignoring and assuming part of series title.')
@ -771,19 +815,19 @@ class FileChecker(object):
continue continue
else: else:
issue_year = dc['date'] issue_year = dc['date']
logger.fdebug('year verified as : ' + str(issue_year)) logger.fdebug('year verified as : %s' % issue_year)
if highest_series_pos > dc['position']: highest_series_pos = dc['position'] if highest_series_pos > dc['position']: highest_series_pos = dc['position']
yearposition = dc['position'] yearposition = dc['position']
yearmodposition = dc['mod_position'] yearmodposition = dc['mod_position']
if len(ab) == 4: if len(ab) == 4:
issue_year = ab issue_year = ab
logger.fdebug('year verified as: ' + str(issue_year)) logger.fdebug('year verified as: %s' % issue_year)
possible_years.append({'year': issue_year, possible_years.append({'year': issue_year,
'yearposition': dc['position'], 'yearposition': dc['position'],
'yearmodposition': dc['mod_position']}) 'yearmodposition': dc['mod_position']})
else: else:
issue_year = ab issue_year = ab
logger.fdebug('date verified as: ' + str(issue_year)) logger.fdebug('date verified as: %s' % issue_year)
if len(possible_years) == 1: if len(possible_years) == 1:
issueyear = possible_years[0]['year'] issueyear = possible_years[0]['year']
@ -809,20 +853,35 @@ class FileChecker(object):
logger.fdebug('No year present within title - ignoring as a variable.') logger.fdebug('No year present within title - ignoring as a variable.')
logger.fdebug('highest_series_position: ' + str(highest_series_pos)) logger.fdebug('highest_series_position: %s' % highest_series_pos)
#---2019-11-30 account for scanner Glorith-HD stupid naming conventions
if len(possible_issuenumbers) == 0 and scangroup == 'Glorith-HD':
logger.fdebug('Abnormal formatting detected. Time to fix this shiet, yo.')
if any([yearposition == 0, yearposition is None]):
logger.fdebug('Too stupid of a format. Nope. Not gonna happen - just reinvent the wheel you fooker.')
else:
issposs = yearposition + 1
#logger.fdebug('split_file: %s' % split_file[issposs])
if '(' and ')' in split_file[issposs]:
new_issuenumber = split_file[issposs]
possible_issuenumbers.append({'number': re.sub('[/(/)]', '', split_file[issposs]).strip(),
'position': split_file.index(new_issuenumber, yearposition),
'mod_position': self.char_file_position(modfilename, new_issuenumber, yearmodposition),
'validcountchk': False})
#---end 2019-11-30
issue_number = None issue_number = None
dash_numbers = [] dash_numbers = []
issue_number_position = len(split_file) issue_number_position = len(split_file)
if len(possible_issuenumbers) > 0: if len(possible_issuenumbers) > 0:
logger.fdebug('possible_issuenumbers: ' + str(possible_issuenumbers)) logger.fdebug('possible_issuenumbers: %s' % possible_issuenumbers)
if len(possible_issuenumbers) >= 1: if len(possible_issuenumbers) >= 1:
p = 1 p = 1
if '-' not in split_file[0]: if '-' not in split_file[0]:
finddash = modfilename.find('-') finddash = modfilename.find('-')
if finddash != -1: if finddash != -1:
logger.fdebug('hyphen located at position: ' + str(finddash)) logger.fdebug('hyphen located at position: %s' % finddash)
if yearposition: if yearposition:
logger.fdebug('yearposition: ' + str(yearposition)) logger.fdebug('yearposition: %s' % yearposition)
else: else:
finddash = -1 finddash = -1
logger.fdebug('dash is in first word, not considering for determing issue number.') logger.fdebug('dash is in first word, not considering for determing issue number.')
@ -841,7 +900,7 @@ class FileChecker(object):
elif pis['validcountchk'] == True: elif pis['validcountchk'] == True:
issue_number = pis['number'] issue_number = pis['number']
issue_number_position = pis['position'] issue_number_position = pis['position']
logger.fdebug('Issue verified and detected as part of a numeric count sequnce: ' + issue_number) logger.fdebug('Issue verified and detected as part of a numeric count sequnce: %s' % issue_number)
if highest_series_pos > pis['position']: highest_series_pos = pis['position'] if highest_series_pos > pis['position']: highest_series_pos = pis['position']
break break
elif pis['mod_position'] > finddash and finddash != -1: elif pis['mod_position'] > finddash and finddash != -1:
@ -851,13 +910,18 @@ class FileChecker(object):
'number': pis['number'], 'number': pis['number'],
'position': pis['position']}) 'position': pis['position']})
continue continue
#2019-10-05 fix - if decimal-spaced filename has a series title with a hyphen will include issue # as part of series title
elif yearposition == pis['position']:
logger.info('Already validated year, ignoring as possible issue number: %s' % pis['number'])
continue
#end 2019-10-05
elif yearposition == pis['position']: elif yearposition == pis['position']:
logger.fdebug('Already validated year, ignoring as possible issue number: ' + str(pis['number'])) logger.fdebug('Already validated year, ignoring as possible issue number: %s' % pis['number'])
continue continue
if p == 1: if p == 1:
issue_number = pis['number'] issue_number = pis['number']
issue_number_position = pis['position'] issue_number_position = pis['position']
logger.fdebug('issue number :' + issue_number) #(pis) logger.fdebug('issue number :%s' % issue_number) #(pis)
if highest_series_pos > pis['position'] and issue2year is False: highest_series_pos = pis['position'] if highest_series_pos > pis['position'] and issue2year is False: highest_series_pos = pis['position']
#else: #else:
#logger.fdebug('numeric probably belongs to series title: ' + str(pis)) #logger.fdebug('numeric probably belongs to series title: ' + str(pis))
@ -881,12 +945,12 @@ class FileChecker(object):
fin_pos = dn['position'] fin_pos = dn['position']
if fin_num: if fin_num:
logger.fdebug('Issue number re-corrected to : ' + fin_num) logger.fdebug('Issue number re-corrected to : %s' % fin_num)
issue_number = fin_num issue_number = fin_num
if highest_series_pos > fin_pos: highest_series_pos = fin_pos if highest_series_pos > fin_pos: highest_series_pos = fin_pos
#--- this is new - 2016-09-18 /account for unicode in issue number when issue number is not deteted above #--- this is new - 2016-09-18 /account for unicode in issue number when issue number is not deteted above
logger.fdebug('issue_position: ' + str(issue_number_position)) logger.fdebug('issue_position: %s' % issue_number_position)
if all([issue_number_position == highest_series_pos, 'XCV' in split_file, issue_number is None]): if all([issue_number_position == highest_series_pos, 'XCV' in split_file, issue_number is None]):
for x in list(wrds): for x in list(wrds):
if x != '': if x != '':
@ -903,23 +967,25 @@ class FileChecker(object):
else: else:
logger.info('No issue number present in filename.') logger.info('No issue number present in filename.')
else: else:
logger.fdebug('issue verified as : ' + issue_number) logger.fdebug('issue verified as : %s' % issue_number)
issue_volume = None issue_volume = None
if len(volume_found) > 0: if len(volume_found) > 0:
issue_volume = 'v' + str(volume_found['volume']) issue_volume = 'v' + str(volume_found['volume'])
if all([highest_series_pos + 1 != volume_found['position'], highest_series_pos != volume_found['position'] + 1, sep_volume == False, booktype == 'issue', len(possible_issuenumbers) > 0]): if all([highest_series_pos + 1 != volume_found['position'], highest_series_pos != volume_found['position'] + 1, sep_volume == False, booktype == 'issue', len(possible_issuenumbers) > 0]):
logger.fdebug('Extra item(s) are present between the volume label and the issue number. Checking..') logger.fdebug('Extra item(s) are present between the volume label and the issue number. Checking..')
split_file.insert(int(issue_number_position), split_file.pop(volume_found['position'])) #highest_series_pos-1, split_file.pop(volume_found['position'])) split_file.insert(int(issue_number_position), split_file.pop(volume_found['position'])) #highest_series_pos-1, split_file.pop(volume_found['position']))
logger.fdebug('new split: ' + str(split_file)) logger.fdebug('new split: %s' % split_file)
highest_series_pos = volume_found['position'] -1 highest_series_pos = volume_found['position'] -1
issue_number_position -=1 #2019-10-02 - account for volume BEFORE issue number
if issue_number_position > highest_series_pos:
issue_number_position -=1
else: else:
if highest_series_pos > volume_found['position']: if highest_series_pos > volume_found['position']:
if sep_volume: if sep_volume:
highest_series_pos = volume_found['position'] - 1 highest_series_pos = volume_found['position'] - 1
else: else:
highest_series_pos = volume_found['position'] highest_series_pos = volume_found['position']
logger.fdebug('Volume detected as : ' + issue_volume) logger.fdebug('Volume detected as : %s' % issue_volume)
if all([len(volume_found) == 0, booktype != 'issue']) or all([len(volume_found) == 0, issue_number_position == len(split_file)]): if all([len(volume_found) == 0, booktype != 'issue']) or all([len(volume_found) == 0, issue_number_position == len(split_file)]):
issue_volume = 'v1' issue_volume = 'v1'
@ -946,9 +1012,9 @@ class FileChecker(object):
if len(possible_years) > 1: if len(possible_years) > 1:
for x in sorted(possible_years, key=operator.itemgetter('yearposition'), reverse=False): for x in sorted(possible_years, key=operator.itemgetter('yearposition'), reverse=False):
if x['yearposition'] <= highest_series_pos: if x['yearposition'] <= highest_series_pos:
logger.fdebug('year ' + str(x['year']) + ' is within series title. Ignoring as YEAR value') logger.fdebug('year %s is within series title. Ignoring as YEAR value' % x['year'])
else: else:
logger.fdebug('year ' + str(x['year']) + ' is outside of series title range. Accepting of year.') logger.fdebug('year %s is outside of series title range. Accepting of year.' % x['year'])
issue_year = x['year'] issue_year = x['year']
highest_series_pos = x['yearposition'] highest_series_pos = x['yearposition']
break break
@ -969,7 +1035,13 @@ class FileChecker(object):
alt_issue = None alt_issue = None
try: try:
if yearposition is not None: if yearposition is not None:
tmpval = yearposition - issue_number_position try:
if volume_found['position'] >= issue_number_position:
tmpval = highest_series_pos + (issue_number_position - volume_found['position'])
else:
tmpval = yearposition - issue_number_position
except:
tmpval = yearposition - issue_number_position
else: else:
tmpval = 1 tmpval = 1
except: except:
@ -1075,7 +1147,7 @@ class FileChecker(object):
if '\?' in series_name: if '\?' in series_name:
series_name = re.sub('\?', '', series_name).strip() series_name = re.sub('\?', '', series_name).strip()
logger.fdebug('series title possibly: ' + series_name) logger.fdebug('series title possibly: %s' % series_name)
if splitvalue is not None: if splitvalue is not None:
logger.fdebug('[SPLITVALUE] possible issue title: %s' % splitvalue) logger.fdebug('[SPLITVALUE] possible issue title: %s' % splitvalue)
alt_series = '%s %s' % (series_name, splitvalue) alt_series = '%s %s' % (series_name, splitvalue)
@ -1214,7 +1286,7 @@ class FileChecker(object):
try: try:
if self.AS_ALT[0] != '127372873872871091383 abdkhjhskjhkjdhakajhf': if self.AS_ALT[0] != '127372873872871091383 abdkhjhskjhkjdhakajhf':
logger.fdebug('Possible Alternate Names to match against (if necessary): ' + str(self.AS_Alt)) logger.fdebug('Possible Alternate Names to match against (if necessary): %s' % self.AS_Alt)
except: except:
pass pass
@ -1259,7 +1331,7 @@ class FileChecker(object):
loopchk = [x for x in self.AS_Alt if re.sub('[\|\s]','', x.lower()).strip() == re.sub('[\|\s]','', nspace_seriesname.lower()).strip()] loopchk = [x for x in self.AS_Alt if re.sub('[\|\s]','', x.lower()).strip() == re.sub('[\|\s]','', nspace_seriesname.lower()).strip()]
if len(loopchk) > 0 and loopchk[0] != '': if len(loopchk) > 0 and loopchk[0] != '':
if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE: if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
logger.fdebug('[FILECHECKER] This should be an alternate: ' + str(loopchk)) logger.fdebug('[FILECHECKER] This should be an alternate: %s' % loopchk)
if any(['annual' in series_name.lower(), 'special' in series_name.lower()]): if any(['annual' in series_name.lower(), 'special' in series_name.lower()]):
if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE: if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
logger.fdebug('[FILECHECKER] Annual/Special detected - proceeding') logger.fdebug('[FILECHECKER] Annual/Special detected - proceeding')
@ -1290,40 +1362,40 @@ class FileChecker(object):
enable_annual = False enable_annual = False
if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE: if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
logger.fdebug('[FILECHECKER] Complete matching list of names to this file [' + str(len(loopchk)) + '] : ' + str(loopchk)) logger.fdebug('[FILECHECKER] Complete matching list of names to this file [%s] : %s' % (len(loopchk), loopchk))
for loopit in loopchk: for loopit in loopchk:
#now that we have the list of all possible matches for the watchcomic + alternate search names, we go through the list until we find a match. #now that we have the list of all possible matches for the watchcomic + alternate search names, we go through the list until we find a match.
modseries_name = loopit modseries_name = loopit
if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE: if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
logger.fdebug('[FILECHECKER] AS_Tuple : ' + str(self.AS_Tuple)) logger.fdebug('[FILECHECKER] AS_Tuple : %s' % self.AS_Tuple)
for ATS in self.AS_Tuple: for ATS in self.AS_Tuple:
if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE: if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
logger.fdebug('[FILECHECKER] ' + str(ATS['AS_Alternate']) + ' comparing to ' + nspace_seriesname) logger.fdebug('[FILECHECKER] %s comparing to %s' % (ATS['AS_Alternate'], nspace_seriesname))
if re.sub('\|','', ATS['AS_Alternate'].lower()).strip() == re.sub('\|','', nspace_seriesname.lower()).strip(): if re.sub('\|','', ATS['AS_Alternate'].lower()).strip() == re.sub('\|','', nspace_seriesname.lower()).strip():
if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE: if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
logger.fdebug('[FILECHECKER] Associating ComiciD : ' + str(ATS['ComicID'])) logger.fdebug('[FILECHECKER] Associating ComiciD : %s' % ATS['ComicID'])
annual_comicid = str(ATS['ComicID']) annual_comicid = str(ATS['ComicID'])
modseries_name = ATS['AS_Alternate'] modseries_name = ATS['AS_Alternate']
break break
logger.fdebug('[FILECHECKER] ' + modseries_name + ' - watchlist match on : ' + filename) logger.fdebug('[FILECHECKER] %s - watchlist match on : %s' % (modseries_name, filename))
if enable_annual: if enable_annual:
if annual_comicid is not None: if annual_comicid is not None:
if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE: if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
logger.fdebug('enable annual is on') logger.fdebug('enable annual is on')
logger.fdebug('annual comicid is ' + str(annual_comicid)) logger.fdebug('annual comicid is %s' % annual_comicid)
if 'biannual' in nspace_watchcomic.lower(): if 'biannual' in nspace_watchcomic.lower():
if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE: if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
logger.fdebug('bi annual detected') logger.fdebug('bi annual detected')
justthedigits = 'BiAnnual ' + justthedigits justthedigits = 'BiAnnual %s' % justthedigits
elif 'annual' in nspace_watchcomic.lower(): elif 'annual' in nspace_watchcomic.lower():
if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE: if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
logger.fdebug('annual detected') logger.fdebug('annual detected')
justthedigits = 'Annual ' + justthedigits justthedigits = 'Annual %s' % justthedigits
elif 'special' in nspace_watchcomic.lower(): elif 'special' in nspace_watchcomic.lower():
justthedigits = 'Special ' + justthedigits justthedigits = 'Special %s' % justthedigits
return {'process_status': 'match', return {'process_status': 'match',
'sub': series_info['sub'], 'sub': series_info['sub'],
@ -1405,7 +1477,7 @@ class FileChecker(object):
'filename': fname, 'filename': fname,
'comicsize': comicsize}) 'comicsize': comicsize})
logger.info('there are ' + str(len(filelist)) + ' files.') logger.info('there are %s files.' % len(filelist))
return filelist return filelist
@ -1507,15 +1579,15 @@ class FileChecker(object):
# extract the !!, store it and then remove it so things will continue. # extract the !!, store it and then remove it so things will continue.
as_start = AS_Alternate.find('!!') as_start = AS_Alternate.find('!!')
if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE: if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
logger.fdebug('as_start: ' + str(as_start) + ' --- ' + str(AS_Alternate[as_start:])) logger.fdebug('as_start: %s --- %s' % (as_start, AS_Alternate[as_start:]))
as_end = AS_Alternate.find('##', as_start) as_end = AS_Alternate.find('##', as_start)
if as_end == -1: if as_end == -1:
as_end = len(AS_Alternate) as_end = len(AS_Alternate)
if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE: if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
logger.fdebug('as_start: ' + str(as_end) + ' --- ' + str(AS_Alternate[as_start:as_end])) logger.fdebug('as_start: %s --- %s' % (as_end, AS_Alternate[as_start:as_end]))
AS_ComicID = AS_Alternate[as_start +2:as_end] AS_ComicID = AS_Alternate[as_start +2:as_end]
if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE: if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
logger.fdebug('[FILECHECKER] Extracted comicid for given annual : ' + str(AS_ComicID)) logger.fdebug('[FILECHECKER] Extracted comicid for given annual : %s' % AS_ComicID)
AS_Alternate = re.sub('!!' + str(AS_ComicID), '', AS_Alternate) AS_Alternate = re.sub('!!' + str(AS_ComicID), '', AS_Alternate)
AS_tupled = True AS_tupled = True
as_dyninfo = self.dynamic_replace(AS_Alternate) as_dyninfo = self.dynamic_replace(AS_Alternate)
@ -1611,19 +1683,24 @@ class FileChecker(object):
return dateline return dateline
def validateAndCreateDirectory(dir, create=False, module=None): def validateAndCreateDirectory(dir, create=False, module=None, dmode=None):
if module is None: if module is None:
module = '' module = ''
module += '[DIRECTORY-CHECK]' module += '[DIRECTORY-CHECK]'
if dmode is None:
dirmode = 'comic'
else:
dirmode = dmode
try: try:
if os.path.exists(dir): if os.path.exists(dir):
logger.info(module + ' Found comic directory: ' + dir) logger.info('%s Found %s directory: %s' % (module, dirmode, dir))
return True return True
else: else:
logger.warn(module + ' Could not find comic directory: ' + dir) logger.warn('%s Could not find %s directory: %s' % (module, dirmode, dir))
if create: if create:
if dir.strip(): if dir.strip():
logger.info(module + ' Creating comic directory (' + str(mylar.CONFIG.CHMOD_DIR) + ') : ' + dir) logger.info('%s Creating %s directory (%s) : %s' % (module, dirmode, mylar.CONFIG.CHMOD_DIR, dir))
try: try:
os.umask(0) # this is probably redudant, but it doesn't hurt to clear the umask here. os.umask(0) # this is probably redudant, but it doesn't hurt to clear the umask here.
if mylar.CONFIG.ENFORCE_PERMS: if mylar.CONFIG.ENFORCE_PERMS:
@ -1633,15 +1710,15 @@ def validateAndCreateDirectory(dir, create=False, module=None):
else: else:
os.makedirs(dir.rstrip()) os.makedirs(dir.rstrip())
except OSError as e: except OSError as e:
logger.warn(module + ' Could not create directory: ' + dir + '[' + str(e) + ']. Aborting.') logger.warn('%s Could not create directory: %s [%s]. Aborting' % (module, dir, e))
return False return False
else: else:
return True return True
else: else:
logger.warn(module + ' Provided directory [' + dir + '] is blank. Aborting.') logger.warn('%s Provided directory [%s] is blank. Aborting.' % (module, dir))
return False return False
except OSError as e: except OSError as e:
logger.warn(module + ' Could not create directory: ' + dir + '[' + str(e) + ']. Aborting.') logger.warn('%s Could not create directory: %s [%s]. Aborting.' % (module, dir, e))
return False return False
return False return False
@ -1704,7 +1781,7 @@ def setperms(path, dir=False):
logger.fdebug('Successfully changed permissions [' + str(mylar.CONFIG.CHMOD_DIR) + ' / ' + str(mylar.CONFIG.CHMOD_FILE) + ']') logger.fdebug('Successfully changed permissions [' + str(mylar.CONFIG.CHMOD_DIR) + ' / ' + str(mylar.CONFIG.CHMOD_FILE) + ']')
except OSError: except OSError:
logger.error('Could not change permissions : ' + path + '. Exiting...') logger.error('Could not change permissions : %s. Exiting...' % path)
return return

8
mylar/getcomics.py
View File

@ -375,6 +375,14 @@ class GC(object):
#write the filename to the db for tracking purposes... #write the filename to the db for tracking purposes...
myDB.upsert('ddl_info', {'filename': filename, 'remote_filesize': remote_filesize}, {'id': id}) myDB.upsert('ddl_info', {'filename': filename, 'remote_filesize': remote_filesize}, {'id': id})
if mylar.CONFIG.DDL_LOCATION is not None and not os.path.isdir(mylar.CONFIG.DDL_LOCATION):
checkdirectory = mylar.filechecker.validateAndCreateDirectory(mylar.CONFIG.DDL_LOCATION, True)
if not checkdirectory:
logger.warn('[ABORTING] Error trying to validate/create DDL download directory: %s.' % mylar.CONFIG.DDL_LOCATION)
return ({"success": False,
"filename": filename,
"path": None})
path = os.path.join(mylar.CONFIG.DDL_LOCATION, filename) path = os.path.join(mylar.CONFIG.DDL_LOCATION, filename)
if t.headers.get('content-encoding') == 'gzip': #.get('Content-Encoding') == 'gzip': if t.headers.get('content-encoding') == 'gzip': #.get('Content-Encoding') == 'gzip':

25
mylar/helpers.py
View File

@ -2476,7 +2476,8 @@ def issue_find_ids(ComicName, ComicID, pack, IssueNumber):
tmp_annuals = pack[pack.find('Annual'):] tmp_annuals = pack[pack.find('Annual'):]
tmp_ann = re.sub('[annual/annuals/+]', '', tmp_annuals.lower()).strip() tmp_ann = re.sub('[annual/annuals/+]', '', tmp_annuals.lower()).strip()
tmp_pack = re.sub('[annual/annuals/+]', '', pack.lower()).strip() tmp_pack = re.sub('[annual/annuals/+]', '', pack.lower()).strip()
pack_issues = range(int(tmp_pack[:tmp_pack.find('-')]),int(tmp_pack[tmp_pack.find('-')+1:])+1) pack_issues_numbers = re.findall(r'\d+', tmp_pack)
pack_issues = range(int(pack_issues_numbers[0]),int(pack_issues_numbers[1])+1)
annualize = True annualize = True
issues = {} issues = {}
@ -2939,6 +2940,14 @@ def weekly_info(week=None, year=None, current=None):
weeknumber = 51 weeknumber = 51
year = 2018 year = 2018
#monkey patch for 2019/2020 - week 52/week 0
if all([weeknumber == 52, c_weeknumber == 51, c_weekyear == 2019]):
weeknumber = 0
year = 2020
elif all([weeknumber == 52, c_weeknumber == 0, c_weekyear == 2020]):
weeknumber = 51
year = 2019
#view specific week (prev_week, next_week) #view specific week (prev_week, next_week)
startofyear = date(year,1,1) startofyear = date(year,1,1)
week0 = startofyear - timedelta(days=startofyear.isoweekday()) week0 = startofyear - timedelta(days=startofyear.isoweekday())
@ -2959,6 +2968,14 @@ def weekly_info(week=None, year=None, current=None):
weeknumber = 51 weeknumber = 51
year = 2018 year = 2018
#monkey patch for 2019/2020 - week 52/week 0
if all([weeknumber == 52, c_weeknumber == 51, c_weekyear == 2019]) or all([weeknumber == '52', year == '2019']):
weeknumber = 0
year = 2020
elif all([weeknumber == 52, c_weeknumber == 0, c_weekyear == 2020]):
weeknumber = 51
year = 2019
stweek = datetime.datetime.strptime(todaydate.strftime('%Y-%m-%d'), '%Y-%m-%d') stweek = datetime.datetime.strptime(todaydate.strftime('%Y-%m-%d'), '%Y-%m-%d')
startweek = stweek - timedelta(days = (stweek.weekday() + 1) % 7) startweek = stweek - timedelta(days = (stweek.weekday() + 1) % 7)
midweek = startweek + timedelta(days = 3) midweek = startweek + timedelta(days = 3)
@ -3110,6 +3127,7 @@ def postprocess_main(queue):
time.sleep(5) time.sleep(5)
elif mylar.APILOCK is False and queue.qsize() >= 1: #len(queue) > 1: elif mylar.APILOCK is False and queue.qsize() >= 1: #len(queue) > 1:
pp = None
item = queue.get(True) item = queue.get(True)
logger.info('Now loading from post-processing queue: %s' % item) logger.info('Now loading from post-processing queue: %s' % item)
if item == 'exit': if item == 'exit':
@ -3124,6 +3142,11 @@ def postprocess_main(queue):
pp = pprocess.post_process() pp = pprocess.post_process()
time.sleep(5) #arbitrary sleep to let the process attempt to finish pp'ing time.sleep(5) #arbitrary sleep to let the process attempt to finish pp'ing
if pp is not None:
if pp['mode'] == 'stop':
#reset the lock so any subsequent items can pp and not keep the queue locked up.
mylar.APILOCK = False
if mylar.APILOCK is True: if mylar.APILOCK is True:
logger.info('Another item is post-processing still...') logger.info('Another item is post-processing still...')
time.sleep(15) time.sleep(15)

3
mylar/notifiers.py
View File

@ -30,6 +30,7 @@ import requests
import smtplib import smtplib
from email.mime.multipart import MIMEMultipart from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText from email.mime.text import MIMEText
from email.utils import formatdate, make_msgid
# This was obviously all taken from headphones with great appreciation :) # This was obviously all taken from headphones with great appreciation :)
@ -383,6 +384,8 @@ class EMAIL:
msg['From'] = str(self.emailfrom) msg['From'] = str(self.emailfrom)
msg['To'] = str(self.emailto) msg['To'] = str(self.emailto)
msg['Subject'] = subject msg['Subject'] = subject
msg['Date'] = formatdate()
msg['Message-ID'] = make_msgid('mylar')
msg.attach(MIMEText(message, 'plain')) msg.attach(MIMEText(message, 'plain'))
if self.emailenc is 1: if self.emailenc is 1:

55
mylar/search.py
View File

@ -1122,6 +1122,9 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDa
continue continue
else: else:
logger.fdebug('match_check: %s' % filecomic) logger.fdebug('match_check: %s' % filecomic)
if filecomic['process_status'] == 'fail':
logger.fdebug('%s was not a match to %s (%s)' % (cleantitle, ComicName, SeriesYear))
continue
elif booktype != parsed_comic['booktype']: elif booktype != parsed_comic['booktype']:
logger.fdebug('Booktypes do not match. Looking for %s, this is a %s. Ignoring this result.' % (booktype, parsed_comic['booktype'])) logger.fdebug('Booktypes do not match. Looking for %s, this is a %s. Ignoring this result.' % (booktype, parsed_comic['booktype']))
continue continue
@ -1148,39 +1151,38 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDa
fndcomicversion = None fndcomicversion = None
if parsed_comic['series_volume'] is not None: if parsed_comic['series_volume'] is not None:
versionfound = "yes" versionfound = "yes"
if len(parsed_comic['series_volume'][1:]) == 4 and parsed_comic['series_volume'][1:].isdigit(): #v2013 if len(parsed_comic['series_volume'][1:]) == 4 and parsed_comic['series_volume'][1:].isdigit(): #v2013
logger.fdebug("[Vxxxx] Version detected as %s" % (parsed_comic['series_volume'])) logger.fdebug("[Vxxxx] Version detected as %s" % (parsed_comic['series_volume']))
vers4year = "yes" #re.sub("[^0-9]", " ", str(ct)) #remove the v vers4year = "yes" #re.sub("[^0-9]", " ", str(ct)) #remove the v
fndcomicversion = parsed_comic['series_volume']
elif len(parsed_comic['series_volume'][1:]) == 1 and parsed_comic['series_volume'][1:].isdigit(): #v2
logger.fdebug("[Vx] Version detected as %s" % parsed_comic['series_volume'])
vers4vol = parsed_comic['series_volume']
fndcomicversion = parsed_comic['series_volume']
elif parsed_comic['series_volume'][1:].isdigit() and len(parsed_comic['series_volume']) < 4:
logger.fdebug('[Vxxx] Version detected as %s' % parsed_comic['series_volume'])
vers4vol = parsed_comic['series_volume']
fndcomicversion = parsed_comic['series_volume']
elif parsed_comic['series_volume'].isdigit() and len(parsed_comic['series_volume']) <=4:
# this stuff is necessary for 32P volume manipulation
if len(parsed_comic['series_volume']) == 4:
vers4year = "yes"
fndcomicversion = parsed_comic['series_volume'] fndcomicversion = parsed_comic['series_volume']
elif len(parsed_comic['series_volume'][1:]) == 1 and parsed_comic['series_volume'][1:].isdigit(): #v2 elif len(parsed_comic['series_volume']) == 1:
logger.fdebug("[Vx] Version detected as %s" % parsed_comic['series_volume'])
vers4vol = parsed_comic['series_volume'] vers4vol = parsed_comic['series_volume']
fndcomicversion = parsed_comic['series_volume'] fndcomicversion = parsed_comic['series_volume']
elif parsed_comic['series_volume'][1:].isdigit() and len(parsed_comic['series_volume']) < 4: elif len(parsed_comic['series_volume']) < 4:
logger.fdebug('[Vxxx] Version detected as %s' % parsed_comic['series_volume'])
vers4vol = parsed_comic['series_volume'] vers4vol = parsed_comic['series_volume']
fndcomicversion = parsed_comic['series_volume'] fndcomicversion = parsed_comic['series_volume']
elif parsed_comic['series_volume'].isdigit() and len(parsed_comic['series_volume']) <=4: else:
# this stuff is necessary for 32P volume manipulation logger.fdebug("error - unknown length for : %s" % parsed_comic['series_volume'])
if len(parsed_comic['series_volume']) == 4:
vers4year = "yes"
fndcomicversion = parsed_comic['series_volume']
elif len(parsed_comic['series_volume']) == 1:
vers4vol = parsed_comic['series_volume']
fndcomicversion = parsed_comic['series_volume']
elif len(parsed_comic['series_volume']) < 4:
vers4vol = parsed_comic['series_volume']
fndcomicversion = parsed_comic['series_volume']
else:
logger.fdebug("error - unknown length for : %s" % parsed_comic['series_volume'])
yearmatch = "false" yearmatch = "false"
if vers4vol != "no" or vers4year != "no": if vers4vol != "no" or vers4year != "no":
logger.fdebug("Series Year not provided but Series Volume detected of %s. Bypassing Year Match." % fndcomicversion) logger.fdebug("Series Year not provided but Series Volume detected of %s. Bypassing Year Match." % fndcomicversion)
yearmatch = "true" yearmatch = "true"
elif ComVersChk == 0: elif ComVersChk == 0 and parsed_comic['issue_year'] is None:
logger.fdebug("Series version detected as V1 (only series in existance with that title). Bypassing Year/Volume check") logger.fdebug("Series version detected as V1 (only series in existance with that title). Bypassing Year/Volume check")
yearmatch = "true" yearmatch = "true"
elif any([UseFuzzy == "0", UseFuzzy == "2", UseFuzzy is None, IssDateFix != "no"]) and parsed_comic['issue_year'] is not None: elif any([UseFuzzy == "0", UseFuzzy == "2", UseFuzzy is None, IssDateFix != "no"]) and parsed_comic['issue_year'] is not None:
@ -1529,7 +1531,12 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDa
issinfo = mylar.COMICINFO['pack_issuelist'] issinfo = mylar.COMICINFO['pack_issuelist']
if issinfo is not None: if issinfo is not None:
#we need to get EVERY issue ID within the pack and update the log to reflect that they're being downloaded via a pack. #we need to get EVERY issue ID within the pack and update the log to reflect that they're being downloaded via a pack.
logger.fdebug('Found matching comic within pack...preparing to send to Updater with IssueIDs: %s and nzbname of %s' % (issueid_info, nzbname))
try:
logger.fdebug('Found matching comic within pack...preparing to send to Updater with IssueIDs: %s and nzbname of %s' % (issueid_info, nzbname))
except NameError:
logger.fdebug('Did not find issueid_info')
#because packs need to have every issue that's not already Downloaded in a Snatched status, throw it to the updater here as well. #because packs need to have every issue that's not already Downloaded in a Snatched status, throw it to the updater here as well.
for isid in issinfo['issues']: for isid in issinfo['issues']:
updater.nzblog(isid['issueid'], nzbname, ComicName, SARC=SARC, IssueArcID=IssueArcID, id=nzbid, prov=tmpprov, oneoff=oneoff) updater.nzblog(isid['issueid'], nzbname, ComicName, SARC=SARC, IssueArcID=IssueArcID, id=nzbid, prov=tmpprov, oneoff=oneoff)

5
mylar/updater.py
View File

@ -856,8 +856,7 @@ def foundsearch(ComicID, IssueID, mode=None, down=None, provider=None, SARC=None
pass pass
myDB.upsert("oneoffhistory", newValue, ctlVal) myDB.upsert("oneoffhistory", newValue, ctlVal)
logger.info('%s Updated the status (Snatched) complete for %s Issue: %s' % (module, ComicName, IssueNum))
logger.info(module + ' Updated the status (Snatched) complete for ' + ComicName + ' Issue: ' + str(IssueNum))
else: else:
if down == 'PP': if down == 'PP':
logger.info(module + ' Setting status to Post-Processed in history.') logger.info(module + ' Setting status to Post-Processed in history.')
@ -917,7 +916,7 @@ def foundsearch(ComicID, IssueID, mode=None, down=None, provider=None, SARC=None
newVal['year'] = pullinfo['year'] newVal['year'] = pullinfo['year']
myDB.upsert("oneoffhistory", newVal, ctlVal) myDB.upsert("oneoffhistory", newVal, ctlVal)
logger.info(module + ' Updating Status (' + downstatus + ') now complete for ' + ComicName + ' issue: ' + IssueNum) logger.info('%s Updating Status (%s) now completed for %s issue: %s' % (module, downstatus, ComicName, IssueNum))
return return
def forceRescan(ComicID, archive=None, module=None, recheck=False): def forceRescan(ComicID, archive=None, module=None, recheck=False):

6
mylar/webserve.py
View File

@ -2580,7 +2580,7 @@ class WebInterface(object):
interval = str(mylar.CONFIG.DOWNLOAD_SCAN_INTERVAL) + ' mins' interval = str(mylar.CONFIG.DOWNLOAD_SCAN_INTERVAL) + ' mins'
if 'version' in jb['JobName'].lower(): if 'version' in jb['JobName'].lower():
status = mylar.VERSION_STATUS status = mylar.VERSION_STATUS
interval = str(mylar.CONFIG.CHECK_GITHUB_INTERVAL) + 'mins' interval = str(mylar.CONFIG.CHECK_GITHUB_INTERVAL) + ' mins'
if status != jb['Status'] and not('rss' in jb['JobName'].lower()): if status != jb['Status'] and not('rss' in jb['JobName'].lower()):
status = jb['Status'] status = jb['Status']
@ -5370,14 +5370,14 @@ class WebInterface(object):
checked_configs = ['enable_https', 'launch_browser', 'syno_fix', 'auto_update', 'annuals_on', 'api_enabled', 'nzb_startup_search', checked_configs = ['enable_https', 'launch_browser', 'syno_fix', 'auto_update', 'annuals_on', 'api_enabled', 'nzb_startup_search',
'enforce_perms', 'sab_to_mylar', 'torrent_local', 'torrent_seedbox', 'rtorrent_ssl', 'rtorrent_verify', 'rtorrent_startonload', 'enforce_perms', 'sab_to_mylar', 'torrent_local', 'torrent_seedbox', 'rtorrent_ssl', 'rtorrent_verify', 'rtorrent_startonload',
'enable_torrents', 'enable_rss', 'nzbsu', 'nzbsu_verify', 'enable_torrents', 'enable_rss', 'nzbsu', 'nzbsu_verify',
'dognzb', 'dognzb_verify', 'experimental', 'enable_torrent_search', 'enable_public', 'enable_32p', 'enable_torznab', 'dognzb', 'dognzb_verify', 'experimental', 'enable_torrent_search', 'enable_32p', 'enable_torznab',
'newznab', 'use_minsize', 'use_maxsize', 'ddump', 'failed_download_handling', 'sab_client_post_processing', 'nzbget_client_post_processing', 'newznab', 'use_minsize', 'use_maxsize', 'ddump', 'failed_download_handling', 'sab_client_post_processing', 'nzbget_client_post_processing',
'failed_auto', 'post_processing', 'enable_check_folder', 'enable_pre_scripts', 'enable_snatch_script', 'enable_extra_scripts', 'failed_auto', 'post_processing', 'enable_check_folder', 'enable_pre_scripts', 'enable_snatch_script', 'enable_extra_scripts',
'enable_meta', 'cbr2cbz_only', 'ct_tag_cr', 'ct_tag_cbl', 'ct_cbz_overwrite', 'rename_files', 'replace_spaces', 'zero_level', 'enable_meta', 'cbr2cbz_only', 'ct_tag_cr', 'ct_tag_cbl', 'ct_cbz_overwrite', 'rename_files', 'replace_spaces', 'zero_level',
'lowercase_filenames', 'autowant_upcoming', 'autowant_all', 'comic_cover_local', 'alternate_latest_series_covers', 'cvinfo', 'snatchedtorrent_notify', 'lowercase_filenames', 'autowant_upcoming', 'autowant_all', 'comic_cover_local', 'alternate_latest_series_covers', 'cvinfo', 'snatchedtorrent_notify',
'prowl_enabled', 'prowl_onsnatch', 'pushover_enabled', 'pushover_onsnatch', 'boxcar_enabled', 'prowl_enabled', 'prowl_onsnatch', 'pushover_enabled', 'pushover_onsnatch', 'boxcar_enabled',
'boxcar_onsnatch', 'pushbullet_enabled', 'pushbullet_onsnatch', 'telegram_enabled', 'telegram_onsnatch', 'slack_enabled', 'slack_onsnatch', 'boxcar_onsnatch', 'pushbullet_enabled', 'pushbullet_onsnatch', 'telegram_enabled', 'telegram_onsnatch', 'slack_enabled', 'slack_onsnatch',
'email_enabled', 'email_enc', 'email_ongrab', 'email_onpost', 'opds_enable', 'opds_authentication', 'opds_metainfo', 'enable_ddl', 'deluge_pause'] 'email_enabled', 'email_enc', 'email_ongrab', 'email_onpost', 'opds_enable', 'opds_authentication', 'opds_metainfo', 'enable_ddl', 'deluge_pause'] #enable_public
for checked_config in checked_configs: for checked_config in checked_configs:
if checked_config not in kwargs: if checked_config not in kwargs:

18
post-processing/autoProcessComics.py
View File

@ -3,14 +3,26 @@ import os.path
import ConfigParser import ConfigParser
import urllib2 import urllib2
import urllib import urllib
import platform
try: try:
import requests import requests
use_requests = True use_requests = True
except ImportError: except ImportError:
print "Requests module not found on system. I'll revert so this will work, but you probably should install " print '''Requests module not found on system. I'll revert so this will work, but you probably should install
print "requests to bypass this in the future (ie. pip install requests)" requests to bypass this in the future (i.e. pip install requests)'''
use_requests = False use_requests = False
use_win32api = False
if platform.system() == 'Windows':
try:
import win32api
use_win32api = True
except ImportError:
print '''The win32api module was not found on this system. While it's fine to run without it, you're
running a Windows-based OS, so it would benefit you to install it. It enables ComicRN to better
work with file paths beyond the 260 character limit. Run "pip install pypiwin32".'''
apc_version = "2.04" apc_version = "2.04"
def processEpisode(dirName, nzbName=None): def processEpisode(dirName, nzbName=None):
@ -18,6 +30,8 @@ def processEpisode(dirName, nzbName=None):
return processIssue(dirName, nzbName) return processIssue(dirName, nzbName)
def processIssue(dirName, nzbName=None, failed=False, comicrn_version=None): def processIssue(dirName, nzbName=None, failed=False, comicrn_version=None):
if use_win32api is True:
dirName = win32api.GetShortPathName(dirName)
config = ConfigParser.ConfigParser() config = ConfigParser.ConfigParser()
configFilename = os.path.join(os.path.dirname(sys.argv[0]), "autoProcessComics.cfg") configFilename = os.path.join(os.path.dirname(sys.argv[0]), "autoProcessComics.cfg")