From 81f8a4b76d7b255e4352ffa96fa1f76c47a06cf0 Mon Sep 17 00:00:00 2001 From: evilhero Date: Fri, 29 Jan 2016 12:38:32 -0500 Subject: [PATCH] FIX:(#1198) Updated py-unrar2 to latest version + patched to fix date format errors which was causing problems with ComicTagger and converting cbr-to-cbz, FIX: small typos in Config and Manage pages. --- data/interfaces/default/config.html | 4 +- data/interfaces/default/manage.html | 2 +- lib/comictaggerlib/UnRAR2/README.md | 11 +++ lib/comictaggerlib/UnRAR2/__init__.py | 9 ++- lib/comictaggerlib/UnRAR2/test_UnRAR2.py | 46 ++++++++++++ lib/comictaggerlib/UnRAR2/unix.py | 89 ++++++++++++++++++------ lib/comictaggerlib/UnRAR2/windows.py | 35 ++++++++-- mylar/cmtagmylar.py | 4 ++ 8 files changed, 166 insertions(+), 34 deletions(-) create mode 100644 lib/comictaggerlib/UnRAR2/README.md diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 79219dfc..f4ca70db 100755 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -312,8 +312,8 @@
- - This is *ONLY* required if Mylar and SABnzbd are on seperate machines, otherwise don't touch it + + This is *ONLY* required if Mylar and SABnzbd are on separate machines, otherwise don't touch it
diff --git a/data/interfaces/default/manage.html b/data/interfaces/default/manage.html index 993d5050..050928fe 100755 --- a/data/interfaces/default/manage.html +++ b/data/interfaces/default/manage.html @@ -116,7 +116,7 @@ Force Options Force Check for Wanted Issues Force Update Active Comics - Check for mylar Updates + Check for Mylar Updates
diff --git a/lib/comictaggerlib/UnRAR2/README.md b/lib/comictaggerlib/UnRAR2/README.md new file mode 100644 index 00000000..04c8f82f --- /dev/null +++ b/lib/comictaggerlib/UnRAR2/README.md @@ -0,0 +1,11 @@ +pyUnRAR2 is a Python (ctypes-based) wrapper around the free UnRAR.dll (Windows) or command-line unrar binary (Unix). + +It is an modified version of Jimmy Retzlaff's pyUnRAR - more simple, +stable and foolproof, with Unix support added. +Notice that it has INCOMPATIBLE interface. + +It enables reading and unpacking of archives created with the RAR/WinRAR archivers. + +Downloads: +https://drive.google.com/folderview?id=0B6L2WW4Bg4jvMnFtbjRKOUtDQTQ&usp=sharing#list + diff --git a/lib/comictaggerlib/UnRAR2/__init__.py b/lib/comictaggerlib/UnRAR2/__init__.py index a913fcb6..516a82f4 100644 --- a/lib/comictaggerlib/UnRAR2/__init__.py +++ b/lib/comictaggerlib/UnRAR2/__init__.py @@ -33,7 +33,7 @@ similar to the C interface provided by UnRAR. There is also a higher level interface which makes some common operations easier. """ -__version__ = '0.99.3' +__version__ = '0.99.6' try: WindowsError @@ -74,8 +74,6 @@ class RarInfo(object): self.size = data['size'] self.datetime = data['datetime'] self.comment = data['comment'] - - def __str__(self): try : @@ -159,6 +157,11 @@ class RarFile(RarFileImplementation): checker = condition2checker(condition) return RarFileImplementation.extract(self, checker, path, withSubpath, overwrite) + def get_volume(self): + """Determine which volume is it in a multi-volume archive. Returns None if it's not a + multi-volume archive, 0-based volume number otherwise.""" + return RarFileImplementation.get_volume(self) + def condition2checker(condition): """Converts different condition types to callback""" if type(condition) in [str, unicode]: diff --git a/lib/comictaggerlib/UnRAR2/test_UnRAR2.py b/lib/comictaggerlib/UnRAR2/test_UnRAR2.py index e86ba2c1..03a4f5b6 100644 --- a/lib/comictaggerlib/UnRAR2/test_UnRAR2.py +++ b/lib/comictaggerlib/UnRAR2/test_UnRAR2.py @@ -15,6 +15,7 @@ def cleanup(dir='test'): # basic test cleanup() rarc = UnRAR2.RarFile('test.rar') +assert rarc.get_volume() == None rarc.infolist() assert rarc.comment == "This is a test." for info in rarc.infoiter(): @@ -28,6 +29,21 @@ del rarc assert (str(saveinfo)=="""""") cleanup() +# shell-unsafe-name test +cleanup() +rarc = UnRAR2.RarFile('[test].rar') +rarc.infolist() +for info in rarc.infoiter(): + saveinfo = info + assert (str(info)=="""""") + break +rarc.extract() +assert os.path.exists('[test].txt') +del rarc +assert (str(saveinfo)=="""""") +cleanup() + + # extract all the files in test.rar cleanup() UnRAR2.RarFile('test.rar').extract() @@ -108,6 +124,15 @@ except IncorrectRARPassword: assert not os.path.exists('test'+os.sep+'top_secret_xxx_file.txt') assert errored cleanup() +errored = False +try: + UnRAR2.RarFile('test_protected_files.rar').extract() +except IncorrectRARPassword: + errored = True +assert not os.path.exists('test'+os.sep+'top_secret_xxx_file.txt') +assert errored +cleanup() + # extract files from an archive with protected headers cleanup() @@ -122,6 +147,27 @@ except IncorrectRARPassword: assert not os.path.exists('test'+os.sep+'top_secret_xxx_file.txt') assert errored cleanup() +errored = False +try: + UnRAR2.RarFile('test_protected_headers.rar').extract() +except IncorrectRARPassword: + errored = True +assert not os.path.exists('test'+os.sep+'top_secret_xxx_file.txt') +assert errored +cleanup() + +# check volume number +cleanup() +rarc1 = UnRAR2.RarFile('test_volumes.part1.rar') +assert rarc1.get_volume() == 0 +rarc2 = UnRAR2.RarFile('test_volumes.part2.rar') +assert rarc2.get_volume() == 1 +cleanup() +rarc1 = UnRAR2.RarFile('test_volumes_old.rar') +assert rarc1.get_volume() == 0 +rarc2 = UnRAR2.RarFile('test_volumes_old.r00') +assert rarc2.get_volume() == 1 +cleanup() # make sure docstring examples are working import doctest diff --git a/lib/comictaggerlib/UnRAR2/unix.py b/lib/comictaggerlib/UnRAR2/unix.py index bd9ee859..d80837dc 100644 --- a/lib/comictaggerlib/UnRAR2/unix.py +++ b/lib/comictaggerlib/UnRAR2/unix.py @@ -48,7 +48,7 @@ def call_unrar(params): pass if rar_executable_cached is None: raise UnpackerNotInstalled("No suitable RAR unpacker installed") - + assert type(params) == list, "params must be list" args = [rar_executable_cached] + params try: @@ -62,25 +62,25 @@ class RarFileImplementation(object): def init(self, password=None): global rar_executable_version self.password = password - - + + stdoutdata, stderrdata = self.call('v', []).communicate() - + for line in stderrdata.splitlines(): if line.strip().startswith("Cannot open"): raise FileOpenError if line.find("CRC failed")>=0: - raise IncorrectRARPassword + raise IncorrectRARPassword accum = [] source = iter(stdoutdata.splitlines()) line = '' - while not (line.startswith('UNRAR')): + while (line.find('RAR ') == -1): line = source.next() signature = line # The code below is mighty flaky # and will probably crash on localized versions of RAR # but I see no safe way to rewrite it using a CLI tool - if signature.startswith("UNRAR 4"): + if signature.find("RAR 4") > -1: rar_executable_version = 4 while not (line.startswith('Comment:') or line.startswith('Pathname/Comment')): if line.strip().endswith('is not RAR archive'): @@ -94,7 +94,7 @@ class RarFileImplementation(object): self.comment = '\n'.join(accum[:-1]) else: self.comment = None - elif signature.startswith("UNRAR 5"): + elif signature.find("RAR 5") > -1: rar_executable_version = 5 line = source.next() while not line.startswith('Archive:'): @@ -107,14 +107,14 @@ class RarFileImplementation(object): else: self.comment = None else: - raise UnpackerNotInstalled("Unsupported RAR version, expected 4.x or 5.x, found: " + raise UnpackerNotInstalled("Unsupported RAR version, expected 4.x or 5.x, found: " + signature.split(" ")[1]) - - + + def escaped_password(self): return '-' if self.password == None else self.password - - + + def call(self, cmd, options=[], files=[]): options2 = options + ['p'+self.escaped_password()] soptions = ['-'+x for x in options2] @@ -136,7 +136,7 @@ class RarFileImplementation(object): if line.strip().endswith('is not RAR archive'): raise InvalidRARArchive if line.startswith("CRC failed") or line.startswith("Checksum error"): - raise IncorrectRARPassword + raise IncorrectRARPassword line = source.next() line = source.next() i = 0 @@ -153,8 +153,13 @@ class RarFileImplementation(object): data['size'] = int(fields[0]) attr = fields[5] data['isdir'] = 'd' in attr.lower() - data['datetime'] = time.strptime(fields[3]+" "+fields[4], '%d-%m-%y %H:%M') + try: + data['datetime'] = time.strptime(fields[3]+" "+fields[4], '%d-%m-%y %H:%M') + except ValueError: + data['datetime'] = time.strptime(fields[3]+" "+fields[4], '%Y-%m-%d %H:%M') + data['comment'] = None + data['volume'] = None yield data accum = [] i += 1 @@ -168,12 +173,17 @@ class RarFileImplementation(object): data['size'] = int(fields[1]) attr = fields[0] data['isdir'] = 'd' in attr.lower() - data['datetime'] = time.strptime(fields[2]+" "+fields[3], '%d-%m-%y %H:%M') + try: + data['datetime'] = time.strptime(fields[2]+" "+fields[3], '%d-%m-%y %H:%M') + except ValueError: + data['datetime'] = time.strptime(fields[2]+" "+fields[3], '%Y-%m-%d %H:%M') + data['comment'] = None + data['volume'] = None yield data i += 1 line = source.next() - + def read_files(self, checker): res = [] @@ -184,7 +194,7 @@ class RarFileImplementation(object): res.append((info, pipe.read())) return res - + def extract(self, checker, path, withSubpath, overwrite): res = [] command = 'x' @@ -209,10 +219,45 @@ class RarFileImplementation(object): proc = self.call(command, options, names) stdoutdata, stderrdata = proc.communicate() if stderrdata.find("CRC failed")>=0 or stderrdata.find("Checksum error")>=0: - raise IncorrectRARPassword - return res - + raise IncorrectRARPassword + return res + def destruct(self): pass - + def get_volume(self): + command = "v" if rar_executable_version == 4 else "l" + stdoutdata, stderrdata = self.call(command, ['c-']).communicate() + + for line in stderrdata.splitlines(): + if line.strip().startswith("Cannot open"): + raise FileOpenError + + source = iter(stdoutdata.splitlines()) + line = '' + while not line.startswith('-----------'): + if line.strip().endswith('is not RAR archive'): + raise InvalidRARArchive + if line.startswith("CRC failed") or line.startswith("Checksum error"): + raise IncorrectRARPassword + line = source.next() + line = source.next() + if rar_executable_version == 4: + while not line.startswith('-----------'): + line = source.next() + line = source.next() + items = line.strip().split() + if len(items)>4 and items[4]=="volume": + return int(items[5]) - 1 + else: + return None + + elif rar_executable_version == 5: + while not line.startswith('-----------'): + line = source.next() + line = source.next() + items = line.strip().split() + if items[1]=="volume": + return int(items[2]) - 1 + else: + return None diff --git a/lib/comictaggerlib/UnRAR2/windows.py b/lib/comictaggerlib/UnRAR2/windows.py index bb92481b..19cca3d7 100644 --- a/lib/comictaggerlib/UnRAR2/windows.py +++ b/lib/comictaggerlib/UnRAR2/windows.py @@ -25,7 +25,7 @@ from __future__ import generators import ctypes, ctypes.wintypes -import os, os.path, sys +import os, os.path, sys, re import Queue import time @@ -43,6 +43,7 @@ ERAR_EREAD = 18 ERAR_EWRITE = 19 ERAR_SMALL_BUF = 20 ERAR_UNKNOWN = 21 +ERAR_MISSING_PASSWORD = 22 RAR_OM_LIST = 0 RAR_OM_EXTRACT = 1 @@ -66,6 +67,9 @@ dll_name = "unrar.dll" if architecture_bits == 64: dll_name = "x64\\unrar64.dll" +volume_naming1 = re.compile("\.r([0-9]{2})$") +volume_naming2 = re.compile("\.([0-9]{3}).rar$") +volume_naming3 = re.compile("\.part([0-9]+).rar$") try: unrar = ctypes.WinDLL(os.path.join(os.path.split(__file__)[0], 'UnRARDLL', dll_name)) @@ -188,7 +192,7 @@ class RarInfoIterator(object): self.index = 0 self.headerData = RARHeaderDataEx() self.res = RARReadHeaderEx(self.arc._handle, ctypes.byref(self.headerData)) - if self.res==ERAR_BAD_DATA: + if self.res in [ERAR_BAD_DATA, ERAR_MISSING_PASSWORD]: raise IncorrectRARPassword self.arc.lockStatus = "locked" self.arc.needskip = False @@ -208,7 +212,7 @@ class RarInfoIterator(object): data = {} data['index'] = self.index - data['filename'] = self.headerData.FileName + data['filename'] = self.headerData.FileNameW data['datetime'] = DosDateTimeToTimeTuple(self.headerData.FileTime) data['isdir'] = ((self.headerData.Flags & 0xE0) == 0xE0) data['size'] = self.headerData.UnpSize + (self.headerData.UnpSizeHigh << 32) @@ -251,7 +255,8 @@ class RarFileImplementation(object): RARSetPassword(self._handle, password) self.lockStatus = "ready" - + + self.isVolume = archiveData.Flags & 1 def destruct(self): @@ -277,7 +282,7 @@ class RarFileImplementation(object): c_callback = UNRARCALLBACK(reader._callback) RARSetCallback(self._handle, c_callback, 1) tmpres = RARProcessFile(self._handle, RAR_TEST, None, None) - if tmpres==ERAR_BAD_DATA: + if tmpres in [ERAR_BAD_DATA, ERAR_MISSING_PASSWORD]: raise IncorrectRARPassword self.needskip = False res.append((info, reader.get_result())) @@ -299,11 +304,29 @@ class RarFileImplementation(object): target = checkres if overwrite or (not os.path.exists(target)): tmpres = RARProcessFile(self._handle, RAR_EXTRACT, None, target) - if tmpres==ERAR_BAD_DATA: + if tmpres in [ERAR_BAD_DATA, ERAR_MISSING_PASSWORD]: raise IncorrectRARPassword self.needskip = False res.append(info) return res + def get_volume(self): + if not self.isVolume: + return None + headerData = RARHeaderDataEx() + res = RARReadHeaderEx(self._handle, ctypes.byref(headerData)) + arcName = headerData.ArcNameW + match3 = volume_naming3.search(arcName) + if match3 != None: + return int(match3.group(1)) - 1 + match2 = volume_naming3.search(arcName) + if match2 != None: + return int(match2.group(1)) + match1 = volume_naming1.search(arcName) + if match1 != None: + return int(match1.group(1)) + 1 + return 0 + + diff --git a/mylar/cmtagmylar.py b/mylar/cmtagmylar.py index 3a0e7796..c5d179e9 100644 --- a/mylar/cmtagmylar.py +++ b/mylar/cmtagmylar.py @@ -238,6 +238,10 @@ def run(dirName, nzbName=None, issueid=None, comversion=None, manual=None, filen initial_ctrun = False elif initial_ctrun and 'Archive is not a RAR' in out: initial_ctrun = False + elif initial_ctrun: + logger.warn(module + '[COMIC-TAGGER][CBR-TO-CBZ] Failed to convert cbr to cbz - check permissions on folder : ' + mylar.CACHE_DIR + ' and/or the location where Mylar is trying to tag the files from.') + initial_ctrun = False + return 'fail' elif 'Cannot find' in out: logger.warn(module + '[COMIC-TAGGER] Unable to locate file: ' + filename) file_error = 'file not found||' + filename