From 3945ba4a0d76c21d9fabcb304a3ff39b30bd2397 Mon Sep 17 00:00:00 2001 From: evilhero Date: Fri, 16 Jan 2015 14:40:08 -0500 Subject: [PATCH] Many updates & bug fixes - too many too list here (check changelog @ http://forum.mylarcomics.com/viewtopic.php?f=2&t=545) --- data/interfaces/default/comicdetails.html | 18 +- data/interfaces/default/config.html | 18 +- data/interfaces/default/importresults.html | 96 ++--- .../default/importresults_popup.html | 107 ++++++ data/interfaces/default/index.html | 2 +- data/interfaces/default/readlist.html | 25 +- data/interfaces/default/weeklypull.html | 21 +- .../PKG-INFO | 240 ++++++++++++ .../SOURCES.txt | 16 + .../dependency_links.txt | 1 + .../installed-files.txt | 13 + .../top_level.txt | 2 + .../zip-safe | 1 + lib/ConcurrentLogHandler/cloghandler.py | 349 ++++++++++++++++++ lib/ConcurrentLogHandler/portalocker.py | 141 +++++++ lib/ConcurrentLogHandler/stresstest.py | 287 ++++++++++++++ mylar/PostProcessor.py | 12 +- mylar/__init__.py | 129 ++++--- mylar/cmtagmylar.py | 19 +- mylar/filechecker.py | 194 +++++++--- mylar/findcomicfeed.py | 4 +- mylar/helpers.py | 126 ++++--- mylar/importer.py | 12 + mylar/librarysync.py | 9 +- mylar/logger.py | 43 ++- mylar/mb.py | 3 +- mylar/newpull.py | 9 +- mylar/search.py | 2 +- mylar/updater.py | 291 ++++++--------- mylar/webserve.py | 271 ++++++++++---- 30 files changed, 1977 insertions(+), 484 deletions(-) create mode 100755 data/interfaces/default/importresults_popup.html create mode 100755 lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/PKG-INFO create mode 100755 lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/SOURCES.txt create mode 100755 lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/dependency_links.txt create mode 100755 lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/installed-files.txt create mode 100755 lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/top_level.txt create mode 100755 lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/zip-safe create mode 100755 lib/ConcurrentLogHandler/cloghandler.py create mode 100755 lib/ConcurrentLogHandler/portalocker.py create mode 100755 lib/ConcurrentLogHandler/stresstest.py diff --git a/data/interfaces/default/comicdetails.html b/data/interfaces/default/comicdetails.html index ea240045..3c4f23d6 100755 --- a/data/interfaces/default/comicdetails.html +++ b/data/interfaces/default/comicdetails.html @@ -511,19 +511,23 @@ <% if annual['Status'] == 'Skipped': - grade = 'Z' + agrade = 'Z' elif annual['Status'] == 'Wanted': - grade = 'X' + agrade = 'X' elif annual['Status'] == 'Snatched': - grade = 'C' + agrade = 'C' elif annual['Status'] == 'Downloaded': - grade = 'A' + agrade = 'A' elif annual['Status'] == 'Archived': - grade = 'A' + agrade = 'A' + elif annual['Status'] == 'Ignored': + agrade = 'A' + elif annual['Status'] == 'Failed': + agrade = 'C' else: - grade = 'A' + agrade = 'A' %> - + <% if annual['Int_IssueNumber'] is None: diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index c15cf096..c53cda41 100755 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -598,10 +598,20 @@
Duplicate Handling -
- -
- +
+ + +
Failed Download Handling diff --git a/data/interfaces/default/importresults.html b/data/interfaces/default/importresults.html index 4d7be802..b79dd1ea 100755 --- a/data/interfaces/default/importresults.html +++ b/data/interfaces/default/importresults.html @@ -34,11 +34,13 @@
-
- - (After importing, Rename the files to configuration settings) - -
+ %if mylar.RENAME_FILES: +
+ + (After importing, Rename the files to configuration settings) + +
+ %endif
(Use existing Metadata to better locate series for import) @@ -59,8 +61,8 @@
- -

To be Imported

+ +

To be Imported

(green indicates confirmed on watchlist)
@@ -74,8 +76,8 @@ - %if results: - %for result in results: + %if results: + %for result in results: <% if result['DisplayName'] is None: displayname = result['ComicName'] @@ -83,42 +85,54 @@ displayname = result['DisplayName'] endif %> - - - - - + + + + + + + + <% + myDB = db.DBConnection() + files = myDB.action("SELECT * FROM importresults WHERE ComicName=?", [result['ComicName']]) + %> + %endfor + %else: + + - - - - <% - myDB = db.DBConnection() - files = myDB.action("SELECT * FROM importresults WHERE ComicName=?", [result['ComicName']]) - - %> - %endfor - %else: - - - %endif - -
- ${displayname}${result['ComicYear']}${result['IssueCount']}${result['Status']} - %if result['WatchMatch'] is not None: - - %endif +
+ ${displayname}${result['ComicYear']}${result['IssueCount']} + %if result['ComicID']: + ${result['Status']} + %else: + ${result['Status']} + %endif + %if result['WatchMatch'] is not None: + + %endif + ${result['ImportDate']} + %if result['Status'] == 'Not Imported': + [Import] + %endif + [Remove] + %if result['implog'] is not None: + [Log] + %endif + %if result['SRID'] is not None and result['Status'] != 'Imported': + [Select] + %endif +
+
There are no results to display
${result['ImportDate']}[Import] - [Remove] - - %if result['implog'] is not None: - [Log] - %endif -
There are no results to display
+ + %endif + +

Already on Watchlist

-
(you need to CONFIRM the match before doing an import!) +
(you need to CONFIRM the match before doing an import!)
@@ -163,8 +177,6 @@
- - <%def name="javascriptIncludes()"> diff --git a/data/interfaces/default/importresults_popup.html b/data/interfaces/default/importresults_popup.html new file mode 100755 index 00000000..85598205 --- /dev/null +++ b/data/interfaces/default/importresults_popup.html @@ -0,0 +1,107 @@ +<%inherit file="base.html" /> +<%! + import mylar + from mylar.helpers import checked + +%> +<%def name="headerIncludes()"> + + + +<%def name="body()"> +
+

Search resultsImport Search results for : ${searchtext}

+
+
+ + + + + + + + + + + + + %if searchresults: + %for result in searchresults: + <% + if result['comicyear'] == '2015': + grade = 'A' + else: + grade = 'Z' + if result['haveit'] != "No": + grade = 'H'; + %> + + + %if result['deck'] == 'None' or result['deck'] is None: + + %else: + + %endif + + + + %if result['haveit'] == 'No': + <% + calledby = "web-import" + %> + + %else: + + %endif + + %endfor + %endif + +
Comic NamePublisherYearIssues
${result['name']}${result['name']}${result['publisher']}${result['comicyear']}${result['issues']}Add this ComicAlready in Library
+
+ + +<%def name="headIncludes()"> + + + +<%def name="javascriptIncludes()"> + + + + diff --git a/data/interfaces/default/index.html b/data/interfaces/default/index.html index 113a300a..42b9b32d 100755 --- a/data/interfaces/default/index.html +++ b/data/interfaces/default/index.html @@ -35,7 +35,7 @@ ${comic['ComicPublisher']} ${comic['ComicName']} - ${comic['ComicYear']} + ${comic['ComicYear']} # ${comic['LatestIssue']} ${comic['LatestDate']}
${comic['haveissues']}/${comic['totalissues']}
diff --git a/data/interfaces/default/readlist.html b/data/interfaces/default/readlist.html index 97f073c7..5de5e65d 100755 --- a/data/interfaces/default/readlist.html +++ b/data/interfaces/default/readlist.html @@ -33,7 +33,7 @@


- + @@ -61,17 +61,26 @@ %for item in readlist: <% - if item['Status'] == 'Downloaded': + if item['Status'] == 'Skipped': + grade = 'Z' + elif item['Status'] == 'Wanted': + grade = 'X' + elif item['Status'] == 'Snatched': + grade = 'C' + elif item['Status'] == 'Downloaded': grade = 'A' + elif item['Status'] == 'Archived': + grade = 'A' + elif item['Status'] == 'Ignored': + grade = 'A' + elif item['Status'] == 'Failed': + grade = 'C' elif item['Status'] == 'Read': grade = 'C' elif item['Status'] == 'Not Watched': grade = 'X' - elif item['Status'] == 'Wanted': - grade = 'Y' else: - grade = 'U' - + grade = 'A' %> @@ -98,9 +107,9 @@ ${item['Status']} %if item['Status'] is None or item['Status'] == None: - Grab it + Grab it %elif item['Status'] == 'Snatched': - Retry + Retry %endif diff --git a/data/interfaces/default/weeklypull.html b/data/interfaces/default/weeklypull.html index 0442a839..d75e2b59 100755 --- a/data/interfaces/default/weeklypull.html +++ b/data/interfaces/default/weeklypull.html @@ -63,19 +63,26 @@ grade = 'C' else: grade = 'A' + if weekly['AUTOWANT'] == True: + grade = 'H' + %> %if pullfilter is True: ${weekly['PUBLISHER']} ${weekly['COMIC']} ${weekly['ISSUE']} - ${weekly['STATUS']} - %if weekly['STATUS'] == 'Skipped': - add series - one off - %if weekly['ISSUE'] == '1' or weekly['ISSUE'] == '0': - Watch - %endif + %if weekly['AUTOWANT']: + Auto-Want + %else: + ${weekly['STATUS']} + %if weekly['STATUS'] == 'Skipped': + %if weekly['ISSUE'] == '1' or weekly['ISSUE'] == '0': + Watch + %endif + add series + one off + %endif %endif %endif diff --git a/lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/PKG-INFO b/lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/PKG-INFO new file mode 100755 index 00000000..45984f9b --- /dev/null +++ b/lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/PKG-INFO @@ -0,0 +1,240 @@ +Metadata-Version: 1.1 +Name: ConcurrentLogHandler +Version: 0.9.1 +Summary: Concurrent logging handler (drop-in replacement for RotatingFileHandler) +Home-page: http://launchpad.net/python-concurrent-log-handler +Author: Lowell Alleman +Author-email: lowell87@gmail.com +License: http://www.apache.org/licenses/LICENSE-2.0 +Description: + Overview + ======== + This module provides an additional log handler for Python's standard logging + package (PEP 282). This handler will write log events to log file which is + rotated when the log file reaches a certain size. Multiple processes can + safely write to the same log file concurrently. + + Details + ======= + .. _portalocker: http://code.activestate.com/recipes/65203/ + + The ``ConcurrentRotatingFileHandler`` class is a drop-in replacement for + Python's standard log handler ``RotatingFileHandler``. This module uses file + locking so that multiple processes can concurrently log to a single file without + dropping or clobbering log events. This module provides a file rotation scheme + like with ``RotatingFileHanler``. Extra care is taken to ensure that logs + can be safely rotated before the rotation process is started. (This module works + around the file rename issue with ``RotatingFileHandler`` on Windows, where a + rotation failure means that all subsequent log events are dropped). + + This module attempts to preserve log records at all cost. This means that log + files will grow larger than the specified maximum (rotation) size. So if disk + space is tight, you may want to stick with ``RotatingFileHandler``, which will + strictly adhere to the maximum file size. + + If you have multiple instances of a script (or multiple scripts) all running at + the same time and writing to the same log file, then *all* of the scripts should + be using ``ConcurrentRotatingFileHandler``. You should not attempt to mix + and match ``RotatingFileHandler`` and ``ConcurrentRotatingFileHandler``. + + This package bundles `portalocker`_ to deal with file locking. Please be aware + that portalocker only supports Unix (posix) an NT platforms at this time, and + therefore this package only supports those platforms as well. + + Installation + ============ + Use the following command to install this package:: + + pip install ConcurrentLogHandler + + If you are installing from source, you can use:: + + python setup.py install + + + Examples + ======== + + Simple Example + -------------- + Here is a example demonstrating how to use this module directly (from within + Python code):: + + from logging import getLogger, INFO + from cloghandler import ConcurrentRotatingFileHandler + import os + + log = getLogger() + # Use an absolute path to prevent file rotation trouble. + logfile = os.path.abspath("mylogfile.log") + # Rotate log after reaching 512K, keep 5 old copies. + rotateHandler = ConcurrentRotatingFileHandler(logfile, "a", 512*1024, 5) + log.addHandler(rotateHandler) + log.setLevel(INFO) + + log.info("Here is a very exciting log message, just for you") + + + Automatic fallback example + -------------------------- + If you are distributing your code and you are unsure if the + `ConcurrentLogHandler` package has been installed everywhere your code will run, + Python makes it easy to gracefully fallback to the built in + `RotatingFileHandler`, here is an example:: + + try: + from cloghandler import ConcurrentRotatingFileHandler as RFHandler + except ImportError: + # Next 2 lines are optional: issue a warning to the user + from warnings import warn + warn("ConcurrentLogHandler package not installed. Using builtin log handler") + from logging.handlers import RotatingFileHandler as RFHandler + + log = getLogger() + rotateHandler = RFHandler("/path/to/mylogfile.log", "a", 1048576, 15) + log.addHandler(rotateHandler) + + + + Config file example + ------------------- + This example shows you how to use this log handler with the logging config file + parser. This allows you to keep your logging configuration code separate from + your application code. + + Example config file: ``logging.ini``:: + + [loggers] + keys=root + + [handlers] + keys=hand01 + + [formatters] + keys=form01 + + [logger_root] + level=NOTSET + handlers=hand01 + + [handler_hand01] + class=handlers.ConcurrentRotatingFileHandler + level=NOTSET + formatter=form01 + args=("rotating.log", "a", 512*1024, 5) + + [formatter_form01] + format=%(asctime)s %(levelname)s %(message)s + + Example Python code: ``app.py``:: + + import logging, logging.config + import cloghandler + + logging.config.fileConfig("logging.ini") + log = logging.getLogger() + log.info("Here is a very exciting log message, just for you") + + + Change Log + ========== + + .. _Red Hat Bug #858912: https://bugzilla.redhat.com/show_bug.cgi?id=858912 + .. _Python Bug #15960: http://bugs.python.org/issue15960 + .. _LP Bug 1199332: https://bugs.launchpad.net/python-concurrent-log-handler/+bug/1199332 + .. _LP Bug 1199333: https://bugs.launchpad.net/python-concurrent-log-handler/+bug/1199333 + + + - 0.9.1: Bug fixes - `LP Bug 1199332`_ and `LP Bug 1199333`_. + * More gracefully handle out of disk space scenarios. Prevent release() from + throwing an exception. + * Handle logging.shutdown() in Python 2.7+. Close the lock file stream via + close(). + * Big thanks to Dan Callaghan for forwarding these issues and patches. + + - 0.9.0: Now requires Python 2.6+ + * Revamp file opening/closing and file-locking internals (inspired by + feedback from Vinay Sajip.) + * Add the 'delay' parameter (delayed log file opening) to better match the + core logging functionality in more recent version of Python. + * For anyone still using Python 2.3-2.5, please use the latest 0.8.x release + + - 0.8.6: Fixed packaging bug with test script + * Fix a small packaging bug from the 0.8.5 release. (Thanks to Björn Häuser + for bringing this to my attention.) + * Updated stresstest.py to always use the correct python version when + launching sub-processes instead of the system's default "python". + + - 0.8.5: Fixed ValueError: I/O operation on closed file + * Thanks to Vince Carney, Arif Kasim, Matt Drew, Nick Coghlan, and + Dan Callaghan for bug reports. Bugs can now be filled here: + https://bugs.launchpad.net/python-concurrent-log-handler. Bugs resolved + `Red Hat Bug #858912`_ and `Python Bug #15960`_ + * Updated ez_setup.py to 0.7.7 + * Updated portalocker to 0.3 (now maintained by Rick van Hattem) + * Initial Python 3 support (needs more testing) + * Fixed minor spelling mistakes + + - 0.8.4: Fixed lock-file naming issue + * Resolved a minor issue where lock-files would be improperly named if the + log file contained ".log" in the middle of the log name. For example, if + you log file was "/var/log/mycompany.logging.mysource.log", the lock file + would be named "/var/log/mycompany.ging.mysource.lock", which is not correct. + Thanks to Dirk Rothe for pointing this out. Since this introduce a slight + lock-file behavior difference, make sure all concurrent writers are updated + to 0.8.4 at the same time if this issue effects you. + * Updated ez_setup.py to 0.6c11 + + - 0.8.3: Fixed a log file rotation bug and updated docs + * Fixed a bug that happens after log rotation when multiple processes are + witting to the same log file. Each process ends up writing to their own + log file ("log.1" or "log.2" instead of "log"). The fix is simply to reopen + the log file and check the size again. I do not believe this bug results in + data loss; however, this certainly was not the desired behavior. (A big + thanks goes to Oliver Tonnhofer for finding, documenting, and providing a + patch for this bug.) + * Cleanup the docs. (aka "the page you are reading right now") I fixed some + silly mistakes and typos... who writes this stuff? + + - 0.8.2: Minor bug fix release (again) + * Found and resolved another issue with older logging packages that do not + support encoding. + + - 0.8.1: Minor bug fix release + * Now importing "codecs" directly; I found some slight differences in the + logging module in different Python 2.4.x releases that caused the module to + fail to load. + + - 0.8.0: Minor feature release + * Add better support for using ``logging.config.fileConfig()``. This class + is now available using ``class=handlers.ConcurrentRotatingFileHandler``. + * Minor changes in how the ``filename`` parameter is handled when given a + relative path. + + - 0.7.4: Minor bug fix + * Fixed a typo in the package description (incorrect class name) + * Added a change log; which you are reading now. + * Fixed the ``close()`` method to no longer assume that stream is still + open. + + To-do + ===== + * This module has had minimal testing in a multi-threaded process. I see no + reason why this should be an issue, but no stress-testing has been done in a + threaded situation. If this is important to you, you could always add + threading support to the ``stresstest.py`` script and send me the patch. + + +Keywords: logging,windows,linux,unix,rotate,portalocker +Platform: nt +Platform: posix +Classifier: Development Status :: 4 - Beta +Classifier: Topic :: System :: Logging +Classifier: Operating System :: POSIX +Classifier: Operating System :: Microsoft :: Windows +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: License :: OSI Approved :: Apache Software License diff --git a/lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/SOURCES.txt b/lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/SOURCES.txt new file mode 100755 index 00000000..02eee052 --- /dev/null +++ b/lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/SOURCES.txt @@ -0,0 +1,16 @@ +.bzrignore +LICENSE +README +do_release.sh +ez_setup.py +pre_commit.sh +setup.cfg +setup.py +stresstest.py +src/cloghandler.py +src/portalocker.py +src/ConcurrentLogHandler.egg-info/PKG-INFO +src/ConcurrentLogHandler.egg-info/SOURCES.txt +src/ConcurrentLogHandler.egg-info/dependency_links.txt +src/ConcurrentLogHandler.egg-info/top_level.txt +src/ConcurrentLogHandler.egg-info/zip-safe \ No newline at end of file diff --git a/lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/dependency_links.txt b/lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/dependency_links.txt new file mode 100755 index 00000000..8b137891 --- /dev/null +++ b/lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/installed-files.txt b/lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/installed-files.txt new file mode 100755 index 00000000..936d185c --- /dev/null +++ b/lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/installed-files.txt @@ -0,0 +1,13 @@ +../cloghandler.py +../portalocker.py +../cloghandler.pyc +../portalocker.pyc +../../../../tests/stresstest.py +../../../../docs/README +../../../../docs/LICENSE +./ +SOURCES.txt +zip-safe +PKG-INFO +dependency_links.txt +top_level.txt diff --git a/lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/top_level.txt b/lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/top_level.txt new file mode 100755 index 00000000..4e964b7c --- /dev/null +++ b/lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/top_level.txt @@ -0,0 +1,2 @@ +cloghandler +portalocker diff --git a/lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/zip-safe b/lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/zip-safe new file mode 100755 index 00000000..8b137891 --- /dev/null +++ b/lib/ConcurrentLogHandler/ConcurrentLogHandler-0.9.1-py2.7.egg-info/zip-safe @@ -0,0 +1 @@ + diff --git a/lib/ConcurrentLogHandler/cloghandler.py b/lib/ConcurrentLogHandler/cloghandler.py new file mode 100755 index 00000000..a4df08a9 --- /dev/null +++ b/lib/ConcurrentLogHandler/cloghandler.py @@ -0,0 +1,349 @@ +# Copyright 2013 Lowell Alleman +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" cloghandler.py: A smart replacement for the standard RotatingFileHandler + +ConcurrentRotatingFileHandler: This class is a log handler which is a drop-in +replacement for the python standard log handler 'RotateFileHandler', the primary +difference being that this handler will continue to write to the same file if +the file cannot be rotated for some reason, whereas the RotatingFileHandler will +strictly adhere to the maximum file size. Unfortunately, if you are using the +RotatingFileHandler on Windows, you will find that once an attempted rotation +fails, all subsequent log messages are dropped. The other major advantage of +this module is that multiple processes can safely write to a single log file. + +To put it another way: This module's top priority is preserving your log +records, whereas the standard library attempts to limit disk usage, which can +potentially drop log messages. If you are trying to determine which module to +use, there are number of considerations: What is most important: strict disk +space usage or preservation of log messages? What OSes are you supporting? Can +you afford to have processes blocked by file locks? + +Concurrent access is handled by using file locks, which should ensure that log +messages are not dropped or clobbered. This means that a file lock is acquired +and released for every log message that is written to disk. (On Windows, you may +also run into a temporary situation where the log file must be opened and closed +for each log message.) This can have potentially performance implications. In my +testing, performance was more than adequate, but if you need a high-volume or +low-latency solution, I suggest you look elsewhere. + +This module currently only support the 'nt' and 'posix' platforms due to the +usage of the portalocker module. I do not have access to any other platforms +for testing, patches are welcome. + +See the README file for an example usage of this module. + +This module supports Python 2.6 and later. + +""" + + +__version__ = '0.9.1' +__revision__ = 'lowell87@gmail.com-20130711022321-doutxl7zyzuwss5a 2013-07-10 22:23:21 -0400 [0]' +__author__ = "Lowell Alleman" +__all__ = [ + "ConcurrentRotatingHandler", +] + + +import os +import sys +from random import randint +from logging import Handler, LogRecord +from logging.handlers import BaseRotatingHandler + +try: + import codecs +except ImportError: + codecs = None + + + +# Question/TODO: Should we have a fallback mode if we can't load portalocker / +# we should still be better off than with the standard RotattingFileHandler +# class, right? We do some rename checking... that should prevent some file +# clobbering that the builtin class allows. + +# sibling module than handles all the ugly platform-specific details of file locking +from portalocker import lock, unlock, LOCK_EX, LOCK_NB, LockException + + +# Workaround for handleError() in Python 2.7+ where record is written to stderr +class NullLogRecord(LogRecord): + def __init__(self): + pass + def __getattr__(self, attr): + return None + +class ConcurrentRotatingFileHandler(BaseRotatingHandler): + """ + Handler for logging to a set of files, which switches from one file to the + next when the current file reaches a certain size. Multiple processes can + write to the log file concurrently, but this may mean that the file will + exceed the given size. + """ + def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, + encoding=None, debug=True, delay=0): + """ + Open the specified file and use it as the stream for logging. + + By default, the file grows indefinitely. You can specify particular + values of maxBytes and backupCount to allow the file to rollover at + a predetermined size. + + Rollover occurs whenever the current log file is nearly maxBytes in + length. If backupCount is >= 1, the system will successively create + new files with the same pathname as the base file, but with extensions + ".1", ".2" etc. appended to it. For example, with a backupCount of 5 + and a base file name of "app.log", you would get "app.log", + "app.log.1", "app.log.2", ... through to "app.log.5". The file being + written to is always "app.log" - when it gets filled up, it is closed + and renamed to "app.log.1", and if files "app.log.1", "app.log.2" etc. + exist, then they are renamed to "app.log.2", "app.log.3" etc. + respectively. + + If maxBytes is zero, rollover never occurs. + + On Windows, it is not possible to rename a file that is currently opened + by another process. This means that it is not possible to rotate the + log files if multiple processes is using the same log file. In this + case, the current log file will continue to grow until the rotation can + be completed successfully. In order for rotation to be possible, all of + the other processes need to close the file first. A mechanism, called + "degraded" mode, has been created for this scenario. In degraded mode, + the log file is closed after each log message is written. So once all + processes have entered degraded mode, the net rotation attempt should + be successful and then normal logging can be resumed. Using the 'delay' + parameter may help reduce contention in some usage patterns. + + This log handler assumes that all concurrent processes logging to a + single file will are using only this class, and that the exact same + parameters are provided to each instance of this class. If, for + example, two different processes are using this class, but with + different values for 'maxBytes' or 'backupCount', then odd behavior is + expected. The same is true if this class is used by one application, but + the RotatingFileHandler is used by another. + """ + # Absolute file name handling done by FileHandler since Python 2.5 + BaseRotatingHandler.__init__(self, filename, mode, encoding, delay) + self.delay = delay + self._rotateFailed = False + self.maxBytes = maxBytes + self.backupCount = backupCount + self._open_lockfile() + # For debug mode, swap out the "_degrade()" method with a more a verbose one. + if debug: + self._degrade = self._degrade_debug + + def _open_lockfile(self): + # Use 'file.lock' and not 'file.log.lock' (Only handles the normal "*.log" case.) + if self.baseFilename.endswith(".log"): + lock_file = self.baseFilename[:-4] + else: + lock_file = self.baseFilename + lock_file += ".lock" + self.stream_lock = open(lock_file,"w") + + def _open(self, mode=None): + """ + Open the current base file with the (original) mode and encoding. + Return the resulting stream. + + Note: Copied from stdlib. Added option to override 'mode' + """ + if mode is None: + mode = self.mode + if self.encoding is None: + stream = open(self.baseFilename, mode) + else: + stream = codecs.open(self.baseFilename, mode, self.encoding) + return stream + + def _close(self): + """ Close file stream. Unlike close(), we don't tear anything down, we + expect the log to be re-opened after rotation.""" + if self.stream: + try: + if not self.stream.closed: + # Flushing probably isn't technically necessary, but it feels right + self.stream.flush() + self.stream.close() + finally: + self.stream = None + + def acquire(self): + """ Acquire thread and file locks. Re-opening log for 'degraded' mode. + """ + # handle thread lock + Handler.acquire(self) + # Issue a file lock. (This is inefficient for multiple active threads + # within a single process. But if you're worried about high-performance, + # you probably aren't using this log handler.) + if self.stream_lock: + # If stream_lock=None, then assume close() was called or something + # else weird and ignore all file-level locks. + if self.stream_lock.closed: + # Daemonization can close all open file descriptors, see + # https://bugzilla.redhat.com/show_bug.cgi?id=952929 + # Try opening the lock file again. Should we warn() here?!? + try: + self._open_lockfile() + except Exception: + self.handleError(NullLogRecord()) + # Don't try to open the stream lock again + self.stream_lock = None + return + lock(self.stream_lock, LOCK_EX) + # Stream will be opened as part by FileHandler.emit() + + def release(self): + """ Release file and thread locks. If in 'degraded' mode, close the + stream to reduce contention until the log files can be rotated. """ + try: + if self._rotateFailed: + self._close() + except Exception: + self.handleError(NullLogRecord()) + finally: + try: + if self.stream_lock and not self.stream_lock.closed: + unlock(self.stream_lock) + except Exception: + self.handleError(NullLogRecord()) + finally: + # release thread lock + Handler.release(self) + + def close(self): + """ + Close log stream and stream_lock. """ + try: + self._close() + if not self.stream_lock.closed: + self.stream_lock.close() + finally: + self.stream_lock = None + Handler.close(self) + + def _degrade(self, degrade, msg, *args): + """ Set degrade mode or not. Ignore msg. """ + self._rotateFailed = degrade + del msg, args # avoid pychecker warnings + + def _degrade_debug(self, degrade, msg, *args): + """ A more colorful version of _degade(). (This is enabled by passing + "debug=True" at initialization). + """ + if degrade: + if not self._rotateFailed: + sys.stderr.write("Degrade mode - ENTERING - (pid=%d) %s\n" % + (os.getpid(), msg % args)) + self._rotateFailed = True + else: + if self._rotateFailed: + sys.stderr.write("Degrade mode - EXITING - (pid=%d) %s\n" % + (os.getpid(), msg % args)) + self._rotateFailed = False + + def doRollover(self): + """ + Do a rollover, as described in __init__(). + """ + self._close() + if self.backupCount <= 0: + # Don't keep any backups, just overwrite the existing backup file + # Locking doesn't much matter here; since we are overwriting it anyway + self.stream = self._open("w") + return + try: + # Determine if we can rename the log file or not. Windows refuses to + # rename an open file, Unix is inode base so it doesn't care. + + # Attempt to rename logfile to tempname: There is a slight race-condition here, but it seems unavoidable + tmpname = None + while not tmpname or os.path.exists(tmpname): + tmpname = "%s.rotate.%08d" % (self.baseFilename, randint(0,99999999)) + try: + # Do a rename test to determine if we can successfully rename the log file + os.rename(self.baseFilename, tmpname) + except (IOError, OSError): + exc_value = sys.exc_info()[1] + self._degrade(True, "rename failed. File in use? " + "exception=%s", exc_value) + return + + # Q: Is there some way to protect this code from a KeboardInterupt? + # This isn't necessarily a data loss issue, but it certainly does + # break the rotation process during stress testing. + + # There is currently no mechanism in place to handle the situation + # where one of these log files cannot be renamed. (Example, user + # opens "logfile.3" in notepad); we could test rename each file, but + # nobody's complained about this being an issue; so the additional + # code complexity isn't warranted. + for i in range(self.backupCount - 1, 0, -1): + sfn = "%s.%d" % (self.baseFilename, i) + dfn = "%s.%d" % (self.baseFilename, i + 1) + if os.path.exists(sfn): + #print "%s -> %s" % (sfn, dfn) + if os.path.exists(dfn): + os.remove(dfn) + os.rename(sfn, dfn) + dfn = self.baseFilename + ".1" + if os.path.exists(dfn): + os.remove(dfn) + os.rename(tmpname, dfn) + #print "%s -> %s" % (self.baseFilename, dfn) + self._degrade(False, "Rotation completed") + finally: + # Re-open the output stream, but if "delay" is enabled then wait + # until the next emit() call. This could reduce rename contention in + # some usage patterns. + if not self.delay: + self.stream = self._open() + + def shouldRollover(self, record): + """ + Determine if rollover should occur. + + For those that are keeping track. This differs from the standard + library's RotatingLogHandler class. Because there is no promise to keep + the file size under maxBytes we ignore the length of the current record. + """ + del record # avoid pychecker warnings + # Is stream is not yet open, skip rollover check. (Check will occur on + # next message, after emit() calls _open()) + if self.stream is None: + return False + if self._shouldRollover(): + # If some other process already did the rollover (which is possible + # on Unix) the file our stream may now be named "log.1", thus + # triggering another rollover. Avoid this by closing and opening + # "log" again. + self._close() + self.stream = self._open() + return self._shouldRollover() + return False + + def _shouldRollover(self): + if self.maxBytes > 0: # are we rolling over? + self.stream.seek(0, 2) #due to non-posix-compliant Windows feature + if self.stream.tell() >= self.maxBytes: + return True + else: + self._degrade(False, "Rotation done or not needed at this time") + return False + + +# Publish this class to the "logging.handlers" module so that it can be use +# from a logging config file via logging.config.fileConfig(). +import logging.handlers +logging.handlers.ConcurrentRotatingFileHandler = ConcurrentRotatingFileHandler diff --git a/lib/ConcurrentLogHandler/portalocker.py b/lib/ConcurrentLogHandler/portalocker.py new file mode 100755 index 00000000..5c0694ec --- /dev/null +++ b/lib/ConcurrentLogHandler/portalocker.py @@ -0,0 +1,141 @@ +# portalocker.py - Cross-platform (posix/nt) API for flock-style file locking. +# Requires python 1.5.2 or better. +"""Cross-platform (posix/nt) API for flock-style file locking. + +Synopsis: + + import portalocker + file = open("somefile", "r+") + portalocker.lock(file, portalocker.LOCK_EX) + file.seek(12) + file.write("foo") + file.close() + +If you know what you're doing, you may choose to + + portalocker.unlock(file) + +before closing the file, but why? + +Methods: + + lock( file, flags ) + unlock( file ) + +Constants: + + LOCK_EX + LOCK_SH + LOCK_NB + +Exceptions: + + LockException + +Notes: + +For the 'nt' platform, this module requires the Python Extensions for Windows. +Be aware that this may not work as expected on Windows 95/98/ME. + +History: + +I learned the win32 technique for locking files from sample code +provided by John Nielsen in the documentation +that accompanies the win32 modules. + +Author: Jonathan Feinberg , + Lowell Alleman , + Rick van Hattem +Version: 0.3 +URL: https://github.com/WoLpH/portalocker +""" + + +__all__ = [ + "lock", + "unlock", + "LOCK_EX", + "LOCK_SH", + "LOCK_NB", + "LockException", +] + +import os + +class LockException(Exception): + # Error codes: + LOCK_FAILED = 1 + +if os.name == 'nt': + import win32con + import win32file + import pywintypes + LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK + LOCK_SH = 0 # the default + LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY + # is there any reason not to reuse the following structure? + __overlapped = pywintypes.OVERLAPPED() +elif os.name == 'posix': + import fcntl + LOCK_EX = fcntl.LOCK_EX + LOCK_SH = fcntl.LOCK_SH + LOCK_NB = fcntl.LOCK_NB +else: + raise RuntimeError("PortaLocker only defined for nt and posix platforms") + +if os.name == 'nt': + def lock(file, flags): + hfile = win32file._get_osfhandle(file.fileno()) + try: + win32file.LockFileEx(hfile, flags, 0, -0x10000, __overlapped) + except pywintypes.error, exc_value: + # error: (33, 'LockFileEx', 'The process cannot access the file because another process has locked a portion of the file.') + if exc_value[0] == 33: + raise LockException(LockException.LOCK_FAILED, exc_value[2]) + else: + # Q: Are there exceptions/codes we should be dealing with here? + raise + + def unlock(file): + hfile = win32file._get_osfhandle(file.fileno()) + try: + win32file.UnlockFileEx(hfile, 0, -0x10000, __overlapped) + except pywintypes.error, exc_value: + if exc_value[0] == 158: + # error: (158, 'UnlockFileEx', 'The segment is already unlocked.') + # To match the 'posix' implementation, silently ignore this error + pass + else: + # Q: Are there exceptions/codes we should be dealing with here? + raise + +elif os.name == 'posix': + def lock(file, flags): + try: + fcntl.flock(file.fileno(), flags) + except IOError, exc_value: + # The exception code varies on different systems so we'll catch + # every IO error + raise LockException(*exc_value) + + def unlock(file): + fcntl.flock(file.fileno(), fcntl.LOCK_UN) + + + +if __name__ == '__main__': + from time import time, strftime, localtime + import sys + import portalocker + + log = open('log.txt', "a+") + portalocker.lock(log, portalocker.LOCK_EX) + + timestamp = strftime("%m/%d/%Y %H:%M:%S\n", localtime(time())) + log.write( timestamp ) + + print "Wrote lines. Hit enter to release lock." + dummy = sys.stdin.readline() + + log.close() + diff --git a/lib/ConcurrentLogHandler/stresstest.py b/lib/ConcurrentLogHandler/stresstest.py new file mode 100755 index 00000000..dde5531e --- /dev/null +++ b/lib/ConcurrentLogHandler/stresstest.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python +""" stresstest.py: A stress-tester for ConcurrentRotatingFileHandler + +This utility spawns a bunch of processes that all try to concurrently write to +the same file. This is pretty much the worst-case scenario for my log handler. +Once all of the processes have completed writing to the log file, the output is +compared to see if any log messages have been lost. + +In the future, I may also add in support for testing with each process having +multiple threads. + + +""" + +__version__ = '$Id$' +__author__ = 'Lowell Alleman' + + +import os +import sys +from subprocess import call, Popen, STDOUT +from time import sleep + +ROTATE_COUNT = 5000 + +# local lib; for testing +from cloghandler import ConcurrentRotatingFileHandler + +class RotateLogStressTester: + def __init__(self, sharedfile, uniquefile, name="LogStressTester", logger_delay=0): + self.sharedfile = sharedfile + self.uniquefile = uniquefile + self.name = name + self.writeLoops = 100000 + self.rotateSize = 128 * 1024 + self.rotateCount = ROTATE_COUNT + self.random_sleep_mode = False + self.debug = False + self.logger_delay = logger_delay + + def getLogHandler(self, fn): + """ Override this method if you want to test a different logging handler + class. """ + return ConcurrentRotatingFileHandler(fn, 'a', self.rotateSize, + self.rotateCount, delay=self.logger_delay, + debug=self.debug) + # To run the test with the standard library's RotatingFileHandler: + # from logging.handlers import RotatingFileHandler + # return RotatingFileHandler(fn, 'a', self.rotateSize, self.rotateCount) + + def start(self): + from logging import getLogger, FileHandler, Formatter, DEBUG + self.log = getLogger(self.name) + self.log.setLevel(DEBUG) + + formatter = Formatter('%(asctime)s [%(process)d:%(threadName)s] %(levelname)-8s %(name)s: %(message)s') + # Unique log handler (single file) + handler = FileHandler(self.uniquefile, "w") + handler.setLevel(DEBUG) + handler.setFormatter(formatter) + self.log.addHandler(handler) + + # If you suspect that the diff stuff isn't working, un comment the next + # line. You should see this show up once per-process. + # self.log.info("Here is a line that should only be in the first output.") + + # Setup output used for testing + handler = self.getLogHandler(self.sharedfile) + handler.setLevel(DEBUG) + handler.setFormatter(formatter) + self.log.addHandler(handler) + + # If this ever becomes a real "Thread", then remove this line: + self.run() + + def run(self): + c = 0 + from random import choice, randint + # Use a bunch of random quotes, numbers, and severity levels to mix it up a bit! + msgs = ["I found %d puppies", "There are %d cats in your hatz", + "my favorite number is %d", "I am %d years old.", "1 + 1 = %d", + "%d/0 = DivideByZero", "blah! %d thingies!", "8 15 16 23 48 %d", + "the worlds largest prime number: %d", "%d happy meals!"] + logfuncts = [self.log.debug, self.log.info, self.log.warn, self.log.error] + + self.log.info("Starting to write random log message. Loop=%d", self.writeLoops) + while c <= self.writeLoops: + c += 1 + msg = choice(msgs) + logfunc = choice(logfuncts) + logfunc(msg, randint(0,99999999)) + + if self.random_sleep_mode and c % 1000 == 0: + # Sleep from 0-15 seconds + s = randint(1,15) + print("PID %d sleeping for %d seconds" % (os.getpid(), s)) + sleep(s) + # break + self.log.info("Done witting random log messages.") + +def iter_lognames(logfile, count): + """ Generator for log file names based on a rotation scheme """ + for i in range(count -1, 0, -1): + yield "%s.%d" % (logfile, i) + yield logfile + +def iter_logs(iterable, missing_ok=False): + """ Generator to extract log entries from shared log file. """ + for fn in iterable: + if os.path.exists(fn): + for line in open(fn): + yield line + elif not missing_ok: + raise ValueError("Missing log file %s" % fn) + +def combine_logs(combinedlog, iterable, mode="w"): + """ write all lines (iterable) into a single log file. """ + fp = open(combinedlog, mode) + for chunk in iterable: + fp.write(chunk) + fp.close() + + + +from optparse import OptionParser +parser = OptionParser(usage="usage: %prog", + version=__version__, + description="Stress test the cloghandler module.") +parser.add_option("--log-calls", metavar="NUM", + action="store", type="int", default=50000, + help="Number of logging entries to write to each log file. " + "Default is %default") +parser.add_option("--random-sleep-mode", + action="store_true", default=False) +parser.add_option("--debug", + action="store_true", default=False) +parser.add_option("--logger-delay", + action="store_true", default=False, + help="Enable the 'delay' mode in the logger class. " + "This means that the log file will be opened on demand.") + + +def main_client(args): + (options, args) = parser.parse_args(args) + if len(args) != 2: + raise ValueError("Require 2 arguments. We have %d args" % len(args)) + (shared, client) = args + + if os.path.isfile(client): + sys.stderr.write("Already a client using output file %s\n" % client) + sys.exit(1) + tester = RotateLogStressTester(shared, client, logger_delay=options.logger_delay) + tester.random_sleep_mode = options.random_sleep_mode + tester.debug = options.debug + tester.writeLoops = options.log_calls + tester.start() + print("We are done pid=%d" % os.getpid()) + + + +class TestManager: + class ChildProc(object): + """ Very simple child container class.""" + __slots__ = [ "popen", "sharedfile", "clientfile" ] + def __init__(self, **kwargs): + self.update(**kwargs) + def update(self, **kwargs): + for key, val in kwargs.items(): + setattr(self, key, val) + + def __init__(self): + self.tests = [] + + def launchPopen(self, *args, **kwargs): + proc = Popen(*args, **kwargs) + cp = self.ChildProc(popen=proc) + self.tests.append(cp) + return cp + + def wait(self, check_interval=3): + """ Wait for all child test processes to complete. """ + print("Waiting while children are out running and playing!") + while True: + sleep(check_interval) + waiting = [] + for cp in self.tests: + if cp.popen.poll() is None: + waiting.append(cp.popen.pid) + if not waiting: + break + print("Waiting on %r " % waiting) + print("All children have stopped.") + + def checkExitCodes(self): + for cp in self.tests: + if cp.popen.poll() != 0: + return False + return True + + + +def unified_diff(a,b, out=sys.stdout): + import difflib + ai = open(a).readlines() + bi = open(b).readlines() + for line in difflib.unified_diff(ai, bi, a, b): + out.write(line) + + + +def main_runner(args): + parser.add_option("--processes", metavar="NUM", + action="store", type="int", default=3, + help="Number of processes to spawn. Default: %default") + parser.add_option("--delay", metavar="secs", + action="store", type="float", default=2.5, + help="Wait SECS before spawning next processes. " + "Default: %default") + parser.add_option("-p", "--path", metavar="DIR", + action="store", default="test", + help="Path to a temporary directory. Default: '%default'") + + + this_script = args[0] + (options, args) = parser.parse_args(args) + options.path = os.path.abspath(options.path) + if not os.path.isdir(options.path): + os.makedirs(options.path) + + manager = TestManager() + shared = os.path.join(options.path, "shared.log") + for client_id in range(options.processes): + client = os.path.join(options.path, "client.log_client%s.log" % client_id) + cmdline = [ sys.executable, this_script, "client", shared, client, + "--log-calls=%d" % options.log_calls ] + if options.random_sleep_mode: + cmdline.append("--random-sleep-mode") + if options.debug: + cmdline.append("--debug") + if options.logger_delay: + cmdline.append("--logger-delay") + + child = manager.launchPopen(cmdline) + child.update(sharedfile=shared, clientfile=client) + sleep(options.delay) + + # Wait for all of the subprocesses to exit + manager.wait() + # Check children exit codes + if not manager.checkExitCodes(): + sys.stderr.write("One or more of the child process has failed.\n" + "Aborting test.\n") + sys.exit(2) + + client_combo = os.path.join(options.path, "client.log.combo") + shared_combo = os.path.join(options.path, "shared.log.combo") + + # Combine all of the log files... + client_files = [ child.clientfile for child in manager.tests ] + + if False: + def sort_em(iterable): + return iterable + else: + sort_em = sorted + + print("Writing out combined client logs...") + combine_logs(client_combo, sort_em(iter_logs(client_files))) + print("done.") + + print("Writing out combined shared logs...") + shared_log_files = iter_lognames(shared, ROTATE_COUNT) + log_lines = iter_logs(shared_log_files, missing_ok=True) + combine_logs(shared_combo, sort_em(log_lines)) + print("done.") + + print("Running internal diff: (If the next line is 'end of diff', then the stress test passed!)") + unified_diff(client_combo, shared_combo) + print(" --- end of diff ----") + + + +if __name__ == '__main__': + if len(sys.argv) > 1 and sys.argv[1].lower() == "client": + main_client(sys.argv[2:]) + else: + main_runner(sys.argv) diff --git a/mylar/PostProcessor.py b/mylar/PostProcessor.py index fab7c38e..aa049e35 100755 --- a/mylar/PostProcessor.py +++ b/mylar/PostProcessor.py @@ -421,12 +421,14 @@ class PostProcessor(object): # At this point, let's just drop it into the Comic Location folder and forget about it.. if 'S' in sandwich: self._log("One-off STORYARC mode enabled for Post-Processing for " + str(sarc)) - logger.info(module + 'One-off STORYARC mode enabled for Post-Processing for ' + str(sarc)) + logger.info(module + ' One-off STORYARC mode enabled for Post-Processing for ' + str(sarc)) if mylar.STORYARCDIR: storyarcd = os.path.join(mylar.DESTINATION_DIR, "StoryArcs", sarc) self._log("StoryArc Directory set to : " + storyarcd) + logger.info(module + ' Story Arc Directory set to : ' + storyarcd) else: self._log("Grab-Bag Directory set to : " + mylar.GRABBAG_DIR) + logger.info(module + ' Story Arc Directory set to : ' + mylar.GRABBAG_DIR) else: self._log("One-off mode enabled for Post-Processing. All I'm doing is moving the file untouched into the Grab-bag directory.") @@ -497,12 +499,13 @@ class PostProcessor(object): grab_dst = os.path.join(grdst, ofilename) self._log("Destination Path : " + grab_dst) + logger.info(module + ' Destination Path : ' + grab_dst) grab_src = os.path.join(self.nzb_folder, ofilename) self._log("Source Path : " + grab_src) logger.info(module + ' Source Path : ' + grab_src) - logger.info(module + ' Moving ' + str(ofilename) + ' into directory : ' + str(grdst)) + logger.info(module + ' Moving ' + str(ofilename) + ' into directory : ' + str(grab_dst)) try: shutil.move(grab_src, grab_dst) @@ -623,6 +626,7 @@ class PostProcessor(object): iss_find = issuenum.find('.') iss_b4dec = issuenum[:iss_find] iss_decval = issuenum[iss_find+1:] + if iss_decval.endswith('.'): iss_decval = iss_decval[:-1] if int(iss_decval) == 0: iss = iss_b4dec issdec = int(iss_decval) @@ -771,9 +775,9 @@ class PostProcessor(object): try: import cmtagmylar if ml is None: - pcheck = cmtagmylar.run(self.nzb_folder, issueid=issueid) + pcheck = cmtagmylar.run(self.nzb_folder, issueid=issueid, comversion=comversion) else: - pcheck = cmtagmylar.run(self.nzb_folder, issueid=issueid, manual="yes", filename=ml['ComicLocation']) + pcheck = cmtagmylar.run(self.nzb_folder, issueid=issueid, comversion=comversion, manual="yes", filename=ml['ComicLocation']) except ImportError: logger.fdebug(module + ' comictaggerlib not found on system. Ensure the ENTIRE lib directory is located within mylar/lib/comictaggerlib/') diff --git a/mylar/__init__.py b/mylar/__init__.py index 44c1f766..b2e150e0 100755 --- a/mylar/__init__.py +++ b/mylar/__init__.py @@ -80,6 +80,12 @@ CFG = None CONFIG_VERSION = None DB_FILE = None +DBCHOICE = None + +#these are used depending on dbchoice. +DBUSER = None +DBPASS = None +DBNAME = None LOG_DIR = None LOG_LIST = [] @@ -142,7 +148,7 @@ DOWNLOAD_SCAN_INTERVAL = 5 CHECK_FOLDER = None ENABLE_CHECK_FOLDER = False INTERFACE = None - +DUPECONSTRAINT = None PREFERRED_QUALITY = 0 CORRECT_METADATA = False MOVE_FILES = False @@ -266,6 +272,8 @@ GRABBAG_DIR = None HIGHCOUNT = 0 READ2FILENAME = 0 STORYARCDIR = 0 +COPY2ARCDIR = 0 + CVAPIFIX = 0 CVURL = None WEEKFOLDER = 0 @@ -364,10 +372,10 @@ def initialize(): with INIT_LOCK: - global __INITIALIZED__, COMICVINE_API, DEFAULT_CVAPI, CVAPI_COUNT, CVAPI_TIME, CVAPI_MAX, FULL_PATH, PROG_DIR, VERBOSE, DAEMON, COMICSORT, DATA_DIR, CONFIG_FILE, CFG, CONFIG_VERSION, LOG_DIR, CACHE_DIR, MAX_LOGSIZE, LOGVERBOSE, OLDCONFIG_VERSION, OS_DETECT, OS_LANG, OS_ENCODING, \ + global __INITIALIZED__, DBCHOICE, DBUSER, DBPASS, DBNAME, COMICVINE_API, DEFAULT_CVAPI, CVAPI_COUNT, CVAPI_TIME, CVAPI_MAX, FULL_PATH, PROG_DIR, VERBOSE, DAEMON, COMICSORT, DATA_DIR, CONFIG_FILE, CFG, CONFIG_VERSION, LOG_DIR, CACHE_DIR, MAX_LOGSIZE, LOGVERBOSE, OLDCONFIG_VERSION, OS_DETECT, OS_LANG, OS_ENCODING, \ queue, HTTP_PORT, HTTP_HOST, HTTP_USERNAME, HTTP_PASSWORD, HTTP_ROOT, HTTPS_FORCE_ON, API_ENABLED, API_KEY, LAUNCH_BROWSER, GIT_PATH, SAFESTART, \ CURRENT_VERSION, LATEST_VERSION, CHECK_GITHUB, CHECK_GITHUB_ON_STARTUP, CHECK_GITHUB_INTERVAL, USER_AGENT, DESTINATION_DIR, MULTIPLE_DEST_DIRS, CREATE_FOLDERS, \ - DOWNLOAD_DIR, USENET_RETENTION, SEARCH_INTERVAL, NZB_STARTUP_SEARCH, INTERFACE, AUTOWANT_ALL, AUTOWANT_UPCOMING, ZERO_LEVEL, ZERO_LEVEL_N, COMIC_COVER_LOCAL, HIGHCOUNT, \ + DOWNLOAD_DIR, USENET_RETENTION, SEARCH_INTERVAL, NZB_STARTUP_SEARCH, INTERFACE, DUPECONSTRAINT, AUTOWANT_ALL, AUTOWANT_UPCOMING, ZERO_LEVEL, ZERO_LEVEL_N, COMIC_COVER_LOCAL, HIGHCOUNT, \ LIBRARYSCAN, LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, NZB_DOWNLOADER, USE_SABNZBD, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, SAB_PRIORITY, SAB_DIRECTORY, USE_BLACKHOLE, BLACKHOLE_DIR, ADD_COMICS, COMIC_DIR, IMP_MOVE, IMP_RENAME, IMP_METADATA, \ USE_NZBGET, NZBGET_HOST, NZBGET_PORT, NZBGET_USERNAME, NZBGET_PASSWORD, NZBGET_CATEGORY, NZBGET_PRIORITY, NZBGET_DIRECTORY, NZBSU, NZBSU_UID, NZBSU_APIKEY, DOGNZB, DOGNZB_UID, DOGNZB_APIKEY, \ NEWZNAB, NEWZNAB_NAME, NEWZNAB_HOST, NEWZNAB_APIKEY, NEWZNAB_UID, NEWZNAB_ENABLED, EXTRA_NEWZNABS, NEWZNAB_EXTRA, \ @@ -378,7 +386,7 @@ def initialize(): ENABLE_RSS, RSS_CHECKINTERVAL, RSS_LASTRUN, FAILED_DOWNLOAD_HANDLING, FAILED_AUTO, ENABLE_TORRENT_SEARCH, ENABLE_KAT, KAT_PROXY, ENABLE_CBT, CBT_PASSKEY, SNATCHEDTORRENT_NOTIFY, \ PROWL_ENABLED, PROWL_PRIORITY, PROWL_KEYS, PROWL_ONSNATCH, NMA_ENABLED, NMA_APIKEY, NMA_PRIORITY, NMA_ONSNATCH, PUSHOVER_ENABLED, PUSHOVER_PRIORITY, PUSHOVER_APIKEY, PUSHOVER_USERKEY, PUSHOVER_ONSNATCH, BOXCAR_ENABLED, BOXCAR_ONSNATCH, BOXCAR_TOKEN, \ PUSHBULLET_ENABLED, PUSHBULLET_APIKEY, PUSHBULLET_DEVICEID, PUSHBULLET_ONSNATCH, LOCMOVE, NEWCOM_DIR, FFTONEWCOM_DIR, \ - PREFERRED_QUALITY, MOVE_FILES, RENAME_FILES, LOWERCASE_FILENAMES, USE_MINSIZE, MINSIZE, USE_MAXSIZE, MAXSIZE, CORRECT_METADATA, FOLDER_FORMAT, FILE_FORMAT, REPLACE_CHAR, REPLACE_SPACES, ADD_TO_CSV, CVINFO, LOG_LEVEL, POST_PROCESSING, POST_PROCESSING_SCRIPT, SEARCH_DELAY, GRABBAG_DIR, READ2FILENAME, STORYARCDIR, CVURL, CVAPIFIX, CHECK_FOLDER, ENABLE_CHECK_FOLDER, \ + PREFERRED_QUALITY, MOVE_FILES, RENAME_FILES, LOWERCASE_FILENAMES, USE_MINSIZE, MINSIZE, USE_MAXSIZE, MAXSIZE, CORRECT_METADATA, FOLDER_FORMAT, FILE_FORMAT, REPLACE_CHAR, REPLACE_SPACES, ADD_TO_CSV, CVINFO, LOG_LEVEL, POST_PROCESSING, POST_PROCESSING_SCRIPT, SEARCH_DELAY, GRABBAG_DIR, READ2FILENAME, STORYARCDIR, COPY2ARCDIR, CVURL, CVAPIFIX, CHECK_FOLDER, ENABLE_CHECK_FOLDER, \ COMIC_LOCATION, QUAL_ALTVERS, QUAL_SCANNER, QUAL_TYPE, QUAL_QUALITY, ENABLE_EXTRA_SCRIPTS, EXTRA_SCRIPTS, ENABLE_PRE_SCRIPTS, PRE_SCRIPTS, PULLNEW, ALT_PULL, COUNT_ISSUES, COUNT_HAVES, COUNT_COMICS, SYNO_FIX, CHMOD_FILE, CHMOD_DIR, ANNUALS_ON, CV_ONLY, CV_ONETIMER, WEEKFOLDER, UMASK if __INITIALIZED__: @@ -404,6 +412,11 @@ def initialize(): HTTP_PORT = 8090 CONFIG_VERSION = check_setting_str(CFG, 'General', 'config_version', '') + DBCHOICE = check_setting_str(CFG, 'General', 'dbchoice', '') + DBUSER = check_setting_str(CFG, 'General', 'dbuser', '') + DBPASS = check_setting_str(CFG, 'General', 'dbpass', '') + DBNAME = check_setting_str(CFG, 'General', 'dbname', '') + COMICVINE_API = check_setting_str(CFG, 'General', 'comicvine_api', '') if not COMICVINE_API: COMICVINE_API = None @@ -455,6 +468,7 @@ def initialize(): CHECK_FOLDER = check_setting_str(CFG, 'General', 'check_folder', '') ENABLE_CHECK_FOLDER = bool(check_setting_int(CFG, 'General', 'enable_check_folder', 0)) INTERFACE = check_setting_str(CFG, 'General', 'interface', 'default') + DUPECONSTRAINT = check_setting_str(CFG, 'General', 'dupeconstraint', 'filesize') AUTOWANT_ALL = bool(check_setting_int(CFG, 'General', 'autowant_all', 0)) AUTOWANT_UPCOMING = bool(check_setting_int(CFG, 'General', 'autowant_upcoming', 1)) COMIC_COVER_LOCAL = bool(check_setting_int(CFG, 'General', 'comic_cover_local', 0)) @@ -494,6 +508,7 @@ def initialize(): if not HIGHCOUNT: HIGHCOUNT = 0 READ2FILENAME = bool(check_setting_int(CFG, 'General', 'read2filename', 0)) STORYARCDIR = bool(check_setting_int(CFG, 'General', 'storyarcdir', 0)) + COPY2ARCDIR = bool(check_setting_int(CFG, 'General', 'copy2arcdir', 0)) PROWL_ENABLED = bool(check_setting_int(CFG, 'Prowl', 'prowl_enabled', 0)) PROWL_KEYS = check_setting_str(CFG, 'Prowl', 'prowl_keys', '') PROWL_ONSNATCH = bool(check_setting_int(CFG, 'Prowl', 'prowl_onsnatch', 0)) @@ -833,14 +848,18 @@ def initialize(): logger.initLogger(verbose=VERBOSE) #logger.mylar_log.initLogger(verbose=VERBOSE) # verbatim back the logger being used since it's now started. -# if LOGTYPE == 'clog': -# logprog = 'Concurrent Log Handler' -# else: -# logprog = 'Rotational Log Handler (default)' -# logger.fdebug('ConcurrentLogHandler package not installed. Using builtin log handler for Rotational logs (default)') -# logger.fdebug('[Windows Users] If you are experiencing log file locking, you should install the ConcurrentLogHandler ( https://pypi.python.org/pypi/ConcurrentLogHandler/0.8.7 )') + if LOGTYPE == 'clog': + logprog = 'Concurrent Rotational Log Handler' + else: + logprog = 'Rotational Log Handler (default)' -# logger.fdebug('Logger set to use : ' + logprog) + logger.fdebug('Logger set to use : ' + logprog) + if LOGTYPE == 'log' and platform.system() == 'Windows': + logger.fdebug('ConcurrentLogHandler package not installed. Using builtin log handler for Rotational logs (default)') + logger.fdebug('[Windows Users] If you are experiencing log file locking and want this auto-enabled, you need to install Python Extensions for Windows ( http://sourceforge.net/projects/pywin32/ )') + + # verbatim DB module. + logger.info('[DB Module] Loading : ' + DBCHOICE + ' as the database module to use.') # Put the cache dir in the data dir for now if not CACHE_DIR: @@ -1062,6 +1081,11 @@ def config_write(): new_config.encoding = 'UTF8' new_config['General'] = {} new_config['General']['config_version'] = CONFIG_VERSION + new_config['General']['dbchoice'] = DBCHOICE + new_config['General']['dbuser'] = DBUSER + new_config['General']['dbpass'] = DBPASS + new_config['General']['dbname'] = DBNAME + if COMICVINE_API is None or COMICVINE_API == '': new_config['General']['comicvine_api'] = COMICVINE_API else: @@ -1113,6 +1137,7 @@ def config_write(): new_config['General']['download_scan_interval'] = DOWNLOAD_SCAN_INTERVAL new_config['General']['check_folder'] = CHECK_FOLDER new_config['General']['interface'] = INTERFACE + new_config['General']['dupeconstraint'] = DUPECONSTRAINT new_config['General']['autowant_all'] = int(AUTOWANT_ALL) new_config['General']['autowant_upcoming'] = int(AUTOWANT_UPCOMING) new_config['General']['preferred_quality'] = int(PREFERRED_QUALITY) @@ -1137,6 +1162,7 @@ def config_write(): new_config['General']['highcount'] = HIGHCOUNT new_config['General']['read2filename'] = int(READ2FILENAME) new_config['General']['storyarcdir'] = int(STORYARCDIR) + new_config['General']['copy2arcdir'] = int(COPY2ARCDIR) new_config['General']['use_minsize'] = int(USE_MINSIZE) new_config['General']['minsize'] = MINSIZE new_config['General']['use_maxsize'] = int(USE_MAXSIZE) @@ -1352,18 +1378,24 @@ def start(): started = True def dbcheck(): + #if DBCHOICE == 'postgresql': + # import psycopg2 + # conn = psycopg2.connect(database=DBNAME, user=DBUSER, password=DBPASS) + # c_error = 'psycopg2.DatabaseError' + #else: - conn=sqlite3.connect(DB_FILE) + conn = sqlite3.connect(DB_FILE) + c_error = 'sqlite3.OperationalError' c=conn.cursor() - c.execute('CREATE TABLE IF NOT EXISTS comics (ComicID TEXT UNIQUE, ComicName TEXT, ComicSortName TEXT, ComicYear TEXT, DateAdded TEXT, Status TEXT, IncludeExtras INTEGER, Have INTEGER, Total INTEGER, ComicImage TEXT, ComicPublisher TEXT, ComicLocation TEXT, ComicPublished TEXT, LatestIssue TEXT, LatestDate TEXT, Description TEXT, QUALalt_vers TEXT, QUALtype TEXT, QUALscanner TEXT, QUALquality TEXT, LastUpdated TEXT, AlternateSearch TEXT, UseFuzzy TEXT, ComicVersion TEXT, SortOrder INTEGER, ForceContinuing INTEGER, ComicName_Filesafe TEXT)') - c.execute('CREATE TABLE IF NOT EXISTS issues (IssueID TEXT, ComicName TEXT, IssueName TEXT, Issue_Number TEXT, DateAdded TEXT, Status TEXT, Type TEXT, ComicID, ArtworkURL Text, ReleaseDate TEXT, Location TEXT, IssueDate TEXT, Int_IssueNumber INT, ComicSize TEXT, AltIssueNumber TEXT, IssueDate_Edit TEXT)') + c.execute('CREATE TABLE IF NOT EXISTS comics (ComicID TEXT UNIQUE, ComicName TEXT, ComicSortName TEXT, ComicYear TEXT, DateAdded TEXT, Status TEXT, IncludeExtras INTEGER, Have INTEGER, Total INTEGER, ComicImage TEXT, ComicPublisher TEXT, ComicLocation TEXT, ComicPublished TEXT, LatestIssue TEXT, LatestDate TEXT, Description TEXT, QUALalt_vers TEXT, QUALtype TEXT, QUALscanner TEXT, QUALquality TEXT, LastUpdated TEXT, AlternateSearch TEXT, UseFuzzy TEXT, ComicVersion TEXT, SortOrder INTEGER, DetailURL TEXT, ForceContinuing INTEGER, ComicName_Filesafe TEXT)') + c.execute('CREATE TABLE IF NOT EXISTS issues (IssueID TEXT, ComicName TEXT, IssueName TEXT, Issue_Number TEXT, DateAdded TEXT, Status TEXT, Type TEXT, ComicID TEXT, ArtworkURL Text, ReleaseDate TEXT, Location TEXT, IssueDate TEXT, Int_IssueNumber INT, ComicSize TEXT, AltIssueNumber TEXT, IssueDate_Edit TEXT)') c.execute('CREATE TABLE IF NOT EXISTS snatched (IssueID TEXT, ComicName TEXT, Issue_Number TEXT, Size INTEGER, DateAdded TEXT, Status TEXT, FolderName TEXT, ComicID TEXT, Provider TEXT)') c.execute('CREATE TABLE IF NOT EXISTS upcoming (ComicName TEXT, IssueNumber TEXT, ComicID TEXT, IssueID TEXT, IssueDate TEXT, Status TEXT, DisplayComicName TEXT)') c.execute('CREATE TABLE IF NOT EXISTS nzblog (IssueID TEXT, NZBName TEXT, SARC TEXT, PROVIDER TEXT, ID TEXT)') c.execute('CREATE TABLE IF NOT EXISTS weekly (SHIPDATE text, PUBLISHER text, ISSUE text, COMIC VARCHAR(150), EXTRA text, STATUS text)') # c.execute('CREATE TABLE IF NOT EXISTS sablog (nzo_id TEXT, ComicName TEXT, ComicYEAR TEXT, ComicIssue TEXT, name TEXT, nzo_complete TEXT)') - c.execute('CREATE TABLE IF NOT EXISTS importresults (impID TEXT, ComicName TEXT, ComicYear TEXT, Status TEXT, ImportDate TEXT, ComicFilename TEXT, ComicLocation TEXT, WatchMatch TEXT, DisplayName TEXT, SRID TEXT)') + c.execute('CREATE TABLE IF NOT EXISTS importresults (impID TEXT, ComicName TEXT, ComicYear TEXT, Status TEXT, ImportDate TEXT, ComicFilename TEXT, ComicLocation TEXT, WatchMatch TEXT, DisplayName TEXT, SRID TEXT, ComicID TEXT, IssueID TEXT)') c.execute('CREATE TABLE IF NOT EXISTS readlist (IssueID TEXT, ComicName TEXT, Issue_Number TEXT, Status TEXT, DateAdded TEXT, Location TEXT, inCacheDir TEXT, SeriesYear TEXT, ComicID TEXT)') c.execute('CREATE TABLE IF NOT EXISTS readinglist(StoryArcID TEXT, ComicName TEXT, IssueNumber TEXT, SeriesYear TEXT, IssueYEAR TEXT, StoryArc TEXT, TotalIssues TEXT, Status TEXT, inCacheDir TEXT, Location TEXT, IssueArcID TEXT, ReadingOrder INT, IssueID TEXT, ComicID TEXT, StoreDate TEXT, IssueDate TEXT, Publisher TEXT, IssuePublisher TEXT, IssueName TEXT)') c.execute('CREATE TABLE IF NOT EXISTS annuals (IssueID TEXT, Issue_Number TEXT, IssueName TEXT, IssueDate TEXT, Status TEXT, ComicID TEXT, GCDComicID TEXT, Location TEXT, ComicSize TEXT, Int_IssueNumber INT, ComicName TEXT, ReleaseDate TEXT, ReleaseComicID TEXT, ReleaseComicName TEXT, IssueDate_Edit TEXT)') @@ -1383,49 +1415,49 @@ def dbcheck(): try: c.execute('SELECT LastUpdated from comics') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE comics ADD COLUMN LastUpdated TEXT') try: c.execute('SELECT QUALalt_vers from comics') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE comics ADD COLUMN QUALalt_vers TEXT') try: c.execute('SELECT QUALtype from comics') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE comics ADD COLUMN QUALtype TEXT') try: c.execute('SELECT QUALscanner from comics') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE comics ADD COLUMN QUALscanner TEXT') try: c.execute('SELECT QUALquality from comics') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE comics ADD COLUMN QUALquality TEXT') try: c.execute('SELECT AlternateSearch from comics') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE comics ADD COLUMN AlternateSearch TEXT') try: c.execute('SELECT ComicVersion from comics') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE comics ADD COLUMN ComicVersion TEXT') try: c.execute('SELECT SortOrder from comics') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE comics ADD COLUMN SortOrder INTEGER') try: c.execute('SELECT UseFuzzy from comics') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE comics ADD COLUMN UseFuzzy TEXT') try: c.execute('SELECT DetailURL from comics') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE comics ADD COLUMN DetailURL TEXT') try: @@ -1443,12 +1475,12 @@ def dbcheck(): try: c.execute('SELECT ComicSize from issues') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE issues ADD COLUMN ComicSize TEXT') try: c.execute('SELECT inCacheDir from issues') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE issues ADD COLUMN inCacheDIR TEXT') try: @@ -1466,27 +1498,27 @@ def dbcheck(): try: c.execute('SELECT WatchMatch from importresults') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE importresults ADD COLUMN WatchMatch TEXT') try: c.execute('SELECT IssueCount from importresults') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE importresults ADD COLUMN IssueCount TEXT') try: c.execute('SELECT ComicLocation from importresults') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE importresults ADD COLUMN ComicLocation TEXT') try: c.execute('SELECT ComicFilename from importresults') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE importresults ADD COLUMN ComicFilename TEXT') try: c.execute('SELECT impID from importresults') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE importresults ADD COLUMN impID TEXT') try: @@ -1503,31 +1535,42 @@ def dbcheck(): c.execute('SELECT SRID from importresults') except: c.execute('ALTER TABLE importresults ADD COLUMN SRID TEXT') + + try: + c.execute('SELECT ComicID from importresults') + except: + c.execute('ALTER TABLE importresults ADD COLUMN ComicID TEXT') + + try: + c.execute('SELECT IssueID from importresults') + except: + c.execute('ALTER TABLE importresults ADD COLUMN IssueID TEXT') + ## -- Readlist Table -- try: c.execute('SELECT inCacheDIR from readlist') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE readlist ADD COLUMN inCacheDIR TEXT') try: c.execute('SELECT Location from readlist') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE readlist ADD COLUMN Location TEXT') try: c.execute('SELECT IssueDate from readlist') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE readlist ADD COLUMN IssueDate TEXT') try: c.execute('SELECT SeriesYear from readlist') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE readlist ADD COLUMN SeriesYear TEXT') try: c.execute('SELECT ComicID from readlist') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE readlist ADD COLUMN ComicID TEXT') @@ -1676,7 +1719,7 @@ def dbcheck(): #value in the sql so we can display it in the details screen for everyone to wonder at. try: c.execute('SELECT not_updated_db from comics') - except sqlite3.OperationalError: + except c_error: c.execute('ALTER TABLE comics ADD COLUMN not_updated_db TEXT') # -- not implemented just yet ;) @@ -1686,12 +1729,12 @@ def dbcheck(): # MetaData will hold the MetaData itself in tuple format # try: # c.execute('SELECT MetaData_Present from comics') -# except sqlite3.OperationalError: +# except c_error: # c.execute('ALTER TABLE importresults ADD COLUMN MetaData_Present TEXT') # try: # c.execute('SELECT MetaData from importresults') -# except sqlite3.OperationalError: +# except c_error: # c.execute('ALTER TABLE importresults ADD COLUMN MetaData TEXT') #let's delete errant comics that are stranded (ie. Comicname = Comic ID: ) @@ -1710,7 +1753,7 @@ def dbcheck(): def csv_load(): # for redudant module calls..include this. - conn=sqlite3.connect(DB_FILE) + conn = sqlite3.connect(DB_FILE) c=conn.cursor() c.execute('DROP TABLE IF EXISTS exceptions') @@ -1752,11 +1795,11 @@ def csv_load(): for row in creader: try: - c.execute("INSERT INTO exceptions VALUES (?,?,?,?);", row) + #print row.split(',') + c.execute("INSERT INTO exceptions VALUES (?,?,?,?)", row) except Exception, e: #print ("Error - invald arguments...-skipping") pass - pass csvfile.close() i+=1 diff --git a/mylar/cmtagmylar.py b/mylar/cmtagmylar.py index 2114bd45..221e8cef 100644 --- a/mylar/cmtagmylar.py +++ b/mylar/cmtagmylar.py @@ -18,7 +18,7 @@ import mylar from mylar import logger from mylar.helpers import cvapi_check -def run (dirName, nzbName=None, issueid=None, manual=None, filename=None, module=None): +def run (dirName, nzbName=None, issueid=None, comversion=None, manual=None, filename=None, module=None): if module is None: module = '' module += '[META-TAGGER]' @@ -153,11 +153,19 @@ def run (dirName, nzbName=None, issueid=None, manual=None, filename=None, module if fcount > 1: logger.fdebug(module + ' More than one cbr/cbz within path, performing Post-Process on first file detected: ' + f) break - shutil.move( f, comicpath ) + if f.endswith('.cbz'): + logger.fdebug(module + ' .cbz file detected. Excluding from temporary directory move at this time.') + comicpath = downloadpath + else: + shutil.move( f, comicpath ) filename = f #just the filename itself fcount+=1 else: # if the filename is identical to the parent folder, the entire subfolder gets copied since it's the first match, instead of just the file + #if os.path.isfile(filename): + #if the filename doesn't exist - force the path assuming it's the 'download path' + filename = os.path.join(downloadpath, filename) + logger.fdebug(module + ' The path where the file is that I was provided is probably wrong - modifying it to : ' + filename) shutil.move( filename, comicpath ) try: @@ -217,7 +225,7 @@ def run (dirName, nzbName=None, issueid=None, manual=None, filename=None, module shutil.rmtree( comicpath ) logger.fdebug(module + ' Successfully removed temporary directory: ' + comicpath) else: - loggger.fdebug(module + ' Unable to remove temporary directory since it is identical to the download location : ' + comicpath) + logger.fdebug(module + ' Unable to remove temporary directory since it is identical to the download location : ' + comicpath) logger.fdebug(module + ' new filename : ' + base) nfilename = base @@ -301,7 +309,10 @@ def run (dirName, nzbName=None, issueid=None, manual=None, filename=None, module logger.fdebug(module + ' absDirName: ' + os.path.abspath(dirName)) ##set up default comictagger options here. - tagoptions = [ "-s", "--verbose" ] + if comversion is None or comversion == '': + comversion = 1 + cvers = 'volume=' + str(comversion) + tagoptions = [ "-s", "--verbose", "-m", cvers ] ## check comictagger version - less than 1.15.beta - take your chances. if sys_type == 'windows': diff --git a/mylar/filechecker.py b/mylar/filechecker.py index 31fd6703..6e9726ca 100755 --- a/mylar/filechecker.py +++ b/mylar/filechecker.py @@ -102,6 +102,7 @@ def listFiles(dir,watchcomic,Publisher,AlternateSearch=None,manual=None,sarc=Non #print item #subname = os.path.join(basedir, item) + subname = item subname = re.sub('\_', ' ', subname) @@ -115,6 +116,17 @@ def listFiles(dir,watchcomic,Publisher,AlternateSearch=None,manual=None,sarc=Non vers4year = "no" vers4vol = "no" + digitchk = 0 + + if sarc and mylar.READ2FILENAME: + logger.info('subname: ' + subname) + removest = subname.find('-') # the - gets removed above so we test for the first blank space... + logger.info('removest: ' + str(removest)) + logger.info('removestdig: ' + str(subname[:removest-1])) + if subname[:removest].isdigit() and removest == 3: + subname = subname[4:] + logger.info('subname set to : ' + subname) + for subit in subsplit: if subit[0].lower() == 'v': @@ -213,8 +225,8 @@ def listFiles(dir,watchcomic,Publisher,AlternateSearch=None,manual=None,sarc=Non subthis = re.sub('\s+',' ', subthis) logger.fdebug('[FILECHECKER] sub-cleaned: ' + subthis) #we need to make sure the file is part of the correct series or else will match falsely - if watchname not in subthis: - logger.fdebug('[FILECHECKER] this is a false match. Ignoring this result.') + if watchname.lower() not in subthis.lower(): + logger.fdebug('[FILECHECKER] ' + watchname + ' this is a false match to ' + subthis + ' - Ignoring this result.') continue subthis = subthis[len(watchname):] #remove watchcomic #we need to now check the remainder of the string for digits assuming it's a possible year @@ -265,8 +277,8 @@ def listFiles(dir,watchcomic,Publisher,AlternateSearch=None,manual=None,sarc=Non subthis = re.sub('\s+',' ', subthis) logger.fdebug('[FILECHECKER] sub-cleaned: ' + subthis) #we need to make sure the file is part of the correct series or else will match falsely - if watchname not in subthis: - logger.fdebug('[FILECHECKER] this is a false match. Ignoring this result.') + if watchname.lower() not in subthis.lower(): + logger.fdebug('[FILECHECKER] ' + watchname + ' this is a false match to ' + subthis + ' - Ignoring this result.') continue subthis = subthis[len(watchname):].strip() #remove watchcomic #we need to now check the remainder of the string for digits assuming it's a possible year @@ -379,7 +391,7 @@ def listFiles(dir,watchcomic,Publisher,AlternateSearch=None,manual=None,sarc=Non if nono in subname: subcnt = subname.count(nono) charpos = indices(subname,nono) # will return a list of char positions in subname - #print "charpos: " + str(charpos) + logger.fdebug("charpos: " + str(charpos)) if nono == '-': i=0 while (i < len(charpos)): @@ -408,20 +420,29 @@ def listFiles(dir,watchcomic,Publisher,AlternateSearch=None,manual=None,sarc=Non #logger.fdebug('[FILECHECKER] (str(nono) + " detected " + str(subcnt) + " times.") # segment '.' having a . by itself will denote the entire string which we don't want elif nono == '.': + logger.fdebug('[FILECHECKER] Decimal check.') x = 0 fndit = 0 dcspace = 0 - while x < subcnt: - fndit = subname.find(nono, fndit) - if subname[fndit-1:fndit].isdigit() and subname[fndit+1:fndit+2].isdigit(): - logger.fdebug('[FILECHECKER] decimal issue detected.') - dcspace+=1 - x+=1 - if dcspace == 1: - nonocount = nonocount + subcnt + dcspace - else: - subname = re.sub('\.', ' ', subname) - nonocount = nonocount + subcnt - 1 #(remove the extension from the length) + while (x < len(charpos)): + for x,j in enumerate(charpos): + fndit = j + logger.fdebug('fndit: ' + str(fndit)) + logger.fdebug('isdigit1: ' + subname[fndit-1:fndit]) + logger.fdebug('isdigit2: ' + subname[fndit+1:fndit+2]) + if subname[fndit-1:fndit].isdigit() and subname[fndit+1:fndit+2].isdigit(): + logger.fdebug('[FILECHECKER] decimal issue detected.') + dcspace+=1 + else: + subname = subname[:fndit] + ' ' + subname[fndit+1:] + nonocount+=1 + x+=1 + nonocount += (subcnt + dcspace) + #if dcspace == 1: + # nonocount = nonocount + subcnt + dcspace + #else: + # subname = re.sub('\.', ' ', subname) + # nonocount = nonocount + subcnt - 1 #(remove the extension from the length) else: #this is new - if it's a symbol seperated by a space on each side it drags in an extra char. x = 0 @@ -581,9 +602,12 @@ def listFiles(dir,watchcomic,Publisher,AlternateSearch=None,manual=None,sarc=Non logger.fdebug('[FILECHECKER] ' + str(j) + ' is >= ' + str(len(subname)) + ' .End reached. ignoring remainder.') break elif subname[j:] == '-': - if j <= len(subname) and subname[j+1].isdigit(): - logger.fdebug('[FILECHECKER] negative issue detected.') - #detneg = "yes" + try: + if j <= len(subname) and subname[j+1].isdigit(): + logger.fdebug('[FILECHECKER] negative issue detected.') + #detneg = "yes" + except IndexError: + logger.fdebug('[FILECHECKER] There was a problem parsing the information from this filename: ' + comicpath) elif j > findtitlepos: if subname[j:] == '#': if subname[j+1].isdigit(): @@ -593,6 +617,7 @@ def listFiles(dir,watchcomic,Publisher,AlternateSearch=None,manual=None,sarc=Non elif ('-' in watchcomic or '.' in watchcomic) and j < len(watchcomic): logger.fdebug('[FILECHECKER] - appears in series title, ignoring.') else: + digitchk = subname[j:] logger.fdebug('[FILECHECKER] special character appears outside of title - ignoring @ position: ' + str(charpos[i])) nonocount-=1 @@ -602,11 +627,11 @@ def listFiles(dir,watchcomic,Publisher,AlternateSearch=None,manual=None,sarc=Non else: jtd_len = len(cchk)# + nonocount - if sarc and mylar.READ2FILENAME: - removest = subname.find(' ') # the - gets removed above so we test for the first blank space... - if subname[:removest].isdigit(): - jtd_len += removest + 1 # +1 to account for space in place of - - logger.fdebug('[FILECHECKER] adjusted jtd_len to : ' + str(removest) + ' because of story-arc reading order tags') +# if sarc and mylar.READ2FILENAME: +# removest = subname.find(' ') # the - gets removed above so we test for the first blank space... +# if subname[:removest].isdigit(): +# jtd_len += removest + 1 # +1 to account for space in place of - +# logger.fdebug('[FILECHECKER] adjusted jtd_len to : ' + str(removest) + ' because of story-arc reading order tags') logger.fdebug('[FILECHECKER] nonocount [' + str(nonocount) + '] cchk [' + cchk + '] length [' + str(len(cchk)) + ']') @@ -637,6 +662,68 @@ def listFiles(dir,watchcomic,Publisher,AlternateSearch=None,manual=None,sarc=Non logger.fdebug('[FILECHECKER] after title removed from SUBNAME [' + justthedigits_1 + ']') + titlechk = False + + if digitchk: + try: + #do the issue title check here + logger.fdebug('[FILECHECKER] Possible issue title is : ' + str(digitchk)) + # see if it can float the digits + try: + st = digitchk.find('.') + logger.fdebug('st:' + str(st)) + st_d = digitchk[:st] + logger.fdebug('st_d:' + str(st_d)) + st_e = digitchk[st+1:] + logger.fdebug('st_e:' + str(st_e)) + #x = int(float(st_d)) + #logger.fdebug('x:' + str(x)) + #validity check + if helpers.is_number(st_d): + #x2 = int(float(st_e)) + if helpers.is_number(st_e): + logger.fdebug('[FILECHECKER] This is a decimal issue.') + else: raise ValueError + else: raise ValueError + except ValueError, e: + if digitchk.startswith('.'): + pass + else: + if len(justthedigits_1) >= len(digitchk): + logger.fdebug('[FILECHECKER] Removing issue title.') + justthedigits_1 = re.sub(digitchk,'', justthedigits_1).strip() + logger.fdebug('[FILECHECKER] After issue title removed [' + justthedigits_1 + ']') + titlechk = True + hyphensplit = digitchk + issue_firstword = digitchk.split()[0] + splitit = subname.split() + splitst = len(splitit) + logger.fdebug('[FILECHECKER] splitit :' + str(splitit)) + logger.fdebug('[FILECHECKER] splitst :' + str(len(splitit))) + orignzb = item + except: + #test this out for manual post-processing items like original sin 003.3 - thor and loki 002... +#*************************************************************************************** +# need to assign digitchk here for issues that don't have a title and fail the above try. +#*************************************************************************************** + try: + logger.fdebug('[FILECHECKER] justthedigits_1 len : ' + str(len(justthedigits_1))) + logger.fdebug('[FILECHECKER] digitchk len : ' + str(len(digitchk))) + if len(justthedigits_1) >= len(digitchk): + logger.fdebug('[FILECHECKER] Removing issue title.') + justthedigits_1 = re.sub(digitchk,'', justthedigits_1).strip() + logger.fdebug('[FILECHECKER] After issue title removed [' + justthedigits_1 + ']') + titlechk = True + hyphensplit = digitchk + issue_firstword = digitchk.split()[0] + splitit = subname.split() + splitst = len(splitit) + logger.info('[FILECHECKER] splitit :' + str(splitit)) + logger.info('[FILECHECKER] splitst :' + str(len(splitit))) + orignzb = item + except: + pass #(revert this back if above except doesn't work) + #remove the title if it appears #findtitle = justthedigits.find('-') #if findtitle > 0 and detneg == "no": @@ -771,31 +858,13 @@ def listFiles(dir,watchcomic,Publisher,AlternateSearch=None,manual=None,sarc=Non #print ("there are " + str(lenm) + " words.") cnt = 0 yearmatch = "none" - #vers4year = "no" - #vers4vol = "no" logger.fdebug('[FILECHECKER] subsplit : ' + str(subsplit)) - #for ct in subsplit: - # if ct.lower().startswith('v') and ct[1:].isdigit(): - # logger.fdebug('[FILECHECKER] possible versioning..checking') - # #we hit a versioning # - account for it - # if ct[1:].isdigit(): - # if len(ct[1:]) == 4: #v2013 - # logger.fdebug('[FILECHECKER] Version detected as ' + str(ct)) - # vers4year = "yes" #re.sub("[^0-9]", " ", str(ct)) #remove the v - # break - # else: - # if len(ct) < 4: - # logger.fdebug('[FILECHECKER] Version detected as ' + str(ct)) - # vers4vol = str(ct) - # break - # logger.fdebug('[FILECHECKER] false version detection..ignoring.') - versionmatch = "false" if vers4year is not "no" or vers4vol is not "no": - if comicvolume: #is not "None" and comicvolume is not None: + if comicvolume: D_ComicVersion = re.sub("[^0-9]", "", comicvolume) if D_ComicVersion == '': D_ComicVersion = 0 @@ -833,9 +902,10 @@ def listFiles(dir,watchcomic,Publisher,AlternateSearch=None,manual=None,sarc=Non # logger.fdebug('[FILECHECKER] Series version detected as V1 (only series in existance with that title). Bypassing year check') # yearmatch = "true" # break - if subnm[cnt][:-2] == '19' or subnm[cnt][:-2] == '20': + if (subnm[cnt].startswith('19') or subnm[cnt].startswith('20')) and len(subnm[cnt]) == 4: logger.fdebug('[FILECHECKER] year detected: ' + str(subnm[cnt])) result_comyear = subnm[cnt] +##### - checking to see what removing this does for the masses if int(result_comyear) <= int(maxyear) and int(result_comyear) >= int(comyear): logger.fdebug('[FILECHECKER] ' + str(result_comyear) + ' is within the series range of ' + str(comyear) + '-' + str(maxyear)) #still possible for incorrect match if multiple reboots of series end/start in same year @@ -843,8 +913,9 @@ def listFiles(dir,watchcomic,Publisher,AlternateSearch=None,manual=None,sarc=Non break else: logger.fdebug('[FILECHECKER] ' + str(result_comyear) + ' - not right - year not within series range of ' + str(comyear) + '-' + str(maxyear)) - yearmatch = "false" + yearmatch = "false" #set to true for mass push check. break +##### - end check cnt+=1 if versionmatch == "false": if yearmatch == "false": @@ -873,6 +944,32 @@ def listFiles(dir,watchcomic,Publisher,AlternateSearch=None,manual=None,sarc=Non #if the sub has an annual, let's remove it from the modwatch as well modwatchcomic = re.sub('annual', '', modwatchcomic.lower()) + isstitle_chk = False + + if titlechk: + issuetitle = helpers.get_issue_title(ComicID=manual['ComicID'], IssueNumber=justthedigits) + + if issuetitle: + vals = [] + watchcomic_split = watchcomic.split() + vals = mylar.search.IssueTitleCheck(issuetitle, watchcomic_split, splitit, splitst, issue_firstword, hyphensplit, orignzb=item) + print 'vals: ' + str(vals) + if vals: + if vals[0]['status'] == 'continue': + continue + else: + logger.fdebug('Issue title status returned of : ' + str(vals[0]['status'])) # will either be OK or pass. + splitit = vals[0]['splitit'] + splitst = vals[0]['splitst'] + isstitle_chk = vals[0]['isstitle_chk'] + possibleissue_num = vals[0]['possibleissue_num'] + #if the issue title was present and it contained a numeric, it will pull that as the issue incorrectly + if isstitle_chk == True: + justthedigits = possibleissue_num + subname = re.sub(' '.join(vals[0]['isstitle_removal']),'',subname).strip() + else: + logger.fdebug('No issue title.') + #tmpitem = item[:jtd_len] # if it's an alphanumeric with a space, rejoin, so we can remove it cleanly just below this. substring_removal = None @@ -895,6 +992,13 @@ def listFiles(dir,watchcomic,Publisher,AlternateSearch=None,manual=None,sarc=Non logger.fdebug('[FILECHECKER] sub_removed: ' + str(sub_removed)) split_sub = sub_removed.rsplit(' ',1)[0].split(' ') #removes last word (assuming it's the issue#) split_mod = modwatchcomic.replace('_', ' ').split() #batman + i = 0 + newc = '' + while (i < len(split_mod)): + newc += split_sub[i] + ' ' + i+=1 + if newc: + split_sub = newc.strip().split() logger.fdebug('[FILECHECKER] split_sub: ' + str(split_sub)) logger.fdebug('[FILECHECKER] split_mod: ' + str(split_mod)) diff --git a/mylar/findcomicfeed.py b/mylar/findcomicfeed.py index 45a93e7c..17b73bd1 100755 --- a/mylar/findcomicfeed.py +++ b/mylar/findcomicfeed.py @@ -129,8 +129,8 @@ def Startit(searchName, searchIssue, searchYear, ComicVersion, IssDateFix): if cName.lower().startswith('for'): pass else: - #this is the crap we ignore. Continue - logger.fdebug('this starts with FOR : ' + str(subs) + '. This is not present in the series - ignoring.') + #this is the crap we ignore. Continue (commented else, as it spams the logs) + #logger.fdebug('this starts with FOR : ' + str(subs) + '. This is not present in the series - ignoring.') continue logger.fdebug('match.') if IssDateFix != "no": diff --git a/mylar/helpers.py b/mylar/helpers.py index f3958ed3..71d1bc9c 100755 --- a/mylar/helpers.py +++ b/mylar/helpers.py @@ -518,7 +518,7 @@ def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=N '$Annual': 'Annual' } - extensions = ('.cbr', '.cbz') + extensions = ('.cbr', '.cbz', '.cb7') if ofilename.lower().endswith(extensions): path, ext = os.path.splitext(ofilename) @@ -574,17 +574,16 @@ def ComicSort(comicorder=None,sequence=None,imported=None): # if it's on startup, load the sql into a tuple for use to avoid record-locking i = 0 import logger - #if mylar.DBCHOICE == 'postgresql': - # import db_postgresql as db - # myDB = db.DBConnection() - # comicsort = myDB.select("SELECT * FROM comics ORDER BY ComicSortName COLLATE ?", [mylar.OS_LANG]) - #else: - # import db - # myDB = db.DBConnection() - # comicsort = myDB.select("SELECT * FROM comics ORDER BY ComicSortName COLLATE NOCASE") - import db - myDB = db.DBConnection() - comicsort = myDB.select("SELECT * FROM comics ORDER BY ComicSortName COLLATE NOCASE") + if mylar.DBCHOICE == 'postgresql': + import db_postgresql as db + myDB = db.DBConnection() + oscollate = mylar.OS_LANG + '.UTF8' + logger.info('OS_LANG: ' + oscollate) + comicsort = myDB.select('SELECT * FROM comics ORDER BY ComicSortName')# COLLATE "%s"', [oscollate]) + else: + import db + myDB = db.DBConnection() + comicsort = myDB.select("SELECT * FROM comics ORDER BY ComicSortName COLLATE NOCASE") comicorderlist = [] comicorder = {} comicidlist = [] @@ -1077,17 +1076,15 @@ def havetotals(refreshit=None): if refreshit is None: - #if mylar.DBCHOICE == 'postgresql': - # import db_postgresql as db - # myDB = db.DBConnection() - # comiclist = myDB.select("SELECT * from comics order by ComicSortName COLLATE ?",[mylar.OS_LANG]) - #else: - # import db - # myDB = db.DBConnection() - # comiclist = myDB.select('SELECT * from comics order by ComicSortName COLLATE NOCASE') - import db - myDB = db.DBConnection() - comiclist = myDB.select('SELECT * from comics order by ComicSortName COLLATE NOCASE') + if mylar.DBCHOICE == 'postgresql': + import db_postgresql as db + myDB = db.DBConnection() + print 'here' + comiclist = myDB.select('SELECT * from comics order by ComicSortName')# COLLATE "%s"',[mylar.OS_LANG]) + else: + import db + myDB = db.DBConnection() + comiclist = myDB.select('SELECT * from comics order by ComicSortName COLLATE NOCASE') else: comiclist = [] myDB = db.DBConnection() @@ -1140,7 +1137,10 @@ def havetotals(refreshit=None): percent = 0 totalissuess = '?' - if comic['ComicPublished'] is None or comic['ComicPublished'] == '': + if comic['LatestDate'] is None: + logger.warn(comic['ComicName'] + ' has not finished loading. Nulling some values so things display properly until they can populate.') + recentstatus = 'Loading' + elif comic['ComicPublished'] is None or comic['ComicPublished'] == '' or comic['LatestDate'] is None: recentstatus = 'Unknown' elif comic['ForceContinuing'] == 1: recentstatus = 'Continuing' @@ -1481,13 +1481,17 @@ def get_issue_title(IssueID=None, ComicID=None, IssueNumber=None): if IssueID: issue = myDB.selectone('SELECT * FROM issues WHERE IssueID=?', [IssueID]).fetchone() if issue is None: - logger.warn('Unable to locate given IssueID within the db.') - return None + issue = myDB.selectone('SELECT * FROM annuals WHERE IssueID=?', [IssueID]).fetchone() + if issue is None: + logger.fdebug('Unable to locate given IssueID within the db. Assuming Issue Title is None.') + return None else: issue = myDB.selectone('SELECT * FROM issues WHERE ComicID=? AND Int_IssueNumber=?', [ComicID, issuedigits(IssueNumber)]).fetchone() if issue is None: - logger.warn('Unable to locate given IssueID within the db.') - return None + issue = myDB.selectone('SELECT * FROM annuals WHERE IssueID=?', [IssueID]).fetchone() + if issue is None: + logger.fdebug('Unable to locate given IssueID within the db. Assuming Issue Title is None.') + return None return issue['IssueName'] @@ -1530,7 +1534,7 @@ def duplicate_filecheck(filename, ComicID=None, IssueID=None, StoryArcID=None): import db, logger myDB = db.DBConnection() - logger.info('duplicate check for ' + filename) + logger.info('[DUPECHECK] Duplicate check for ' + filename) filesz = os.path.getsize(filename) if IssueID: @@ -1538,33 +1542,71 @@ def duplicate_filecheck(filename, ComicID=None, IssueID=None, StoryArcID=None): if dupchk is None: dupchk = myDB.selectone("SELECT * FROM annuals WHERE IssueID=?", [IssueID]).fetchone() if dupchk is None: - logger.info('Unable to find corresponding Issue within the DB. Do you still have the series on your watchlist?') + logger.info('[DUPECHECK] Unable to find corresponding Issue within the DB. Do you still have the series on your watchlist?') return if any( [ dupchk['Status'] == 'Downloaded', dupchk['Status'] == 'Archived' ] ): - logger.info('Existing Status already set to ' + dupchk['Status']) + logger.info('[DUPECHECK] Existing Status already set to ' + dupchk['Status']) dupsize = dupchk['ComicSize'] + cid = [] if dupsize is None: - logger.info('Existing filesize is 0 bytes as I cannot locate the orginal entry - it is probably archived.') - rtnval = "dupe" + logger.info('[DUPECHECK] Existing filesize is 0 bytes as I cannot locate the orginal entry - it is probably archived.') + logger.fdebug('[DUPECHECK] Checking series for unrefreshed series syndrome (USS).') + havechk = myDB.selectone('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone() + if havechk: + if havechk['Have'] >= havechk['Total']: + logger.info('[DUPECHECK] Series has invalid issue totals [' + str(havechk['Have']) + '/' + str(havechk['Total']) + '] Attempting to Refresh & continue post-processing this issue.') + cid.append(ComicID) + logger.fdebug('[DUPECHECK] ComicID: ' + str(ComicID)) + mylar.updater.dbUpdate(ComicIDList=cid,calledfrom='dupechk') + return duplicate_filecheck(filename, ComicID, IssueID, StoryArcID) + else: + rtnval = "dupe" + else: + rtnval = "dupe" else: - logger.info('Existing file :' + dupchk['Location'] + ' has a filesize of : ' + str(dupsize) + ' bytes.') + logger.info('[DUPECHECK] Existing file :' + dupchk['Location'] + ' has a filesize of : ' + str(dupsize) + ' bytes.') #keywords to force keep / delete #this will be eventually user-controlled via the GUI once the options are enabled. - if int(dupsize) >= filesz: - logger.info('Existing filesize is greater than : ' + str(filesz) + ' bytes.') - rtnval = "dupe" - elif int(dupsize) == 0: - logger.info('Existing filesize is 0 as I cannot locate the original entry. Will assume it is Archived already.') + if int(dupsize) == 0: + logger.info('[DUPECHECK] Existing filesize is 0 as I cannot locate the original entry. Will assume it is Archived already.') rtnval = "dupe" else: - logger.info('Existing filesize is less than : ' + str(filesz) + ' bytes. Checking configuration if I should keep this or not.') - rtnval = "write" + logger.fdebug('[DUPECHECK] Based on duplication preferences I will retain based on : ' + mylar.DUPECONSTRAINT) + if 'cbr' in mylar.DUPECONSTRAINT or 'cbz' in mylar.DUPECONSTRAINT: + if 'cbr' in mylar.DUPECONSTRAINT: + #this has to be configured in config - either retain cbr or cbz. + if dupchk['Location'].endswith('.cbz'): + #keep dupechk['Location'] + logger.info('[DUPECHECK-CBR PRIORITY] [#' + dupchk['Issue_Number'] + '] Retaining currently scanned in file : ' + dupchk['Location']) + rtnval = "dupe" + else: + #keep filename + logger.info('[DUPECHECK-CBR PRIORITY] [#' + dupchk['Issue_Number'] + '] Retaining newly scanned in file : ' + filename) + rtnval = "write" + + elif 'cbz' in mylar.DUPECONSTRAINT: + if dupchk['Location'].endswith('.cbr'): + #keep dupchk['Location'] + logger.info('[DUPECHECK-CBZ PRIORITY] [#' + dupchk['Issue_Number'] + '] Retaining currently scanned in filename : ' + dupchk['Location']) + rtnval = "dupe" + else: + #keep filename + logger.info('[DUPECHECK-CBZ PRIORITY] [#' + dupchk['Issue_Number'] + '] Retaining newly scanned in filename : ' + filename) + rtnval = "write" + + if mylar.DUPECONSTRAINT == 'filesize': + if filesz <= dupsize: + logger.info('[DUPECHECK-FILESIZE PRIORITY] [#' + dupchk['Issue_Number'] + '] Retaining currently scanned in filename : ' + filename) + rtnval = "dupe" + else: + logger.info('[DUPECHECK-FILESIZE PRIORITY] [#' + dupchk['Issue_Number'] + '] Retaining newly scanned in filename : ' + dupchk['Location']) + rtnval = "write" else: - logger.info('Duplication detection returned no hits. This is not a duplicate of anything currently on your watchlist.') + logger.info('[DUPECHECK] Duplication detection returned no hits. This is not a duplicate of anything that I have scanned in as of yet.') rtnval = "write" return rtnval diff --git a/mylar/importer.py b/mylar/importer.py index 09910f12..2433040d 100755 --- a/mylar/importer.py +++ b/mylar/importer.py @@ -598,6 +598,18 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None,c logger.info('Returning to Future-Check module to complete the add & remove entry.') return + if imported == 'yes': + logger.info('Successfully imported : ' + comic['ComicName']) + #now that it's moved / renamed ... we remove it from importResults or mark as completed. + + results = myDB.select("SELECT * from importresults WHERE ComicName=?", [ogcname]) + if results is not None: + for result in results: + controlValue = {"impID": result['impid']} + newValue = {"Status": "Imported", + "ComicID": comicid} + myDB.upsert("importresults", newValue, controlValue) + if calledfrom == 'addbyid': logger.info('Sucessfully added ' + comic['ComicName'] + ' (' + str(SeriesYear) + ') by directly using the ComicVine ID') return diff --git a/mylar/librarysync.py b/mylar/librarysync.py index 27d54103..f1929c55 100755 --- a/mylar/librarysync.py +++ b/mylar/librarysync.py @@ -269,10 +269,11 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None) else: logger.fdebug("false decimal represent. Chunking to extra word.") cn = cn + cs[i] + " " - break - issue = cs[i] - logger.fdebug("issue detected : " + str(issue)) - idetected = 'yes' + #break + else: + issue = cs[i] + logger.fdebug("issue detected : " + str(issue)) + idetected = 'yes' elif '\#' in cs[i] or decimaldetect == 'yes': logger.fdebug("issue detected: " + str(cs[i])) diff --git a/mylar/logger.py b/mylar/logger.py index 7ae737c1..af000816 100755 --- a/mylar/logger.py +++ b/mylar/logger.py @@ -15,12 +15,13 @@ import os import sys -import logging +#import logging import traceback import threading +import platform import mylar -from logging import handlers +from logging import getLogger, INFO, DEBUG, StreamHandler, Formatter, Handler from mylar import helpers @@ -29,9 +30,9 @@ FILENAME = 'mylar.log' MAX_FILES = 5 # Mylar logger -logger = logging.getLogger('mylar') +logger = getLogger('mylar') -class LogListHandler(logging.Handler): +class LogListHandler(Handler): """ Log handler for Web UI. """ @@ -42,6 +43,24 @@ class LogListHandler(logging.Handler): mylar.LOG_LIST.insert(0, (helpers.now(), message, record.levelname, record.threadName)) def initLogger(verbose=1): + #concurrentLogHandler/0.8.7 (to deal with windows locks) + #since this only happens on windows boxes, if it's nix/mac use the default logger. + if platform.system() == 'Windows': + #set the path to the lib here - just to make sure it can detect cloghandler & portalocker. + import sys + sys.path.append(os.path.join(mylar.PROG_DIR, 'lib')) + + try: + from ConcurrentLogHandler.cloghandler import ConcurrentRotatingFileHandler as RFHandler + mylar.LOGTYPE = 'clog' + except ImportError: + mylar.LOGTYPE = 'log' + from logging.handlers import RotatingFileHandler as RFHandler + else: + mylar.LOGTYPE = 'log' + from logging.handlers import RotatingFileHandler as RFHandler + + if mylar.MAX_LOGSIZE: MAX_SIZE = mylar.MAX_LOGSIZE else: @@ -58,14 +77,14 @@ def initLogger(verbose=1): # Configure the logger to accept all messages logger.propagate = False - logger.setLevel(logging.DEBUG)# if verbose == 2 else logging.INFO) + logger.setLevel(DEBUG)# if verbose == 2 else logging.INFO) # Setup file logger filename = os.path.join(mylar.LOG_DIR, FILENAME) - file_formatter = logging.Formatter('%(asctime)s - %(levelname)-7s :: %(threadName)s : %(message)s', '%d-%b-%Y %H:%M:%S') - file_handler = handlers.RotatingFileHandler(filename, maxBytes=MAX_SIZE, backupCount=MAX_FILES) - file_handler.setLevel(logging.DEBUG) + file_formatter = Formatter('%(asctime)s - %(levelname)-7s :: %(threadName)s : %(message)s', '%d-%b-%Y %H:%M:%S') + file_handler = RFHandler(filename, "a", maxBytes=MAX_SIZE, backupCount=MAX_FILES) + file_handler.setLevel(DEBUG) file_handler.setFormatter(file_formatter) logger.addHandler(file_handler) @@ -79,20 +98,20 @@ def initLogger(verbose=1): #else: # loglist_handler.setLevel(logging.INFO) #-- - loglist_handler.setLevel(logging.INFO) + loglist_handler.setLevel(INFO) logger.addHandler(loglist_handler) # Setup console logger if verbose: - console_formatter = logging.Formatter('%(asctime)s - %(levelname)s :: %(threadName)s : %(message)s', '%d-%b-%Y %H:%M:%S') - console_handler = logging.StreamHandler() + console_formatter = Formatter('%(asctime)s - %(levelname)s :: %(threadName)s : %(message)s', '%d-%b-%Y %H:%M:%S') + console_handler = StreamHandler() console_handler.setFormatter(console_formatter) #print 'verbose is ' + str(verbose) #if verbose == 2: # console_handler.setLevel(logging.DEBUG) #else: # console_handler.setLevel(logging.INFO) - console_handler.setLevel(logging.INFO) + console_handler.setLevel(INFO) logger.addHandler(console_handler) diff --git a/mylar/mb.py b/mylar/mb.py index 5417a576..8ac8c29a 100755 --- a/mylar/mb.py +++ b/mylar/mb.py @@ -374,7 +374,8 @@ def findComic(name, mode, issue, limityear=None, explicit=None, type=None): }) #logger.fdebug('year: ' + str(xmlYr) + ' - constraint met: ' + str(xmlTag) + '[' + str(xmlYr) + '] --- 4050-' + str(xmlid)) else: - logger.fdebug('year: ' + str(xmlYr) + ' - contraint not met. Has to be within ' + str(limityear)) + pass + #logger.fdebug('year: ' + str(xmlYr) + ' - contraint not met. Has to be within ' + str(limityear)) n+=1 #search results are limited to 100 and by pagination now...let's account for this. countResults = countResults + 100 diff --git a/mylar/newpull.py b/mylar/newpull.py index 334263b3..d6ef5e99 100755 --- a/mylar/newpull.py +++ b/mylar/newpull.py @@ -51,7 +51,6 @@ def newpull(): #918 - Marvel Comics #952 - Comics & Graphic Novels # - Magazines - #print ("titlet: " + str(headt)) findurl_link = headt.findAll('a', href=True)[0] urlID = findurl_link.findNext(text=True) issue_link = findurl_link['href'] @@ -66,13 +65,13 @@ def newpull(): #logger.fdebug('publisher:' + str(isspublisher)) found_iss = headt.findAll('td') if "Home/1/1/71/920" in issue_link: - logger.fdebug('Ignoring - menu option.') + #logger.fdebug('Ignoring - menu option.') return if "PREVIEWS" in headt: - logger.fdebug('Ignoring: ' + found_iss[0]) + #logger.fdebug('Ignoring: ' + found_iss[0]) break if "MAGAZINES" in headt: - logger.fdebug('End.') + #logger.fdebug('End.') endthis = True break if len(found_iss) > 0: @@ -104,7 +103,7 @@ def newpull(): if pl['publisher'] == oldpub: exceptln = str(pl['ID']) + "\t" + str(pl['name']) + "\t" + str(pl['price']) else: - exceptln = pl['publisher'] + exceptln = pl['publisher'] + "\n" + str(pl['ID']) + "\t" + str(pl['name']) + "\t" + str(pl['price']) for lb in breakhtml: exceptln = re.sub(lb,'', exceptln).strip() diff --git a/mylar/search.py b/mylar/search.py index 271187cb..967afb78 100755 --- a/mylar/search.py +++ b/mylar/search.py @@ -1386,7 +1386,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None): result = myDB.selectone('SELECT * FROM annuals where IssueID=?', [issueid]).fetchone() mode = 'want_ann' if result is None: - logger.info("Unable to locate IssueID - you probably should delete/refresh the series.") + logger.fdebug("Unable to locate IssueID - you probably should delete/refresh the series.") return ComicID = result['ComicID'] comic = myDB.selectone('SELECT * FROM comics where ComicID=?', [ComicID]).fetchone() diff --git a/mylar/updater.py b/mylar/updater.py index 593bdbf1..3feaac3a 100755 --- a/mylar/updater.py +++ b/mylar/updater.py @@ -51,8 +51,8 @@ def dbUpdate(ComicIDList=None, calledfrom=None): else: if CV_EXcomicid['variloop'] == '99': mismatch = "yes" - if ComicID[:1] == "G": threading.Thread(target=importer.GCDimport, args=[ComicID]).start() - else: threading.Thread(target=importer.addComictoDB, args=[ComicID,mismatch]).start() + if ComicID[:1] == "G": mylar.importer.GCDimport(ComicID) + else: importer.addComictoDB(ComicID,mismatch) else: if mylar.CV_ONETIMER == 1: logger.fdebug("CV_OneTimer option enabled...") @@ -624,16 +624,16 @@ def forceRescan(ComicID,archive=None,module=None): if archive is None: tmpval = filechecker.listFiles(dir=rescan['ComicLocation'], watchcomic=rescan['ComicName'], Publisher=rescan['ComicPublisher'], AlternateSearch=altnames) comiccnt = int(tmpval['comiccount']) - logger.info('comiccnt is:' + str(comiccnt)) + logger.fdebug(module + 'comiccnt is:' + str(comiccnt)) fca.append(tmpval) if mylar.MULTIPLE_DEST_DIRS is not None and mylar.MULTIPLE_DEST_DIRS != 'None' and os.path.join(mylar.MULTIPLE_DEST_DIRS, os.path.basename(rescan['ComicLocation'])) != rescan['ComicLocation']: - logger.info('multiple_dest_dirs:' + mylar.MULTIPLE_DEST_DIRS) - logger.info('dir: ' + rescan['ComicLocation']) - logger.info('os.path.basename: ' + os.path.basename(rescan['ComicLocation'])) + logger.fdebug(module + 'multiple_dest_dirs:' + mylar.MULTIPLE_DEST_DIRS) + logger.fdebug(module + 'dir: ' + rescan['ComicLocation']) + logger.fdebug(module + 'os.path.basename: ' + os.path.basename(rescan['ComicLocation'])) pathdir = os.path.join(mylar.MULTIPLE_DEST_DIRS, os.path.basename(rescan['ComicLocation'])) logger.info(module + ' Now checking files for ' + rescan['ComicName'] + ' (' + str(rescan['ComicYear']) + ') in :' + pathdir ) tmpv = filechecker.listFiles(dir=pathdir, watchcomic=rescan['ComicName'], Publisher=rescan['ComicPublisher'], AlternateSearch=altnames) - logger.info('tmpv filecount: ' + str(tmpv['comiccount'])) + logger.fdebug(module + 'tmpv filecount: ' + str(tmpv['comiccount'])) comiccnt += int(tmpv['comiccount']) fca.append(tmpv) else: @@ -712,199 +712,142 @@ def forceRescan(ComicID,archive=None,module=None): break temploc= tmpfc['JusttheDigits'].replace('_', ' ') -# temploc = tmpfc['ComicFilename'].replace('_', ' ') temploc = re.sub('[\#\']', '', temploc) logger.fdebug(module + ' temploc: ' + str(temploc)) if 'annual' not in temploc.lower(): #remove the extension here - extensions = ('.cbr','.cbz') + extensions = ('.cbr','.cbz','.cb7') if temploc.lower().endswith(extensions): logger.fdebug(module + ' Removed extension for issue: ' + str(temploc)) temploc = temploc[:-4] -# deccnt = str(temploc).count('.') -# if deccnt > 1: - #logger.fdebug('decimal counts are :' + str(deccnt)) - #if the file is formatted with '.' in place of spaces we need to adjust. - #before replacing - check to see if digits on either side of decimal and if yes, DON'T REMOVE -# occur=1 -# prevstart = 0 -# digitfound = "no" -# decimalfound = "no" -# tempreconstruct = '' -# while (occur <= deccnt): -# n = occur -# start = temploc.find('.') -# while start >=0 and n > 1: -# start = temploc.find('.', start+len('.')) -# n-=1 -# #logger.fdebug('occurance ' + str(occur) + ' of . at position: ' + str(start)) -# if temploc[prevstart:start].isdigit(): -# if digitfound == "yes": -# #logger.fdebug('this is a decimal, assuming decimal issue.') -# decimalfound = "yes" -# reconst = "." + temploc[prevstart:start] + " " -# else: -# #logger.fdebug('digit detected.') -# digitfound = "yes" -# reconst = temploc[prevstart:start] -# else: -# reconst = temploc[prevstart:start] + " " -# #logger.fdebug('word: ' + reconst) -# tempreconstruct = tempreconstruct + reconst -# #logger.fdebug('tempreconstruct is : ' + tempreconstruct) -# prevstart = (start+1) -# occur+=1 -# #logger.fdebug('word: ' + temploc[prevstart:]) -# tempreconstruct = tempreconstruct + " " + temploc[prevstart:] -# #logger.fdebug('final filename to use is : ' + str(tempreconstruct)) -# temploc = tempreconstruct - #logger.fdebug("checking " + str(temploc)) - #fcnew_b4 = shlex.split(str(temploc)) fcnew_af = re.findall('[^\()]+', temploc) fcnew = shlex.split(fcnew_af[0]) fcn = len(fcnew) n = 0 - while (n <= iscnt): - som = 0 + while True: try: reiss = reissues[n] except IndexError: break -# int_iss, iss_except = helpers.decimal_issue(reiss['Issue_Number']) int_iss = helpers.issuedigits(reiss['Issue_Number']) issyear = reiss['IssueDate'][:4] old_status = reiss['Status'] issname = reiss['IssueName'] - #logger.fdebug('integer_issue:' + str(int_iss) + ' ... status: ' + str(old_status)) - #if comic in format of "SomeSeries 5(c2c)(2013).cbr" whatever...it'll die. - #can't distinguish the 5(c2c) to tell it's the issue #... fnd_iss_except = 'None' - #print ("Issue, int_iss, iss_except: " + str(reiss['Issue_Number']) + "," + str(int_iss) + "," + str(iss_except)) - - - while (som < fcn): - #counts get buggered up when the issue is the last field in the filename - ie. '50.cbr' - #logger.fdebug('checking word - ' + str(fcnew[som])) - if ".cbr" in fcnew[som].lower(): - fcnew[som] = fcnew[som].replace(".cbr", "") - elif ".cbz" in fcnew[som].lower(): - fcnew[som] = fcnew[som].replace(".cbz", "") - if "(c2c)" in fcnew[som].lower(): - fcnew[som] = fcnew[som].replace("(c2c)", " ") - get_issue = shlex.split(str(fcnew[som])) - if fcnew[som] != " ": - fcnew[som] = get_issue[0] - - - if som+1 < len(fcnew) and len(fcnew[som+1]) == 2: - #print "fcnew[som+1]: " + str(fcnew[som+1]) - #print "fcnew[som]: " + str(fcnew[som]) - if 'au' in fcnew[som+1].lower(): - #if the 'AU' is in 005AU vs 005 AU it will yield different results. - fcnew[som] = fcnew[som] + 'AU' - fcnew[som+1] = '93939999919190933' - logger.info(module + ' AU Detected seperate from issue - combining and continuing') - elif 'ai' in fcnew[som+1].lower(): - #if the 'AI' is in 005AI vs 005 AI it will yield different results. - fcnew[som] = fcnew[som] + 'AI' - fcnew[som+1] = '93939999919190933' - logger.info(module + ' AI Detected seperate from issue - combining and continuing') - - #sometimes scanners refuse to use spaces between () and lump the issue right at the start - #mylar assumes it's all one word in this case..let's dump the brackets. - fcdigit = helpers.issuedigits(fcnew[som]) + fcdigit = helpers.issuedigits(temploc) - #logger.fdebug("fcdigit: " + str(fcdigit)) - #logger.fdebug("int_iss: " + str(int_iss)) + if int(fcdigit) == int_iss: + logger.fdebug(module + ' [' + str(reiss['IssueID']) + '] Issue match - fcdigit: ' + str(fcdigit) + ' ... int_iss: ' + str(int_iss)) - if int(fcdigit) == int_iss: - logger.fdebug(module + ' [' + str(reiss['IssueID']) + '] Issue match - fcdigit: ' + str(fcdigit) + ' ... int_iss: ' + str(int_iss)) - - if '-' in temploc and temploc.find(reiss['Issue_Number']) > temploc.find('-'): - logger.fdebug(module + ' I have detected a possible Title in the filename') - logger.fdebug(module + ' the issue # has occured after the -, so I assume that it is part of the Title') - break - - multiplechk = False - - for d in issuedupechk: - if int(d['fcdigit']) == int(fcdigit): - if len(mc_issue) > 1: - logger.fdebug('[Initial Check] multiple check issue detected - more than one issue with identical numbering for series.') - for mi in mc_issue: - if (mi['IssueYear'] in tmpfc['ComicFilename']):# and (int(d['issueyear']) == int(mi['IssueYear'])) and (d['fcdigit'] == mi['Int_IssueNumber']): - logger.fdebug('[Initial Check] detected : ' + str(mi['IssueYear']) + ' within filename.') - multiplechk = False - issuedupe = "no" - break - else: - logger.fdebug('[Initial Check] ' + str(mi['Int_IssueNumber']) + ' - did not detect year within filename - expecting (' + str(mi['IssueYear']) + '). Assuming this is the identical numbered issue.') - multiplechk = True - if multiplechk == False: break - - - if multiplechk == True: - logger.fdebug(module + ' Duplicate issue detected - not counting this: ' + str(tmpfc['ComicFilename'])) - #logger.fdebug(module + ' is a duplicate of ' + d['filename']) - #logger.fdebug('fcdigit:' + str(fcdigit) + ' === dupedigit: ' + str(d['fcdigit'])) - issuedupe = "yes" - break - - if issuedupe == "no": - - foundchk = False - - #make sure we are adding the correct issue. - for d in issuedupechk: - if int(d['fcdigit']) == int(fcdigit): - if len(mc_issue) > 1 and multiplechk == False: - #if len(mc_issue) > 1 and multiplechk != False and any d['Int_IssueNumber'] == int_iss for d in mc_issue): - for mi in mc_issue: - logger.fdebug('[DupeCheck]' + str(mi['IssueID']) + ' comparing to ' + str(d['issueid'])) - if mi['IssueID'] != d['issueid'] and mi['IssueID'] == reiss['IssueID']: - logger.fdebug('Most likely, I should not be marking this as a dupe.') - if (mi['IssueYear'] in tmpfc['ComicFilename']) and (d['fcdigit'] == mi['Int_IssueNumber']): - logger.fdebug('[DupeCheck] detected : ' + str(mi['IssueYear']) + ' within filename. This is an issue that happens to have duplicate numbering and is acceptable') - foundchk = True - break - else: - logger.fdebug('[DupeCheck] ' + str(mi['Int_IssueNumber']) + ': did not detect year (' + str(mi['IssueYear']) + ').') - foundchk = False - else: - foundchk = True - - - if foundchk == True: - logger.fdebug('[DupeCheck] This is not a duplicate. foundchk is : ' + str(foundchk)) - letitgo = True - break - - if foundchk == False: - logger.fdebug(module + ' Matched...issue: ' + rescan['ComicName'] + '#' + reiss['Issue_Number'] + ' --- ' + str(int_iss)) - havefiles+=1 - haveissue = "yes" - isslocation = str(tmpfc['ComicFilename']) - issSize = str(tmpfc['ComicSize']) - logger.fdebug(module + ' .......filename: ' + str(isslocation)) - logger.fdebug(module + ' .......filesize: ' + str(tmpfc['ComicSize'])) - # to avoid duplicate issues which screws up the count...let's store the filename issues then - # compare earlier... - issuedupechk.append({'fcdigit': int(fcdigit), - 'filename': tmpfc['ComicFilename'], - 'issueyear': issyear, - 'issueid': reiss['IssueID']}) - break - #else: - # if the issue # matches, but there is no year present - still match. - # determine a way to match on year if present, or no year (currently). - - if issuedupe == "yes": - logger.fdebug(module + ' I should break out here because of a dupe.') + if '-' in temploc and temploc.find(reiss['Issue_Number']) > temploc.find('-'): + logger.fdebug(module + ' I have detected a possible Title in the filename') + logger.fdebug(module + ' the issue # has occured after the -, so I assume that it is part of the Title') break - som+=1 + + #baseline these to default to normal scanning + multiplechk = False + issuedupe = "no" + foundchk = False + + #check here if muliple identical numbering issues exist for the series + if len(mc_issue) > 1: + for mi in mc_issue: + if mi['Int_IssueNumber'] == int_iss: + if mi['IssueID'] == reiss['IssueID']: + logger.fdebug(module + ' IssueID matches to multiple issues : ' + str(mi['IssueID']) + '. Checking dupe.') + logger.fdebug(module + ' miISSUEYEAR: ' + str(mi['IssueYear']) + ' -- issyear : ' + str(issyear)) + if any(mi['IssueID'] == d['issueid'] for d in issuedupechk): + logger.fdebug(module + ' IssueID already within dupe. Checking next if available.') + multiplechk = True + break + if (mi['IssueYear'] in tmpfc['ComicFilename']) and (issyear == mi['IssueYear']): + logger.fdebug(module + ' Matched to year within filename : ' + str(issyear)) + multiplechk = False + break + else: + logger.fdebug(module + ' Did not match to year within filename : ' + str(issyear)) + multiplechk = True + if multiplechk == True: + n+=1 + continue + + #this will detect duplicate filenames within the same directory. + for di in issuedupechk: + if di['fcdigit'] == fcdigit: + #base off of config - base duplication keep on filesize or file-type (or both) + logger.fdebug('[DUPECHECK] Duplicate issue detected [' + di['filename'] + '] [' + tmpfc['ComicFilename'] + ']') + # mylar.DUPECONSTRAINT = 'filesize' / 'filetype-cbr' / 'filetype-cbz' + logger.fdebug('[DUPECHECK] Based on duplication preferences I will retain based on : ' + mylar.DUPECONSTRAINT) + removedupe = False + if 'cbr' in mylar.DUPECONSTRAINT or 'cbz' in mylar.DUPECONSTRAINT: + if 'cbr' in mylar.DUPECONSTRAINT: + #this has to be configured in config - either retain cbr or cbz. + if tmpfc['ComicFilename'].endswith('.cbz'): + #keep di['filename'] + logger.fdebug('[DUPECHECK-CBR PRIORITY] [#' + reiss['Issue_Number'] + '] Retaining currently scanned in file : ' + di['filename']) + issuedupe = "yes" + break + else: + #keep tmpfc['ComicFilename'] + logger.fdebug('[DUPECHECK-CBR PRIORITY] [#' + reiss['Issue_Number'] + '] Retaining newly scanned in file : ' + tmpfc['ComicFilename']) + removedupe = True + elif 'cbz' in mylar.DUPECONSTRAINT: + if tmpfc['ComicFilename'].endswith('.cbr'): + #keep di['filename'] + logger.fdebug('[DUPECHECK-CBZ PRIORITY] [#' + reiss['Issue_Number'] + '] Retaining currently scanned in filename : ' + di['filename']) + issuedupe = "yes" + break + else: + #keep tmpfc['ComicFilename'] + logger.fdebug('[DUPECHECK-CBZ PRIORITY] [#' + reiss['Issue_Number'] + '] Retaining newly scanned in filename : ' + tmpfc['ComicFilename']) + removedupe = True + + if mylar.DUPECONSTRAINT == 'filesize': + if tmpfc['ComicSize'] <= di['filesize']: + logger.fdebug('[DUPECHECK-FILESIZE PRIORITY] [#' + reiss['Issue_Number'] + '] Retaining currently scanned in filename : ' + di['filename']) + issuedupe = "yes" + break + else: + logger.fdebug('[DUPECHECK-FILESIZE PRIORITY] [#' + reiss['Issue_Number'] + '] Retaining newly scanned in filename : ' + tmpfc['ComicFilename']) + removedupe = True + + if removedupe: + #need to remove the entry from issuedupechk so can add new one. + #tuple(y for y in x if y) for x in a + issuedupe_temp = [] + for x in issuedupechk: + if x['filename'] != di['filename']: + issuedupe_temp.append(x) + issuedupechk = issuedupe_temp + foundchk = False + break + + if issuedupe == "no": + + if foundchk == False: + logger.fdebug(module + ' Matched...issue: ' + rescan['ComicName'] + '#' + reiss['Issue_Number'] + ' --- ' + str(int_iss)) + havefiles+=1 + haveissue = "yes" + isslocation = str(tmpfc['ComicFilename']) + issSize = str(tmpfc['ComicSize']) + logger.fdebug(module + ' .......filename: ' + str(isslocation)) + logger.fdebug(module + ' .......filesize: ' + str(tmpfc['ComicSize'])) + # to avoid duplicate issues which screws up the count...let's store the filename issues then + # compare earlier... + issuedupechk.append({'fcdigit': fcdigit, + 'filename': tmpfc['ComicFilename'], + 'filesize': tmpfc['ComicSize'], + 'issueyear': issyear, + 'issueid': reiss['IssueID']}) + break + + if issuedupe == "yes": + logger.fdebug(module + ' I should break out here because of a dupe.') + break + if haveissue == "yes" or issuedupe == "yes": break n+=1 else: diff --git a/mylar/webserve.py b/mylar/webserve.py index 0997808c..fb86a051 100755 --- a/mylar/webserve.py +++ b/mylar/webserve.py @@ -1,3 +1,4 @@ + # This file is part of Mylar. # # Mylar is free software: you can redistribute it and/or modify @@ -320,12 +321,18 @@ class WebInterface(object): raise cherrypy.HTTPRedirect("comicDetails?ComicID=%s" % comicid) addComic.exposed = True - def addbyid(self,comicid): + def addbyid(self,comicid,calledby=None,imported=None,ogcname=None): mismatch = "no" logger.info('Attempting to add directly by ComicVineID: ' + str(comicid)) if comicid.startswith('4050-'): comicid = re.sub('4050-','', comicid) - threading.Thread(target=importer.addComictoDB, args=[comicid,mismatch,None]).start() - raise cherrypy.HTTPRedirect("comicDetails?ComicID=%s" % comicid) + threading.Thread(target=importer.addComictoDB, args=[comicid,mismatch,None,imported,ogcname]).start() + print calledby + if calledby == True or calledby == 'True': + return + elif calledby == 'web-import': + raise cherrypy.HTTPRedirect("importResults") + else: + raise cherrypy.HTTPRedirect("comicDetails?ComicID=%s" % comicid) addbyid.exposed = True def addStoryArc(self, storyarcname, storyarcyear, storyarcpublisher, storyarcissues, arcid, arclist , desc, image): @@ -512,6 +519,9 @@ class WebInterface(object): myDB.upsert("readinglist", newVals, newCtrl) + #run the Search for Watchlist matches now. + logger.fdebug('Now searching your watchlist for matches belonging to this story arc.') + self.ArcWatchlist(storyarcid) raise cherrypy.HTTPRedirect("detailReadlist?StoryArcID=%s&StoryArcName=%s" % (storyarcid, storyarcname)) addStoryArc.exposed = True @@ -1177,12 +1187,21 @@ class WebInterface(object): else: logger.info(u"Story Arc : " + str(SARC) + " queueing selected issue...") logger.info(u"IssueArcID : " + str(IssueArcID)) + #try to load the issue dates - can now sideload issue details. + dateload = myDB.selectone('SELECT * FROM readinglist WHERE IssueArcID=?', [IssueArcID]).fetchone() + if dateload is None: + IssueDate = None + StoreDate = None + else: + IssueDate = dateload['IssueDate'] + StoreDate = dateload['StoreDate'] + if ComicYear is None: ComicYear = SeriesYear logger.info(u"Marking " + ComicName + " " + ComicIssue + " as wanted...") controlValueDict = {"IssueArcID": IssueArcID} newStatus = {"Status": "Wanted"} myDB.upsert("readinglist", newStatus, controlValueDict) - foundcom, prov = search.search_init(ComicName=ComicName, IssueNumber=ComicIssue, ComicYear=ComicYear, SeriesYear=None, Publisher=None, IssueDate=None, StoreDate=None, IssueID=None, AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=SARC, IssueArcID=IssueArcID) + foundcom, prov = search.search_init(ComicName=ComicName, IssueNumber=ComicIssue, ComicYear=ComicYear, SeriesYear=None, Publisher=None, IssueDate=IssueDate, StoreDate=StoreDate, IssueID=None, AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=SARC, IssueArcID=IssueArcID) if foundcom == "yes": logger.info(u"Downloaded " + ComicName + " #" + ComicIssue + " (" + str(ComicYear) + ")") #raise cherrypy.HTTPRedirect("readlist") @@ -1383,6 +1402,15 @@ class WebInterface(object): def pullist(self): myDB = db.DBConnection() + autowants = myDB.select("SELECT * FROM futureupcoming WHERE Status='Wanted'") + autowant = [] + if autowants: + for aw in autowants: + autowant.append({"ComicName": aw['ComicName'], + "IssueNumber": aw['IssueNumber'], + "Publisher": aw['Publisher'], + "Status": aw['Status'], + "DisplayComicName": aw['DisplayComicName']}) weeklyresults = [] wantedcount = 0 popit = myDB.select("SELECT * FROM sqlite_master WHERE name='weekly' and type='table'") @@ -1397,12 +1425,32 @@ class WebInterface(object): x = weekly['ISSUE'] if x is not None: - weeklyresults.append({ + if not autowant: + weeklyresults.append({ "PUBLISHER" : weekly['PUBLISHER'], "ISSUE" : weekly['ISSUE'], "COMIC" : weekly['COMIC'], - "STATUS" : weekly['STATUS'] + "STATUS" : weekly['STATUS'], + "AUTOWANT" : False }) + else: + if any(x['ComicName'].lower() == weekly['COMIC'].lower() for x in autowant): + weeklyresults.append({ + "PUBLISHER" : weekly['PUBLISHER'], + "ISSUE" : weekly['ISSUE'], + "COMIC" : weekly['COMIC'], + "STATUS" : weekly['STATUS'], + "AUTOWANT" : True + }) + else: + weeklyresults.append({ + "PUBLISHER" : weekly['PUBLISHER'], + "ISSUE" : weekly['ISSUE'], + "COMIC" : weekly['COMIC'], + "STATUS" : weekly['STATUS'], + "AUTOWANT" : False + }) + if weekly['STATUS'] == 'Wanted': wantedcount +=1 @@ -1811,7 +1859,7 @@ class WebInterface(object): comic = myDB.selectone("SELECT * FROM comics WHERE ComicID=?", [comicid]).fetchone() comicdir = comic['ComicLocation'] comicname = comic['ComicName'] - extensions = ('.cbr', '.cbz') + extensions = ('.cbr', '.cbz', '.cb7') issues = myDB.select("SELECT * FROM issues WHERE ComicID=?", [comicid]) if mylar.ANNUALS_ON: issues += myDB.select("SELECT * FROM annuals WHERE ComicID=?", [comicid]) @@ -1986,6 +2034,9 @@ class WebInterface(object): logger.info("Removed " + str(IssueID) + " from Reading List") elif StoryArcID: myDB.action('DELETE from readinglist WHERE StoryArcID=?', [StoryArcID]) + stid = 'S' + str(StoryArcID) + '_%' + #delete from the nzblog so it will always find the most current downloads. Nzblog has issueid, but starts with ArcID + myDB.action('DELETE from nzblog WHERE IssueID LIKE ?', [stid]) logger.info("Removed " + str(StoryArcID) + " from Story Arcs.") elif IssueArcID: myDB.action('DELETE from readinglist WHERE IssueArcID=?', [IssueArcID]) @@ -2239,11 +2290,15 @@ class WebInterface(object): #cycle through the story arcs here for matches on the watchlist if sarc_title != arc['storyarc']: - dstloc = os.path.join(mylar.DESTINATION_DIR, 'StoryArcs', arc['storyarc']) + if mylar.STORYARCDIR: + dstloc = os.path.join(mylar.DESTINATION_DIR, 'StoryArcs', arc['storyarc']) + else: + dstloc = os.path.join(mylar.DESTINATION_DIR, mylar.GRABBAG_DIR) + if os.path.isdir(dstloc): logger.info('Validating Directory (' + dstloc + '). Already exists! Continuing...') else: - logger.fdebug('Updated Directory doesn not exist! - attempting to create now.') + logger.fdebug('Updated Directory does not exist! - attempting to create now.') filechecker.validateAndCreateDirectory(dstloc, True) @@ -2269,8 +2324,8 @@ class WebInterface(object): isschk = myDB.selectone("SELECT * FROM issues WHERE Issue_Number=? AND ComicID=?", [str(GCDissue), comic['ComicID']]).fetchone() else: issue_int = helpers.issuedigits(arc['IssueNumber']) - logger.info('int_issue = ' + str(issue_int)) - isschk = myDB.selectone("SELECT * FROM issues WHERE Int_IssueNumber=? AND ComicID=?", [issue_int, comic['ComicID']]).fetchone() + logger.fdebug('int_issue = ' + str(issue_int)) + isschk = myDB.selectone("SELECT * FROM issues WHERE Int_IssueNumber=? AND ComicID=? AND STATUS !='Snatched'", [issue_int, comic['ComicID']]).fetchone() if isschk is None: logger.fdebug("we matched on name, but issue " + str(arc['IssueNumber']) + " doesn't exist for " + comic['ComicName']) else: @@ -2306,7 +2361,10 @@ class WebInterface(object): "IssueNumber": arc['IssueNumber'], "IssueYear": arc['IssueYear']}) - dstloc = os.path.join(mylar.DESTINATION_DIR, 'StoryArcs', arc['storyarc']) + if mylar.STORYARCDIR: + dstloc = os.path.join(mylar.DESTINATION_DIR, 'StoryArcs', arc['storyarc']) + else: + dstloc = mylar.GRABBAG_DIR logger.fdebug('destination location set to : ' + dstloc) filechk = filechecker.listFiles(dstloc, arc['ComicName'], Publisher=None, sarc='true') @@ -2368,27 +2426,32 @@ class WebInterface(object): if issue['Status'] == 'Downloaded': issloc = os.path.join(m_arc['match_filedirectory'], issue['Location']) logger.fdebug('source location set to : ' + issloc) - dstloc = os.path.join(mylar.DESTINATION_DIR, 'StoryArcs', m_arc['match_storyarc']) + if mylar.STORYARCDIR: + dstloc = os.path.join(mylar.DESTINATION_DIR, 'StoryArcs', m_arc['match_storyarc']) + else: + dstloc = mylar.GRABBAG_DIR + logger.fdebug('destination location set to : ' + dstloc) - logger.fdebug('attempting to copy into StoryArc directory') - #copy into StoryArc directory... - if os.path.isfile(issloc): - if not os.path.isfile(dstloc): - if mylar.READ2FILENAME: - readorder = helpers.renamefile_readingorder(m_arc['match_readingorder']) - dfilename = str(readorder) + "-" + issue['Location'] + if mylar.COPY2ARCDIR: + logger.fdebug('attempting to copy into StoryArc directory') + #copy into StoryArc directory... + if os.path.isfile(issloc): + if not os.path.isfile(dstloc): + if mylar.READ2FILENAME: + readorder = helpers.renamefile_readingorder(m_arc['match_readingorder']) + dfilename = str(readorder) + "-" + issue['Location'] + else: + dfilename = issue['Location'] + + dstloc = os.path.join(dstloc, dfilename) + + logger.fdebug('copying ' + issloc + ' to ' + dstloc) + shutil.copy(issloc, dstloc) else: - dfilename = issue['Location'] - - dstloc = os.path.join(mylar.DESTINATION_DIR, 'StoryArcs', m_arc['match_storyarc'], dfilename) - - logger.fdebug('copying ' + issloc + ' to ' + dstloc) - shutil.copy(issloc, dstloc) + logger.fdebug('destination file exists: ' + dstloc) else: - logger.fdebug('destination file exists: ' + dstloc) - else: - logger.fdebug('source file does not exist: ' + issloc) + logger.fdebug('source file does not exist: ' + issloc) else: logger.fdebug("We don't have " + issue['ComicName'] + " :# " + str(issue['Issue_Number'])) @@ -2398,19 +2461,20 @@ class WebInterface(object): myDB.upsert("readinglist",newVal,ctrlVal) logger.info("Marked " + issue['ComicName'] + " :# " + str(issue['Issue_Number']) + " as Wanted.") + return ArcWatchlist.exposed = True def ReadGetWanted(self, StoryArcID): # this will queue up (ie. make 'Wanted') issues in a given Story Arc that are 'Not Watched' - print StoryArcID + #print StoryArcID stupdate = [] mode = 'story_arc' myDB = db.DBConnection() wantedlist = myDB.select("SELECT * FROM readinglist WHERE StoryArcID=? AND Status is Null", [StoryArcID]) if wantedlist is not None: for want in wantedlist: - print want + #print want issuechk = myDB.selectone("SELECT * FROM issues WHERE IssueID=?", [want['IssueArcID']]).fetchone() SARC = want['StoryArc'] IssueArcID = want['IssueArcID'] @@ -2422,10 +2486,10 @@ class WebInterface(object): issdate = want['IssueDate'] logger.info("-- NOT a watched series queue.") logger.info(want['ComicName'] + " -- #" + str(want['IssueNumber'])) - logger.info(u"Story Arc : " + str(SARC) + " queueing selected issue...") + logger.info(u"Story Arc : " + str(SARC) + " queueing the selected issue...") logger.info(u"IssueArcID : " + str(IssueArcID)) - logger.info('ComicID: ' + str(s_comicid) + ' --- IssueID: ' + str(s_issueid)) - logger.info('StoreDate: ' + str(stdate) + ' --- IssueDate: ' + str(issdate)) + logger.info(u"ComicID: " + s_comicid + " --- IssueID: " + s_issueid) + logger.info(u"StoreDate: " + str(stdate) + " --- IssueDate: " + str(issdate)) foundcom, prov = search.search_init(ComicName=want['ComicName'], IssueNumber=want['IssueNumber'], ComicYear=want['IssueYear'], SeriesYear=want['SeriesYear'], Publisher=want['Publisher'], IssueDate=issdate, StoreDate=stdate, IssueID=None, AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=SARC, IssueArcID=IssueArcID, mode=None, rsscheck=None, ComicID=None) else: # it's a watched series @@ -2436,11 +2500,11 @@ class WebInterface(object): foundcom, prov = search.search_init(ComicName=issuechk['ComicName'], IssueNumber=issuechk['Issue_Number'], ComicYear=issuechk['IssueYear'], SeriesYear=issuechk['SeriesYear'], Publisher=None, IssueDate=None, StoreDate=issuechk['ReleaseDate'], IssueID=issuechk['IssueID'], AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=SARC, IssueArcID=IssueArcID) if foundcom == "yes": - print "sucessfully found." + logger.fdebug('sucessfully found.') #update the status - this is necessary for torrents as they are in 'snatched' status. updater.foundsearch(s_comicid, s_issueid, mode=mode, provider=prov, SARC=SARC, IssueArcID=IssueArcID) else: - print "not sucessfully found." + logger.fdebug('not sucessfully found.') stupdate.append({"Status": "Wanted", "IssueArcID": IssueArcID, "IssueID": "None"}) @@ -2448,7 +2512,7 @@ class WebInterface(object): watchlistchk = myDB.select("SELECT * FROM readinglist WHERE StoryArcID=? AND Status='Wanted'", [StoryArcID]) if watchlistchk is not None: for watchchk in watchlistchk: - print "Watchlist hit - " + str(watchchk['ComicName']) + logger.fdebug('Watchlist hit - ' + str(watchchk['ComicName'])) issuechk = myDB.selectone("SELECT * FROM issues WHERE IssueID=?", [watchchk['IssueArcID']]).fetchone() SARC = watchchk['StoryArc'] IssueArcID = watchchk['IssueArcID'] @@ -2458,9 +2522,21 @@ class WebInterface(object): s_issueid = None logger.fdebug("-- NOT a watched series queue.") logger.fdebug(watchchk['ComicName'] + " -- #" + str(watchchk['IssueNumber'])) - logger.info(u"Story Arc : " + str(SARC) + " queueing selected issue...") + logger.info(u"Story Arc : " + str(SARC) + " queueing up the selected issue...") logger.info(u"IssueArcID : " + str(IssueArcID)) - foundcom, prov = search.search_init(ComicName=watchchk['ComicName'], IssueNumber=watchchk['IssueNumber'], ComicYear=watchchk['IssueYEAR'], SeriesYear=watchchk['SeriesYear'], Publisher=watchchk['ComicPublisher'], IssueDate=None, StoreDate=None, IssueID=None, AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=SARC, IssueArcID=IssueArcID, mode=None, rsscheck=None, ComicID=None) + try: + issueyear = watchchk['IssueYEAR'] + logger.fdebug('issueYEAR : ' + issueyear) + except: + issueyear = watchchk['StoreDate'][:4] + logger.info('issueyear : ' + str(issueyear)) + logger.info('comicname : ' + watchchk['ComicName']) + logger.info('issuenumber : ' + watchchk['IssueNumber']) + logger.info('comicyear : ' + watchchk['SeriesYear']) + logger.info('publisher : ' + watchchk['IssuePublisher']) + logger.info('SARC : ' + SARC) + logger.info('IssueArcID : ' + IssueArcID) + foundcom, prov = search.search_init(ComicName=watchchk['ComicName'], IssueNumber=watchchk['IssueNumber'], ComicYear=issueyear, SeriesYear=watchchk['SeriesYear'], Publisher=watchchk['IssuePublisher'], IssueDate=None, StoreDate=None, IssueID=None, AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=SARC, IssueArcID=IssueArcID, mode=None, rsscheck=None, ComicID=None) else: # it's a watched series s_comicid = issuechk['ComicID'] @@ -2469,23 +2545,22 @@ class WebInterface(object): logger.fdebug(issuechk['ComicName'] + " -- #" + str(issuechk['Issue_Number'])) foundcom,prov = search.search_init(ComicName=issuechk['ComicName'], IssueNumber=issuechk['Issue_Number'], ComicYear=issuechk['IssueYear'], SeriesYear=issuechk['SeriesYear'], Publisher=None, IssueDate=None, StoreDate=issuechk['ReleaseDate'], IssueID=issuechk['IssueID'], AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=SARC, IssueArcID=IssueArcID, mode=None, rsscheck=None, ComicID=None) if foundcom == "yes": - print "sucessfully found." updater.foundsearch(s_comicid, s_issueid, mode=mode, provider=prov, SARC=SARC, IssueArcID=IssueArcID) else: - print "Watchlist issue not sucessfully found." - print "issuearcid: " + str(IssueArcID) - print "issueid: " + str(s_issueid) + logger.fdebug('Watchlist issue not sucessfully found') + logger.fdebug('issuearcid: ' + str(IssueArcID)) + logger.fdebug('issueid: ' + str(s_issueid)) stupdate.append({"Status": "Wanted", "IssueArcID": IssueArcID, "IssueID": s_issueid}) if len(stupdate) > 0: - print str(len(stupdate)) + " issues need to get updated to Wanted Status" + logger.fdebug(str(len(stupdate)) + ' issues need to get updated to Wanted Status') for st in stupdate: ctrlVal = {'IssueArcID': st['IssueArcID']} newVal = {'Status': st['Status']} if st['IssueID']: - print "issueid:" + str(st['IssueID']) + logger.fdebug('issueid:' + str(st['IssueID'])) newVal['IssueID'] = st['IssueID'] myDB.upsert("readinglist", newVal, ctrlVal) ReadGetWanted.exposed = True @@ -2508,12 +2583,18 @@ class WebInterface(object): ReadMassCopy.exposed = True - def importLog(self, ComicName): + def importLog(self, ComicName, SRID=None): myDB = db.DBConnection() - impchk = myDB.selectone("SELECT * FROM importresults WHERE ComicName=?", [ComicName]).fetchone() + impchk = None + if SRID != 'None': + impchk = myDB.selectone("SELECT * FROM importresults WHERE SRID=?", [SRID]).fetchone() + if impchk is None: + logger.error('No associated log found for this ID : ' + SRID) if impchk is None: - logger.error(u"No associated log found for this import : " + ComicName) - return + impchk = myDB.selectone("SELECT * FROM importresults WHERE ComicName=?", [ComicName]).fetchone() + if impchk is None: + logger.error('No associated log found for this ComicName : ' + ComicName) + return implog = impchk['implog'].replace("\n","
\n") return implog @@ -2735,13 +2816,13 @@ class WebInterface(object): deleteimport.exposed = True def preSearchit(self, ComicName, comiclist=None, mimp=0, displaycomic=None): - implog = '' - implog = implog + "imp_rename:" + str(mylar.IMP_RENAME) + "\n" - implog = implog + "imp_move:" + str(mylar.IMP_MOVE) + "\n" if mimp == 0: comiclist = [] comiclist.append(ComicName) for cl in comiclist: + implog = '' + implog = implog + "imp_rename:" + str(mylar.IMP_RENAME) + "\n" + implog = implog + "imp_move:" + str(mylar.IMP_MOVE) + "\n" ComicName = cl logger.info('comicname is :' + ComicName) implog = implog + "comicName: " + str(ComicName) + "\n" @@ -2800,20 +2881,21 @@ class WebInterface(object): if (result['ComicYear'] not in yearRANGE) or (yearRANGE is None): if result['ComicYear'] <> "0000": implog = implog + "adding..." + str(result['ComicYear']) + "\n" - yearRANGE.append(result['ComicYear']) + yearRANGE.append(str(result['ComicYear'])) yearTOP = str(result['ComicYear']) getiss_num = helpers.issuedigits(getiss) miniss_num = helpers.issuedigits(str(minISSUE)) startiss_num = helpers.issuedigits(str(startISSUE)) if int(getiss_num) > int(miniss_num): implog = implog + "issue now set to : " + str(getiss) + " ... it was : " + str(minISSUE) + "\n" + logger.fdebug('Minimum issue now set to : ' + str(getiss) + ' - it was : ' + str(minISSUE)) minISSUE = str(getiss) if int(getiss_num) < int(startiss_num): implog = implog + "issue now set to : " + str(getiss) + " ... it was : " + str(startISSUE) + "\n" + logger.fdebug('Start issue now set to : ' + str(getiss) + ' - it was : ' + str(startISSUE)) startISSUE = str(getiss) - - if helpers.issuedigits(startISSUE) == 1000: # if it's an issue #1, get the year and assume that's the start. - starttheyear = result['ComicYear'] + if helpers.issuedigits(startISSUE) == 1000: # if it's an issue #1, get the year and assume that's the start. + startyear = result['ComicYear'] #taking this outside of the transaction in an attempt to stop db locking. if mylar.IMP_MOVE and movealreadyonlist == "yes": @@ -2826,8 +2908,12 @@ class WebInterface(object): #figure out # of issues and the year range allowable if starttheyear is None: if yearTOP > 0: - maxyear = int(yearTOP) - (int(minISSUE) / 12) - yearRANGE.append(str(maxyear)) + if helpers.int_num(minISSUE) < 1000: + maxyear = int(yearTOP) + else: + maxyear = int(yearTOP) - (int(minISSUE) / 12) + if str(maxyear) not in yearRANGE: + yearRANGE.append(str(maxyear)) implog = implog + "there is a " + str(maxyear) + " year variation based on the 12 issues/year\n" else: implog = implog + "no year detected in any issues...Nulling the value\n" @@ -2838,7 +2924,7 @@ class WebInterface(object): #determine a best-guess to # of issues in series #this needs to be reworked / refined ALOT more. #minISSUE = highest issue #, startISSUE = lowest issue # - numissues = helpers.int_num(minISSUE) - helpers.int_num(startISSUE) + numissues = helpers.int_num(minISSUE) - helpers.int_num(startISSUE) +1 # add 1 to account for one issue itself. #normally minissue would work if the issue #'s started at #1. implog = implog + "the years involved are : " + str(yearRANGE) + "\n" implog = implog + "highest issue # is : " + str(minISSUE) + "\n" @@ -2851,7 +2937,6 @@ class WebInterface(object): #cnvers = cnsplit[cnwords-1] ogcname = ComicName for splitt in cnsplit: - print ("split") if 'v' in str(splitt): implog = implog + "possible versioning detected.\n" if splitt[1:].isdigit(): @@ -2863,8 +2948,8 @@ class WebInterface(object): mode='series' displaycomic = helpers.filesafe(ComicName) - logger.info('displaycomic : ' + displaycomic) - logger.info('comicname : ' + ComicName) + logger.fdebug('displaycomic : ' + displaycomic) + logger.fdebug('comicname : ' + ComicName) if yearRANGE is None: sresults, explicit = mb.findComic(displaycomic, mode, issue=numissues, explicit='all') #ogcname, mode, issue=numissues, explicit='all') #ComicName, mode, issue=numissues) else: @@ -2874,55 +2959,80 @@ class WebInterface(object): if len(sresults) == 1: sr = sresults[0] implog = implog + "only one result...automagik-mode enabled for " + displaycomic + " :: " + str(sr['comicid']) + "\n" + logger.fdebug("only one result...automagik-mode enabled for " + displaycomic + " :: " + str(sr['comicid'])) resultset = 1 # #need to move the files here. elif len(sresults) == 0 or len(sresults) is None: implog = implog + "no results, removing the year from the agenda and re-querying.\n" + logger.fdebug("no results, removing the year from the agenda and re-querying.") sresults, explicit = mb.findComic(ogcname, mode, issue=numissues, explicit='all') #ComicName, mode, issue=numissues) if len(sresults) == 1: sr = sresults[0] implog = implog + "only one result...automagik-mode enabled for " + displaycomic + " :: " + str(sr['comicid']) + "\n" + logger.fdebug("only one result...automagik-mode enabled for " + displaycomic + " :: " + str(sr['comicid'])) resultset = 1 else: resultset = 0 else: implog = implog + "returning results to screen - more than one possibility.\n" + logger.fdebug("Returning results to Select option - more than one possibility, manual intervention required.") resultset = 0 - #write implog to db here. - print "Writing import log to db for viewing pleasure." - ctrlVal = {"ComicName": ComicName} - newVal = {"implog": implog} - myDB.upsert("importresults", newVal, ctrlVal) + #generate random Search Results ID to allow for easier access for viewing logs / search results. + import random + SRID = str(random.randint(100000,999999)) - logger.info('sresults: ' + str(sresults)) + #write implog to db here. + ctrlVal = {"ComicName": ogcname} #{"ComicName": ComicName} + newVal = {"implog": implog, + "SRID": SRID} + myDB.upsert("importresults", newVal, ctrlVal) # store the search results for series that returned more than one result for user to select later / when they want. # should probably assign some random numeric for an id to reference back at some point. - cVal = {"Series": ComicName} for sr in sresults: - nVal = {"results": len(sresults), + cVal = {"SRID": SRID, + "comicid": sr['comicid']} + #should store ogcname in here somewhere to account for naming conversions above. + nVal = {"Series": ComicName, + "results": len(sresults), "publisher": sr['publisher'], "haveit": sr['haveit'], "name": sr['name'], "deck": sr['deck'], "url": sr['url'], "description": sr['description'], - "comicid": sr['comicid'], "comicimage": sr['comicimage'], "issues": sr['issues'], "comicyear": sr['comicyear']} myDB.upsert("searchresults", nVal, cVal) if resultset == 1: + self.addbyid(sr['comicid'], calledby=True, imported='yes', ogcname=ogcname) #implog = implog + "ogcname -- " + str(ogcname) + "\n" - cresults = self.addComic(comicid=sr['comicid'],comicname=sr['name'],comicyear=sr['comicyear'],comicpublisher=sr['publisher'],comicimage=sr['comicimage'],comicissues=sr['issues'],imported='yes',ogcname=ogcname) #imported=comicstoIMP,ogcname=ogcname) - return serve_template(templatename="searchfix.html", title="Error Check", comicname=sr['name'], comicid=sr['comicid'], comicyear=sr['comicyear'], comicimage=sr['comicimage'], comicissues=sr['issues'], cresults=cresults, imported='yes', ogcname=str(ogcname)) + #cresults = self.addComic(comicid=sr['comicid'],comicname=sr['name'],comicyear=sr['comicyear'],comicpublisher=sr['publisher'],comicimage=sr['comicimage'],comicissues=sr['issues'],imported='yes',ogcname=ogcname) #imported=comicstoIMP,ogcname=ogcname) + #return serve_template(templatename="searchfix.html", title="Error Check", comicname=sr['name'], comicid=sr['comicid'], comicyear=sr['comicyear'], comicimage=sr['comicimage'], comicissues=sr['issues'], cresults=cresults, imported='yes', ogcname=str(ogcname)) #else: # return serve_template(templatename="searchresults.html", title='Import Results for: "' + displaycomic + '"',searchresults=sresults, type=type, imported='yes', ogcname=ogcname, name=ogcname, explicit=explicit, serinfo=None) #imported=comicstoIMP, ogcname=ogcname) + #status update. + ctrlVal = {"ComicName": ComicName} + newVal = {"Status": 'Imported', + "SRID": SRID, + "ComicID": sr['comicid']} + myDB.upsert("importresults", newVal, ctrlVal) preSearchit.exposed = True + def importresults_popup(self,SRID,ComicName,imported=None,ogcname=None): + myDB = db.DBConnection() + results = myDB.select("SELECT * FROM searchresults WHERE SRID=?", [SRID]) + if results: + return serve_template(templatename="importresults_popup.html", title="results", searchtext=ComicName, searchresults=results) + else: + logger.warn('There are no search results to view for this entry ' + ComicName + ' [' + str(SRID) + ']. Something is probably wrong.') + return + importresults_popup.exposed = True + def pretty_git(self, br_history): #in order to 'prettify' the history log for display, we need to break it down so it's line by line. br_split = br_history.split("\n") #split it on each commit @@ -2962,7 +3072,6 @@ class WebInterface(object): pretty_git.exposed = True #--- def config(self): - interface_dir = os.path.join(mylar.PROG_DIR, 'data/interfaces/') interface_list = [ name for name in os.listdir(interface_dir) if os.path.isdir(os.path.join(interface_dir, name)) ] #---- @@ -3058,7 +3167,7 @@ class WebInterface(object): "enable_kat" : helpers.checked(mylar.ENABLE_KAT), "enable_cbt" : helpers.checked(mylar.ENABLE_CBT), "cbt_passkey" : mylar.CBT_PASSKEY, - "snatchedtorrent_notify" : mylar.SNATCHEDTORRENT_NOTIFY, + "snatchedtorrent_notify" : helpers.checked(mylar.SNATCHEDTORRENT_NOTIFY), "destination_dir" : mylar.DESTINATION_DIR, "create_folders" : helpers.checked(mylar.CREATE_FOLDERS), "chmod_dir" : mylar.CHMOD_DIR, @@ -3070,6 +3179,7 @@ class WebInterface(object): "use_maxsize" : helpers.checked(mylar.USE_MAXSIZE), "maxsize" : mylar.MAXSIZE, "interface_list" : interface_list, + "dupeconstraint" : mylar.DUPECONSTRAINT, "autowant_all" : helpers.checked(mylar.AUTOWANT_ALL), "autowant_upcoming" : helpers.checked(mylar.AUTOWANT_UPCOMING), "comic_cover_local" : helpers.checked(mylar.COMIC_COVER_LOCAL), @@ -3278,14 +3388,18 @@ class WebInterface(object): raise cherrypy.HTTPRedirect("comicDetails?ComicID=%s" % ComicID) comic_config.exposed = True - def readOptions(self, StoryArcID=None, StoryArcName=None, read2filename=0, storyarcdir=0): + def readOptions(self, StoryArcID=None, StoryArcName=None, read2filename=0, storyarcdir=0, copy2arcdir=0): + print 'initial' print mylar.READ2FILENAME print mylar.STORYARCDIR + print mylar.COPY2ARCDIR mylar.READ2FILENAME = int(read2filename) mylar.STORYARCDIR = int(storyarcdir) + mylar.COPY2ARCDIR = int(copy2arcdir) + print 'after int' print mylar.READ2FILENAME print mylar.STORYARCDIR - + print mylar.COPY2ARCDIR mylar.config_write() #force the check/creation of directory com_location here @@ -3313,7 +3427,7 @@ class WebInterface(object): prowl_enabled=0, prowl_onsnatch=0, prowl_keys=None, prowl_priority=None, nma_enabled=0, nma_apikey=None, nma_priority=0, nma_onsnatch=0, pushover_enabled=0, pushover_onsnatch=0, pushover_apikey=None, pushover_userkey=None, pushover_priority=None, boxcar_enabled=0, boxcar_onsnatch=0, boxcar_token=None, pushbullet_enabled=0, pushbullet_apikey=None, pushbullet_deviceid=None, pushbullet_onsnatch=0, preferred_quality=0, move_files=0, rename_files=0, add_to_csv=1, cvinfo=0, lowercase_filenames=0, folder_format=None, file_format=None, enable_extra_scripts=0, extra_scripts=None, enable_pre_scripts=0, pre_scripts=None, post_processing=0, syno_fix=0, search_delay=None, chmod_dir=0777, chmod_file=0660, cvapifix=0, - tsab=None, destination_dir=None, create_folders=1, replace_spaces=0, replace_char=None, use_minsize=0, minsize=None, use_maxsize=0, maxsize=None, autowant_all=0, autowant_upcoming=0, comic_cover_local=0, zero_level=0, zero_level_n=None, interface=None, **kwargs): + tsab=None, destination_dir=None, create_folders=1, replace_spaces=0, replace_char=None, use_minsize=0, minsize=None, use_maxsize=0, maxsize=None, autowant_all=0, autowant_upcoming=0, comic_cover_local=0, zero_level=0, zero_level_n=None, interface=None, dupeconstraint=None, **kwargs): mylar.COMICVINE_API = comicvine_api mylar.HTTP_HOST = http_host mylar.HTTP_PORT = http_port @@ -3431,6 +3545,7 @@ class WebInterface(object): mylar.AUTOWANT_UPCOMING = autowant_upcoming mylar.COMIC_COVER_LOCAL = comic_cover_local mylar.INTERFACE = interface + mylar.DUPECONSTRAINT = dupeconstraint mylar.ENABLE_EXTRA_SCRIPTS = enable_extra_scripts mylar.EXTRA_SCRIPTS = extra_scripts mylar.ENABLE_PRE_SCRIPTS = enable_pre_scripts