mylar/mylar/helpers.py

#  This file is part of Mylar.
# -*- coding: utf-8 -*-
#
#  Mylar is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  Mylar is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with Mylar.  If not, see <http://www.gnu.org/licenses/>.

import time
from operator import itemgetter
import datetime
from datetime import timedelta, date
import subprocess
import requests
import shlex
import Queue
import json
import re
import sys
import ctypes
import platform
import calendar
import itertools
import shutil
import hashlib
import gzip
import os, errno
from StringIO import StringIO
from apscheduler.triggers.interval import IntervalTrigger

import mylar
import logger
from mylar import db, sabnzbd, nzbget, process, getcomics

def multikeysort(items, columns):

    comparers = [((itemgetter(col[1:].strip()), -1) if col.startswith('-') else (itemgetter(col.strip()), 1)) for col in columns]

    def comparer(left, right):
        for fn, mult in comparers:
            result = cmp(fn(left), fn(right))
            if result:
                return mult * result
        else:
            return 0

    return sorted(items, cmp=comparer)

def checked(variable):
    if variable:
        return 'Checked'
    else:
        return ''

def radio(variable, pos):

    if variable == pos:
        return 'Checked'
    else:
        return ''

def latinToAscii(unicrap):
    """
    From couch potato
    """
    xlate = {0xc0: 'A', 0xc1: 'A', 0xc2: 'A', 0xc3: 'A', 0xc4: 'A', 0xc5: 'A',
        0xc6: 'Ae', 0xc7: 'C',
        0xc8: 'E', 0xc9: 'E', 0xca: 'E', 0xcb: 'E', 0x86: 'e',
        0xcc: 'I', 0xcd: 'I', 0xce: 'I', 0xcf: 'I',
        0xd0: 'Th', 0xd1: 'N',
        0xd2: 'O', 0xd3: 'O', 0xd4: 'O', 0xd5: 'O', 0xd6: 'O', 0xd8: 'O',
        0xd9: 'U', 0xda: 'U', 0xdb: 'U', 0xdc: 'U',
        0xdd: 'Y', 0xde: 'th', 0xdf: 'ss',
        0xe0: 'a', 0xe1: 'a', 0xe2: 'a', 0xe3: 'a', 0xe4: 'a', 0xe5: 'a',
        0xe6: 'ae', 0xe7: 'c',
        0xe8: 'e', 0xe9: 'e', 0xea: 'e', 0xeb: 'e', 0x0259: 'e',
        0xec: 'i', 0xed: 'i', 0xee: 'i', 0xef: 'i',
        0xf0: 'th', 0xf1: 'n',
        0xf2: 'o', 0xf3: 'o', 0xf4: 'o', 0xf5: 'o', 0xf6: 'o', 0xf8: 'o',
        0xf9: 'u', 0xfa: 'u', 0xfb: 'u', 0xfc: 'u',
        0xfd: 'y', 0xfe: 'th', 0xff: 'y',
        0xa1: '!', 0xa2: '{cent}', 0xa3: '{pound}', 0xa4: '{currency}',
        0xa5: '{yen}', 0xa6: '|', 0xa7: '{section}', 0xa8: '{umlaut}',
        0xa9: '{C}', 0xaa: '{^a}', 0xab: '<<', 0xac: '{not}',
        0xad: '-', 0xae: '{R}', 0xaf: '_', 0xb0: '{degrees}',
        0xb1: '{+/-}', 0xb2: '{^2}', 0xb3: '{^3}', 0xb4: "'",
        0xb5: '{micro}', 0xb6: '{paragraph}', 0xb7: '*', 0xb8: '{cedilla}',
        0xb9: '{^1}', 0xba: '{^o}', 0xbb: '>>',
        0xbc: '{1/4}', 0xbd: '{1/2}', 0xbe: '{3/4}', 0xbf: '?',
        0xd7: '*', 0xf7: '/'
        }

    r = ''
    for i in unicrap:
        if xlate.has_key(ord(i)):
            r += xlate[ord(i)]
        elif ord(i) >= 0x80:
            pass
        else:
            r += str(i)
    return r

def convert_milliseconds(ms):

    seconds = ms /1000
    gmtime = time.gmtime(seconds)
    if seconds > 3600:
        minutes = time.strftime("%H:%M:%S", gmtime)
    else:
        minutes = time.strftime("%M:%S", gmtime)

    return minutes

def convert_seconds(s):

    gmtime = time.gmtime(s)
    if s > 3600:
        minutes = time.strftime("%H:%M:%S", gmtime)
    else:
        minutes = time.strftime("%M:%S", gmtime)

    return minutes

def today():
    today = datetime.date.today()
    yyyymmdd = datetime.date.isoformat(today)
    return yyyymmdd

def now():
    now = datetime.datetime.now()
    return now.strftime("%Y-%m-%d %H:%M:%S")

def utctimestamp():
    return time.time()

def bytes_to_mb(bytes):

    mb = int(bytes) /1048576
    size = '%.1f MB' % mb
    return size

def human_size(size_bytes):
    """
    format a size in bytes into a 'human' file size, e.g. bytes, KB, MB, GB, TB, PB
    Note that bytes/KB will be reported in whole numbers but MB and above will have greater precision
    e.g. 1 byte, 43 bytes, 443 KB, 4.3 MB, 4.43 GB, etc
    """
    if size_bytes == 1:
        # because I really hate unnecessary plurals
        return "1 byte"

    suffixes_table = [('bytes', 0), ('KB', 0), ('MB', 1), ('GB', 2), ('TB', 2), ('PB', 2)]

    num = float(0 if size_bytes is None else size_bytes)
    for suffix, precision in suffixes_table:
        if num < 1024.0:
            break
        num /= 1024.0

    if precision == 0:
        formatted_size = "%d" % num
    else:
        formatted_size = str(round(num, ndigits=precision))

    return "%s %s" % (formatted_size, suffix)

def human2bytes(s):
    """
    >>> human2bytes('1M')
    1048576
    >>> human2bytes('1G')
    1073741824
    """
    symbols = ('B', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
    letter = s[-1:].strip().upper()
    num = re.sub(',', '', s[:-1])
    #assert num.isdigit() and letter in symbols
    #use below assert statement to handle sizes with decimal places
    if num != '0':
        assert float(num) and letter in symbols
        num = float(num)
        prefix = {symbols[0]: 1}
        for i, s in enumerate(symbols[1:]):
            prefix[s] = 1 << (i +1) *10
        return int(num * prefix[letter])
    else:
        return 0

def replace_all(text, dic):
    for i, j in dic.iteritems():
        text = text.replace(i, j)
    return text.rstrip()

def cleanName(string):

    pass1 = latinToAscii(string).lower()
    out_string = re.sub('[\/\@\#\$\%\^\*\+\"\[\]\{\}\<\>\=\_]', ' ', pass1).encode('utf-8')

    return out_string

def cleanTitle(title):

    title = re.sub('[\.\-\/\_]', ' ', title).lower()

    # Strip out extra whitespace
    title = ' '.join(title.split())

    title = title.title()

    return title

def extract_logline(s):
    # Default log format
    pattern = re.compile(r'(?P<timestamp>.*?)\s\-\s(?P<level>.*?)\s*\:\:\s(?P<thread>.*?)\s\:\s(?P<message>.*)', re.VERBOSE)
    match = pattern.match(s)
    if match:
        timestamp = match.group("timestamp")
        level = match.group("level")
        thread = match.group("thread")
        message = match.group("message")
        return (timestamp, level, thread, message)
    else:
        return None

def is_number(s):
    try:
        float(s)
    except (ValueError, TypeError):
        return False
    else:
        return True

def decimal_issue(iss):
    iss_find = iss.find('.')
    dec_except = None
    if iss_find == -1:
        #no matches for a decimal, assume we're converting from decimal to int.
        #match for special issues with alphanumeric numbering...
        if 'au' in iss.lower():
            dec_except = 'AU'
            decex = iss.lower().find('au')
            deciss = int(iss[:decex]) * 1000
        else:
            deciss = int(iss) * 1000
    else:
        iss_b4dec = iss[:iss_find]
        iss_decval = iss[iss_find +1:]
        if int(iss_decval) == 0:
            iss = iss_b4dec
            issdec = int(iss_decval)
        else:
            if len(iss_decval) == 1:
                iss = iss_b4dec + "." + iss_decval
                issdec = int(iss_decval) * 10
            else:
                iss = iss_b4dec + "." + iss_decval.rstrip('0')
                issdec = int(iss_decval.rstrip('0')) * 10
        deciss = (int(iss_b4dec) * 1000) + issdec
    return deciss, dec_except

def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=None, annualize=None, arc=False):
            #import db
            myDB = db.DBConnection()
            comicid = str(comicid)   # it's coming in unicoded...

            logger.fdebug(type(comicid))
            logger.fdebug(type(issueid))
            logger.fdebug('comicid: %s' % comicid)
            logger.fdebug('issue# as per cv: %s' % issue)
            # the issue here is a non-decimalized version, we need to see if it's got a decimal and if not, add '.00'
#            iss_find = issue.find('.')
#            if iss_find < 0:
#                # no decimal in issue number
#                iss = str(int(issue)) + ".00"
#            else:
#                iss_b4dec = issue[:iss_find]
#                iss_decval = issue[iss_find+1:]
#                if len(str(int(iss_decval))) == 1:
#                    iss = str(int(iss_b4dec)) + "." + str(int(iss_decval)*10)
#                else:
#                    if issue.endswith(".00"):
#                        iss = issue
#                    else:
#                        iss = str(int(iss_b4dec)) + "." + iss_decval
#            issue = iss

#            print ("converted issue#: " + str(issue))
            logger.fdebug('issueid:' + str(issueid))

            if issueid is None:
                logger.fdebug('annualize is ' + str(annualize))
                if arc:
                    #this has to be adjusted to be able to include story arc issues that span multiple arcs
                    chkissue = myDB.selectone("SELECT * from storyarcs WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone()
                else:
                    chkissue = myDB.selectone("SELECT * from issues WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone()
                    if all([chkissue is None, annualize is None, not mylar.CONFIG.ANNUALS_ON]):
                        chkissue = myDB.selectone("SELECT * from annuals WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone()

                if chkissue is None:
                    #rechk chkissue against int value of issue #
                    if arc:
                        chkissue = myDB.selectone("SELECT * from storyarcs WHERE ComicID=? AND Int_IssueNumber=?", [comicid, issuedigits(issue)]).fetchone()
                    else:
                        chkissue = myDB.selectone("SELECT * from issues WHERE ComicID=? AND Int_IssueNumber=?", [comicid, issuedigits(issue)]).fetchone()
                        if all([chkissue is None, annualize == 'yes', mylar.CONFIG.ANNUALS_ON]):
                            chkissue = myDB.selectone("SELECT * from annuals WHERE ComicID=? AND Int_IssueNumber=?", [comicid, issuedigits(issue)]).fetchone()

                    if chkissue is None:
                        logger.error('Invalid Issue_Number - please validate.')
                        return
                    else:
                        logger.info('Int Issue_number compare found. continuing...')
                        issueid = chkissue['IssueID']
                else:
                    issueid = chkissue['IssueID']

            #use issueid to get publisher, series, year, issue number
            logger.fdebug('issueid is now : ' + str(issueid))
            if arc:
                issuenzb = myDB.selectone("SELECT * from storyarcs WHERE ComicID=? AND IssueID=? AND StoryArc=?", [comicid, issueid, arc]).fetchone()
            else:
                issuenzb = myDB.selectone("SELECT * from issues WHERE ComicID=? AND IssueID=?", [comicid, issueid]).fetchone()
                if issuenzb is None:
                    logger.fdebug('not an issue, checking against annuals')
                    issuenzb = myDB.selectone("SELECT * from annuals WHERE ComicID=? AND IssueID=?", [comicid, issueid]).fetchone()
                    if issuenzb is None:
                        logger.fdebug('Unable to rename - cannot locate issue id within db')
                        return
                    else:
                        annualize = True

            if issuenzb is None:
                logger.fdebug('Unable to rename - cannot locate issue id within db')
                return

            #remap the variables to a common factor.
            if arc:
                issuenum = issuenzb['IssueNumber']
                issuedate = issuenzb['IssueDate']
                publisher = issuenzb['IssuePublisher']
                series = issuenzb['ComicName']
                seriesfilename = series   #Alternate FileNaming is not available with story arcs.
                seriesyear = issuenzb['SeriesYear']
                arcdir = filesafe(issuenzb['StoryArc'])
                if mylar.CONFIG.REPLACE_SPACES:
                    arcdir = arcdir.replace(' ', mylar.CONFIG.REPLACE_CHAR)
                if mylar.CONFIG.STORYARCDIR:
                    storyarcd = os.path.join(mylar.CONFIG.DESTINATION_DIR, "StoryArcs", arcdir)
                    logger.fdebug('Story Arc Directory set to : ' + storyarcd)
                else:
                    logger.fdebug('Story Arc Directory set to : ' + mylar.CONFIG.GRABBAG_DIR)
                    storyarcd = os.path.join(mylar.CONFIG.DESTINATION_DIR, mylar.CONFIG.GRABBAG_DIR)

                comlocation = storyarcd
                comversion = None   #need to populate this.

            else:
                issuenum = issuenzb['Issue_Number']
                issuedate = issuenzb['IssueDate']
                comicnzb= myDB.selectone("SELECT * from comics WHERE comicid=?", [comicid]).fetchone()
                publisher = comicnzb['ComicPublisher']
                series = comicnzb['ComicName']
                if comicnzb['AlternateFileName'] is None or comicnzb['AlternateFileName'] == 'None':
                    seriesfilename = series
                else:
                    seriesfilename = comicnzb['AlternateFileName']
                    logger.fdebug('Alternate File Naming has been enabled for this series. Will rename series title to : ' + seriesfilename)
                seriesyear = comicnzb['ComicYear']
                comlocation = comicnzb['ComicLocation']
                comversion = comicnzb['ComicVersion']

            unicodeissue = issuenum

            if type(issuenum) == unicode:
               vals = {u'\xbd':'.5',u'\xbc':'.25',u'\xbe':'.75',u'\u221e':'9999999999',u'\xe2':'9999999999'}
            else:
               vals = {'\xbd':'.5','\xbc':'.25','\xbe':'.75','\u221e':'9999999999','\xe2':'9999999999'}
            x = [vals[key] for key in vals if key in issuenum]
            if x:
                issuenum = x[0]
                logger.fdebug('issue number formatted: %s' % issuenum)

            #comicid = issuenzb['ComicID']
            #issueno = str(issuenum).split('.')[0]
            issue_except = 'None'
            issue_exceptions = ['AU',
                                'INH',
                                'NOW',
                                'AI',
                                'MU',
                                'HU',
                                'A',
                                'B',
                                'C',
                                'X',
                                'O']
            valid_spaces = ('.', '-')
            for issexcept in issue_exceptions:
                if issexcept.lower() in issuenum.lower():
                    logger.fdebug('ALPHANUMERIC EXCEPTION : [' + issexcept + ']')
                    v_chk = [v for v in valid_spaces if v in issuenum]
                    if v_chk:
                        iss_space = v_chk[0]
                        logger.fdebug('character space denoted as : ' + iss_space)
                    else:
                        logger.fdebug('character space not denoted.')
                        iss_space = ''
#                    if issexcept == 'INH':
#                       issue_except = '.INH'
                    if issexcept == 'NOW':
                       if '!' in issuenum: issuenum = re.sub('\!', '', issuenum)
#                       issue_except = '.NOW'

                    issue_except = iss_space + issexcept
                    logger.fdebug('issue_except denoted as : ' + issue_except)
                    issuenum = re.sub("[^0-9]", "", issuenum)
                    break

#            if 'au' in issuenum.lower() and issuenum[:1].isdigit():
#                issue_except = ' AU'
#            elif 'ai' in issuenum.lower() and issuenum[:1].isdigit():
#                issuenum = re.sub("[^0-9]", "", issuenum)
#                issue_except = ' AI'
#            elif 'inh' in issuenum.lower() and issuenum[:1].isdigit():
#                issuenum = re.sub("[^0-9]", "", issuenum)
#                issue_except = '.INH'
#            elif 'now' in issuenum.lower() and issuenum[:1].isdigit():
#                if '!' in issuenum: issuenum = re.sub('\!', '', issuenum)
#                issuenum = re.sub("[^0-9]", "", issuenum)
#                issue_except = '.NOW'
            if '.' in issuenum:
                iss_find = issuenum.find('.')
                iss_b4dec = issuenum[:iss_find]
                if iss_find == 0:
                    iss_b4dec = '0'
                iss_decval = issuenum[iss_find +1:]
                if iss_decval.endswith('.'):
                    iss_decval = iss_decval[:-1]
                if int(iss_decval) == 0:
                    iss = iss_b4dec
                    issdec = int(iss_decval)
                    issueno = iss
                else:
                    if len(iss_decval) == 1:
                        iss = iss_b4dec + "." + iss_decval
                        issdec = int(iss_decval) * 10
                    else:
                        iss = iss_b4dec + "." + iss_decval.rstrip('0')
                        issdec = int(iss_decval.rstrip('0')) * 10
                    issueno = iss_b4dec
            else:
                iss = issuenum
                issueno = iss
            # issue zero-suppression here
            if mylar.CONFIG.ZERO_LEVEL == "0":
                zeroadd = ""
            else:
                if mylar.CONFIG.ZERO_LEVEL_N  == "none": zeroadd = ""
                elif mylar.CONFIG.ZERO_LEVEL_N == "0x": zeroadd = "0"
                elif mylar.CONFIG.ZERO_LEVEL_N == "00x": zeroadd = "00"

            logger.fdebug('Zero Suppression set to : ' + str(mylar.CONFIG.ZERO_LEVEL_N))
            prettycomiss = None

            if issueno.isalpha():
                logger.fdebug('issue detected as an alpha.')
                prettycomiss = str(issueno)
            else:
                try:
                    x = float(issuenum)
                    #validity check
                    if x < 0:
                        logger.info('I\'ve encountered a negative issue #: %s. Trying to accomodate.' % issueno)
                        prettycomiss = '-' + str(zeroadd) + str(issueno[1:])
                    elif x == 9999999999:
                        logger.fdebug('Infinity issue found.')
                        issuenum = 'infinity'
                    elif x >= 0:
                        pass
                    else:
                        raise ValueError
                except ValueError, e:
                    logger.warn('Unable to properly determine issue number [ %s] - you should probably log this on github for help.' % issueno)
                    return

            if prettycomiss is None and len(str(issueno)) > 0:
                #if int(issueno) < 0:
                #    self._log("issue detected is a negative")
                #    prettycomiss = '-' + str(zeroadd) + str(abs(issueno))
                if int(issueno) < 10:
                    logger.fdebug('issue detected less than 10')
                    if '.' in iss:
                        if int(iss_decval) > 0:
                            issueno = str(iss)
                            prettycomiss = str(zeroadd) + str(iss)
                        else:
                            prettycomiss = str(zeroadd) + str(int(issueno))
                    else:
                        prettycomiss = str(zeroadd) + str(iss)
                    if issue_except != 'None':
                        prettycomiss = str(prettycomiss) + issue_except
                    logger.fdebug('Zero level supplement set to ' + str(mylar.CONFIG.ZERO_LEVEL_N) + '. Issue will be set as : ' + str(prettycomiss))
                elif int(issueno) >= 10 and int(issueno) < 100:
                    logger.fdebug('issue detected greater than 10, but less than 100')
                    if mylar.CONFIG.ZERO_LEVEL_N == "none":
                        zeroadd = ""
                    else:
                        zeroadd = "0"
                    if '.' in iss:
                        if int(iss_decval) > 0:
                            issueno = str(iss)
                            prettycomiss = str(zeroadd) + str(iss)
                        else:
                           prettycomiss = str(zeroadd) + str(int(issueno))
                    else:
                        prettycomiss = str(zeroadd) + str(iss)
                    if issue_except != 'None':
                        prettycomiss = str(prettycomiss) + issue_except
                    logger.fdebug('Zero level supplement set to ' + str(mylar.CONFIG.ZERO_LEVEL_N) + '.Issue will be set as : ' + str(prettycomiss))
                else:
                    logger.fdebug('issue detected greater than 100')
                    if issuenum == 'infinity':
                        prettycomiss = 'infinity'
                    else:
                        if '.' in iss:
                            if int(iss_decval) > 0:
                                issueno = str(iss)
                        prettycomiss = str(issueno)
                    if issue_except != 'None':
                        prettycomiss = str(prettycomiss) + issue_except
                    logger.fdebug('Zero level supplement set to ' + str(mylar.CONFIG.ZERO_LEVEL_N) + '. Issue will be set as : ' + str(prettycomiss))
            elif len(str(issueno)) == 0:
                prettycomiss = str(issueno)
                logger.fdebug('issue length error - cannot determine length. Defaulting to None:  ' + str(prettycomiss))

            logger.fdebug('Pretty Comic Issue is : ' + str(prettycomiss))
            if mylar.CONFIG.UNICODE_ISSUENUMBER:
                logger.fdebug('Setting this to Unicode format as requested: %s' % prettycomiss)
                prettycomiss = unicodeissue

            issueyear = issuedate[:4]
            month = issuedate[5:7].replace('-', '').strip()
            month_name = fullmonth(month)
            if month_name is None:
                month_name = 'None'
            logger.fdebug('Issue Year : ' + str(issueyear))
            logger.fdebug('Publisher: ' + publisher)
            logger.fdebug('Series: ' + series)
            logger.fdebug('Year: '  + str(seriesyear))
            logger.fdebug('Comic Location: ' + comlocation)
            if comversion is None:
                comversion = 'None'
            #if comversion is None, remove it so it doesn't populate with 'None'
            if comversion == 'None':
                chunk_f_f = re.sub('\$VolumeN', '', mylar.CONFIG.FILE_FORMAT)
                chunk_f = re.compile(r'\s+')
                chunk_file_format = chunk_f.sub(' ', chunk_f_f)
                logger.fdebug('No version # found for series, removing from filename')
                logger.fdebug("new format: " + str(chunk_file_format))
            else:
                chunk_file_format = mylar.CONFIG.FILE_FORMAT

            if annualize is None:
                chunk_f_f = re.sub('\$Annual', '', chunk_file_format)
                chunk_f = re.compile(r'\s+')
                chunk_file_format = chunk_f.sub(' ', chunk_f_f)
                logger.fdebug('not an annual - removing from filename paramaters')
                logger.fdebug('new format: ' + str(chunk_file_format))

            else:
                logger.fdebug('chunk_file_format is: ' + str(chunk_file_format))
                if mylar.CONFIG.ANNUALS_ON:
                    if 'annual' in series.lower():
                        if '$Annual' not in chunk_file_format: # and 'annual' not in ofilename.lower():
                        #if it's an annual, but $annual isn't specified in file_format, we need to
                        #force it in there, by default in the format of $Annual $Issue
                            #prettycomiss = "Annual " + str(prettycomiss)
                            logger.fdebug('[%s][ANNUALS-ON][ANNUAL IN SERIES][NO ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss))
                        else:
                            #because it exists within title, strip it then use formatting tag for placement of wording.
                            chunk_f_f = re.sub('\$Annual', '', chunk_file_format)
                            chunk_f = re.compile(r'\s+')
                            chunk_file_format = chunk_f.sub(' ', chunk_f_f)
                            logger.fdebug('[%s][ANNUALS-ON][ANNUAL IN SERIES][ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss))
                    else:
                        if '$Annual' not in chunk_file_format: # and 'annual' not in ofilename.lower():
                        #if it's an annual, but $annual isn't specified in file_format, we need to
                        #force it in there, by default in the format of $Annual $Issue
                            prettycomiss = "Annual %s" % prettycomiss
                            logger.fdebug('[%s][ANNUALS-ON][ANNUAL NOT IN SERIES][NO ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss))
                        else:
                            logger.fdebug('[%s][ANNUALS-ON][ANNUAL NOT IN SERIES][ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss))

                else:
                    #if annuals aren't enabled, then annuals are being tracked as independent series.
                    #annualize will be true since it's an annual in the seriesname.
                    if 'annual' in series.lower():
                        if '$Annual' not in chunk_file_format: # and 'annual' not in ofilename.lower():
                        #if it's an annual, but $annual isn't specified in file_format, we need to
                        #force it in there, by default in the format of $Annual $Issue
                            #prettycomiss = "Annual " + str(prettycomiss)
                            logger.fdebug('[%s][ANNUALS-OFF][ANNUAL IN SERIES][NO ANNUAL FORMAT] prettycomiss: %s' (series, prettycomiss))
                        else:
                            #because it exists within title, strip it then use formatting tag for placement of wording.
                            chunk_f_f = re.sub('\$Annual', '', chunk_file_format)
                            chunk_f = re.compile(r'\s+')
                            chunk_file_format = chunk_f.sub(' ', chunk_f_f)
                            logger.fdebug('[%s][ANNUALS-OFF][ANNUAL IN SERIES][ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss))
                    else:
                        if '$Annual' not in chunk_file_format: # and 'annual' not in ofilename.lower():
                            #if it's an annual, but $annual isn't specified in file_format, we need to
                            #force it in there, by default in the format of $Annual $Issue
                            prettycomiss = "Annual %s" % prettycomiss
                            logger.fdebug('[%s][ANNUALS-OFF][ANNUAL NOT IN SERIES][NO ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss))
                        else:
                            logger.fdebug('[%s][ANNUALS-OFF][ANNUAL NOT IN SERIES][ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss))


                    logger.fdebug('Annual detected within series title of ' + series + '. Not auto-correcting issue #')

            seriesfilename = seriesfilename.encode('ascii', 'ignore').strip()
            filebad = [':', ',', '/', '?', '!', '\'', '\"', '\*'] #in u_comicname or '/' in u_comicname or ',' in u_comicname or '?' in u_comicname:
            for dbd in filebad:
                if dbd in seriesfilename:
                    if any([dbd == '/', dbd == '*']): 
                        repthechar = '-'
                    else:
                        repthechar = ''
                    seriesfilename = seriesfilename.replace(dbd, repthechar)
                    logger.fdebug('Altering series name due to filenaming restrictions: ' + seriesfilename)

            publisher = re.sub('!', '', publisher)

            file_values = {'$Series':    seriesfilename,
                           '$Issue':     prettycomiss,
                           '$Year':      issueyear,
                           '$series':    series.lower(),
                           '$Publisher': publisher,
                           '$publisher': publisher.lower(),
                           '$VolumeY':   'V' + str(seriesyear),
                           '$VolumeN':   comversion,
                           '$monthname': month_name,
                           '$month':     month,
                           '$Annual':    'Annual'
                          }

            extensions = ('.cbr', '.cbz', '.cb7')

            if ofilename.lower().endswith(extensions):
                path, ext = os.path.splitext(ofilename)

            if mylar.CONFIG.FILE_FORMAT == '':
                logger.fdebug('Rename Files is not enabled - keeping original filename.')
                #check if extension is in nzb_name - will screw up otherwise
                if ofilename.lower().endswith(extensions):
                    nfilename = ofilename[:-4]
                else:
                    nfilename = ofilename
            else:
                nfilename = replace_all(chunk_file_format, file_values)
                if mylar.CONFIG.REPLACE_SPACES:
                    #mylar.CONFIG.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
                    nfilename = nfilename.replace(' ', mylar.CONFIG.REPLACE_CHAR)

            nfilename = re.sub('[\,\:]', '', nfilename) + ext.lower()
            logger.fdebug('New Filename: ' + nfilename)

            if mylar.CONFIG.LOWERCASE_FILENAMES:
                nfilename = nfilename.lower()
                dst = os.path.join(comlocation, nfilename)
            else:
                dst = os.path.join(comlocation, nfilename)

            logger.fdebug('Source: ' + ofilename)
            logger.fdebug('Destination: ' + dst)

            rename_this = {"destination_dir": dst,
                           "nfilename": nfilename,
                           "issueid": issueid,
                           "comicid": comicid}

            return rename_this


def apiremove(apistring, type):
    if type == 'nzb':
        value_regex = re.compile("(?<=apikey=)(?P<value>.*?)(?=$)")
        #match = value_regex.search(apistring)
        apiremoved = value_regex.sub("xUDONTNEEDTOKNOWTHISx", apistring)
    else:
        #type = $ to denote end of string
        #type = & to denote up until next api variable
        value_regex1 = re.compile("(?<=%26i=1%26r=)(?P<value>.*?)(?=" + str(type) +")")
        #match = value_regex.search(apistring)
        apiremoved1 = value_regex1.sub("xUDONTNEEDTOKNOWTHISx", apistring)
        value_regex = re.compile("(?<=apikey=)(?P<value>.*?)(?=" + str(type) +")")
        apiremoved = value_regex.sub("xUDONTNEEDTOKNOWTHISx", apiremoved1)

    #need to remove the urlencoded-portions as well in future
    return apiremoved

def remove_apikey(payd, key):
        #payload = some dictionary with payload values
        #key = the key to replace with REDACTED (normally apikey)
    for k,v in payd.items():
        payd[key] = 'REDACTED'

    return payd

def ComicSort(comicorder=None, sequence=None, imported=None):
    if sequence:
        # if it's on startup, load the sql into a tuple for use to avoid record-locking
        i = 0
        #import db
        myDB = db.DBConnection()
        comicsort = myDB.select("SELECT * FROM comics ORDER BY ComicSortName COLLATE NOCASE")
        comicorderlist = []
        comicorder = {}
        comicidlist = []
        if sequence == 'update':
            mylar.COMICSORT['SortOrder'] = None
            mylar.COMICSORT['LastOrderNo'] = None
            mylar.COMICSORT['LastOrderID'] = None
        for csort in comicsort:
            if csort['ComicID'] is None: pass
            if not csort['ComicID'] in comicidlist:
                if sequence == 'startup':
                    comicorderlist.append({
                         'ComicID':             csort['ComicID'],
                         'ComicOrder':           i
                         })
                elif sequence == 'update':
                    comicorderlist.append({
#                    mylar.COMICSORT['SortOrder'].append({
                         'ComicID':             csort['ComicID'],
                         'ComicOrder':           i
                         })

                comicidlist.append(csort['ComicID'])
                i+=1
        if sequence == 'startup':
            if i == 0:
                comicorder['SortOrder'] = ({'ComicID': '99999', 'ComicOrder': 1})
                comicorder['LastOrderNo'] = 1
                comicorder['LastOrderID'] = 99999
            else:
                comicorder['SortOrder'] = comicorderlist
                comicorder['LastOrderNo'] = i -1
                comicorder['LastOrderID'] = comicorder['SortOrder'][i -1]['ComicID']
            if i < 0: i == 0
            logger.info('Sucessfully ordered ' + str(i -1) + ' series in your watchlist.')
            return comicorder
        elif sequence == 'update':
            mylar.COMICSORT['SortOrder'] = comicorderlist
            #print ("i:" + str(i))
            if i == 0:
                placemnt = 1
            else:
                placemnt = int(i -1)
            mylar.COMICSORT['LastOrderNo'] = placemnt
            mylar.COMICSORT['LastOrderID'] = mylar.COMICSORT['SortOrder'][placemnt]['ComicID']
            return
    else:
        # for new series adds, we already know the comicid, so we set the sortorder to an abnormally high #
        # we DO NOT write to the db to avoid record-locking.
        # if we get 2 999's we're in trouble though.
        sortedapp = []
        if comicorder['LastOrderNo'] == '999':
            lastorderval = int(comicorder['LastOrderNo']) + 1
        else:
            lastorderval = 999
        sortedapp.append({
             'ComicID':             imported,
             'ComicOrder':           lastorderval
             })
        mylar.COMICSORT['SortOrder'] = sortedapp
        mylar.COMICSORT['LastOrderNo'] = lastorderval
        mylar.COMICSORT['LastOrderID'] = imported
        return

def fullmonth(monthno):
    #simple numerical to worded month conversion....
    basmonths = {'1': 'January', '2': 'February', '3': 'March', '4': 'April', '5': 'May', '6': 'June', '7': 'July', '8': 'August', '9': 'September', '10': 'October', '11': 'November', '12': 'December'}

    monthconv = None

    for numbs in basmonths:
        if int(numbs) == int(monthno):
            monthconv = basmonths[numbs]

    return monthconv

def updateComicLocation():
    #in order for this to work, the ComicLocation MUST be left at the original location.
    #in the config.ini - set LOCMOVE = 1  (to enable this to run on the NEXT startup)
    #                  - set NEWCOMDIR = new ComicLocation
    #after running, set ComicLocation to new location in Configuration GUI

    #import db
    myDB = db.DBConnection()
    if mylar.CONFIG.NEWCOM_DIR is not None:
        logger.info('Performing a one-time mass update to Comic Location')
        #create the root dir if it doesn't exist
        checkdirectory = mylar.filechecker.validateAndCreateDirectory(mylar.CONFIG.NEWCOM_DIR, create=True)
        if not checkdirectory:
            logger.warn('Error trying to validate/create directory. Aborting this process at this time.')
            return
        dirlist = myDB.select("SELECT * FROM comics")
        comloc = []

        if dirlist is not None:
            for dl in dirlist:

                u_comicnm = dl['ComicName']
                # let's remove the non-standard characters here that will break filenaming / searching.
                comicname_folder = filesafe(u_comicnm)

                publisher = re.sub('!', '', dl['ComicPublisher']) # thanks Boom!
                year = dl['ComicYear']

                if dl['Corrected_Type'] is not None:
                    booktype = dl['Corrected_Type']
                else:
                    booktype = dl['Type']
                if booktype == 'Print' or all([booktype != 'Print', mylar.CONFIG.FORMAT_BOOKTYPE is False]):
                    chunk_fb = re.sub('\$Type', '', mylar.CONFIG.FOLDER_FORMAT)
                    chunk_b = re.compile(r'\s+')
                    chunk_folder_format = chunk_b.sub(' ', chunk_fb)
                else:
                    chunk_folder_format = mylar.CONFIG.FOLDER_FORMAT

                comversion = dl['ComicVersion']
                if comversion is None:
                    comversion = 'None'
                #if comversion is None, remove it so it doesn't populate with 'None'
                if comversion == 'None':
                    chunk_f_f = re.sub('\$VolumeN', '', mylar.CONFIG.FOLDER_FORMAT)
                    chunk_f = re.compile(r'\s+')
                    folderformat = chunk_f.sub(' ', chunk_f_f)
                else:
                    folderformat = mylar.CONFIG.FOLDER_FORMAT

                #do work to generate folder path

                values = {'$Series':        comicname_folder,
                          '$Publisher':     publisher,
                          '$Year':          year,
                          '$series':        comicname_folder.lower(),
                          '$publisher':     publisher.lower(),
                          '$VolumeY':       'V' + str(year),
                          '$VolumeN':       comversion,
                          '$Annual':        'Annual',
                          '$Type':          booktype
                          }

                #set the paths here with the seperator removed allowing for cross-platform altering.
                ccdir = re.sub(r'[\\|/]', '%&', mylar.CONFIG.NEWCOM_DIR)
                ddir = re.sub(r'[\\|/]', '%&', mylar.CONFIG.DESTINATION_DIR)
                dlc = re.sub(r'[\\|/]', '%&', dl['ComicLocation'])

                if mylar.CONFIG.FFTONEWCOM_DIR:
                    #if this is enabled (1) it will apply the Folder_Format to all the new dirs
                    if mylar.CONFIG.FOLDER_FORMAT == '':
                        comlocation = re.sub(ddir, ccdir, dlc).strip()
                    else:
                        first = replace_all(folderformat, values)
                        if mylar.CONFIG.REPLACE_SPACES:
                            #mylar.CONFIG.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
                            first = first.replace(' ', mylar.CONFIG.REPLACE_CHAR)
                        comlocation = os.path.join(mylar.CONFIG.NEWCOM_DIR, first).strip()

                else:
                    #DESTINATION_DIR = /mnt/mediavg/Comics
                    #NEWCOM_DIR = /mnt/mediavg/Comics/Comics-1
                    #dl['ComicLocation'] = /mnt/mediavg/Comics/Batman-(2011)
                    comlocation = re.sub(ddir, ccdir, dlc).strip()

                #regenerate the new path location so that it's os.dependent now.
                com_done = re.sub('%&', os.sep.encode('unicode-escape'), comlocation).strip()

                comloc.append({"comlocation":  com_done,
                               "origlocation": dl['ComicLocation'],
                               "comicid":      dl['ComicID']})

            if len(comloc) > 0:
                #give the information about what we're doing.
                if mylar.CONFIG.FFTONEWCOM_DIR:
                    logger.info('FFTONEWCOM_DIR is enabled. Applying the existing folder format to ALL directories regardless of existing location paths')
                else:
                    logger.info('FFTONEWCOM_DIR is not enabled. I will keep existing subdirectory paths, and will only change the actual Comic Location in the path.')
                    logger.fdebug(' (ie. /mnt/Comics/Marvel/Hush-(2012) to /mnt/mynewLocation/Marvel/Hush-(2012) ')

                #do the deed.
                for cl in comloc:
                    ctrlVal = {"ComicID":      cl['comicid']}
                    newVal = {"ComicLocation": cl['comlocation']}
                    myDB.upsert("Comics", newVal, ctrlVal)
                    logger.fdebug('Updated : ' + cl['origlocation'] + ' .: TO :. ' + cl['comlocation'])
                logger.info('Updated ' + str(len(comloc)) + ' series to a new Comic Location as specified in the config.ini')
            else:
                logger.fdebug('Failed in updating the Comic Locations. Check Folder Format string and/or log the issue.')
        else:
            logger.info('There are no series in your watchlist to Update the locations. Not updating anything at this time.')
        #set the value to 0 here so we don't keep on doing this...
        mylar.CONFIG.LOCMOVE = 0
        #mylar.config_write()
    else:
        logger.info('No new ComicLocation path specified - not updating. Set NEWCOMD_DIR in config.ini')
        #raise cherrypy.HTTPRedirect("config")
    return

def cleanhtml(raw_html):
    #cleanr = re.compile('<.*?>')
    #cleantext = re.sub(cleanr, '', raw_html)
    #return cleantext
    from bs4 import BeautifulSoup

    VALID_TAGS = ['div', 'p']

    soup = BeautifulSoup(raw_html, "html.parser")

    for tag in soup.findAll('p'):
        if tag.name not in VALID_TAGS:
            tag.replaceWith(tag.renderContents())
    flipflop = soup.renderContents()
    print flipflop
    return flipflop


def issuedigits(issnum):
    #import db

    int_issnum = None

    try:
        tst = issnum.isdigit()
    except:
        try:
            isstest = str(issnum)
            tst = isstest.isdigit()
        except:
            return 9999999999
        else:
            issnum = str(issnum)

    if issnum.isdigit():
        int_issnum = int(issnum) * 1000
    else:
        #count = 0
        #for char in issnum:
        #    if char.isalpha():
        #        count += 1
        #if count > 5:
        #    logger.error('This is not an issue number - not enough numerics to parse')
        #    int_issnum = 999999999999999
        #    return int_issnum
        try:
            if 'au' in issnum.lower() and issnum[:1].isdigit():
                int_issnum = (int(issnum[:-2]) * 1000) + ord('a') + ord('u')
            elif 'ai' in issnum.lower() and issnum[:1].isdigit():
                int_issnum = (int(issnum[:-2]) * 1000) + ord('a') + ord('i')
            elif 'inh' in issnum.lower() or 'now' in issnum.lower():
                remdec = issnum.find('.')  #find the decimal position.
                if remdec == -1:
                #if no decimal, it's all one string
                #remove the last 3 characters from the issue # (INH)
                    int_issnum = (int(issnum[:-3]) * 1000) + ord('i') + ord('n') + ord('h')
                else:
                    int_issnum = (int(issnum[:-4]) * 1000) + ord('i') + ord('n') + ord('h')
            elif 'now' in issnum.lower():
                if '!' in issnum: issnum = re.sub('\!', '', issnum)
                remdec = issnum.find('.')  #find the decimal position.
                if remdec == -1:
                #if no decimal, it's all one string
                #remove the last 3 characters from the issue # (NOW)
                    int_issnum = (int(issnum[:-3]) * 1000) + ord('n') + ord('o') + ord('w')
                else:
                    int_issnum = (int(issnum[:-4]) * 1000) + ord('n') + ord('o') + ord('w')
            elif 'mu' in issnum.lower():
                remdec = issnum.find('.')
                if remdec == -1:
                    int_issnum = (int(issnum[:-2]) * 1000) + ord('m') + ord('u')
                else:
                    int_issnum = (int(issnum[:-3]) * 1000) + ord('m') + ord('u')
            elif 'hu' in issnum.lower():
                remdec = issnum.find('.')  #find the decimal position.
                if remdec == -1:
                    int_issnum = (int(issnum[:-2]) * 1000) + ord('h') + ord('u')
                else:
                    int_issnum = (int(issnum[:-3]) * 1000) + ord('h') + ord('u')

        except ValueError as e:
            logger.error('[' + issnum + '] Unable to properly determine the issue number. Error: %s', e)
            return 9999999999

        if int_issnum is not None:
            return int_issnum

        #try:
        #    issnum.decode('ascii')
        #    logger.fdebug('ascii character.')
        #except:
        #    logger.fdebug('Unicode character detected: ' + issnum)
        #else: issnum.decode(mylar.SYS_ENCODING).decode('utf-8')
        if type(issnum) == str:
            try:
                issnum = issnum.decode('utf-8')
            except:
                issnum = issnum.decode('windows-1252')

        if type(issnum) == unicode:
            vals = {u'\xbd':.5,u'\xbc':.25,u'\xbe':.75,u'\u221e':9999999999,u'\xe2':9999999999}
        else:
            vals = {'\xbd':.5,'\xbc':.25,'\xbe':.75,'\u221e':9999999999,'\xe2':9999999999}

        x = [vals[key] for key in vals if key in issnum]

        if x:
            chk = re.sub('[^0-9]', '', issnum).strip()
            if len(chk) == 0:
                int_issnum = x[0] * 1000
            else:
                int_issnum = (int(re.sub('[^0-9]', '', issnum).strip()) + x[0]) * 1000
            #logger.fdebug('int_issnum: ' + str(int_issnum))
        else:
            if any(['.' in issnum, ',' in issnum]):
                #logger.fdebug('decimal detected.')
                if ',' in issnum: issnum = re.sub(',', '.', issnum)
                issst = str(issnum).find('.')
                if issst == 0:
                    issb4dec = 0
                else:
                    issb4dec = str(issnum)[:issst]
                decis = str(issnum)[issst +1:]
                if len(decis) == 1:
                    decisval = int(decis) * 10
                    issaftdec = str(decisval)
                elif len(decis) == 2:
                    decisval = int(decis)
                    issaftdec = str(decisval)
                else:
                    decisval = decis
                    issaftdec = str(decisval)
                #if there's a trailing decimal (ie. 1.50.) and it's either intentional or not, blow it away.
                if issaftdec[-1:] == '.':
                    issaftdec = issaftdec[:-1]
                try:
                    int_issnum = (int(issb4dec) * 1000) + (int(issaftdec) * 10)
                except ValueError:
                    #logger.fdebug('This has no issue # for me to get - Either a Graphic Novel or one-shot.')
                    int_issnum = 999999999999999
            else:
                try:
                    x = float(issnum)
                    #logger.info(x)
                    #validity check
                    if x < 0:
                        #logger.info("I've encountered a negative issue #: " + str(issnum) + ". Trying to accomodate.")
                        int_issnum = (int(x) *1000) - 1
                    elif bool(x):
                        logger.fdebug('Infinity issue found.')
                        int_issnum = 9999999999 * 1000
                    else: raise ValueError
                except ValueError, e:
                    #this will account for any alpha in a issue#, so long as it doesn't have decimals.
                    x = 0
                    tstord = None
                    issno = None
                    invchk = "false"
                    if issnum.lower() != 'preview':
                        while (x < len(issnum)):
                            if issnum[x].isalpha():
                            #take first occurance of alpha in string and carry it through
                                tstord = issnum[x:].rstrip()
                                tstord = re.sub('[\-\,\.\+]', '', tstord).rstrip()
                                issno = issnum[:x].rstrip()
                                issno = re.sub('[\-\,\.\+]', '', issno).rstrip()
                                try:
                                    isschk = float(issno)
                                except ValueError, e:
                                    if len(issnum) == 1 and issnum.isalpha():
                                        break
                                    logger.fdebug('[' + issno + '] Invalid numeric for issue - cannot be found. Ignoring.')
                                    issno = None
                                    tstord = None
                                    invchk = "true"
                                break
                            x+=1
                    if tstord is not None and issno is not None:
                        a = 0
                        ordtot = 0
                        if len(issnum) == 1 and issnum.isalpha():
                            int_issnum = ord(tstord.lower())
                        else:
                            while (a < len(tstord)):
                                ordtot += ord(tstord[a].lower())  #lower-case the letters for simplicty
                                a+=1
                            int_issnum = (int(issno) * 1000) + ordtot
                    elif invchk == "true":
                        if any([issnum.lower() == 'fall', issnum.lower() == 'spring', issnum.lower() == 'summer', issnum.lower() == 'winter']):
                            inu = 0
                            ordtot = 0
                            while (inu < len(issnum)):
                                ordtot += ord(issnum[inu].lower())  #lower-case the letters for simplicty
                                inu+=1
                            int_issnum = ordtot
                        else:
                            logger.fdebug('this does not have an issue # that I can parse properly.')
                            return 999999999999999
                    else:
                        if issnum == '9-5':
                            issnum = u'9\xbd'
                            logger.fdebug('issue: 9-5 is an invalid entry. Correcting to : ' + issnum)
                            int_issnum = (9 * 1000) + (.5 * 1000)
                        elif issnum == '112/113':
                            int_issnum = (112 * 1000) + (.5 * 1000)
                        elif issnum == '14-16':
                            int_issnum = (15 * 1000) + (.5 * 1000)
                        elif issnum.lower() == 'preview':
                            inu = 0
                            ordtot = 0
                            while (inu < len(issnum)):
                                ordtot += ord(issnum[inu].lower())  #lower-case the letters for simplicty
                                inu+=1
                            int_issnum = ordtot
                        else:
                            logger.error(issnum + ' this has an alpha-numeric in the issue # which I cannot account for.')
                            return 999999999999999

    return int_issnum


def checkthepub(ComicID):
    #import db
    myDB = db.DBConnection()
    publishers = ['marvel', 'dc', 'darkhorse']
    pubchk = myDB.selectone("SELECT * FROM comics WHERE ComicID=?", [ComicID]).fetchone()
    if pubchk is None:
        logger.fdebug('No publisher information found to aid in determining series..defaulting to base check of 55 days.')
        return mylar.CONFIG.BIGGIE_PUB
    else:
        for publish in publishers:
            if publish in pubchk['ComicPublisher'].lower():
                #logger.fdebug('Biggie publisher detected - ' + pubchk['ComicPublisher'])
                return mylar.CONFIG.BIGGIE_PUB

        #logger.fdebug('Indie publisher detected - ' + pubchk['ComicPublisher'])
        return mylar.CONFIG.INDIE_PUB

def annual_update():
    #import db
    myDB = db.DBConnection()
    annuallist = myDB.select('SELECT * FROM annuals')
    if annuallist is None:
        logger.info('no annuals to update.')
        return

    cnames = []
    #populate the ComicName field with the corresponding series name from the comics table.
    for ann in annuallist:
        coms = myDB.selectone('SELECT * FROM comics WHERE ComicID=?', [ann['ComicID']]).fetchone()
        cnames.append({'ComicID':     ann['ComicID'],
                       'ComicName':   coms['ComicName']
                      })

    #write in a seperate loop to avoid db locks
    i=0
    for cns in cnames:
        ctrlVal = {"ComicID":      cns['ComicID']}
        newVal = {"ComicName":     cns['ComicName']}
        myDB.upsert("annuals", newVal, ctrlVal)
        i+=1

    logger.info(str(i) + ' series have been updated in the annuals table.')
    return

def replacetheslash(data):
    # this is necessary for the cache directory to display properly in IE/FF.
    # os.path.join will pipe in the '\' in windows, which won't resolve
    # when viewing through cherrypy - so convert it and viola.
    if platform.system() == "Windows":
        slashreplaced = data.replace('\\', '/')
    else:
        slashreplaced = data
    return slashreplaced

def urlretrieve(urlfile, fpath):
    chunk = 4096
    f = open(fpath, "w")
    while 1:
        data = urlfile.read(chunk)
        if not data:
            print "done."
            break
        f.write(data)
        print "Read %s bytes"%len(data)

def renamefile_readingorder(readorder):
    logger.fdebug('readingorder#: ' + str(readorder))
    if int(readorder) < 10: readord = "00" + str(readorder)
    elif int(readorder) >= 10 and int(readorder) < 99: readord = "0" + str(readorder)
    else: readord = str(readorder)

    return readord

def latestdate_fix():
    #import db
    datefix = []
    cnupdate = []
    myDB = db.DBConnection()
    comiclist = myDB.select('SELECT * FROM comics')
    if comiclist is None:
        logger.fdebug('No Series in watchlist to correct latest date')
        return
    for cl in comiclist:
        if cl['ComicName_Filesafe'] is None:
            cnupdate.append({"comicid":  cl['ComicID'],
                            "comicname_filesafe": filesafe(cl['ComicName'])})
        latestdate = cl['LatestDate']
        #logger.fdebug("latestdate:  " + str(latestdate))
        try:
            if latestdate[8:] == '':
                #logger.fdebug("invalid date " + str(latestdate) + " appending 01 for day to avoid errors")
                if len(latestdate) <= 7:
                    finddash = latestdate.find('-')
                    #logger.info('dash found at position ' + str(finddash))
                    if finddash != 4:  #format of mm-yyyy
                        lat_month = latestdate[:finddash]
                        lat_year = latestdate[finddash +1:]
                    else:  #format of yyyy-mm
                        lat_month = latestdate[finddash +1:]
                        lat_year = latestdate[:finddash]

                    latestdate = (lat_year) + '-' + str(lat_month) + '-01'
                    datefix.append({"comicid":    cl['ComicID'],
                                    "latestdate": latestdate})
                    #logger.info('latest date: ' + str(latestdate))
        except:
            datefix.append({"comicid":    cl['ComicID'],
                            "latestdate": '0000-00-00'})

    #now we fix.
    if len(datefix) > 0:
       logger.info('Preparing to correct/fix ' + str(len(datefix)) + ' series that have incorrect values given for the Latest Date field.')
       for df in datefix:
          newCtrl = {"ComicID":    df['comicid']}
          newVal = {"LatestDate":  df['latestdate']}
          myDB.upsert("comics", newVal, newCtrl)
    if len(cnupdate) > 0:
       logger.info('Preparing to update ' + str(len(cnupdate)) + ' series on your watchlist for use with non-ascii characters')
       for cn in cnupdate:
          newCtrl = {"ComicID":           cn['comicid']}
          newVal = {"ComicName_Filesafe": cn['comicname_filesafe']}
          myDB.upsert("comics", newVal, newCtrl)

    return

def upgrade_dynamic():
    #import db
    dynamic_comiclist = []
    myDB = db.DBConnection()
    #update the comicdb to include the Dynamic Names (and any futher changes as required)
    clist = myDB.select('SELECT * FROM Comics')
    for cl in clist:
        cl_d = mylar.filechecker.FileChecker(watchcomic=cl['ComicName'])
        cl_dyninfo = cl_d.dynamic_replace(cl['ComicName'])
        dynamic_comiclist.append({'DynamicComicName': re.sub('[\|\s]','', cl_dyninfo['mod_seriesname'].lower()).strip(),
                             'ComicID':          cl['ComicID']})

    if len(dynamic_comiclist) > 0:
        for dl in dynamic_comiclist:
            CtrlVal = {"ComicID": dl['ComicID']}
            newVal = {"DynamicComicName": dl['DynamicComicName']}
            myDB.upsert("Comics", newVal, CtrlVal)

    #update the storyarcsdb to include the Dynamic Names (and any futher changes as required)
    dynamic_storylist = []
    rlist = myDB.select('SELECT * FROM storyarcs WHERE StoryArcID is not NULL')
    for rl in rlist:
        rl_d = mylar.filechecker.FileChecker(watchcomic=rl['ComicName'])
        rl_dyninfo = cl_d.dynamic_replace(rl['ComicName'])
        dynamic_storylist.append({'DynamicComicName': re.sub('[\|\s]','', rl_dyninfo['mod_seriesname'].lower()).strip(),
                                  'IssueArcID':          rl['IssueArcID']})

    if len(dynamic_storylist) > 0:
        for ds in dynamic_storylist:
            CtrlVal = {"IssueArcID": ds['IssueArcID']}
            newVal = {"DynamicComicName": ds['DynamicComicName']}
            myDB.upsert("storyarcs", newVal, CtrlVal)   

    logger.info('Finished updating ' + str(len(dynamic_comiclist)) + ' / ' + str(len(dynamic_storylist)) + ' entries within the db.')
    mylar.CONFIG.DYNAMIC_UPDATE = 4
    mylar.CONFIG.writeconfig()
    return

def checkFolder(folderpath=None):
    from mylar import PostProcessor

    queue = Queue.Queue()
    #monitor a selected folder for 'snatched' files that haven't been processed
    if folderpath is None:
        logger.info('Checking folder ' + mylar.CONFIG.CHECK_FOLDER + ' for newly snatched downloads')
        path = mylar.CONFIG.CHECK_FOLDER
    else:
        logger.info('Submitted folder ' + folderpath + ' for direct folder post-processing')
        path = folderpath

    PostProcess = PostProcessor.PostProcessor('Manual Run', path, queue=queue)
    vals = PostProcess.Process()
    return

def LoadAlternateSearchNames(seriesname_alt, comicid):
    #seriesname_alt = db.comics['AlternateSearch']
    AS_Alt = []
    Alternate_Names = {}
    alt_count = 0

    #logger.fdebug('seriesname_alt:' + str(seriesname_alt))
    if seriesname_alt is None or seriesname_alt == 'None':
        return "no results"
    else:
        chkthealt = seriesname_alt.split('##')
        if chkthealt == 0:
            AS_Alternate = seriesname_alt
            AS_Alt.append(seriesname_alt)
        for calt in chkthealt:
            AS_Alter = re.sub('##', '', calt)
            u_altsearchcomic = AS_Alter.encode('ascii', 'ignore').strip()
            AS_formatrem_seriesname = re.sub('\s+', ' ', u_altsearchcomic)
            if AS_formatrem_seriesname[:1] == ' ': AS_formatrem_seriesname = AS_formatrem_seriesname[1:]

            AS_Alt.append({"AlternateName": AS_formatrem_seriesname})
            alt_count+=1

        Alternate_Names['AlternateName'] = AS_Alt
        Alternate_Names['ComicID'] = comicid
        Alternate_Names['Count'] = alt_count
        logger.info('AlternateNames returned:' + str(Alternate_Names))

        return Alternate_Names

def havetotals(refreshit=None):
        #import db

        comics = []
        myDB = db.DBConnection()

        if refreshit is None:
            if mylar.CONFIG.ANNUALS_ON:
                comiclist = myDB.select('SELECT comics.*, COUNT(totalAnnuals.IssueID) AS TotalAnnuals FROM comics LEFT JOIN annuals as totalAnnuals on totalAnnuals.ComicID = comics.ComicID GROUP BY comics.ComicID order by comics.ComicSortName COLLATE NOCASE')
            else:
                comiclist = myDB.select('SELECT * FROM comics GROUP BY ComicID order by ComicSortName COLLATE NOCASE')
        else:
            comiclist = []
            comicref = myDB.selectone('SELECT comics.ComicID AS ComicID, comics.Have AS Have, comics.Total as Total, COUNT(totalAnnuals.IssueID) AS TotalAnnuals FROM comics LEFT JOIN annuals as totalAnnuals on totalAnnuals.ComicID = comics.ComicID WHERE comics.ComicID=? GROUP BY comics.ComicID', [refreshit]).fetchone()
            #refreshit is the ComicID passed from the Refresh Series to force/check numerical have totals
            comiclist.append({"ComicID":      comicref['ComicID'],
                              "Have":         comicref['Have'],
                              "Total":        comicref['Total'],
                              "TotalAnnuals": comicref['TotalAnnuals']})

        for comic in comiclist:
            #--not sure about this part
            #if comic['Total'] is None:
            #    if refreshit is not None:
            #        logger.fdebug(str(comic['ComicID']) + ' has no issuedata available. Forcing complete Refresh/Rescan')
            #        return True
            #    else:
            #        continue
            try:
                totalissues = comic['Total']
#                if mylar.CONFIG.ANNUALS_ON:
#                    totalissues += comic['TotalAnnuals']
                haveissues = comic['Have']
            except TypeError:
                logger.warning('[Warning] ComicID: ' + str(comic['ComicID']) + ' is incomplete - Removing from DB. You should try to re-add the series.')
                myDB.action("DELETE from COMICS WHERE ComicID=? AND ComicName LIKE 'Comic ID%'", [comic['ComicID']])
                myDB.action("DELETE from ISSUES WHERE ComicID=? AND ComicName LIKE 'Comic ID%'", [comic['ComicID']])
                continue

            if not haveissues:
                havetracks = 0

            if refreshit is not None:
                if haveissues > totalissues:
                    return True   # if it's 5/4, send back to updater and don't restore previous status'
                else:
                    return False  # if it's 5/5 or 4/5, send back to updater and restore previous status'

            try:
                percent = (haveissues *100.0) /totalissues
                if percent > 100:
                    percent = 101
            except (ZeroDivisionError, TypeError):
                percent = 0
                totalissues = '?'

            if comic['LatestDate'] is None:
                logger.warn(comic['ComicName'] + ' has not finished loading. Nulling some values so things display properly until they can populate.')
                recentstatus = 'Loading'
            elif comic['ComicPublished'] is None or comic['ComicPublished'] == '' or comic['LatestDate'] is None:
                recentstatus = 'Unknown'
            elif comic['ForceContinuing'] == 1:
                recentstatus = 'Continuing'
            elif 'present' in comic['ComicPublished'].lower() or (today()[:4] in comic['LatestDate']):
                latestdate = comic['LatestDate']
                #pull-list f'd up the date by putting '15' instead of '2015' causing 500 server errors
                if '-' in latestdate[:3]:
                    st_date = latestdate.find('-')
                    st_remainder = latestdate[st_date+1:]
                    st_year = latestdate[:st_date]
                    year = '20' + st_year
                    latestdate = str(year) + '-' + str(st_remainder)
                    #logger.fdebug('year set to: ' + latestdate)
                c_date = datetime.date(int(latestdate[:4]), int(latestdate[5:7]), 1)
                n_date = datetime.date.today()
                recentchk = (n_date - c_date).days
                if comic['NewPublish'] is True:
                    recentstatus = 'Continuing'
                else:
                    #do this just incase and as an extra measure of accuracy hopefully.
                    if recentchk < 55:
                        recentstatus = 'Continuing'
                    else:
                        recentstatus = 'Ended'
            else:
                recentstatus = 'Ended'

            if recentstatus == 'Loading':
                cpub = comic['ComicPublished']
            else:
                try:
                    cpub = re.sub('(N)', '', comic['ComicPublished']).strip()
                except Exception as e:
                    logger.warn('[Error: %s] No Publisher found for %s - you probably want to Refresh the series when you get a chance.' % (e, comic['ComicName']))
                    cpub = None

            comics.append({"ComicID":         comic['ComicID'],
                           "ComicName":       comic['ComicName'],
                           "ComicSortName":   comic['ComicSortName'],
                           "ComicPublisher":  comic['ComicPublisher'],
                           "ComicYear":       comic['ComicYear'],
                           "ComicImage":      comic['ComicImage'],
                           "LatestIssue":     comic['LatestIssue'],
                           "LatestDate":      comic['LatestDate'],
                           "ComicPublished":  cpub,
                           "Status":          comic['Status'],
                           "recentstatus":    recentstatus,
                           "percent":         percent,
                           "totalissues":     totalissues,
                           "haveissues":      haveissues,
                           "DateAdded":       comic['LastUpdated'],
                           "Type":            comic['Type'],
                           "Corrected_Type":   comic['Corrected_Type']})

        return comics

def filesafe(comic):
    import unicodedata
    if u'\u2014' in comic:
        comic = re.sub(u'\u2014', ' - ', comic)
    try:
        u_comic = unicodedata.normalize('NFKD', comic).encode('ASCII', 'ignore').strip()
    except TypeError:
        u_comic = comic.encode('ASCII', 'ignore').strip()

    comicname_filesafe = re.sub('[\:\'\"\,\?\!\\\]', '', u_comic)
    comicname_filesafe = re.sub('[\/\*]', '-', comicname_filesafe)

    return comicname_filesafe

def IssueDetails(filelocation, IssueID=None, justinfo=False):
    import zipfile
    from xml.dom.minidom import parseString

    issuedetails = []
    issuetag = None

    if justinfo is False:
        dstlocation = os.path.join(mylar.CONFIG.CACHE_DIR, 'temp.zip')


        if filelocation.endswith('.cbz'):
            logger.fdebug('CBZ file detected. Checking for .xml within file')
            shutil.copy(filelocation, dstlocation)
        else:
            logger.fdebug('filename is not a cbz : ' + filelocation)
            return

        cover = "notfound"
        pic_extensions = ('.jpg','.png','.webp')
        modtime = os.path.getmtime(dstlocation)
        low_infile = 999999

        try:
            with zipfile.ZipFile(dstlocation, 'r') as inzipfile:
                for infile in sorted(inzipfile.namelist()):
                    tmp_infile = re.sub("[^0-9]","", infile).strip()
                    if tmp_infile == '':
                        pass
                    elif int(tmp_infile) < int(low_infile):
                        low_infile = tmp_infile
                        low_infile_name = infile
                    if infile == 'ComicInfo.xml':
                        logger.fdebug('Extracting ComicInfo.xml to display.')
                        dst = os.path.join(mylar.CONFIG.CACHE_DIR, 'ComicInfo.xml')
                        data = inzipfile.read(infile)
                        #print str(data)
                        issuetag = 'xml'
                    #looks for the first page and assumes it's the cover. (Alternate covers handled later on)
                    elif any(['000.' in infile, '00.' in infile]) and infile.endswith(pic_extensions) and cover == "notfound":
                        logger.fdebug('Extracting primary image ' + infile + ' as coverfile for display.')
                        local_file = open(os.path.join(mylar.CONFIG.CACHE_DIR, 'temp.jpg'), "wb")
                        local_file.write(inzipfile.read(infile))
                        local_file.close
                        cover = "found"
                    elif any(['00a' in infile, '00b' in infile, '00c' in infile, '00d' in infile, '00e' in infile]) and infile.endswith(pic_extensions) and cover == "notfound":
                        logger.fdebug('Found Alternate cover - ' + infile + ' . Extracting.')
                        altlist = ('00a', '00b', '00c', '00d', '00e')
                        for alt in altlist:
                            if alt in infile:
                                local_file = open(os.path.join(mylar.CONFIG.CACHE_DIR, 'temp.jpg'), "wb")
                                local_file.write(inzipfile.read(infile))
                                local_file.close
                                cover = "found"
                                break

                    elif (any(['001.jpg' in infile, '001.png' in infile, '001.webp' in infile, '01.jpg' in infile, '01.png' in infile, '01.webp' in infile]) or all(['0001' in infile, infile.endswith(pic_extensions)]) or all(['01' in infile, infile.endswith(pic_extensions)])) and cover == "notfound":
                        logger.fdebug('Extracting primary image ' + infile + ' as coverfile for display.')
                        local_file = open(os.path.join(mylar.CONFIG.CACHE_DIR, 'temp.jpg'), "wb")
                        local_file.write(inzipfile.read(infile))
                        local_file.close
                        cover = "found"

                if cover != "found":
                    logger.fdebug('Invalid naming sequence for jpgs discovered. Attempting to find the lowest sequence and will use as cover (it might not work). Currently : ' + str(low_infile))
                    local_file = open(os.path.join(mylar.CONFIG.CACHE_DIR, 'temp.jpg'), "wb")
                    logger.fdebug('infile_name used for displaying: %s' % low_infile_name)
                    local_file.write(inzipfile.read(low_infile_name))
                    local_file.close
                    cover = "found"                

        except:
            logger.info('ERROR. Unable to properly retrieve the cover for displaying. It\'s probably best to re-tag this file.')
            return

        ComicImage = os.path.join('cache', 'temp.jpg?' +str(modtime))
        IssueImage = replacetheslash(ComicImage)

    else:
        IssueImage = "None"
        try:
            with zipfile.ZipFile(filelocation, 'r') as inzipfile:
                for infile in sorted(inzipfile.namelist()):
                    if infile == 'ComicInfo.xml':
                        logger.fdebug('Found ComicInfo.xml - now retrieving information.')
                        data = inzipfile.read(infile)
                        issuetag = 'xml'
                        break
        except:
            logger.info('ERROR. Unable to properly retrieve the cover for displaying. It\'s probably best to re-tag this file.')
            return


    if issuetag is None:
        data = None
        try:
            dz = zipfile.ZipFile(filelocation, 'r')
            data = dz.comment
        except:
            logger.warn('Unable to extract comment field from zipfile.')
            return
        else:
            if data:
                issuetag = 'comment'
            else:
                logger.warn('No metadata available in zipfile comment field.')
                return   

    logger.info('Tag returned as being: ' + str(issuetag))

    #logger.info('data:' + str(data))

    if issuetag == 'xml':
        #import easy to use xml parser called minidom:
        dom = parseString(data)

        results = dom.getElementsByTagName('ComicInfo')
        for result in results:
            try:
                issue_title = result.getElementsByTagName('Title')[0].firstChild.wholeText
            except:
                issue_title = "None"
            try:
                series_title = result.getElementsByTagName('Series')[0].firstChild.wholeText
            except:
                series_title = "None"
            try:
                series_volume = result.getElementsByTagName('Volume')[0].firstChild.wholeText
            except:
                series_volume = "None"
            try:
                issue_number = result.getElementsByTagName('Number')[0].firstChild.wholeText
            except:
                issue_number = "None"
            try:
                summary = result.getElementsByTagName('Summary')[0].firstChild.wholeText
            except:
                summary = "None"

            if '*List' in summary:
                summary_cut = summary.find('*List')
                summary = summary[:summary_cut]
                #check here to see if Covers exist as they will probably be misnamed when trying to determine the actual cover
                # (ie. 00a.jpg / 00d.jpg  - when there's a Cover A or a Cover D listed)
            try:
                notes = result.getElementsByTagName('Notes')[0].firstChild.wholeText  #IssueID is in here
            except:
                notes = "None"
            try:
                year = result.getElementsByTagName('Year')[0].firstChild.wholeText
            except:
                year = "None"
            try:
                month = result.getElementsByTagName('Month')[0].firstChild.wholeText
            except:
                month = "None"
            try:
                day = result.getElementsByTagName('Day')[0].firstChild.wholeText
            except:
                day = "None"
            try:
                writer = result.getElementsByTagName('Writer')[0].firstChild.wholeText
            except:
                writer = "None"
            try:
                penciller = result.getElementsByTagName('Penciller')[0].firstChild.wholeText
            except:
                penciller = "None"
            try:
                inker = result.getElementsByTagName('Inker')[0].firstChild.wholeText
            except:
                inker = "None"
            try:
                colorist = result.getElementsByTagName('Colorist')[0].firstChild.wholeText
            except:
                colorist = "None"
            try:
                letterer = result.getElementsByTagName('Letterer')[0].firstChild.wholeText
            except:
                letterer = "None"
            try:
                cover_artist = result.getElementsByTagName('CoverArtist')[0].firstChild.wholeText
            except:
                cover_artist = "None"
            try:
                editor = result.getElementsByTagName('Editor')[0].firstChild.wholeText
            except:
                editor = "None"
            try:
                publisher = result.getElementsByTagName('Publisher')[0].firstChild.wholeText
            except:
                publisher = "None"
            try:
                webpage = result.getElementsByTagName('Web')[0].firstChild.wholeText
            except:
                webpage = "None"
            try:
                pagecount = result.getElementsByTagName('PageCount')[0].firstChild.wholeText
            except:
                pagecount = 0

            #not used atm.
            #to validate a front cover if it's tagged as one within the zip (some do this)
            #i = 0
            #try:
            #    pageinfo = result.getElementsByTagName('Page')[0].attributes
            #    if pageinfo: pageinfo_test == True
            #except:
            #    pageinfo_test = False

            #if pageinfo_test:
            #    while (i < int(pagecount)):
            #        pageinfo = result.getElementsByTagName('Page')[i].attributes
            #        attrib = pageinfo.getNamedItem('Image')
            #        #logger.fdebug('Frontcover validated as being image #: ' + str(attrib.value))
            #        att = pageinfo.getNamedItem('Type')
            #        #logger.fdebug('pageinfo: ' + str(pageinfo))
            #        if att.value == 'FrontCover':
            #            #logger.fdebug('FrontCover detected. Extracting.')
            #            break
            #        i+=1

    elif issuetag == 'comment':
        logger.info('CBL Tagging.')
        stripline = 'Archive:  ' + filelocation
        data = re.sub(stripline, '', data.encode("utf-8")).strip()
        if data is None or data == '':
            return
        import ast
        ast_data = ast.literal_eval(str(data))
        lastmodified = ast_data['lastModified']

        dt = ast_data['ComicBookInfo/1.0']
        try:
            publisher = dt['publisher']
        except:
            publisher = None
        try:
            year = dt['publicationYear']
        except:
            year = None
        try:
            month = dt['publicationMonth']
        except:
            month = None
        try:
            day = dt['publicationDay']
        except:
            day = None
        try:
            issue_title = dt['title']
        except:
            issue_title = None
        try:
            series_title = dt['series']
        except:
            series_title = None
        try:
            issue_number = dt['issue']
        except:
            issue_number = None
        try:
            summary = dt['comments']
        except:
            summary = "None"

        editor = "None"
        colorist = "None"
        artist = "None"
        writer = "None"
        letterer = "None"
        cover_artist = "None"
        penciller = "None"
        inker = "None"

        try:
            series_volume = dt['volume']
        except:
            series_volume = None

        try:
            t = dt['credits']
        except:
            editor = None
            colorist = None
            artist = None
            writer = None
            letterer = None
            cover_artist = None
            penciller = None
            inker = None
            
        else:
            for cl in dt['credits']:
                if cl['role'] == 'Editor':
                    if editor == "None": editor = cl['person']
                    else: editor += ', ' + cl['person']
                elif cl['role'] == 'Colorist':
                    if colorist == "None": colorist = cl['person']
                    else: colorist += ', ' + cl['person']
                elif cl['role'] == 'Artist':
                    if artist == "None": artist = cl['person']
                    else: artist += ', ' + cl['person']
                elif cl['role'] == 'Writer':
                    if writer == "None": writer = cl['person']
                    else: writer += ', ' + cl['person']
                elif cl['role'] == 'Letterer':
                    if letterer == "None": letterer = cl['person']
                    else: letterer += ', ' + cl['person']
                elif cl['role'] == 'Cover':
                    if cover_artist == "None": cover_artist = cl['person']
                    else: cover_artist += ', ' + cl['person']
                elif cl['role'] == 'Penciller':
                    if penciller == "None": penciller = cl['person']
                    else: penciller += ', ' + cl['person']
                elif cl['role'] == 'Inker':
                    if inker == "None": inker = cl['person']
                    else: inker += ', ' + cl['person']

        try:
            notes = dt['notes']
        except:
            notes = "None"
        try:
            webpage = dt['web']
        except:
            webpage = "None"
        try:
            pagecount = dt['pagecount']
        except:
            pagecount = "None"

    else:
        logger.warn('Unable to locate any metadata within cbz file. Tag this file and try again if necessary.')
        return

    issuedetails.append({"title":        issue_title,
                         "series":       series_title,
                         "volume":       series_volume,
                         "issue_number": issue_number,
                         "summary":      summary,
                         "notes":        notes,
                         "year":         year,
                         "month":        month,
                         "day":          day,
                         "writer":       writer,
                         "penciller":    penciller,
                         "inker":        inker,
                         "colorist":     colorist,
                         "letterer":     letterer,
                         "cover_artist": cover_artist,
                         "editor":       editor,
                         "publisher":    publisher,
                         "webpage":      webpage,
                         "pagecount":    pagecount,
                         "IssueImage":   IssueImage})

    return issuedetails

def get_issue_title(IssueID=None, ComicID=None, IssueNumber=None, IssueArcID=None):
    #import db
    myDB = db.DBConnection()
    if IssueID:
        issue = myDB.selectone('SELECT * FROM issues WHERE IssueID=?', [IssueID]).fetchone()
        if issue is None:
            issue = myDB.selectone('SELECT * FROM annuals WHERE IssueID=?', [IssueID]).fetchone()
            if issue is None:
                logger.fdebug('Unable to locate given IssueID within the db. Assuming Issue Title is None.')
                return None
    else:
        issue = myDB.selectone('SELECT * FROM issues WHERE ComicID=? AND Int_IssueNumber=?', [ComicID, issuedigits(IssueNumber)]).fetchone()
        if issue is None:
            issue = myDB.selectone('SELECT * FROM annuals WHERE IssueID=?', [IssueID]).fetchone()
            if issue is None:
                if IssueArcID:
                    issue = myDB.selectone('SELECT * FROM readlist WHERE IssueArcID=?', [IssueArcID]).fetchone()
                    if issue is None:
                        logger.fdebug('Unable to locate given IssueID within the db. Assuming Issue Title is None.')
                        return None
                else:
                    logger.fdebug('Unable to locate given IssueID within the db. Assuming Issue Title is None.')
                    return None

    return issue['IssueName']

def int_num(s):
    try:
        return int(s)
    except ValueError:
        return float(s)

def listPull(weeknumber, year):
    #import db
    library = {}
    myDB = db.DBConnection()
    # Get individual comics
    list = myDB.select("SELECT ComicID FROM Weekly WHERE weeknumber=? AND year=?", [weeknumber,year])
    for row in list:
        library[row['ComicID']] = row['ComicID']
    return library

def listLibrary(comicid=None):
    #import db
    library = {}
    myDB = db.DBConnection()
    if comicid is None:
        if mylar.CONFIG.ANNUALS_ON is True:
            list = myDB.select("SELECT a.comicid, b.releasecomicid, a.status FROM Comics AS a LEFT JOIN annuals AS b on a.comicid=b.comicid group by a.comicid")
        else:
            list = myDB.select("SELECT comicid, status FROM Comics group by comicid")
    else:
        if mylar.CONFIG.ANNUALS_ON is True:
            list = myDB.select("SELECT a.comicid, b.releasecomicid, a.status FROM Comics AS a LEFT JOIN annuals AS b on a.comicid=b.comicid WHERE a.comicid=? group by a.comicid", [re.sub('4050-', '', comicid).strip()])
        else:
            list = myDB.select("SELECT comicid, status FROM Comics WHERE comicid=? group by comicid", [re.sub('4050-', '', comicid).strip()])

    for row in list:
        library[row['ComicID']] = {'comicid':        row['ComicID'],
                                   'status':         row['Status']}
        try:
            if row['ReleaseComicID'] is not None:
                library[row['ReleaseComicID']] = {'comicid':   row['ComicID'],
                                                  'status':    row['Status']}
        except:
            pass

    return library

def listStoryArcs():
    #import db
    library = {}
    myDB = db.DBConnection()
    # Get Distinct Arc IDs
    #list = myDB.select("SELECT DISTINCT(StoryArcID) FROM storyarcs");
    #for row in list:
    #    library[row['StoryArcID']] = row['StoryArcID']
    # Get Distinct CV Arc IDs
    list = myDB.select("SELECT DISTINCT(CV_ArcID) FROM storyarcs");
    for row in list:
        library[row['CV_ArcID']] = {'comicid': row['CV_ArcID']}
    return library

def listoneoffs(weeknumber, year):
    #import db
    library = []
    myDB = db.DBConnection()
    # Get Distinct one-off issues from the pullist that have already been downloaded / snatched
    list = myDB.select("SELECT DISTINCT(IssueID), Status, ComicID, ComicName, Status, IssueNumber FROM oneoffhistory WHERE weeknumber=? and year=? AND Status='Downloaded' OR Status='Snatched'", [weeknumber, year])
    for row in list:
        library.append({'IssueID':     row['IssueID'],
                        'ComicID':     row['ComicID'],
                        'ComicName':   row['ComicName'],
                        'IssueNumber': row['IssueNumber'],
                        'Status':      row['Status'],
                        'weeknumber':  weeknumber,
                        'year':        year})
    return library

def manualArc(issueid, reading_order, storyarcid):
    #import db
    if issueid.startswith('4000-'):
        issueid = issueid[5:]

    myDB = db.DBConnection()

    arc_chk = myDB.select("SELECT * FROM storyarcs WHERE StoryArcID=? AND NOT Manual is 'deleted'", [storyarcid])
    storyarcname = arc_chk[0]['StoryArc']
    storyarcissues = arc_chk[0]['TotalIssues']

    iss_arcids = []
    for issarc in arc_chk:
        iss_arcids.append({"IssueArcID":     issarc['IssueArcID'],
                           "IssueID":        issarc['IssueID'],
                           "Manual":         issarc['Manual'],
                           "ReadingOrder":   issarc['ReadingOrder']})


    arc_results = mylar.cv.getComic(comicid=None, type='issue', issueid=None, arcid=storyarcid, arclist='M' + str(issueid))
    arcval = arc_results['issuechoice'][0]
    comicname = arcval['ComicName']
    st_d = mylar.filechecker.FileChecker(watchcomic=comicname)
    st_dyninfo = st_d.dynamic_replace(comicname)
    dynamic_name = re.sub('[\|\s]','', st_dyninfo['mod_seriesname'].lower()).strip()
    issname = arcval['Issue_Name']
    issid = str(arcval['IssueID'])
    comicid = str(arcval['ComicID'])
    cidlist = str(comicid)
    st_issueid = None
    manual_mod = 'added'
    new_readorder = []
    for aid in iss_arcids:
        if aid['IssueID'] == issid:
            logger.info('Issue already exists for storyarc [IssueArcID:' + aid['IssueArcID'] + '][Manual:' + aid['Manual'])
            st_issueid = aid['IssueArcID']
            manual_mod = aid['Manual']

        if reading_order is None:
            #if no reading order is given, drop in the last spot.
            reading_order = len(iss_arcids) + 1
        if int(aid['ReadingOrder']) >= int(reading_order):
            reading_seq = int(aid['ReadingOrder']) + 1
        else:
            reading_seq = int(aid['ReadingOrder'])

        new_readorder.append({'IssueArcID':   aid['IssueArcID'],
                              'IssueID':      aid['IssueID'],
                              'ReadingOrder': reading_seq})

    import random
    if st_issueid is None:
        st_issueid = str(storyarcid) + "_" + str(random.randint(1000,9999))
    issnum = arcval['Issue_Number']
    issdate = str(arcval['Issue_Date'])
    storedate = str(arcval['Store_Date'])
    int_issnum = issuedigits(issnum)

    comicid_results = mylar.cv.getComic(comicid=None, type='comicyears', comicidlist=cidlist)
    seriesYear = 'None'
    issuePublisher = 'None'
    seriesVolume = 'None'

    if issname is None:
        IssueName = 'None'
    else:
        IssueName = issname[:70]

    for cid in comicid_results:
        if cid['ComicID'] == comicid:
            seriesYear = cid['SeriesYear']
            issuePublisher = cid['Publisher']
            seriesVolume = cid['Volume']
            #assume that the arc is the same
            storyarcpublisher = issuePublisher
            break


    newCtrl = {"IssueID":           issid,
               "StoryArcID":        storyarcid}
    newVals = {"ComicID":           comicid,
               "IssueArcID":        st_issueid,
               "StoryArc":          storyarcname,
               "ComicName":         comicname,
               "Volume":            seriesVolume,
               "DynamicComicName":  dynamic_name,
               "IssueName":         IssueName,
               "IssueNumber":       issnum,
               "Publisher":         storyarcpublisher,
               "TotalIssues":       str(int(storyarcissues) +1),
               "ReadingOrder":      int(reading_order),  #arbitrarily set it to the last reading order sequence # just to see if it works.
               "IssueDate":         issdate,
               "ReleaseDate":       storedate,
               "SeriesYear":        seriesYear,
               "IssuePublisher":    issuePublisher,
               "CV_ArcID":          storyarcid,
               "Int_IssueNumber":   int_issnum,
               "Manual":            manual_mod}

    myDB.upsert("storyarcs", newVals, newCtrl)

    #now we resequence the reading-order to accomdate the change.
    logger.info('Adding the new issue into the reading order & resequencing the order to make sure there are no sequence drops...')
    new_readorder.append({'IssueArcID':   st_issueid,
                          'IssueID':      issid,
                          'ReadingOrder': int(reading_order)})

    newrl = 0
    for rl in sorted(new_readorder, key=itemgetter('ReadingOrder'), reverse=False):
        if rl['ReadingOrder'] - 1 != newrl:
            rorder = newrl + 1
            logger.fdebug(rl['IssueID'] + ' - changing reading order seq to : ' + str(rorder))
        else:
            rorder = rl['ReadingOrder']
            logger.fdebug(rl['IssueID'] + ' - setting reading order seq to : ' + str(rorder))

        rl_ctrl = {"IssueID":           rl['IssueID'],
                   "IssueArcID":        rl['IssueArcID'],
                   "StoryArcID":        storyarcid}
        r1_new = {"ReadingOrder":       rorder}
        newrl = rorder

        myDB.upsert("storyarcs", r1_new, rl_ctrl)

    #check to see if the issue exists already so we can set the status right away.
    iss_chk = myDB.selectone('SELECT * FROM issues where issueid = ?', [issueid]).fetchone()
    if iss_chk is None:
        logger.info('Issue is not currently in your watchlist. Setting status to Skipped')
        status_change = 'Skipped'
    else:
        status_change = iss_chk['Status']
        logger.info('Issue currently exists in your watchlist. Setting status to ' + status_change)
        myDB.upsert("storyarcs", {'Status': status_change}, newCtrl)

    return

def listIssues(weeknumber, year):
    #import db
    library = []
    myDB = db.DBConnection()
    # Get individual issues
    list = myDB.select("SELECT issues.Status, issues.ComicID, issues.IssueID, issues.ComicName, issues.IssueDate, issues.ReleaseDate, weekly.publisher, issues.Issue_Number from weekly, issues where weekly.IssueID = issues.IssueID and weeknumber = ? and year = ?", [int(weeknumber), year])
    for row in list:
        if row['ReleaseDate'] is None:
            tmpdate = row['IssueDate']
        else:
            tmpdate = row['ReleaseDate']
        library.append({'ComicID': row['ComicID'],
                        'Status':  row['Status'],
                        'IssueID': row['IssueID'],
                        'ComicName': row['ComicName'],
                        'Publisher': row['publisher'],
                        'Issue_Number': row['Issue_Number'],
                        'IssueYear': tmpdate})

    # Add the annuals
    if mylar.CONFIG.ANNUALS_ON:
        list = myDB.select("SELECT annuals.Status, annuals.ComicID, annuals.ReleaseComicID, annuals.IssueID, annuals.ComicName, annuals.ReleaseDate, annuals.IssueDate, weekly.publisher, annuals.Issue_Number from weekly, annuals where weekly.IssueID = annuals.IssueID and weeknumber = ? and year = ?", [int(weeknumber), year])
        for row in list:
            if row['ReleaseDate'] is None:
                tmpdate = row['IssueDate']
            else:
                tmpdate = row['ReleaseDate']
            library.append({'ComicID': row['ComicID'],
                            'Status':  row['Status'],
                            'IssueID': row['IssueID'],
                            'ComicName': row['ComicName'],
                            'Publisher': row['publisher'],
                            'Issue_Number': row['Issue_Number'],
                            'IssueYear': tmpdate})

    #tmplist = library
    #librarylist = []
    #for liblist in tmplist:
    #    lb = myDB.select('SELECT ComicVersion, Type, ComicYear, ComicID from comics WHERE ComicID=?', [liblist['ComicID']])
    #    librarylist.append(liblist)
    #    librarylist.update({'Comic_Volume': lb['ComicVersion'],
    #                        'ComicYear': lb['ComicYear'],
    #                        'ComicType': lb['Type']})
    return library

def incr_snatched(ComicID):
    #import db
    myDB = db.DBConnection()
    incr_count = myDB.selectone("SELECT Have FROM Comics WHERE ComicID=?", [ComicID]).fetchone()
    logger.fdebug('Incrementing HAVE count total to : ' + str(incr_count['Have'] + 1))
    newCtrl = {"ComicID":    ComicID}
    newVal = {"Have":  incr_count['Have'] + 1}
    myDB.upsert("comics", newVal, newCtrl)
    return

def duplicate_filecheck(filename, ComicID=None, IssueID=None, StoryArcID=None, rtnval=None):
    #filename = the filename in question that's being checked against
    #comicid = the comicid of the series that's being checked for duplication
    #issueid = the issueid of the issue that's being checked for duplication
    #storyarcid = the storyarcid of the issue that's being checked for duplication.
    #rtnval = the return value of a previous duplicate_filecheck that's re-running against new values
    #
    #import db
    myDB = db.DBConnection()

    logger.info('[DUPECHECK] Duplicate check for ' + filename)
    try:
        filesz = os.path.getsize(filename)
    except OSError as e:
        logger.warn('[DUPECHECK] File cannot be located in location specified. Something has moved or altered the name.')
        logger.warn('[DUPECHECK] Make sure if you are using ComicRN, you do not have Completed Download Handling enabled (or vice-versa). Aborting')
        return

    if IssueID:
        dupchk = myDB.selectone("SELECT * FROM issues WHERE IssueID=?", [IssueID]).fetchone()
    if dupchk is None:
        dupchk = myDB.selectone("SELECT * FROM annuals WHERE IssueID=?", [IssueID]).fetchone()
        if dupchk is None:
            logger.info('[DUPECHECK] Unable to find corresponding Issue within the DB. Do you still have the series on your watchlist?')
            return

    series = myDB.selectone("SELECT * FROM comics WHERE ComicID=?", [dupchk['ComicID']]).fetchone()

    #if it's a retry and the file was already snatched, the status is Snatched and won't hit the dupecheck.
    #rtnval will be one of 3: 
    #'write' - write new file
    #'dupe_file' - do not write new file as existing file is better quality
    #'dupe_src' - write new file, as existing file is a lesser quality (dupe)

    if dupchk['Status'] == 'Downloaded' or dupchk['Status'] == 'Archived':
        try:
            dupsize = dupchk['ComicSize']
        except:
            logger.info('[DUPECHECK] Duplication detection returned no hits as this is a new Snatch. This is not a duplicate.')
            rtnval = {'action':  "write"}

        logger.info('[DUPECHECK] Existing Status already set to ' + dupchk['Status'])
        cid = []
        if dupsize is None:
            logger.info('[DUPECHECK] Existing filesize is 0 bytes as I cannot locate the orginal entry - it is probably archived.')
            logger.fdebug('[DUPECHECK] Checking series for unrefreshed series syndrome (USS).')
            havechk = myDB.selectone('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone()
            if havechk:
                if havechk['Have'] > havechk['Total']:
                    logger.info('[DUPECHECK] Series has invalid issue totals [' + str(havechk['Have']) + '/' + str(havechk['Total']) + '] Attempting to Refresh & continue post-processing this issue.')
                    cid.append(ComicID)
                    logger.fdebug('[DUPECHECK] ComicID: ' + str(ComicID))
                    mylar.updater.dbUpdate(ComicIDList=cid, calledfrom='dupechk')
                    return duplicate_filecheck(filename, ComicID, IssueID, StoryArcID)
                else:
                    if rtnval is not None:
                        if rtnval['action'] == 'dont_dupe':
                            logger.fdebug('[DUPECHECK] File is Archived but no file can be located within the db at the specified location. Assuming this was a manual archival and will not post-process this issue.')
                        return rtnval
                    else:
                        rtnval = {'action':  "dont_dupe"}
                        #file is Archived, but no entry exists in the db for the location. Assume Archived, and don't post-process.
                        #quick rescan of files in dir, then rerun the dup check again...
                        mylar.updater.forceRescan(ComicID)
                        chk1 = duplicate_filecheck(filename, ComicID, IssueID, StoryArcID, rtnval)
                        rtnval = chk1
            else:
                rtnval = {'action':  "dupe_file",
                          'to_dupe': os.path.join(series['ComicLocation'], dupchk['Location'])}
        else:
            logger.info('[DUPECHECK] Existing file within db :' + dupchk['Location'] + ' has a filesize of : ' + str(dupsize) + ' bytes.')

            #keywords to force keep / delete
            #this will be eventually user-controlled via the GUI once the options are enabled.

            if int(dupsize) == 0:
                logger.info('[DUPECHECK] Existing filesize is 0 as I cannot locate the original entry.')
                if dupchk['Status'] == 'Archived':
                    logger.info('[DUPECHECK] Assuming issue is Archived.')
                    rtnval = {'action':  "dupe_file",
                              'to_dupe': filename}
                    return rtnval
                else:
                    logger.info('[DUPECHECK] Assuming 0-byte file - this one is gonna get hammered.')

            logger.fdebug('[DUPECHECK] Based on duplication preferences I will retain based on : ' + mylar.CONFIG.DUPECONSTRAINT)

            tmp_dupeconstraint = mylar.CONFIG.DUPECONSTRAINT

            if any(['cbr' in mylar.CONFIG.DUPECONSTRAINT, 'cbz' in mylar.CONFIG.DUPECONSTRAINT]):
                if 'cbr' in mylar.CONFIG.DUPECONSTRAINT:
                    if filename.endswith('.cbr'):
                        #this has to be configured in config - either retain cbr or cbz.
                        if dupchk['Location'].endswith('.cbr'):
                            logger.info('[DUPECHECK-CBR PRIORITY] [#' + dupchk['Issue_Number'] + '] BOTH files are in cbr format. Retaining the larger filesize of the two.')
                            tmp_dupeconstraint = 'filesize'
                        else:
                            #keep filename
                            logger.info('[DUPECHECK-CBR PRIORITY] [#' + dupchk['Issue_Number'] + '] Retaining newly scanned in file : ' + filename)
                            rtnval = {'action':  "dupe_src",
                                      'to_dupe': os.path.join(series['ComicLocation'], dupchk['Location'])}
                    else:
                        if dupchk['Location'].endswith('.cbz'):
                            logger.info('[DUPECHECK-CBR PRIORITY] [#' + dupchk['Issue_Number'] + '] BOTH files are in cbz format. Retaining the larger filesize of the two.')
                            tmp_dupeconstraint = 'filesize'
                        else:
                            #keep filename
                            logger.info('[DUPECHECK-CBR PRIORITY] [#' + dupchk['Issue_Number'] + '] Retaining newly scanned in file : ' + dupchk['Location'])
                            rtnval = {'action':  "dupe_file",
                                      'to_dupe': filename}

                elif 'cbz' in mylar.CONFIG.DUPECONSTRAINT:
                    if filename.endswith('.cbr'):
                        if dupchk['Location'].endswith('.cbr'):
                            logger.info('[DUPECHECK-CBZ PRIORITY] [#' + dupchk['Issue_Number'] + '] BOTH files are in cbr format. Retaining the larger filesize of the two.')
                            tmp_dupeconstraint = 'filesize'
                        else:
                            #keep filename
                            logger.info('[DUPECHECK-CBZ PRIORITY] [#' + dupchk['Issue_Number'] + '] Retaining currently scanned in filename : ' + dupchk['Location'])
                            rtnval = {'action':  "dupe_file",
                                      'to_dupe': filename}
                    else:
                        if dupchk['Location'].endswith('.cbz'):
                            logger.info('[DUPECHECK-CBZ PRIORITY] [#' + dupchk['Issue_Number'] + '] BOTH files are in cbz format. Retaining the larger filesize of the two.')
                            tmp_dupeconstraint = 'filesize'
                        else:
                            #keep filename
                            logger.info('[DUPECHECK-CBZ PRIORITY] [#' + dupchk['Issue_Number'] + '] Retaining newly scanned in filename : ' + filename)
                            rtnval = {'action':  "dupe_src",
                                      'to_dupe': os.path.join(series['ComicLocation'], dupchk['Location'])}

            if mylar.CONFIG.DUPECONSTRAINT == 'filesize' or tmp_dupeconstraint == 'filesize':
                if filesz <= int(dupsize) and int(dupsize) != 0:
                    logger.info('[DUPECHECK-FILESIZE PRIORITY] [#' + dupchk['Issue_Number'] + '] Retaining currently scanned in filename : ' + dupchk['Location'])
                    rtnval = {'action':  "dupe_file",
                              'to_dupe': filename}
                else:
                    logger.info('[DUPECHECK-FILESIZE PRIORITY] [#' + dupchk['Issue_Number'] + '] Retaining newly scanned in filename : ' + filename)
                    rtnval = {'action':  "dupe_src",
                              'to_dupe': os.path.join(series['ComicLocation'], dupchk['Location'])}

    else:
        logger.info('[DUPECHECK] Duplication detection returned no hits. This is not a duplicate of anything that I have scanned in as of yet.')
        rtnval = {'action':  "write"}
    return rtnval

def create_https_certificates(ssl_cert, ssl_key):
    """
    Create a pair of self-signed HTTPS certificares and store in them in
    'ssl_cert' and 'ssl_key'. Method assumes pyOpenSSL is installed.

    This code is stolen from SickBeard (http://github.com/midgetspy/Sick-Beard).
    """

    from OpenSSL import crypto
    from certgen import createKeyPair, createCertRequest, createCertificate, \
        TYPE_RSA, serial

    # Create the CA Certificate
    cakey = createKeyPair(TYPE_RSA, 2048)
    careq = createCertRequest(cakey, CN="Certificate Authority")
    cacert = createCertificate(careq, (careq, cakey), serial, (0, 60 * 60 * 24 * 365 * 10)) # ten years

    pkey = createKeyPair(TYPE_RSA, 2048)
    req = createCertRequest(pkey, CN="Mylar")
    cert = createCertificate(req, (cacert, cakey), serial, (0, 60 * 60 * 24 * 365 * 10)) # ten years

    # Save the key and certificate to disk
    try:
        with open(ssl_key, "w") as fp:
            fp.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey))
        with open(ssl_cert, "w") as fp:
            fp.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert))
    except IOError as e:
        logger.error("Error creating SSL key and certificate: %s", e)
        return False

    return True

def torrent_create(site, linkid, alt=None):
    if any([site == '32P', site == 'TOR']):
        pass
    #elif site == 'TPSE':
    #    if alt is None:
    #        url = mylar.TPSEURL + 'torrent/' + str(linkid) + '.torrent'
    #    else:
    #        url = mylar.TPSEURL + 'torrent/' + str(linkid) + '.torrent'
    elif site == 'DEM':
        url = mylar.DEMURL + 'files/download/' + str(linkid) + '/'
    elif site == 'WWT':
        url = mylar.WWTURL + 'download.php'

    return url

def parse_32pfeed(rssfeedline):
    KEYS_32P = {}
    if mylar.CONFIG.ENABLE_32P and len(rssfeedline) > 1:
        userid_st = rssfeedline.find('&user')
        userid_en = rssfeedline.find('&', userid_st +1)
        if userid_en == -1:
            USERID_32P = rssfeedline[userid_st +6:]
        else:
            USERID_32P = rssfeedline[userid_st +6:userid_en]

        auth_st = rssfeedline.find('&auth')
        auth_en = rssfeedline.find('&', auth_st +1)
        if auth_en == -1:
            AUTH_32P = rssfeedline[auth_st +6:]
        else:
            AUTH_32P = rssfeedline[auth_st +6:auth_en]

        authkey_st = rssfeedline.find('&authkey')
        authkey_en = rssfeedline.find('&', authkey_st +1)
        if authkey_en == -1:
            AUTHKEY_32P = rssfeedline[authkey_st +9:]
        else:
            AUTHKEY_32P = rssfeedline[authkey_st +9:authkey_en]

        KEYS_32P = {"user":    USERID_32P,
                    "auth":    AUTH_32P,
                    "authkey": AUTHKEY_32P,
                    "passkey": mylar.CONFIG.PASSKEY_32P}

    return KEYS_32P

def humanize_time(amount, units = 'seconds'):

    def process_time(amount, units):

        INTERVALS = [   1, 60,
                        60*60,
                        60*60*24,
                        60*60*24*7,
                        60*60*24*7*4,
                        60*60*24*7*4*12,
                        60*60*24*7*4*12*100,
                        60*60*24*7*4*12*100*10]
        NAMES = [('second', 'seconds'),
                 ('minute', 'minutes'),
                 ('hour', 'hours'),
                 ('day', 'days'),
                 ('week', 'weeks'),
                 ('month', 'months'),
                 ('year', 'years'),
                 ('century', 'centuries'),
                 ('millennium', 'millennia')]

        result = []

        unit = map(lambda a: a[1], NAMES).index(units)
        # Convert to seconds
        amount = amount * INTERVALS[unit]

        for i in range(len(NAMES)-1, -1, -1):
            a = amount // INTERVALS[i]
            if a > 0:
                result.append( (a, NAMES[i][1 % a]) )
                amount -= a * INTERVALS[i]

        return result

    rd = process_time(int(amount), units)
    cont = 0
    for u in rd:
        if u[0] > 0:
            cont += 1

    buf = ''
    i = 0
    for u in rd:
        if u[0] > 0:
            buf += "%d %s" % (u[0], u[1])
            cont -= 1

        if i < (len(rd)-1):
            if cont > 1:
                buf += ", "
            else:
                buf += " and "

        i += 1

    return buf

def issue_status(IssueID):
    #import db
    myDB = db.DBConnection()

    IssueID = str(IssueID)

    logger.fdebug('[ISSUE-STATUS] Issue Status Check for %s' % IssueID)

    isschk = myDB.selectone("SELECT * FROM issues WHERE IssueID=?", [IssueID]).fetchone()
    if isschk is None:
        isschk = myDB.selectone("SELECT * FROM annuals WHERE IssueID=?", [IssueID]).fetchone()
        if isschk is None:
            isschk = myDB.selectone("SELECT * FROM storyarcs WHERE IssueArcID=?", [IssueID]).fetchone()
            if isschk is None:
                logger.warn('Unable to retrieve IssueID from db. This is a problem. Aborting.')
                return False

    if any([isschk['Status'] == 'Downloaded', isschk['Status'] == 'Snatched']):
        return True
    else:
        return False

def crc(filename):
    #memory in lieu of speed (line by line)
    #prev = 0
    #for eachLine in open(filename,"rb"):
    #    prev = zlib.crc32(eachLine, prev)
    #return "%X"%(prev & 0xFFFFFFFF)

    #speed in lieu of memory (file into memory entirely)
    #return "%X" % (zlib.crc32(open(filename, "rb").read()) & 0xFFFFFFFF)
    filename = filename.encode(mylar.SYS_ENCODING)
    return hashlib.md5(filename).hexdigest()

def issue_find_ids(ComicName, ComicID, pack, IssueNumber):
    #import db

    myDB = db.DBConnection()

    issuelist = myDB.select("SELECT * FROM issues WHERE ComicID=?", [ComicID])

    if 'Annual' not in pack:
        packlist = [x.strip() for x in pack.split(',')]
        plist = []
        pack_issues = []
        for pl in packlist:
            if '-' in pl:
                plist.append(range(int(pl[:pl.find('-')]),int(pl[pl.find('-')+1:])+1))
            else:
                plist.append(int(pl))

        for pi in plist:
            if type(pi) == list:
                for x in pi:
                    pack_issues.append(x)
            else:
                pack_issues.append(pi)

        pack_issues.sort()
        annualize = False
    else:
        #remove the annuals wording
        tmp_annuals = pack[pack.find('Annual'):]
        tmp_ann = re.sub('[annual/annuals/+]', '', tmp_annuals.lower()).strip()
        tmp_pack = re.sub('[annual/annuals/+]', '', pack.lower()).strip() 
        pack_issues_numbers = re.findall(r'\d+', tmp_pack)
        pack_issues = range(int(pack_issues_numbers[0]),int(pack_issues_numbers[1])+1)
        annualize = True

    issues = {}
    issueinfo = []

    Int_IssueNumber = issuedigits(IssueNumber)
    valid = False

    for iss in pack_issues:
       int_iss = issuedigits(iss)
       for xb in issuelist:
           if xb['Status'] != 'Downloaded':
               if xb['Int_IssueNumber'] == int_iss:
                   issueinfo.append({'issueid':      xb['IssueID'],
                                     'int_iss':      int_iss,
                                     'issuenumber':  xb['Issue_Number']})
                   break

    for x in issueinfo:
       if Int_IssueNumber == x['int_iss']:
           valid = True
           break

    issues['issues'] = issueinfo

    if len(issues['issues']) == len(pack_issues):
        logger.info('Complete issue count of ' + str(len(pack_issues)) + ' issues are available within this pack for ' + ComicName)
    else:
        logger.info('Issue counts are not complete (not a COMPLETE pack) for ' + ComicName)

    issues['issue_range'] = pack_issues
    issues['valid'] = valid
    return issues

def conversion(value):
    if type(value) == str:
        try:
            value = value.decode('utf-8')
        except:
            value = value.decode('windows-1252')
    return value

def clean_url(url):
    leading = len(url) - len(url.lstrip(' '))
    ending = len(url) - len(url.rstrip(' '))
    if leading >= 1:
        url = url[leading:]
    if ending >=1:
        url = url[:-ending]
    return url

def chunker(seq, size):
    #returns a list from a large group of tuples by size (ie. for group in chunker(seq, 3))
    return [seq[pos:pos + size] for pos in xrange(0, len(seq), size)]

def cleanHost(host, protocol = True, ssl = False, username = None, password = None):
    """  Return a cleaned up host with given url options set
            taken verbatim from CouchPotato
    Changes protocol to https if ssl is set to True and http if ssl is set to false.
    >>> cleanHost("localhost:80", ssl=True)
    'https://localhost:80/'
    >>> cleanHost("localhost:80", ssl=False)
    'http://localhost:80/'

    Username and password is managed with the username and password variables
    >>> cleanHost("localhost:80", username="user", password="passwd")
    'http://user:passwd@localhost:80/'

    Output without scheme (protocol) can be forced with protocol=False
    >>> cleanHost("localhost:80", protocol=False)
    'localhost:80'
    """

    if not '://' in host and protocol:
        host = ('https://' if ssl else 'http://') + host

    if not protocol:
        host = host.split('://', 1)[-1]

    if protocol and username and password:
        try:
            auth = re.findall('^(?:.+?//)(.+?):(.+?)@(?:.+)$', host)
            if auth:
                log.error('Cleanhost error: auth already defined in url: %s, please remove BasicAuth from url.', host)
            else:
                host = host.replace('://', '://%s:%s@' % (username, password), 1)
        except:
            pass

    host = host.rstrip('/ ')
    if protocol:
        host += '/'

    return host

def checkthe_id(comicid=None, up_vals=None):
    #import db
    myDB = db.DBConnection()
    if not up_vals:
        chk = myDB.selectone("SELECT * from ref32p WHERE ComicID=?", [comicid]).fetchone()
        if chk is None:
           return None
        else:
           #if updated time hasn't been set or it's > 24 hrs, blank the entry so we can make sure we pull an updated groupid from 32p
           if chk['Updated'] is None:
               logger.fdebug('Reference found for 32p - but the id has never been verified after populating. Verifying it is still the right id before proceeding.')
               return None
           else:
               c_obj_date = datetime.datetime.strptime(chk['Updated'], "%Y-%m-%d %H:%M:%S")
               n_date = datetime.datetime.now()
               absdiff = abs(n_date - c_obj_date)
               hours = (absdiff.days * 24 * 60 * 60 + absdiff.seconds) / 3600.0
               if hours >= 24:
                   logger.fdebug('Reference found for 32p - but older than 24hours since last checked. Verifying it is still the right id before proceeding.')
                   return None
               else:
                   return {'id':     chk['ID'],
                           'series': chk['Series']}

    else:
        ctrlVal = {'ComicID':     comicid}
        newVal =  {'Series':      up_vals[0]['series'],
                   'ID':          up_vals[0]['id'],
                   'Updated':     now()}
        myDB.upsert("ref32p", newVal, ctrlVal)

def updatearc_locs(storyarcid, issues):
    #import db
    myDB = db.DBConnection()
    issuelist = []
    for x in issues:
        issuelist.append(x['IssueID'])
    tmpsql = "SELECT a.comicid, a.comiclocation, b.comicid, b.status, b.issueid, b.location FROM comics as a INNER JOIN issues as b ON a.comicid = b.comicid WHERE b.issueid in ({seq})".format(seq=','.join(['?'] *(len(issuelist))))
    chkthis = myDB.select(tmpsql, issuelist)
    update_iss = []
    if chkthis is None:
        return
    else:
        for chk in chkthis:
            if chk['Status'] == 'Downloaded':
                pathsrc = os.path.join(chk['ComicLocation'], chk['Location'])
                if not os.path.exists(pathsrc):
                    try:
                        if all([mylar.CONFIG.MULTIPLE_DEST_DIRS is not None, mylar.CONFIG.MULTIPLE_DEST_DIRS != 'None', os.path.join(mylar.CONFIG.MULTIPLE_DEST_DIRS, os.path.basename(chk['ComicLocation'])) != chk['ComicLocation'], os.path.exists(os.path.join(mylar.CONFIG.MULTIPLE_DEST_DIRS, os.path.basename(chk['ComicLocation'])))]):
                            pathsrc = os.path.join(mylar.CONFIG.MULTIPLE_DEST_DIRS, os.path.basename(chk['ComicLocation']), chk['Location'])
                        else:
                            logger.fdebug(module + ' file does not exist in location: ' + pathdir + '. Cannot valid location - some options will not be available for this item.')
                            continue
                    except:
                        continue

#                update_iss.append({'IssueID':    chk['IssueID'],
#                                   'Location':   pathdir})
                arcinfo = None
                for la in issues:
                    if la['IssueID'] == chk['IssueID']:
                        arcinfo = la
                        break

                if arcinfo is None:
                    continue

                if arcinfo['Publisher'] is None:
                    arcpub = arcinfo['IssuePublisher']
                else:
                    arcpub = arcinfo['Publisher']

                grdst = arcformat(arcinfo['StoryArc'], spantheyears(arcinfo['StoryArcID']), arcpub)
                if grdst is not None:
                    logger.info('grdst:' + grdst)
                    #send to renamer here if valid.
                    dfilename = chk['Location']
                    if mylar.CONFIG.RENAME_FILES:
                        renamed_file = rename_param(arcinfo['ComicID'], arcinfo['ComicName'], arcinfo['IssueNumber'], chk['Location'], issueid=arcinfo['IssueID'], arc=arcinfo['StoryArc'])
                        if renamed_file:
                            dfilename = renamed_file['nfilename']

                    if mylar.CONFIG.READ2FILENAME:
                        #logger.fdebug('readingorder#: ' + str(arcinfo['ReadingOrder']))
                        #if int(arcinfo['ReadingOrder']) < 10: readord = "00" + str(arcinfo['ReadingOrder'])
                        #elif int(arcinfo['ReadingOrder']) >= 10 and int(arcinfo['ReadingOrder']) <= 99: readord = "0" + str(arcinfo['ReadingOrder'])
                        #else: readord = str(arcinfo['ReadingOrder'])
                        readord = renamefile_readingorder(arcinfo['ReadingOrder'])
                        dfilename = str(readord) + "-" + dfilename

                    pathdst = os.path.join(grdst, dfilename)

                    logger.fdebug('Destination Path : ' + pathdst)
                    logger.fdebug('Source Path : ' + pathsrc)
                    if not os.path.isdir(grdst):
                        logger.fdebug('[ARC-DIRECTORY] Arc directory doesn\'t exist. Creating: %s' % grdst)
                        mylar.filechecker.validateAndCreateDirectory(grdst, create=True)

                    if not os.path.isfile(pathdst):
                        logger.info('[' + mylar.CONFIG.ARC_FILEOPS.upper() + '] ' + pathsrc + ' into directory : ' + pathdst)

                        try:
                            #need to ensure that src is pointing to the series in order to do a soft/hard-link properly
                            fileoperation = file_ops(pathsrc, pathdst, arc=True)
                            if not fileoperation:
                                raise OSError
                        except (OSError, IOError):
                            logger.fdebug('[' + mylar.CONFIG.ARC_FILEOPS.upper() + '] Failure ' + pathsrc + ' - check directories and manually re-run.')
                            continue
                    updateloc = pathdst
                else:
                    updateloc = pathsrc

                update_iss.append({'IssueID':    chk['IssueID'],
                                   'Location':   updateloc})

    for ui in update_iss:
        logger.info(ui['IssueID'] + ' to update location to: ' + ui['Location'])
        myDB.upsert("storyarcs", {'Location': ui['Location']}, {'IssueID': ui['IssueID'], 'StoryArcID': storyarcid})


def spantheyears(storyarcid):
    #import db
    myDB = db.DBConnection()

    totalcnt = myDB.select("SELECT * FROM storyarcs WHERE StoryArcID=?", [storyarcid])
    lowyear = 9999
    maxyear = 0
    for la in totalcnt:
        if la['IssueDate'] is None or la['IssueDate'] == '0000-00-00':
            continue
        else:
            if int(la['IssueDate'][:4]) > maxyear:
                maxyear = int(la['IssueDate'][:4])
            if int(la['IssueDate'][:4]) < lowyear:
                lowyear = int(la['IssueDate'][:4])

    if maxyear == 0:
        spanyears = la['SeriesYear']
    elif lowyear == maxyear:
        spanyears = str(maxyear)
    else:
        spanyears = str(lowyear) + ' - ' + str(maxyear) #la['SeriesYear'] + ' - ' + str(maxyear)
    return spanyears

def arcformat(arc, spanyears, publisher):
    arcdir = filesafe(arc)
    if publisher is None:
        publisher = 'None'

    values = {'$arc':         arcdir,
              '$spanyears':   spanyears,
              '$publisher':   publisher}

    tmp_folderformat = mylar.CONFIG.ARC_FOLDERFORMAT

    if publisher == 'None':
        chunk_f_f = re.sub('\$publisher', '', tmp_folderformat)
        chunk_f = re.compile(r'\s+')
        tmp_folderformat = chunk_f.sub(' ', chunk_f_f)


    if any([tmp_folderformat == '', tmp_folderformat is None]):
        arcpath = arcdir
    else:
        arcpath = replace_all(tmp_folderformat, values)

    if mylar.CONFIG.REPLACE_SPACES:
        arcpath = arcpath.replace(' ', mylar.CONFIG.REPLACE_CHAR)

    if arcpath.startswith('/'):
        arcpath = arcpath[1:]
    elif arcpath.startswith('//'):
        arcpath = arcpath[2:]

    if mylar.CONFIG.STORYARCDIR is True:
        dstloc = os.path.join(mylar.CONFIG.DESTINATION_DIR, 'StoryArcs', arcpath)
    elif mylar.CONFIG.COPY2ARCDIR is True:
        logger.warn('Story arc directory is not configured. Defaulting to grabbag directory: ' + mylar.CONFIG.GRABBAG_DIR)
        dstloc = os.path.join(mylar.CONFIG.GRABBAG_DIR, arcpath)
    else:
        dstloc = None

    return dstloc

def torrentinfo(issueid=None, torrent_hash=None, download=False, monitor=False):
    #import db
    from base64 import b16encode, b32decode

    #check the status of the issueid to make sure it's in Snatched status and was grabbed via torrent.
    if issueid:
        myDB = db.DBConnection()
        cinfo = myDB.selectone('SELECT a.Issue_Number, a.ComicName, a.Status, b.Hash from issues as a inner join snatched as b ON a.IssueID=b.IssueID WHERE a.IssueID=?', [issueid]).fetchone()
        if cinfo is None:
            logger.warn('Unable to locate IssueID of : ' + issueid)
            snatch_status = 'ERROR'

        if cinfo['Status'] != 'Snatched' or cinfo['Hash'] is None:
            logger.warn(cinfo['ComicName'] + ' #' + cinfo['Issue_Number'] + ' is currently in a ' + cinfo['Status'] + ' Status.')
            snatch_status = 'ERROR'

        torrent_hash = cinfo['Hash']

    logger.fdebug("Working on torrent: " + torrent_hash)
    if len(torrent_hash) == 32:
       torrent_hash = b16encode(b32decode(torrent_hash))

    if not len(torrent_hash) == 40:
       logger.error("Torrent hash is missing, or an invalid hash value has been passed")
       snatch_status = 'ERROR'
    else:
        if mylar.USE_RTORRENT:
            import test
            rp = test.RTorrent()
            torrent_info = rp.main(torrent_hash, check=True)
        elif mylar.USE_DELUGE:
            #need to set the connect here as well....
            import torrent.clients.deluge as delu
            dp = delu.TorrentClient()
            if not dp.connect(mylar.CONFIG.DELUGE_HOST, mylar.CONFIG.DELUGE_USERNAME, mylar.CONFIG.DELUGE_PASSWORD):
                logger.warn('Not connected to Deluge!')

            torrent_info = dp.get_torrent(torrent_hash)
        else:
            snatch_status = 'ERROR'
            return

    logger.info('torrent_info: %s' % torrent_info)

    if torrent_info is False or len(torrent_info) == 0:
        logger.warn('torrent returned no information. Check logs - aborting auto-snatch at this time.')
        snatch_status = 'ERROR'
    else:
        if mylar.USE_DELUGE:
            torrent_status = torrent_info['is_finished']
            torrent_files = torrent_info['num_files']
            torrent_folder = torrent_info['save_path']
            torrent_info['total_filesize'] = torrent_info['total_size']
            torrent_info['upload_total'] = torrent_info['total_uploaded']
            torrent_info['download_total'] = torrent_info['total_payload_download']
            torrent_info['time_started'] = torrent_info['time_added']

        elif mylar.USE_RTORRENT:
            torrent_status = torrent_info['completed']
            torrent_files = len(torrent_info['files'])
            torrent_folder = torrent_info['folder']

        if all([torrent_status is True, download is True]):
            if not issueid: 
                torrent_info['snatch_status'] = 'STARTING...'
                #yield torrent_info

            import shlex, subprocess
            logger.info('Torrent is completed and status is currently Snatched. Attempting to auto-retrieve.')
            with open(mylar.CONFIG.AUTO_SNATCH_SCRIPT, 'r') as f:
                first_line = f.readline()

            if mylar.CONFIG.AUTO_SNATCH_SCRIPT.endswith('.sh'):
                shell_cmd = re.sub('#!', '', first_line)
                if shell_cmd == '' or shell_cmd is None:
                    shell_cmd = '/bin/bash'
            else:
                shell_cmd = sys.executable

            curScriptName = shell_cmd + ' ' + str(mylar.CONFIG.AUTO_SNATCH_SCRIPT).decode("string_escape")
            if torrent_files > 1:
                downlocation = torrent_folder.encode('utf-8')
            else:
                if mylar.USE_DELUGE:
                    downlocation = os.path.join(torrent_folder.encode('utf-8'), torrent_info['files'][0]['path'])
                else:
                    downlocation = torrent_info['files'][0].encode('utf-8')

            autosnatch_env = os.environ.copy()
            autosnatch_env['downlocation'] = re.sub("'", "\\'",downlocation)

            #these are pulled from the config and are the ssh values to use to retrieve the data
            autosnatch_env['host'] = mylar.CONFIG.PP_SSHHOST
            autosnatch_env['port'] = mylar.CONFIG.PP_SSHPORT
            autosnatch_env['user'] = mylar.CONFIG.PP_SSHUSER
            autosnatch_env['localcd'] = mylar.CONFIG.PP_SSHLOCALCD
            #bash won't accept None, so send check and send empty strings for the 2 possible None values if needed
            if mylar.CONFIG.PP_SSHKEYFILE is not None:
                autosnatch_env['keyfile'] = mylar.CONFIG.PP_SSHKEYFILE
            else:
                autosnatch_env['keyfile'] = ''
            if mylar.CONFIG.PP_SSHPASSWD is not None:
                autosnatch_env['passwd'] = mylar.CONFIG.PP_SSHPASSWD
            else:
                autosnatch_env['passwd'] = ''


            #downlocation = re.sub("\'", "\\'", downlocation)
            #downlocation = re.sub("&", "\&", downlocation)

            script_cmd = shlex.split(curScriptName, posix=False) # + [downlocation]
            logger.fdebug(u"Executing command " +str(script_cmd))
            try:
                p = subprocess.Popen(script_cmd, env=dict(autosnatch_env), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=mylar.PROG_DIR)
                out, err = p.communicate()
                logger.fdebug(u"Script result: " + out)
            except OSError, e:
                logger.warn(u"Unable to run extra_script: " + e)
                snatch_status = 'ERROR'
            else:
                if 'Access failed: No such file' in out:
                    logger.fdebug('Not located in location it is supposed to be in - probably has been moved by some script and I got the wrong location due to timing. Trying again...')
                    snatch_status = 'IN PROGRESS'
                else:
                    snatch_status = 'COMPLETED'
                torrent_info['completed'] = torrent_status
                torrent_info['files'] = torrent_files
                torrent_info['folder'] = torrent_folder

        else:
            if download is True:
                snatch_status = 'IN PROGRESS'
            elif monitor is True:
                #pause the torrent, copy it to the cache folder, unpause the torrent and return the complete path to the cache location
                if mylar.USE_DELUGE:
                    pauseit = dp.stop_torrent(torrent_hash)
                    if pauseit is False:
                        logger.warn('Unable to pause torrent - cannot run post-process on item at this time.')
                        snatch_status = 'MONITOR FAIL'
                    else:
                        try:
                            new_filepath = os.path.join(torrent_path, '.copy')
                            logger.fdebug('New_Filepath: %s' % new_filepath)
                            shutil.copy(torrent_path, new_filepath)
                            torrent_info['copied_filepath'] = new_filepath
                        except:
                            logger.warn('Unexpected Error: %s' % sys.exc_info()[0])
                            logger.warn('Unable to create temporary directory to perform meta-tagging. Processing cannot continue with given item at this time.')
                            torrent_info['copied_filepath'] = torrent_path
                            SNATCH_STATUS = 'MONITOR FAIL'
                        else:
                            startit = dp.start_torrent(torrent_hash)
                            SNATCH_STATUS = 'MONITOR COMPLETE'
            else:
                snatch_status = 'NOT SNATCHED'

    torrent_info['snatch_status'] = snatch_status
    return torrent_info

def weekly_info(week=None, year=None, current=None):
    #find the current week and save it as a reference point.
    todaydate = datetime.datetime.today()
    current_weeknumber = todaydate.strftime("%U")
    if current is not None:
        c_weeknumber = int(current[:current.find('-')])
        c_weekyear = int(current[current.find('-')+1:])
    else:
        c_weeknumber = week
        c_weekyear = year

    if week:
        weeknumber = int(week)
        year = int(year)

        #monkey patch for 2018/2019 - week 52/week 0
        if all([weeknumber == 52, c_weeknumber == 51, c_weekyear == 2018]):
            weeknumber = 0
            year = 2019
        elif all([weeknumber == 52, c_weeknumber == 0, c_weekyear == 2019]):
            weeknumber = 51
            year = 2018

        #monkey patch for 2019/2020 - week 52/week 0
        if all([weeknumber == 52, c_weeknumber == 51, c_weekyear == 2019]):
            weeknumber = 0
            year = 2020
        elif all([weeknumber == 52, c_weeknumber == 0, c_weekyear == 2020]):
            weeknumber = 51
            year = 2019

        #view specific week (prev_week, next_week)
        startofyear = date(year,1,1)
        week0 = startofyear - timedelta(days=startofyear.isoweekday())
        stweek = datetime.datetime.strptime(week0.strftime('%Y-%m-%d'), '%Y-%m-%d')
        startweek = stweek + timedelta(weeks = weeknumber)
        midweek = startweek + timedelta(days = 3)
        endweek = startweek + timedelta(days = 6)
    else:
        #find the given week number for the current day
        weeknumber = current_weeknumber
        year = todaydate.strftime("%Y")

        #monkey patch for 2018/2019 - week 52/week 0
        if all([weeknumber == 52, c_weeknumber == 51, c_weekyear == 2018]):
            weeknumber = 0
            year = 2019
        elif all([weeknumber == 52, c_weeknumber == 0, c_weekyear == 2019]):
            weeknumber = 51
            year = 2018

        #monkey patch for 2019/2020 - week 52/week 0
        if all([weeknumber == 52, c_weeknumber == 51, c_weekyear == 2019]) or all([weeknumber == '52', year == '2019']):
            weeknumber = 0
            year = 2020
        elif all([weeknumber == 52, c_weeknumber == 0, c_weekyear == 2020]):
            weeknumber = 51
            year = 2019

        stweek = datetime.datetime.strptime(todaydate.strftime('%Y-%m-%d'), '%Y-%m-%d')
        startweek = stweek - timedelta(days = (stweek.weekday() + 1) % 7)
        midweek = startweek + timedelta(days = 3)
        endweek = startweek + timedelta(days = 6)

    prev_week = int(weeknumber) - 1
    prev_year = year
    if prev_week < 0:
        prev_week = 52
        prev_year = int(year) - 1

    next_week = int(weeknumber) + 1
    next_year = year
    if next_week > 52:
        next_year = int(year) + 1
        next_week = datetime.date(int(next_year),1,1).strftime("%U")

    date_fmt = "%B %d, %Y"
    try:
        con_startweek = u"" + startweek.strftime(date_fmt).decode('utf-8')
        con_endweek = u"" + endweek.strftime(date_fmt).decode('utf-8')
    except:
        con_startweek = u"" + startweek.strftime(date_fmt).decode('cp1252')
        con_endweek = u"" + endweek.strftime(date_fmt).decode('cp1252')

    if mylar.CONFIG.WEEKFOLDER_LOC is not None:
        weekdst = mylar.CONFIG.WEEKFOLDER_LOC
    else:
        weekdst = mylar.CONFIG.DESTINATION_DIR

    if mylar.SCHED_WEEKLY_LAST is not None:
        weekly_stamp = datetime.datetime.fromtimestamp(mylar.SCHED_WEEKLY_LAST)
        weekly_last = weekly_stamp.replace(microsecond=0)
    else:
        weekly_last = 'None'

    weekinfo = {'weeknumber':         weeknumber,
                'startweek':          con_startweek,
                'midweek':            midweek.strftime('%Y-%m-%d'),
                'endweek':            con_endweek,
                'year':               year,
                'prev_weeknumber':    prev_week,
                'prev_year':          prev_year,
                'next_weeknumber':    next_week,
                'next_year':          next_year,
                'current_weeknumber': current_weeknumber,
                'last_update':        weekly_last}

    if weekdst is not None:
        if mylar.CONFIG.WEEKFOLDER_FORMAT == 0:
            weekn = weeknumber
            if len(str(weekn)) == 1:
                weekn = '%s%s' % ('0', str(weekn))
            weekfold = os.path.join(weekdst, '%s-%s' % (weekinfo['year'], weekn))
        else:
            weekfold = os.path.join(weekdst, str( str(weekinfo['midweek']) ))
    else:
        weekfold = None

    weekinfo['week_folder'] = weekfold

    return weekinfo

def latestdate_update():
    #import db
    myDB = db.DBConnection()
    ccheck = myDB.select('SELECT a.ComicID, b.IssueID, a.LatestDate, b.ReleaseDate, b.Issue_Number from comics as a left join issues as b on a.comicid=b.comicid where a.LatestDate < b.ReleaseDate or a.LatestDate like "%Unknown%" group by a.ComicID')
    if ccheck is None or len(ccheck) == 0:
        return
    logger.info('Now preparing to update ' + str(len(ccheck)) + ' series that have out-of-date latest date information.')
    ablist = []
    for cc in ccheck:
        ablist.append({'ComicID':     cc['ComicID'],
                       'LatestDate':  cc['ReleaseDate'],
                       'LatestIssue': cc['Issue_Number']})

    #forcibly set the latest date and issue number to the most recent.
    for a in ablist:
        logger.info(a)
        newVal = {'LatestDate':         a['LatestDate'],
                  'LatestIssue':        a['LatestIssue']}
        ctrlVal = {'ComicID':           a['ComicID']}
        logger.info('updating latest date for : ' + a['ComicID'] + ' to ' + a['LatestDate'] + ' #' + a['LatestIssue'])
        myDB.upsert("comics", newVal, ctrlVal)

def ddl_downloader(queue):
    myDB = db.DBConnection()
    while True:
        if mylar.DDL_LOCK is True:
            time.sleep(5)

        elif mylar.DDL_LOCK is False and queue.qsize() >= 1:
            item = queue.get(True)
            if item == 'exit':
                logger.info('Cleaning up workers for shutdown')
                break
            logger.info('Now loading request from DDL queue: %s' % item['series'])

            #write this to the table so we have a record of what's going on.
            ctrlval = {'id':      item['id']}
            val = {'status':       'Downloading',
                   'updated_date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M')}
            myDB.upsert('ddl_info', val, ctrlval)

            ddz = getcomics.GC()
            ddzstat = ddz.downloadit(item['id'], item['link'], item['mainlink'], item['resume'])

            if ddzstat['success'] is True:
                tdnow = datetime.datetime.now()
                nval = {'status':  'Completed',
                        'updated_date': tdnow.strftime('%Y-%m-%d %H:%M')}
                myDB.upsert('ddl_info', nval, ctrlval)

            if all([ddzstat['success'] is True, mylar.CONFIG.POST_PROCESSING is True]):
                try:
                    if ddzstat['filename'] is None:
                        logger.info('%s successfully downloaded - now initiating post-processing.' % (os.path.basename(ddzstat['path'])))
                        mylar.PP_QUEUE.put({'nzb_name':     os.path.basename(ddzstat['path']),
                                            'nzb_folder':   ddzstat['path'],
                                            'failed':       False,
                                            'issueid':      None,
                                            'comicid':      item['comicid'],
                                            'apicall':      True,
                                            'ddl':          True})
                    else:
                        logger.info('%s successfully downloaded - now initiating post-processing.' % (ddzstat['filename']))
                        mylar.PP_QUEUE.put({'nzb_name':     ddzstat['filename'],
                                            'nzb_folder':   ddzstat['path'],
                                            'failed':       False,
                                            'issueid':      item['issueid'],
                                            'comicid':      item['comicid'],
                                            'apicall':      True,
                                            'ddl':          True})
                except Exception as e:
                    logger.info('process error: %s [%s]' %(e, ddzstat))
            elif all([ddzstat['success'] is True, mylar.CONFIG.POST_PROCESSING is False]):
                logger.info('File successfully downloaded. Post Processing is not enabled - item retained here: %s' % os.path.join(ddzstat['path'],ddzstat['filename']))
            else:
                logger.info('[Status: %s] Failed to download: %s ' % (ddzstat['success'], ddzstat))
                nval = {'status':  'Failed',
                        'updated_date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M')}
                myDB.upsert('ddl_info', nval, ctrlval)
        else:
            time.sleep(5)

def postprocess_main(queue):
    while True:
        if mylar.APILOCK is True:
            time.sleep(5)

        elif mylar.APILOCK is False and queue.qsize() >= 1: #len(queue) > 1:
            pp = None
            item = queue.get(True)
            logger.info('Now loading from post-processing queue: %s' % item)
            if item == 'exit':
                logger.info('Cleaning up workers for shutdown')
                break

            if mylar.APILOCK is False:
                try:
                    pprocess = process.Process(item['nzb_name'], item['nzb_folder'], item['failed'], item['issueid'], item['comicid'], item['apicall'], item['ddl'])
                except:
                    pprocess = process.Process(item['nzb_name'], item['nzb_folder'], item['failed'], item['issueid'], item['comicid'], item['apicall'])
                pp = pprocess.post_process()
                time.sleep(5) #arbitrary sleep to let the process attempt to finish pp'ing

            if pp is not None:
                if pp['mode'] == 'stop':
                    #reset the lock so any subsequent items can pp and not keep the queue locked up.
                    mylar.APILOCK = False

            if mylar.APILOCK is True:
                logger.info('Another item is post-processing still...')
                time.sleep(15)
                #mylar.PP_QUEUE.put(item)
        else:
            time.sleep(5)

def search_queue(queue):
    while True:
        if mylar.SEARCHLOCK is True:
            time.sleep(5)

        elif mylar.SEARCHLOCK is False and queue.qsize() >= 1: #len(queue) > 1:
            item = queue.get(True)
            if item == 'exit':
                logger.info('[SEARCH-QUEUE] Cleaning up workers for shutdown')
                break

            logger.info('[SEARCH-QUEUE] Now loading item from search queue: %s' % item)
            if mylar.SEARCHLOCK is False:
                ss_queue = mylar.search.searchforissue(item['issueid'])
                time.sleep(5) #arbitrary sleep to let the process attempt to finish pp'ing

            if mylar.SEARCHLOCK is True:
                logger.fdebug('[SEARCH-QUEUE] Another item is currently being searched....')
                time.sleep(15)
        else:
            time.sleep(5)


def worker_main(queue):
    while True:
        if queue.qsize() >= 1:
            item = queue.get(True)
            logger.info('Now loading from queue: ' + item)
            if item == 'exit':
                logger.info('Cleaning up workers for shutdown')
                break
            snstat = torrentinfo(torrent_hash=item, download=True)
            if snstat['snatch_status'] == 'IN PROGRESS':
                logger.info('Still downloading in client....let us try again momentarily.')
                time.sleep(30)
                mylar.SNATCHED_QUEUE.put(item)
            elif any([snstat['snatch_status'] == 'MONITOR FAIL', snstat['snatch_status'] == 'MONITOR COMPLETE']):
                logger.info('File copied for post-processing - submitting as a direct pp.')
                threading.Thread(target=self.checkFolder, args=[os.path.abspath(os.path.join(snstat['copied_filepath'], os.pardir))]).start()
        else:
            time.sleep(15)

def nzb_monitor(queue):
    while True:
        if queue.qsize() >= 1:
            item = queue.get(True)
            if item == 'exit':
                logger.info('Cleaning up workers for shutdown')
                break
            logger.info('Now loading from queue: %s' % item)
            if all([mylar.USE_SABNZBD is True, mylar.CONFIG.SAB_CLIENT_POST_PROCESSING is True]):
                nz = sabnzbd.SABnzbd(item)
                nzstat = nz.processor()
            elif all([mylar.USE_NZBGET is True, mylar.CONFIG.NZBGET_CLIENT_POST_PROCESSING is True]):
                nz = nzbget.NZBGet()
                nzstat = nz.processor(item)
            else:
                logger.warn('There are no NZB Completed Download handlers enabled. Not sending item to completed download handling...')
                break

            if any([nzstat['status'] == 'file not found', nzstat['status'] == 'double-pp']):
                logger.warn('Unable to complete post-processing call due to not finding file in the location provided. [%s]' % item)
            elif nzstat['status'] is False:
                logger.info('Could not find NZBID %s in the downloader\'s queue. I will requeue this item for post-processing...' % item['NZBID'])
                time.sleep(5)
                mylar.NZB_QUEUE.put(item)
            elif nzstat['status'] is True:
                if nzstat['failed'] is False:
                    logger.info('File successfully downloaded - now initiating completed downloading handling.')
                else:
                    logger.info('File failed either due to being corrupt or incomplete - now initiating completed failed downloading handling.')
                try:
                    mylar.PP_QUEUE.put({'nzb_name':     nzstat['name'],
                                        'nzb_folder':   nzstat['location'],
                                        'failed':       nzstat['failed'],
                                        'issueid':      nzstat['issueid'],
                                        'comicid':      nzstat['comicid'],
                                        'apicall':      nzstat['apicall'],
                                        'ddl':          False})
                    #cc = process.Process(nzstat['name'], nzstat['location'], failed=nzstat['failed'])
                    #nzpp = cc.post_process()
                except Exception as e:
                    logger.info('process error: %s' % e)
        else:
            time.sleep(5)

def script_env(mode, vars):
    #mode = on-snatch, pre-postprocess, post-postprocess
    #var = dictionary containing variables to pass
    mylar_env = os.environ.copy()
    if mode == 'on-snatch':
        runscript = mylar.CONFIG.SNATCH_SCRIPT
        if 'torrentinfo' in vars:
            if 'hash' in vars['torrentinfo']:
                mylar_env['mylar_release_hash'] = vars['torrentinfo']['hash'] 
            if 'torrent_filename' in vars['torrentinfo']:
                mylar_env['mylar_torrent_filename'] = vars['torrentinfo']['torrent_filename']
            if 'name' in vars['torrentinfo']:
                mylar_env['mylar_release_name'] = vars['torrentinfo']['name']
            if 'folder' in vars['torrentinfo']:
                mylar_env['mylar_release_folder'] = vars['torrentinfo']['folder']
            if 'label' in vars['torrentinfo']:
                mylar_env['mylar_release_label'] = vars['torrentinfo']['label']
            if 'total_filesize' in vars['torrentinfo']:
                mylar_env['mylar_release_filesize'] = str(vars['torrentinfo']['total_filesize'])
            if 'time_started' in vars['torrentinfo']:
                mylar_env['mylar_release_start'] = str(vars['torrentinfo']['time_started'])
            if 'filepath' in vars['torrentinfo']:
                mylar_env['mylar_torrent_file'] = str(vars['torrentinfo']['filepath'])
            else:
                try:
                    mylar_env['mylar_release_files'] = '|'.join(vars['torrentinfo']['files'])
                except TypeError:
                    mylar_env['mylar_release_files'] = '|'.join(json.dumps(vars['torrentinfo']['files']))
        elif 'nzbinfo' in vars:
            mylar_env['mylar_release_id'] = vars['nzbinfo']['id']
            if 'client_id' in vars['nzbinfo']:
                mylar_env['mylar_client_id'] = vars['nzbinfo']['client_id']
            mylar_env['mylar_release_nzbname'] = vars['nzbinfo']['nzbname']
            mylar_env['mylar_release_link'] = vars['nzbinfo']['link']
            mylar_env['mylar_release_nzbpath'] = vars['nzbinfo']['nzbpath']
            if 'blackhole' in vars['nzbinfo']:
                mylar_env['mylar_release_blackhole'] = vars['nzbinfo']['blackhole']
        mylar_env['mylar_release_provider'] = vars['provider']
        if 'comicinfo' in vars:
            try:
                if vars['comicinfo']['comicid'] is not None:
                    mylar_env['mylar_comicid'] = vars['comicinfo']['comicid']  #comicid/issueid are unknown for one-offs (should be fixable tho)
                else:
                    mylar_env['mylar_comicid'] = 'None'
            except:
                pass
            try:
                if vars['comicinfo']['issueid'] is not None:
                    mylar_env['mylar_issueid'] = vars['comicinfo']['issueid']
                else:
                    mylar_env['mylar_issueid'] = 'None'
            except:
                pass
            try:
                if vars['comicinfo']['issuearcid'] is not None:
                    mylar_env['mylar_issuearcid'] = vars['comicinfo']['issuearcid']
                else:
                    mylar_env['mylar_issuearcid'] = 'None'
            except:
                pass
            mylar_env['mylar_comicname'] = vars['comicinfo']['comicname']
            mylar_env['mylar_issuenumber'] = str(vars['comicinfo']['issuenumber'])
            try:
                mylar_env['mylar_comicvolume'] = str(vars['comicinfo']['volume'])
            except:
                pass
            try:
                mylar_env['mylar_seriesyear'] = str(vars['comicinfo']['seriesyear'])
            except:
                pass
            try:
                mylar_env['mylar_issuedate'] = str(vars['comicinfo']['issuedate'])
            except:
                pass

        mylar_env['mylar_release_pack'] = str(vars['pack'])
        if vars['pack'] is True:
            if vars['pack_numbers'] is not None:
                mylar_env['mylar_release_pack_numbers'] = vars['pack_numbers']
            if vars['pack_issuelist'] is not None:
                mylar_env['mylar_release_pack_issuelist'] = vars['pack_issuelist']
        mylar_env['mylar_method'] = vars['method']
        mylar_env['mylar_client'] = vars['clientmode']

    elif mode == 'post-process':
        #to-do
        runscript = mylar.CONFIG.EXTRA_SCRIPTS
    elif mode == 'pre-process':
        #to-do
        runscript = mylar.CONFIG.PRE_SCRIPTS

    logger.fdebug('Initiating ' + mode + ' script detection.')
    with open(runscript, 'r') as f:
        first_line = f.readline()

    if runscript.endswith('.sh'):
        shell_cmd = re.sub('#!', '', first_line)
        if shell_cmd == '' or shell_cmd is None:
            shell_cmd = '/bin/bash'
    else:
        shell_cmd = sys.executable

    curScriptName = shell_cmd + ' ' + runscript.decode("string_escape")
    logger.fdebug("snatch script detected...enabling: " + str(curScriptName))

    script_cmd = shlex.split(curScriptName)
    logger.fdebug(u"Executing command " +str(script_cmd))
    try:
        subprocess.call(script_cmd, env=dict(mylar_env))
    except OSError, e:
        logger.warn(u"Unable to run extra_script: " + str(script_cmd))
        return False
    else:
        return True

def get_the_hash(filepath):
    import bencode
    # Open torrent file
    torrent_file = open(filepath, "rb")
    metainfo = bencode.decode(torrent_file.read())
    info = metainfo['info']
    thehash = hashlib.sha1(bencode.encode(info)).hexdigest().upper()
    logger.info('Hash of file : ' + thehash)
    return {'hash':     thehash}

def disable_provider(site, newznab=False):
    logger.info('Temporarily disabling %s due to not responding' % site)
    if newznab is True:
        tmplist = []
        for ti in mylar.CONFIG.EXTRA_NEWZNABS:
            tmpnewz = list(ti)
            if tmpnewz[0] == site:
                tmpnewz[5] = '0'
            tmplist.append(tuple(tmpnewz))
        mylar.CONFIG.EXTRA_NEWZNABS = tmplist
    else:
        if site == 'nzbsu':
            mylar.CONFIG.NZBSU = False
        elif site == 'dognzb':
            mylar.CONFIG.DOGNZB = False
        elif site == 'experimental':
            mylar.CONFIG.EXPERIMENTAL = False
        elif site == '32P':
            mylar.CONFIG.ENABLE_32P = False

def date_conversion(originaldate):
    c_obj_date = datetime.datetime.strptime(originaldate, "%Y-%m-%d %H:%M:%S")
    n_date = datetime.datetime.now()
    absdiff = abs(n_date - c_obj_date)
    hours = (absdiff.days * 24 * 60 * 60 + absdiff.seconds) / 3600.0
    return hours

def job_management(write=False, job=None, last_run_completed=None, current_run=None, status=None):
        jobresults = []

        #import db
        myDB = db.DBConnection()

        if job is None:
            dbupdate_newstatus = 'Waiting'
            dbupdate_nextrun = None
            if mylar.CONFIG.ENABLE_RSS is True:
                rss_newstatus = 'Waiting'
            else:
                rss_newstatus = 'Paused'
            rss_nextrun = None
            weekly_newstatus = 'Waiting'
            weekly_nextrun = None
            search_newstatus = 'Waiting'
            search_nextrun = None
            version_newstatus = 'Waiting'
            version_nextrun = None
            if mylar.CONFIG.ENABLE_CHECK_FOLDER is True:
                monitor_newstatus = 'Waiting'
            else:
                monitor_newstatus = 'Paused'
            monitor_nextrun = None

            job_info = myDB.select('SELECT DISTINCT * FROM jobhistory')
            #set default values if nothing has been ran yet
            for ji in job_info:
                if 'update' in ji['JobName'].lower():
                    if mylar.SCHED_DBUPDATE_LAST is None:
                        mylar.SCHED_DBUPDATE_LAST = ji['prev_run_timestamp']
                    dbupdate_newstatus = ji['status']
                    mylar.UPDATER_STATUS = dbupdate_newstatus
                    dbupdate_nextrun = ji['next_run_timestamp']
                elif 'search' in ji['JobName'].lower():
                    if mylar.SCHED_SEARCH_LAST is None:
                        mylar.SCHED_SEARCH_LAST = ji['prev_run_timestamp']
                    search_newstatus = ji['status']
                    mylar.SEARCH_STATUS = search_newstatus
                    search_nextrun = ji['next_run_timestamp']
                elif 'rss' in ji['JobName'].lower():
                    if mylar.SCHED_RSS_LAST is None:
                        mylar.SCHED_RSS_LAST = ji['prev_run_timestamp']
                    rss_newstatus = ji['status']
                    mylar.RSS_STATUS = rss_newstatus
                    rss_nextrun = ji['next_run_timestamp']
                elif 'weekly' in ji['JobName'].lower():
                    if mylar.SCHED_WEEKLY_LAST is None:
                        mylar.SCHED_WEEKLY_LAST = ji['prev_run_timestamp']
                    weekly_newstatus = ji['status']
                    mylar.WEEKLY_STATUS = weekly_newstatus
                    weekly_nextrun = ji['next_run_timestamp']
                elif 'version' in ji['JobName'].lower():
                    if mylar.SCHED_VERSION_LAST is None:
                        mylar.SCHED_VERSION_LAST = ji['prev_run_timestamp']
                    version_newstatus = ji['status']
                    mylar.VERSION_STATUS = version_newstatus
                    version_nextrun = ji['next_run_timestamp']
                elif 'monitor' in ji['JobName'].lower():
                    if mylar.SCHED_MONITOR_LAST is None:
                        mylar.SCHED_MONITOR_LAST = ji['prev_run_timestamp']
                    monitor_newstatus = ji['status']
                    mylar.MONITOR_STATUS = monitor_newstatus
                    monitor_nextrun = ji['next_run_timestamp']

            monitors = {'weekly': mylar.SCHED_WEEKLY_LAST,
                        'monitor': mylar.SCHED_MONITOR_LAST,
                        'search': mylar.SCHED_SEARCH_LAST,
                        'dbupdater': mylar.SCHED_DBUPDATE_LAST,
                        'version': mylar.SCHED_VERSION_LAST,
                        'rss': mylar.SCHED_RSS_LAST}

            #this is for initial startup
            for jb in mylar.SCHED.get_jobs():
                #logger.fdebug('jb: %s' % jb)
                jobinfo = str(jb)
                if 'Status Updater' in jobinfo.lower():
                    continue
                elif 'update' in jobinfo.lower():
                    prev_run_timestamp = mylar.SCHED_DBUPDATE_LAST
                    newstatus = dbupdate_newstatus
                    mylar.UPDATER_STATUS = newstatus
                elif 'search' in jobinfo.lower():
                    prev_run_timestamp = mylar.SCHED_SEARCH_LAST
                    newstatus = search_newstatus
                    mylar.SEARCH_STATUS = newstatus
                elif 'rss' in jobinfo.lower():
                    prev_run_timestamp = mylar.SCHED_RSS_LAST
                    newstatus = rss_newstatus
                    mylar.RSS_STATUS = newstatus
                elif 'weekly' in jobinfo.lower():
                    prev_run_timestamp = mylar.SCHED_WEEKLY_LAST
                    newstatus = weekly_newstatus
                    mylar.WEEKLY_STATUS = newstatus
                elif 'version' in jobinfo.lower():
                    prev_run_timestamp = mylar.SCHED_VERSION_LAST
                    newstatus = version_newstatus
                    mylar.VERSION_STATUS = newstatus
                elif 'monitor' in jobinfo.lower():
                    prev_run_timestamp = mylar.SCHED_MONITOR_LAST
                    newstatus = monitor_newstatus
                    mylar.MONITOR_STATUS = newstatus

                jobname = jobinfo[:jobinfo.find('(')-1].strip()
                #logger.fdebug('jobinfo: %s' % jobinfo)
                try:
                    jobtimetmp = jobinfo.split('at: ')[1].split('.')[0].strip()
                except:
                    continue
                #logger.fdebug('jobtimetmp: %s' % jobtimetmp)
                jobtime = float(calendar.timegm(datetime.datetime.strptime(jobtimetmp[:-1], '%Y-%m-%d %H:%M:%S %Z').timetuple()))
                #logger.fdebug('jobtime: %s' % jobtime)

                if prev_run_timestamp is not None:
                    prev_run_time_utc = datetime.datetime.utcfromtimestamp(float(prev_run_timestamp))
                    prev_run_time_utc = prev_run_time_utc.replace(microsecond=0)
                else:
                    prev_run_time_utc = None
                #logger.fdebug('prev_run_time: %s' % prev_run_timestamp)
                #logger.fdebug('prev_run_time type: %s' % type(prev_run_timestamp))
                jobresults.append({'jobname': jobname,
                                   'next_run_datetime': datetime.datetime.utcfromtimestamp(jobtime),
                                   'prev_run_datetime': prev_run_time_utc,
                                   'next_run_timestamp': jobtime,
                                   'prev_run_timestamp': prev_run_timestamp,
                                   'status': newstatus})

        if not write:
            if len(jobresults) == 0:
                return monitors
            else:
                return jobresults
        else:
            if job is None:
                for x in jobresults:
                    updateCtrl = {'JobName':  x['jobname']}
                    updateVals = {'next_run_timestamp': x['next_run_timestamp'],
                                  'prev_run_timestamp': x['prev_run_timestamp'],
                                  'next_run_datetime': x['next_run_datetime'],
                                  'prev_run_datetime': x['prev_run_datetime'],
                                  'status': x['status']}

                    myDB.upsert('jobhistory', updateVals, updateCtrl)
            else:
                #logger.fdebug('Updating info - job: %s' % job)
                #logger.fdebug('Updating info - last run: %s' % last_run_completed)
                #logger.fdebug('Updating info - status: %s' % status)
                updateCtrl = {'JobName':  job}
                if current_run is not None:
                    pr_datetime = datetime.datetime.utcfromtimestamp(current_run)
                    pr_datetime = pr_datetime.replace(microsecond=0)
                    updateVals = {'prev_run_timestamp': current_run,
                                  'prev_run_datetime': pr_datetime,
                                  'status':  status}
                    #logger.info('updateVals: %s' % updateVals)
                elif last_run_completed is not None:
                    if any([job == 'DB Updater', job == 'Auto-Search', job == 'RSS Feeds', job == 'Weekly Pullist', job == 'Check Version', job == 'Folder Monitor']):
                        jobstore = None
                        for jbst in mylar.SCHED.get_jobs():
                            jb = str(jbst)
                            if 'Status Updater' in jb.lower():
                               continue
                            elif job == 'DB Updater' and 'update' in jb.lower():
                                nextrun_stamp = utctimestamp() + (int(mylar.DBUPDATE_INTERVAL) * 60)
                                jobstore = jbst
                                break
                            elif job == 'Auto-Search' and 'search' in jb.lower():
                                nextrun_stamp = utctimestamp() + (mylar.CONFIG.SEARCH_INTERVAL * 60)
                                jobstore = jbst
                                break
                            elif job == 'RSS Feeds' and 'rss' in jb.lower():
                                nextrun_stamp = utctimestamp() + (int(mylar.CONFIG.RSS_CHECKINTERVAL) * 60)
                                mylar.SCHED_RSS_LAST = last_run_completed
                                jobstore = jbst
                                break
                            elif job == 'Weekly Pullist' and 'weekly' in jb.lower():
                                if mylar.CONFIG.ALT_PULL == 2:
                                    wkt = 4
                                else:
                                    wkt = 24
                                nextrun_stamp = utctimestamp() + (wkt * 60 * 60)
                                mylar.SCHED_WEEKLY_LAST = last_run_completed
                                jobstore = jbst
                                break
                            elif job == 'Check Version' and 'version' in jb.lower():
                                nextrun_stamp = utctimestamp() + (mylar.CONFIG.CHECK_GITHUB_INTERVAL * 60)
                                jobstore = jbst
                                break
                            elif job == 'Folder Monitor' and 'monitor' in jb.lower():
                                nextrun_stamp = utctimestamp() + (int(mylar.CONFIG.DOWNLOAD_SCAN_INTERVAL) * 60)
                                jobstore = jbst
                                break

                        if jobstore is not None:
                            nextrun_date = datetime.datetime.utcfromtimestamp(nextrun_stamp)
                            jobstore.modify(next_run_time=nextrun_date)
                            nextrun_date = nextrun_date.replace(microsecond=0)
                        else:
                            # if the rss is enabled after startup, we have to re-set it up...
                            nextrun_stamp = utctimestamp() + (int(mylar.CONFIG.RSS_CHECKINTERVAL) * 60)
                            nextrun_date = datetime.datetime.utcfromtimestamp(nextrun_stamp)
                            mylar.SCHED_RSS_LAST = last_run_completed

                    logger.fdebug('ReScheduled job: %s to %s' % (job, nextrun_date))
                    lastrun_comp = datetime.datetime.utcfromtimestamp(last_run_completed)
                    lastrun_comp = lastrun_comp.replace(microsecond=0)
                    #if it's completed, then update the last run time to the ending time of the job
                    updateVals = {'prev_run_timestamp':   last_run_completed,
                                  'prev_run_datetime':    lastrun_comp,
                                  'last_run_completed':   'True',
                                  'next_run_timestamp':   nextrun_stamp,
                                  'next_run_datetime':    nextrun_date,
                                  'status':               status}

                #logger.fdebug('Job update for %s: %s' % (updateCtrl, updateVals))
                myDB.upsert('jobhistory', updateVals, updateCtrl)


def stupidchk():
    #import db
    myDB = db.DBConnection()
    CCOMICS = myDB.select("SELECT COUNT(*) FROM comics WHERE Status='Active'")
    ens = myDB.select("SELECT COUNT(*) FROM comics WHERE Status='Loading' OR Status='Paused'")
    mylar.COUNT_COMICS = CCOMICS[0][0]
    mylar.EN_OOMICS = ens[0][0]

def newznab_test(name, host, ssl, apikey):
    from xml.dom.minidom import parseString, Element
    params = {'t':       'search',
              'apikey':  apikey,
              'o':       'xml'}

    if host[:-1] == '/':
        host = host + 'api'
    else:
        host = host + '/api'
    headers = {'User-Agent': str(mylar.USER_AGENT)}
    logger.info('host: %s' % host)
    try:
        r = requests.get(host, params=params, headers=headers, verify=bool(ssl))
    except Exception as e:
        logger.warn('Unable to connect: %s' % e)
        return
    else:
        try:
            data = parseString(r.content)
        except Exception as e:
            logger.warn('[WARNING] Error attempting to test: %s' % e)

        try:
            error_code = data.getElementsByTagName('error')[0].attributes['code'].value
        except Exception as e:
            logger.info('Connected - Status code returned: %s' % r.status_code)
            if r.status_code == 200:
                return True
            else:
                logger.warn('Received response - Status code returned: %s' % r.status_code)
                return False

        code = error_code
        description = data.getElementsByTagName('error')[0].attributes['description'].value
        logger.info('[ERROR:%s] - %s' % (code, description))
        return False

def torznab_test(name, host, ssl, apikey):
    from xml.dom.minidom import parseString, Element
    params = {'t':       'search',
              'apikey':  apikey,
              'o':       'xml'}

    if host[-1:] == '/':
        host = host[:-1]
    headers = {'User-Agent': str(mylar.USER_AGENT)}
    logger.info('host: %s' % host)
    try:
        r = requests.get(host, params=params, headers=headers, verify=bool(ssl))
    except Exception as e:
        logger.warn('Unable to connect: %s' % e)
        return
    else:
        try:
            data = parseString(r.content)
        except Exception as e:
            logger.warn('[WARNING] Error attempting to test: %s' % e)

        try:
            error_code = data.getElementsByTagName('error')[0].attributes['code'].value
        except Exception as e:
            logger.info('Connected - Status code returned: %s' % r.status_code)
            if r.status_code == 200:
                return True
            else:
                logger.warn('Received response - Status code returned: %s' % r.status_code)
                return False

        code = error_code
        description = data.getElementsByTagName('error')[0].attributes['description'].value
        logger.info('[ERROR:%s] - %s' % (code, description))
        return False

def get_free_space(folder):
    min_threshold = 100000000 #threshold for minimum amount of freespace available (#100mb)
    if platform.system() == "Windows":
        free_bytes = ctypes.c_ulonglong(0)
        ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(folder), None, None, ctypes.pointer(free_bytes))
        dst_freesize = free_bytes.value
    else:
        st = os.statvfs(folder)
        dst_freesize = st.f_bavail * st.f_frsize
    logger.fdebug('[FREESPACE-CHECK] %s has %s free' % (folder, sizeof_fmt(dst_freesize)))
    if min_threshold > dst_freesize:
        logger.warn('[FREESPACE-CHECK] There is only %s space left on %s' % (dst_freesize, folder))
        return False
    else:
        return True

def sizeof_fmt(num, suffix='B'):
    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
        if abs(num) < 1024.0:
            return "%3.1f%s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f%s%s" % (num, 'Yi', suffix)

def getImage(comicid, url, issueid=None):

    if os.path.exists(mylar.CONFIG.CACHE_DIR):
        pass
    else:
        #let's make the dir.
        try:
            os.makedirs(str(mylar.CONFIG.CACHE_DIR))
            logger.info('Cache Directory successfully created at: %s' % mylar.CONFIG.CACHE_DIR)

        except OSError:
            logger.error('Could not create cache dir. Check permissions of cache dir: %s' % mylar.CONFIG.CACHE_DIR)

    coverfile = os.path.join(mylar.CONFIG.CACHE_DIR,  str(comicid) + '.jpg')

    #if cover has '+' in url it's malformed, we need to replace '+' with '%20' to retreive properly.

    #new CV API restriction - one api request / second.(probably unecessary here, but it doesn't hurt)
    if mylar.CONFIG.CVAPI_RATE is None or mylar.CONFIG.CVAPI_RATE < 2:
        time.sleep(2)
    else:
        time.sleep(mylar.CONFIG.CVAPI_RATE)

    logger.info('Attempting to retrieve the comic image for series')
    try:
        r = requests.get(url, params=None, stream=True, verify=mylar.CONFIG.CV_VERIFY, headers=mylar.CV_HEADERS)
    except Exception as e:
        logger.warn('[ERROR: %s] Unable to download image from CV URL link: %s' % (e, url))
        coversize = 0
        statuscode = '400'
    else:
        statuscode = str(r.status_code)
        logger.fdebug('comic image retrieval status code: %s' % statuscode)

        if statuscode != '200':
            logger.warn('Unable to download image from CV URL link: %s [Status Code returned: %s]' % (url, statuscode))
            coversize = 0
        else:
            if r.headers.get('Content-Encoding') == 'gzip':
                buf = StringIO(r.content)
                f = gzip.GzipFile(fileobj=buf)

            with open(coverfile, 'wb') as f:
                for chunk in r.iter_content(chunk_size=1024):
                    if chunk: # filter out keep-alive new chunks
                        f.write(chunk)
                        f.flush()


            statinfo = os.stat(coverfile)
            coversize = statinfo.st_size

    if any([int(coversize) < 10000, statuscode != '200']):
        try:
            if statuscode != '200':
                logger.info('Trying to grab an alternate cover due to problems trying to retrieve the main cover image.')
            else:
                logger.info('Image size invalid [%s bytes] - trying to get alternate cover image.' % coversize)
        except Exception as e:
            logger.info('Image size invalid [%s bytes] - trying to get alternate cover image.' % coversize)

        logger.fdebug('invalid image link is here: %s' % url)

        if os.path.exists(coverfile):
            os.remove(coverfile)

        return 'retry'

def publisherImages(publisher):
    if publisher == 'DC Comics':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-dccomics.png',
                          'publisher_image_alt':   'DC',
                          'publisher_imageH':      '50',
                          'publisher_imageW':      '50'}
    elif publisher == 'Marvel':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-marvel.jpg',
                          'publisher_image_alt':   'Marvel',
                          'publisher_imageH':      '50',
                          'publisher_imageW':      '100'}
    elif publisher == 'Image':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-imagecomics.png',
                          'publisher_image_alt':   'Image',
                          'publisher_imageH':      '100',
                          'publisher_imageW':      '50'}
    elif publisher == 'Dark Horse Comics' or publisher == 'Dark Horse':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-darkhorse.png',
                          'publisher_image_alt':   'DarkHorse',
                          'publisher_imageH':      '100',
                          'publisher_imageW':      '75'}
    elif publisher == 'IDW Publishing':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-idwpublish.png',
                          'publisher_image_alt':   'IDW',
                          'publisher_imageH':      '50',
                          'publisher_imageW':      '100'}
    elif publisher == 'Icon':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-iconcomics.png',
                          'publisher_image_alt':   'Icon',
                          'publisher_imageH':      '50',
                          'publisher_imageW':      '100'}
    elif publisher == 'Red5':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-red5comics.png',
                          'publisher_image_alt':   'Red5',
                          'publisher_imageH':      '50',
                          'publisher_imageW':      '100'}
    elif publisher == 'Vertigo':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-vertigo.png',
                          'publisher_image_alt':   'Vertigo',
                          'publisher_imageH':      '50',
                          'publisher_imageW':      '100'}
    elif publisher == 'Shadowline':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-shadowline.png',
                          'publisher_image_alt':   'Shadowline',
                          'publisher_imageH':      '50',
                          'publisher_imageW':      '150'}
    elif publisher == 'Archie Comics':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-archiecomics.jpg',
                          'publisher_image_alt':   'Archie',
                          'publisher_imageH':      '75',
                          'publisher_imageW':      '75'}
    elif publisher == 'Oni Press':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-onipress.jpg',
                          'publisher_image_alt':   'Oni Press',
                          'publisher_imageH':      '50',
                          'publisher_imageW':      '100'}
    elif publisher == 'Tokyopop':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-tokyopop.jpg',
                          'publisher_image_alt':   'Tokyopop',
                          'publisher_imageH':      '100',
                          'publisher_imageW':      '50'}
    elif publisher == 'Midtown Comics':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-midtowncomics.jpg',
                          'publisher_image_alt':   'Midtown',
                          'publisher_imageH':      '50',
                          'publisher_imageW':      '100'}
    elif publisher == 'Boom! Studios':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-boom.jpg',
                          'publisher_image_alt':   'Boom!',
                          'publisher_imageH':      '50',
                          'publisher_imageW':      '100'}
    elif publisher == 'Skybound':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-skybound.jpg',
                          'publisher_image_alt':   'Skybound',
                          'publisher_imageH':      '50',
                          'publisher_imageW':      '100'}
    elif publisher == 'Dynamite Entertainment':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-dynamite.png',
                          'publisher_image_alt':   'Dynamite',
                          'publisher_imageH':      '50',
                          'publisher_imageW':      '125'}
    elif publisher == 'Top Cow':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-topcow.gif',
                          'publisher_image_alt':   'Top Cow',
                          'publisher_imageH':      '75',
                          'publisher_imageW':      '100'}
    elif publisher == 'Cartoon Books':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-cartoonbooks.jpg',
                          'publisher_image_alt':   'Cartoon Books',
                          'publisher_imageH':      '75',
                          'publisher_imageW':      '90'}
    elif publisher == 'Valiant':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-valiant.png',
                          'publisher_image_alt':   'Valiant',
                          'publisher_imageH':      '100',
                          'publisher_imageW':      '100'}
    elif publisher == 'Action Lab':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-actionlabs.png',
                          'publisher_image_alt':   'Action Lab',
                          'publisher_imageH':      '100',
                          'publisher_imageW':      '100'}
    elif publisher == 'Zenescope Entertainment':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-zenescope.png',
                          'publisher_image_alt':   'Zenescope',
                          'publisher_imageH':      '125',
                          'publisher_imageW':      '125'}
    elif publisher == '2000 ad':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-2000ad.jpg',
                          'publisher_image_alt':   '2000 AD',
                          'publisher_imageH':      '75',
                          'publisher_imageW':      '50'}
    elif publisher == 'Aardvark':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-aardvark.png',
                          'publisher_image_alt':   'Aardvark',
                          'publisher_imageH':      '100',
                          'publisher_imageW':      '100'}
    elif publisher == 'Abstract Studio':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-abstract.jpg',
                          'publisher_image_alt':   'Abstract Studio',
                          'publisher_imageH':      '75',
                          'publisher_imageW':      '50'}
    elif publisher == 'Aftershock Comics':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-aftershock.jpg',
                          'publisher_image_alt':   'Aftershock',
                          'publisher_imageH':      '100',
                          'publisher_imageW':      '75'}
    elif publisher == 'Avatar Press':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-avatarpress.jpg',
                          'publisher_image_alt':   'Avatar Press',
                          'publisher_imageH':      '100',
                          'publisher_imageW':      '75'}
    elif publisher == 'Benitez Productions':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-benitez.png',
                          'publisher_image_alt':   'Benitez',
                          'publisher_imageH':      '75',
                          'publisher_imageW':      '125'}
    elif publisher == 'Boundless Comics':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-boundless.png',
                          'publisher_image_alt':   'Boundless',
                          'publisher_imageH':      '75',
                          'publisher_imageW':      '75'}
    elif publisher == 'Darby Pop':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-darbypop.png',
                          'publisher_image_alt':   'Darby Pop',
                          'publisher_imageH':      '75',
                          'publisher_imageW':      '125'}
    elif publisher == 'Devil\'s Due':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-devilsdue.png',
                          'publisher_image_alt':   'Devil\'s Due',
                          'publisher_imageH':      '75',
                          'publisher_imageW':      '75'}
    elif publisher == 'Joe Books':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-joebooks.png',
                          'publisher_image_alt':   'Joe Books',
                          'publisher_imageH':      '100',
                          'publisher_imageW':      '100'}
    elif publisher == 'Titan Comics':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-titan.png',
                          'publisher_image_alt':   'Titan',
                          'publisher_imageH':      '75',
                          'publisher_imageW':      '75'}
    elif publisher == 'Viz':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-viz.png',
                          'publisher_image_alt':   'Viz',
                          'publisher_imageH':      '50',
                          'publisher_imageW':      '50'}
    elif publisher == 'Warp Graphics':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-warpgraphics.png',
                          'publisher_image_alt':   'Warp Graphics',
                          'publisher_imageH':      '125',
                          'publisher_imageW':      '75'}
    elif publisher == 'Wildstorm':
        comicpublisher = {'publisher_image':       'interfaces/default/images/publisherlogos/logo-wildstorm.png',
                          'publisher_image_alt':   'Wildstorm',
                          'publisher_imageH':      '50',
                          'publisher_imageW':      '100'}
    else:
        comicpublisher = {'publisher_image':       None,
                          'publisher_image_alt':   'Nope',
                          'publisher_imageH':      '0',
                          'publisher_imageW':      '0'}

    return comicpublisher

def lookupthebitches(filelist, folder, nzbname, nzbid, prov, hash, pulldate):
    #import db
    myDB = db.DBConnection()
    watchlist = listLibrary()
    matchlist = []
    #get the weeknumber/year for the pulldate
    dt = datetime.datetime.strptime(pulldate, '%Y-%m-%d')
    weeknumber = dt.strftime("%U")
    year = dt.strftime("%Y")
    for f in filelist:
        file = re.sub(folder, '', f).strip()
        pp = mylar.filechecker.FileChecker(justparse=True, file=file)
        parsedinfo = pp.listFiles()
        if parsedinfo['parse_status'] == 'success':
            dyncheck = re.sub('[\|\s]', '', parsedinfo['dynamic_name'].lower()).strip()
            check = myDB.selectone('SELECT * FROM weekly WHERE DynamicName=? AND weeknumber=? AND year=? AND STATUS<>"Downloaded"', [dyncheck, weeknumber, year]).fetchone()
            if check is not None:
                logger.fdebug('[%s] found match: %s #%s' % (file, check['COMIC'], check['ISSUE']))
                matchlist.append({'comicname':     check['COMIC'],
                                  'issue':         check['ISSUE'],
                                  'comicid':       check['ComicID'],
                                  'issueid':       check['IssueID'],
                                  'dynamicname':   check['DynamicName']})
        else:
            logger.fdebug('[%s] unable to match to the pull: %s' % (file, parsedinfo))

    if len(matchlist) > 0:
        for x in matchlist:
            if all([x['comicid'] not in watchlist, mylar.CONFIG.PACK_0DAY_WATCHLIST_ONLY is False]):
                oneoff = True
                mode = 'pullwant'
            elif all([x['comicid'] not in watchlist, mylar.CONFIG.PACK_0DAY_WATCHLIST_ONLY is True]):
                continue
            else:
                oneoff = False
                mode = 'want'
            mylar.updater.nzblog(x['issueid'], nzbname, x['comicname'], id=nzbid, prov=prov, oneoff=oneoff)
            mylar.updater.foundsearch(x['comicid'], x['issueid'], mode=mode, provider=prov, hash=hash)


def DateAddedFix():
    #import db
    myDB = db.DBConnection()
    DA_A = datetime.datetime.today()
    DateAdded = DA_A.strftime('%Y-%m-%d')
    issues = myDB.select("SELECT IssueID FROM issues WHERE Status='Wanted' and DateAdded is NULL")
    for da in issues:
        myDB.upsert("issues", {'DateAdded': DateAdded}, {'IssueID': da[0]})
    annuals = myDB.select("SELECT IssueID FROM annuals WHERE Status='Wanted' and DateAdded is NULL")
    for an in annuals:
        myDB.upsert("annuals", {'DateAdded': DateAdded}, {'IssueID': an[0]})

def file_ops(path,dst,arc=False,one_off=False):
#    # path = source path + filename
#    # dst = destination path + filename
#    # arc = to denote if the file_operation is being performed as part of a story arc or not where the series exists on the watchlist already
#    # one-off = if the file_operation is being performed where it is either going into the grabbab_dst or story arc folder

#    #get the crc of the file prior to the operation and then compare after to ensure it's complete.
#    crc_check = mylar.filechecker.crc(path)
#    #will be either copy / move

    if any([one_off, arc]):
        action_op = mylar.CONFIG.ARC_FILEOPS
    else:
        action_op = mylar.CONFIG.FILE_OPTS

    if action_op == 'copy' or (arc is True and any([action_op == 'copy', action_op == 'move'])):
        try:
            shutil.copy( path , dst )
#        if crc_check == mylar.filechecker.crc(dst):
        except Exception as e:
            logger.error('[%s] error : %s' % (action_op, e))
            return False
        return True

    elif action_op == 'move':
        try:
            shutil.move( path , dst )
#        if crc_check == mylar.filechecker.crc(dst):
        except Exception as e:
            logger.error('[MOVE] error : %s' % e)
            return False
        return True

    elif any([action_op == 'hardlink', action_op == 'softlink']):
        if 'windows' not in mylar.OS_DETECT.lower():
            # if it's an arc, then in needs to go reverse since we want to keep the src files (in the series directory)
            if action_op == 'hardlink':
                import sys

                # Open a file
                try:
                    fd = os.open( path, os.O_RDWR|os.O_CREAT )
                    os.close( fd )

                    # Now create another copy of the above file.
                    os.link( path, dst )
                    logger.info('Created hard link successfully!!')
                except OSError, e:
                    if e.errno == errno.EXDEV:
                        logger.warn('[' + str(e) + '] Hardlinking failure. Could not create hardlink - dropping down to copy mode so that this operation can complete. Intervention is required if you wish to continue using hardlinks.')
                        try:
                            shutil.copy( path, dst )
                            logger.fdebug('Successfully copied file to : ' + dst) 
                            return True
                        except Exception as e:
                            logger.error('[COPY] error : %s' % e)
                            return False
                    else:
                        logger.warn('[' + str(e) + '] Hardlinking failure. Could not create hardlink - Intervention is required if you wish to continue using hardlinks.')
                        return False

                hardlinks = os.lstat( dst ).st_nlink
                if hardlinks > 1:
                    logger.info('Created hard link [' + str(hardlinks) + '] successfully!! (' + dst + ')')
                else:
                    logger.warn('Hardlink cannot be verified. You should probably verify that it is created properly.')

                return True

            elif action_op ==  'softlink':
                try:
                    #first we need to copy the file to the new location, then create the symlink pointing from new -> original
                    if not arc:
                        shutil.move( path, dst )            
                        if os.path.lexists( path ):
                            os.remove( path )
                        os.symlink( dst, path )
                        logger.fdebug('Successfully created softlink [' + dst + ' --> ' + path + ']')
                    else:
                        os.symlink ( path, dst )
                        logger.fdebug('Successfully created softlink [' + path + ' --> ' + dst + ']')
                except OSError, e:
                    #if e.errno == errno.EEXIST:
                    #    os.remove(dst)
                    #    os.symlink( path, dst )
                    #else:
                    logger.warn('[' + str(e) + '] Unable to create symlink. Dropping down to copy mode so that this operation can continue.')
                    try:
                        shutil.copy( dst, path )
                        logger.fdebug('Successfully copied file [' + dst + ' --> ' + path + ']')
                    except Exception as e:
                        logger.error('[COPY] error : %s' % e)
                        return False

                return True

        else:
            #Not ready just yet.
            pass

            #softlinks = shortcut (normally junctions are called softlinks, but for this it's being called a softlink)
            #hardlinks = MUST reside on the same drive as the original
            #junctions = not used (for directories across same machine only but different drives)

            #option 1
            #this one needs to get tested
            #import ctypes
            #kdll = ctypes.windll.LoadLibrary("kernel32.dll")
            #kdll.CreateSymbolicLinkW(path, dst, 0)

            #option 2
            import lib.winlink as winlink
            if mylar.CONFIG.FILE_OPTS == 'hardlink':
                try:
                    os.system(r'mklink /H dst path')
                    logger.fdebug('Successfully hardlinked file [' + dst + ' --> ' + path + ']')
                except OSError, e:
                    logger.warn('[' + e + '] Unable to create symlink. Dropping down to copy mode so that this operation can continue.')
                    try:
                        shutil.copy( dst, path )
                        logger.fdebug('Successfully copied file [' + dst + ' --> ' + path + ']')
                    except:
                        return False

            elif mylar.CONFIG.FILE_OPTS == 'softlink':  #ie. shortcut.
                try:
                    shutil.move( path, dst )
                    if os.path.lexists( path ):
                        os.remove( path )
                    os.system(r'mklink dst path')
                    logger.fdebug('Successfully created symlink [' + dst + ' --> ' + path + ']')
                except OSError, e:
                    raise e
                    logger.warn('[' + e + '] Unable to create softlink. Dropping down to copy mode so that this operation can continue.')
                    try:
                        shutil.copy( dst, path )
                        logger.fdebug('Successfully copied file [' + dst + ' --> ' + path + ']')
                    except:
                        return False


    else:
        return False

from threading import Thread

class ThreadWithReturnValue(Thread):
    def __init__(self, group=None, target=None, name=None,
                 args=(), kwargs={}, Verbose=None):
        Thread.__init__(self, group, target, name, args, kwargs, Verbose)
        self._return = None
    def run(self):
        if self._Thread__target is not None:
            self._return = self._Thread__target(*self._Thread__args, **self._Thread__kwargs)

    def join(self):
        Thread.join(self)
        return self._return