mylar/mylar/filechecker.py

#/usr/bin/env python
# -*- coding: utf-8 -*-
#  This file is part of Mylar.
#
#  Mylar is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  Mylar is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with Mylar.  If not, see <http://www.gnu.org/licenses/>.

import os
import re
import sys
import glob
import shutil
import operator
import urllib
import logging
import unicodedata
import optparse
from fnmatch import fnmatch

import datetime as dt

import subprocess
from subprocess import CalledProcessError, check_output

import mylar
from mylar import logger, helpers

class FileChecker(object):

    def __init__(self, dir=None, watchcomic=None, Publisher=None, AlternateSearch=None, manual=None, sarc=None, justparse=None, file=None, pp_mode=False):
        #dir = full path to the series Comic Location (manual pp will just be psssing the already parsed filename)
        if dir:
            self.dir = dir
        else:
            self.dir = None

        if watchcomic:
            #watchcomic = unicode name of series that is being searched against
            self.og_watchcomic = watchcomic
            self.watchcomic = re.sub('\?', '', watchcomic).strip()  #strip the ? sepearte since it affects the regex.
            self.watchcomic = re.sub(u'\u2014', ' - ', watchcomic).strip()  #replace the \u2014 with a normal - because this world is f'd up enough to have something like that.
            self.watchcomic = re.sub(u'\u2013', ' - ', watchcomic).strip()  #replace the \u2013 with a normal - because again, people are dumb.
            if type(self.watchcomic) != str:
                self.watchcomic = unicodedata.normalize('NFKD', self.watchcomic).encode('ASCII', 'ignore')
        else:
            self.watchcomic = None

        if Publisher:
            #publisher = publisher of watchcomic
            self.publisher = Publisher
        else:
            self.publisher = None

        #alternatesearch = list of alternate search names
        if AlternateSearch:
            self.AlternateSearch = AlternateSearch
        else:
            self.AlternateSearch = None

        #manual = true / false if it's a manual post-processing run
        if manual:
            self.manual = manual
        else:
            self.manual = None

        #sarc = true / false if it's being run against an existing story-arc
        if sarc:
            self.sarc = sarc
        else:
            self.sarc = None

        #justparse = true/false when manually post-processing, will quickly parse the filename to find
        #the series name in order to query the sql instead of cycling through each series in the watchlist.
        if justparse:
            self.justparse = True
        else:
            self.justparse = False

        #file = parse just one filename (used primarily during import/scan)
        if file:
            self.file = file
            self.justparse = True
        else:
            self.file = None

        if pp_mode:
            self.pp_mode = True
        else:
            self.pp_mode = False

        self.failed_files = []
        self.dynamic_handlers = ['/','-',':',';','\'',',','&','?','!','+','(',')','\u2014','\u2013']
        self.dynamic_replacements = ['and','the']
        self.rippers = ['-empire','-empire-hd','minutemen-','-dcp','Glorith-HD']

        #pre-generate the AS_Alternates now
        AS_Alternates = self.altcheck()
        self.AS_Alt = AS_Alternates['AS_Alt']
        self.AS_Tuple = AS_Alternates['AS_Tuple']

    def listFiles(self):
        comiclist = []
        watchmatch = {}
        dirlist = []
        comiccnt = 0

        if self.file:
            runresults = self.parseit(self.dir, self.file)
            return {'parse_status':        runresults['parse_status'],
                    'sub':                 runresults['sub'],
                    'comicfilename':       runresults['comicfilename'],
                    'comiclocation':       runresults['comiclocation'],
                    'series_name':         runresults['series_name'],
                    'series_name_decoded': runresults['series_name_decoded'],
                    'issueid':             runresults['issueid'],
                    'dynamic_name':        runresults['dynamic_name'],
                    'series_volume':       runresults['series_volume'],
                    'alt_series':          runresults['alt_series'],
                    'alt_issue':           runresults['alt_issue'],
                    'issue_year':          runresults['issue_year'],
                    'issue_number':        runresults['issue_number'],
                    'scangroup':           runresults['scangroup'],
                    'reading_order':       runresults['reading_order'],
                    'booktype':            runresults['booktype']
                    }
        else:
            filelist = self.traverse_directories(self.dir)

            for files in filelist:
                filedir = files['directory']
                filename = files['filename']
                filesize = files['comicsize']
                if filename.startswith('.'):
                    continue

                logger.debug('[FILENAME]: %s' % filename)
                runresults = self.parseit(self.dir, filename, filedir)
                if runresults:
                    try:
                        if runresults['parse_status']:
                            run_status = runresults['parse_status']
                    except:
                        if runresults['process_status']:
                            run_status = runresults['process_status']

                    if any([run_status == 'success', run_status == 'match']):
                        if self.justparse:
                            comiclist.append({
                                    'sub':                 runresults['sub'],
                                    'comicfilename':       runresults['comicfilename'],
                                    'comiclocation':       runresults['comiclocation'],
                                    'series_name':         helpers.conversion(runresults['series_name']),
                                    'series_name_decoded': runresults['series_name_decoded'],
                                    'issueid':             runresults['issueid'],
                                    'alt_series':          helpers.conversion(runresults['alt_series']),
                                    'alt_issue':           runresults['alt_issue'],
                                    'dynamic_name':        runresults['dynamic_name'],
                                    'series_volume':       runresults['series_volume'],
                                    'issue_year':          runresults['issue_year'],
                                    'issue_number':        runresults['issue_number'],
                                    'scangroup':           runresults['scangroup'],
                                    'reading_order':       runresults['reading_order'],
                                    'booktype':            runresults['booktype']
                                    })
                        else:
                            comiclist.append({
                                     'sub':                     runresults['sub'],
                                     'ComicFilename':           runresults['comicfilename'],
                                     'ComicLocation':           runresults['comiclocation'],
                                     'ComicSize':               files['comicsize'],
                                     'ComicName':               helpers.conversion(runresults['series_name']),
                                     'SeriesVolume':            runresults['series_volume'],
                                     'IssueYear':               runresults['issue_year'],
                                     'JusttheDigits':           runresults['justthedigits'],
                                     'AnnualComicID':           runresults['annual_comicid'],
                                     'issueid':                 runresults['issueid'],
                                     'scangroup':               runresults['scangroup'],
                                     'booktype':                runresults['booktype']
                                     })
                        comiccnt +=1
                    else:
                        #failure
                        self.failed_files.append({'parse_status':   'failure',
                                                  'sub':            runresults['sub'],
                                                  'comicfilename':  runresults['comicfilename'],
                                                  'comiclocation':  runresults['comiclocation'],
                                                  'series_name':    helpers.conversion(runresults['series_name']),
                                                  'series_volume':  runresults['series_volume'],
                                                  'alt_series':     helpers.conversion(runresults['alt_series']),
                                                  'alt_issue':      runresults['alt_issue'],
                                                  'issue_year':     runresults['issue_year'],
                                                  'issue_number':   runresults['issue_number'],
                                                  'issueid':        runresults['issueid'],
                                                  'scangroup':      runresults['scangroup'],
                                                  'booktype':       runresults['booktype']
                                                  })

        watchmatch['comiccount'] = comiccnt
        if len(comiclist) > 0:
            watchmatch['comiclist'] = comiclist
        else:
            watchmatch['comiclist'] = []
            
        if len(self.failed_files) > 0:
            logger.info('FAILED FILES: %s' % self.failed_files)

        return watchmatch

    def parseit(self, path, filename, subpath=None):

            #filename = filename.encode('ASCII').decode('utf8')
            path_list = None
            if subpath is None:
                subpath = path
                tmppath = None
                path_list = None
            else:
                logger.fdebug('[CORRECTION] Sub-directory found. Altering path configuration.')
                #basepath the sub if it exists to get the parent folder.
                logger.fdebug('[SUB-PATH] Checking Folder Name for more information.')
                #sub = re.sub(origpath, '', path).strip()})
                logger.fdebug('[SUB-PATH] Original Path : %s' % path)
                logger.fdebug('[SUB-PATH] Sub-directory : %s' % subpath)
                subpath = helpers.conversion(subpath)
                if 'windows' in mylar.OS_DETECT.lower():
                    if path in subpath:
                        ab = len(path)
                        tmppath = subpath[ab:]
                else:
                    tmppath = subpath.replace(path, '').strip()

                path_list = os.path.normpath(tmppath)
                if '/' == path_list[0] or '\\' == path_list[0]:
                    #need to remove any leading slashes so the os join can properly join the components
                    path_list = path_list[1:]
                logger.fdebug('[SUB-PATH] subpath set to : %s' % path_list)


            #parse out the extension for type
            comic_ext = ('.cbr','.cbz','.cb7','.pdf')
            if os.path.splitext(filename)[1].endswith(comic_ext):
                filetype = os.path.splitext(filename)[1]
            else:
                filetype = 'unknown'

            #find the issue number first.
            #split the file and then get all the relevant numbers that could possibly be an issue number.
            #remove the extension.
            modfilename = re.sub(filetype, '', filename).strip()
            reading_order = None

            #if it's a story-arc, make sure to remove any leading reading order #'s
            if self.sarc and mylar.CONFIG.READ2FILENAME:
                removest = modfilename.find('-') # the - gets removed above so we test for the first blank space...
                if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
                   logger.fdebug('[SARC] Checking filename for Reading Order sequence - Reading Sequence Order found #: %s' % modfilename[:removest])
                if modfilename[:removest].isdigit() and removest <= 3:
                    reading_order = {'reading_sequence': str(modfilename[:removest]),
                                     'filename':         filename[removest+1:]}
                    modfilename = modfilename[removest+1:]
                    if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
                        logger.fdebug('[SARC] Removed Reading Order sequence from subname. Now set to : %s' % modfilename)

            #make sure all the brackets are properly spaced apart
            if modfilename.find('\s') == -1:
                #if no spaces exist, assume decimals being used as spacers (ie. nzb name)
                modspacer = '.'
            else:
                modspacer = ' '
            m = re.findall('[^()]+', modfilename)
            cnt = 1
            #2019-12-24----fixed to accomodate naming convention like Amazing Mary Jane (2019) 002.cbr, and to account for brackets properly
            try:
                while cnt < len(m):
                    #logger.fdebug('[m=%s] modfilename.find: %s' % (m[cnt], modfilename[modfilename.find('('+m[cnt]+')')+len(m[cnt])+2]))
                    #logger.fdebug('mod_1: %s' % modfilename.find('('+m[cnt]+')'))
                    if modfilename[modfilename.find('('+m[cnt]+')')-1] != modspacer and modfilename.find('('+m[cnt]+')') != -1:
                        #logger.fdebug('before_space: %s' % modfilename[modfilename.find('('+m[cnt]+')')-1])
                        #logger.fdebug('after_space: %s' % modfilename[modfilename.find('('+m[cnt]+')')+len(m[cnt])+2])
                        modfilename = '%s%s%s' % (modfilename[:modfilename.find('('+m[cnt]+')')], modspacer, modfilename[modfilename.find('('+m[cnt]+')'):])
                    cnt+=1
            except Exception as e:
                #logger.warn('[ERROR] %s' % e)
                pass
            #---end 2019-12-24

            #grab the scanner tags here.
            scangroup = None
            rippers = [x for x in self.rippers if x.lower() in modfilename.lower()]
            if rippers:
                #it's always possible that this could grab something else since tags aren't unique. Try and figure it out.
                if len(rippers) > 0:
                    m = re.findall('[^()]+', modfilename)
                    #--2019-11-30  needed for Glorith naming conventions when it's an nzb name with all formatting removed.
                    if len(m) == 1:
                        spf30 = re.compile(ur"[^.]+", re.UNICODE)
                        #logger.fdebug('spf30: %s' % spf30)
                        split_file30 = spf30.findall(modfilename)
                        #logger.fdebug('split_file30: %s' % split_file30)
                        if len(split_file30) > 3 and 'Glorith-HD' in modfilename:
                            scangroup = 'Glorith-HD'
                            sp_pos = 0
                            for x in split_file30:
                                if sp_pos+1 > len(split_file30):
                                    break
                                if x[-1] == ',' and self.checkthedate(split_file30[sp_pos+1]):
                                    modfilename = re.sub(x, x[:-1], modfilename, count=1)
                                    break
                                sp_pos+=1
                    #-- end 2019-11-30
                    cnt = 1
                    for rp in rippers:
                        while cnt < len(m):
                            if m[cnt] == ' ':
                                pass
                            elif rp.lower() in m[cnt].lower():
                                scangroup = re.sub('[\(\)]', '', m[cnt]).strip()
                                logger.fdebug('Scanner group tag discovered: %s' % scangroup)
                                modfilename = modfilename.replace(m[cnt],'').strip()
                                break
                            cnt +=1

                    modfilename = modfilename.replace('()','').strip()

            issueid = None
            x = modfilename.find('[__')
            if x != -1:
                y = modfilename.find('__]', x)
                if y != -1:
                    issueid = modfilename[x+3:y]
                    logger.fdebug('issueid: %s' % issueid)
                    modfilename = '%s %s'.strip() % (modfilename[:x], modfilename[y+3:])
                    logger.fdebug('issueid %s removed successfully: %s' % (issueid, modfilename))

            #here we take a snapshot of the current modfilename, the intent is that we will remove characters that match
            #as we discover them - namely volume, issue #, years, etc
            #the remaining strings should be the series title and/or issue title if present (has to be detected properly)
            modseries = modfilename

            #try and remove /remember unicode character strings here (multiline ones get seperated/removed in below regex)
            pat = re.compile(u'[\x00-\x7f]{3,}', re.UNICODE)
            replack = pat.sub('XCV', modfilename)
            wrds = replack.split('XCV')
            tmpfilename = modfilename
            if len(wrds) > 1:
                for i in list(wrds):
                    if i != '':
                        tmpfilename = re.sub(i, 'XCV', tmpfilename)

            tmpfilename = ''.join(tmpfilename)
            modfilename = tmpfilename

            sf3 = re.compile(ur"[^,\s_]+", re.UNICODE)
            split_file3 = sf3.findall(modfilename)
            #--2019-11-30
            if len(split_file3) == 1 or all([len(split_file3) == 2, scangroup == 'Glorith-HD']):
            #--end 2019-11-30
                logger.fdebug('Improperly formatted filename - there is no seperation using appropriate characters between wording.')
                sf3 = re.compile(ur"[^,\s_\.]+", re.UNICODE)
                split_file3 = sf3.findall(modfilename)
                logger.fdebug('NEW split_file3: %s' % split_file3)

            ret_sf2 = ' '.join(split_file3)

            sf = re.findall('''\( [^\)]* \) |\[ [^\]]* \] |\[ [^\#]* \]|\S+''', ret_sf2, re.VERBOSE)
            #sf = re.findall('''\( [^\)]* \) |\[ [^\]]* \] |\S+''', ret_sf2, re.VERBOSE)

            ret_sf1 = ' '.join(sf)

            #here we should account for some characters that get stripped out due to the regex's
            #namely, unique characters - known so far: +, &, @
            #c11 = '\+'
            #f11 = '\&'
            #g11 = '\''
            ret_sf1 = re.sub('\+', 'c11', ret_sf1).strip()
            ret_sf1 = re.sub('\&', 'f11', ret_sf1).strip()
            ret_sf1 = re.sub('\'', 'g11', ret_sf1).strip()
            ret_sf1 = re.sub('\@', 'h11', ret_sf1).strip()

            #split_file = re.findall('(?imu)\([\w\s-]+\)|[-+]?\d*\.\d+|\d+[\s]COVERS+|\d{4}-\d{2}-\d{2}|\d+[(th|nd|rd|st)]+|\d+|[\w-]+|#?\d\.\d+|#[\.-]\w+|#[\d*\.\d+|\w+\d+]+|#(?<![\w\d])XCV(?![\w\d])+|#[\w+]|\)', ret_sf1, re.UNICODE)
            split_file = re.findall('(?imu)\([\w\s-]+\)|[-+]?\d*\.\d+|\d+[\s]COVERS+|\d{4}-\d{2}-\d{2}|\d+[(th|nd|rd|st)]+|[\(^\)+]|\d+|[\w-]+|#?\d\.\d+|#[\.-]\w+|#[\d*\.\d+|\w+\d+]+|#(?<![\w\d])XCV(?![\w\d])+|#[\w+]|\)', ret_sf1, re.UNICODE)

            #10-20-2018 ---START -- attempt to detect '01 (of 7.3)'
            #10-20-2018          -- attempt to detect '36p ctc' as one element
            spf = []
            mini = False
            wrdcnt = 0
            for x in split_file:
                if x == 'of':
                    mini = True
                    spf.append(x)
                elif mini is True:
                    mini = False
                    try:
                        logger.fdebug('checking now: %s' % x)
                        if x.lower() == 'infinity':
                            raise Exception
                        if x.isdigit():
                            logger.fdebug('[MINI-SERIES] MAX ISSUES IN SERIES: %s' % x)
                            spf.append('(of %s)' % x)
                        elif float(x) > 0:
                            logger.fdebug('[MINI-DECIMAL SERIES] MAX ISSUES IN SERIES: %s' % x)
                            spf.append('(of %s)' % x)
                    except Exception as e:
                        spf.append(x)

                elif x  == ')' or x == '(':
                    pass
                elif x == 'p' or x == 'ctc':
                    try:
                        if spf[wrdcnt-1].isdigit():
                            logger.debug('THIS SHOULD BE : %s%s' % (spf[wrdcnt-1], x))
                            newline = '%s%s' % (spf[wrdcnt-1], x)
                            spf[wrdcnt -1] = newline
                            #wrdcnt =-1
                        elif spf[wrdcnt-1][-1] == 'p' and spf[wrdcnt-1][:-1].isdigit() and x == 'ctc':
                            logger.fdebug('THIS SHOULD BE : %s%s' % (spf[wrdcnt-1], x))
                            newline = '%s%s' % (spf[wrdcnt-1], x)
                            spf[wrdcnt -1] = newline
                            #wrdcnt =-1
                    except Exception as e:
                        spf.append(x)
                else:
                    spf.append(x)
                wrdcnt +=1

            if len(spf) > 0:
                split_file = spf
                logger.fdebug('NEWLY SPLIT REORGD: %s' % split_file)
            #10-20-2018 ---END

            if len(split_file) == 1:
                logger.fdebug('Improperly formatted filename - there is no seperation using appropriate characters between wording.')
                ret_sf1 = re.sub('\-',' ', ret_sf1).strip()
                split_file = re.findall('(?imu)\([\w\s-]+\)|[-+]?\d*\.\d+|\d+|[\w-]+|#?\d\.\d+|#(?<![\w\d])XCV(?![\w\d])+|\)', ret_sf1, re.UNICODE)


            possible_issuenumbers = []
            volumeprior = False
            volume = None
            volume_found = {}
            datecheck = []
            lastissue_label = None
            lastissue_position = 0
            lastmod_position = 0
            booktype = 'issue'
            #exceptions that are considered alpha-numeric issue numbers
            exceptions = ('NOW', 'AI', 'AU', 'X', 'A', 'B', 'C', 'INH', 'MU', 'HU', 'SUMMER', 'SPRING', 'FALL', 'WINTER', 'PREVIEW')

            #unicode characters, followed by int value 
    #        num_exceptions = [{iss:u'\xbd',val:.5},{iss:u'\xbc',val:.25}, {iss:u'\xe',val:.75}, {iss:u'\221e',val:'infinity'}]

            file_length = 0
            validcountchk = False
            sep_volume = False
            current_pos = -1
            for sf in split_file:
                current_pos +=1
                #the series title will always be first and be AT LEAST one word.
                if split_file.index(sf) >= 0 and not volumeprior:
                    dtcheck = re.sub('[\(\)\,]', '', sf).strip()
                    #if there's more than one date, assume the right-most date is the actual issue date.
                    if any(['19' in dtcheck, '20' in dtcheck]) and not any([dtcheck.lower().startswith('v19'), dtcheck.lower().startswith('v20')]) and len(dtcheck) >=4:
                        logger.fdebug('checking date : %s' % dtcheck)
                        checkdate_response = self.checkthedate(dtcheck)
                        if checkdate_response:
                            logger.fdebug('date: %s' % checkdate_response)
                            datecheck.append({'date':         dtcheck,
                                              'position':     split_file.index(sf),
                                              'mod_position': self.char_file_position(modfilename, sf, lastmod_position)})

                #this handles the exceptions list in the match for alpha-numerics
                test_exception = ''.join([i for i in sf if not i.isdigit()])
                if any([x for x in exceptions if x.lower() == test_exception.lower()]):
                    logger.fdebug('Exception match: %s' % test_exception)
                    if lastissue_label is not None:
                        if lastissue_position == (split_file.index(sf) -1):
                            logger.fdebug('alphanumeric issue number detected as : %s %s' % (lastissue_label,sf))
                            for x in possible_issuenumbers:
                                possible_issuenumbers = []
                                if int(x['position']) != int(lastissue_position):
                                    possible_issuenumbers.append({'number':        x['number'],
                                                                  'position':      x['position'],
                                                                  'mod_position':  x['mod_position'],
                                                                  'validcountchk': x['validcountchk']})

                            possible_issuenumbers.append({'number':       '%s %s' % (lastissue_label, sf),
                                                          'position':     lastissue_position,
                                                          'mod_position': self.char_file_position(modfilename, sf, lastmod_position),
                                                          'validcountchk': validcountchk})
                    else:
                        #if the issue number & alpha character(s) don't have a space seperating them (ie. 15A)
                        #test_exception is the alpha-numeric
                        logger.fdebug('Possible alpha numeric issue (or non-numeric only). Testing my theory.')
                        test_sf = re.sub(test_exception.lower(), '', sf.lower()).strip()
                        logger.fdebug('[%s] Removing possible alpha issue leaves: %s (Should be a numeric)' % (test_exception, test_sf))
                        if test_sf.isdigit():
                            possible_issuenumbers.append({'number':       sf,
                                                          'position':     split_file.index(sf),
                                                          'mod_position': self.char_file_position(modfilename, sf, lastmod_position),
                                                          'validcountchk': validcountchk})
                        else:
                            test_position = modfilename[self.char_file_position(modfilename, sf,lastmod_position)-1]
                            if test_position == '#':
                                possible_issuenumbers.append({'number':       sf,
                                                              'position':     split_file.index(sf),
                                                              'mod_position': self.char_file_position(modfilename, sf, lastmod_position),
                                                              'validcountchk': validcountchk})

                if sf == 'XCV':
#  new 2016-09-19 \ attempt to check for XCV which replaces any unicode above
                    for x in list(wrds):
                        if x != '':
                            tmpissue_number = re.sub('XCV', x, split_file[split_file.index(sf)])
                    logger.info('[SPECIAL-CHARACTER ISSUE] Possible issue # : %s' % tmpissue_number)
                    possible_issuenumbers.append({'number':       sf,
                                                  'position':     split_file.index(sf),
                                                  'mod_position': self.char_file_position(modfilename, sf, lastmod_position),
                                                  'validcountchk': validcountchk})

                count = None
                found = False

                match = re.search('(?<=\sof\s)\d+(?=\s)', sf, re.IGNORECASE)
                if match:
                        logger.fdebug('match')
                        count = match.group()
                        found = True

                if found is False:
                        match = re.search('(?<=\(of\s)\d+(?=\))', sf,  re.IGNORECASE)
                        if match:
                                count = match.group()
                                found = True


                if count:
#                    count = count.lstrip("0")
                    logger.fdebug('Mini-Series Count detected. Maximum issue # set to : %s' % count.lstrip('0'))
                    # if the count was  detected, then it's in a '(of 4)' or whatever pattern
                    # 95% of the time the digit immediately preceding the '(of 4)' is the actual issue #
                    logger.fdebug('Issue Number SHOULD BE: %s' % lastissue_label)
                    validcountchk = True

                match2 = re.search('(\d+[\s])covers', sf, re.IGNORECASE)
                if match2:
                    num_covers = re.sub('[^0-9]', '', match2.group()).strip()
                    #logger.fdebug('%s covers detected within filename' % num_covers)
                    continue

                if all([lastissue_position == (split_file.index(sf) -1), lastissue_label is not None, '#' not in sf, sf != 'p']):
                    #find it in the original file to see if there's a decimal between.
                    findst = lastissue_mod_position+1
                    if findst >= len(modfilename):
                        findst = len(modfilename) -1
                    
                    if modfilename[findst] != '.' or modfilename[findst] != '#': #findst != '.' and findst != '#':
                        if sf.isdigit():
                            seper_num = False
                            for x in datecheck:
                                if x['position'] == split_file.index(sf, lastissue_position):
                                    seper_num = True
                            if seper_num is False:
                                logger.fdebug('2 seperate numbers detected. Assuming 2nd number is the actual issue')

                            #possible_issuenumbers.append({'number':       sf,
                            #                              'position':     split_file.index(sf, lastissue_position), #modfilename.find(sf)})
                            #                              'mod_position': self.char_file_position(modfilename, sf, lastmod_position),
                            #                              'validcountchk': validcountchk})

                            #used to see if the issue is an alpha-numeric (ie. 18.NOW, 50-X, etc)
                            lastissue_position = split_file.index(sf, lastissue_position)
                            lastissue_label = sf
                            lastissue_mod_position = file_length
                        else:
                            pass
                    else:
                        bb = len(lastissue_label) + findst
                        #find current sf 
                        #logger.fdebug('bb: ' + str(bb) + '[' + modfilename[findst:bb] + ']')
                        cf = modfilename.find(sf, file_length)
                        #logger.fdebug('cf: ' + str(cf) + '[' + modfilename[cf:cf+len(sf)] + ']')
                        diff = bb
                        #logger.fdebug('diff: ' + str(bb) + '[' + modfilename[bb] + ']')
                        if modfilename[bb] == '.':
                            #logger.fdebug('decimal detected.')
                            logger.fdebug('[DECiMAL-DETECTION] Issue being stored for validation as : %s' % modfilename[findst:cf+len(sf)])
                            for x in possible_issuenumbers:
                                possible_issuenumbers = []
                                #logger.fdebug('compare: ' + str(x['position']) + ' .. ' + str(lastissue_position))
                                #logger.fdebug('compare: ' + str(x['position']) + ' .. ' + str(split_file.index(sf, lastissue_position)))
                                if int(x['position']) != int(lastissue_position) and int(x['position']) != split_file.index(sf, lastissue_position):
                                    possible_issuenumbers.append({'number':        x['number'],
                                                                  'position':      x['position'],
                                                                  'mod_position':  x['mod_position'],
                                                                  'validcountchk': x['validcountchk']})

                            possible_issuenumbers.append({'number':        modfilename[findst:cf+len(sf)],
                                                          'position':      split_file.index(lastissue_label, lastissue_position),
                                                          'mod_position':  findst,
                                                          'dec_position':  bb,
                                                          'rem_position':  split_file.index(sf),
                                                          'validcountchk': validcountchk})

                        else:
                            if ('#' in sf or sf.isdigit()) or validcountchk:
                                if validcountchk:
                                    #if it's not a decimal but the digits are back-to-back, then it's something else.
                                    possible_issuenumbers.append({'number':        lastissue_label,
                                                                  'position':      lastissue_position,
                                                                  'mod_position':  lastissue_mod_position,
                                                                  'validcountchk': validcountchk})

                                    validcountchk = False
                                #used to see if the issue is an alpha-numeric (ie. 18.NOW, 50-X, etc)
                                lastissue_position = split_file.index(sf, lastissue_position)
                                lastissue_label = sf
                                lastissue_mod_position = file_length

                elif '#' in sf:
                    logger.fdebug('Issue number found: %s' % sf)
                    #pound sign will almost always indicate an issue #, so just assume it's as such.
                    locateiss_st = modfilename.find('#')
                    locateiss_end = modfilename.find(' ', locateiss_st)
                    if locateiss_end == -1:
                        locateiss_end = len(modfilename)
                    if modfilename[locateiss_end-1] == ')':
                        locateiss_end = locateiss_end -1
                    possible_issuenumbers.append({'number':       modfilename[locateiss_st:locateiss_end],
                                                  'position':     split_file.index(sf), #locateiss_st})
                                                  'mod_position': self.char_file_position(modfilename, sf, lastmod_position),
                                                  'validcountchk': validcountchk})

                #now we try to find the series title &/or volume lablel.
                if any( [sf.lower().startswith('v'), sf.lower().startswith('vol'), volumeprior == True, 'volume' in sf.lower(), 'vol' in sf.lower(), 'part' in sf.lower()] ) and sf.lower() not in {'one','two','three','four','five','six'}:
                    if any([ split_file[split_file.index(sf)].isdigit(), split_file[split_file.index(sf)][3:].isdigit(), split_file[split_file.index(sf)][1:].isdigit() ]):
                        if all(identifier in sf for identifier in ['.', 'v']):
                            volume = sf.split('.')[0]
                        else:
                            volume = re.sub("[^0-9]", "", sf)
                        if volumeprior:
                            try:
                                volume_found['position'] = split_file.index(volumeprior_label, current_pos -1) #if this passes, then we're ok, otherwise will try exception
                                logger.fdebug('volume_found: %s' % volume_found['position'])
                                #remove volume numeric from split_file
                                split_file.pop(volume_found['position'])
                                split_file.pop(split_file.index(sf, current_pos-1))
                                #join the previous label to the volume numeric
                                #volume = str(volumeprior_label) + str(volume)
                                #insert the combined info back
                                split_file.insert(volume_found['position'], volumeprior_label + volume)
                                split_file.insert(volume_found['position']+1, '')
                                #volume_found['position'] = split_file.index(sf, current_pos)
                                #logger.fdebug('NEWSPLITFILE: %s' % split_file)
                            except:
                                volumeprior = False
                                volumeprior_label = None
                                sep_volume = False
                                continue
                        else:
                            volume_found['position'] = split_file.index(sf, current_pos)

                        volume_found['volume'] = volume
                        logger.fdebug('volume label detected as : Volume %s @ position: %s' % (volume, volume_found['position']))
                        volumeprior = False
                        volumeprior_label = None
                    elif all(['vol' in sf.lower(), len(sf) == 3]) or all(['vol.' in sf.lower(), len(sf) == 4]):
                        #if there's a space between the vol and # - adjust.
                        volumeprior = True
                        volumeprior_label = sf
                        sep_volume = True
                        logger.fdebug('volume label detected, but vol. number is not adjacent, adjusting scope to include number.')
                    elif 'volume' in sf.lower() or all(['part' in sf.lower(), len(sf) == 4]):
                        volume = re.sub("[^0-9]", "", sf)
                        if volume.isdigit():
                            volume_found['volume'] = volume
                            volume_found['position'] = split_file.index(sf)
                        else:
                            volumeprior = True
                            volumeprior_label = sf
                            sep_volume = True

                    elif any([sf == 'I', sf == 'II', sf == 'III', sf == 'IV']) and volumeprior:
                        volumeprior = False
                        volumeprior_label = None
                        sep_volume = False
                        continue
                else:
                    #reset the sep_volume indicator here in case a false Volume detected above
                    sep_volume = False

                    #check here for numeric or negative number
                    if sf.isdigit() and split_file.index(sf, current_pos) == 0:
                        continue
                    if sf.isdigit():
                        possible_issuenumbers.append({'number':       sf,
                                                      'position':     split_file.index(sf, current_pos), #modfilename.find(sf)})
                                                      'mod_position': self.char_file_position(modfilename, sf, lastmod_position),
                                                      'validcountchk': validcountchk})

                        #used to see if the issue is an alpha-numeric (ie. 18.NOW, 50-X, etc)
                        lastissue_position = split_file.index(sf, current_pos)
                        lastissue_label = sf
                        lastissue_mod_position = file_length
                        #logger.fdebug('possible issue found: %s' % sf)
                    else:
                        try:
                            x = float(sf)
                            #validity check
                            if x < 0:
                                logger.fdebug('I have encountered a negative issue #: %s' % sf)
                                possible_issuenumbers.append({'number':       sf,
                                                              'position':     split_file.index(sf, lastissue_position), #modfilename.find(sf)})
                                                              'mod_position': self.char_file_position(modfilename, sf, lastmod_position),
                                                              'validcountchk': validcountchk})

                                lastissue_position = split_file.index(sf, lastissue_position)
                                lastissue_label = sf
                                lastissue_mod_position = file_length
                            elif x > 0:
                                logger.fdebug('I have encountered a decimal issue #: %s' % sf)
                                possible_issuenumbers.append({'number':       sf,
                                                              'position':     split_file.index(sf, lastissue_position), #modfilename.find(sf)})
                                                              'mod_position': self.char_file_position(modfilename, sf, lastmod_position),
                                                              'validcountchk': validcountchk})

                                lastissue_position = split_file.index(sf, lastissue_position)
                                lastissue_label = sf
                                lastissue_mod_position = file_length
                            else:
                                raise ValueError
                        except ValueError, e:
                           #10-20-2018 - to detect issue numbers such as #000.0000½
                            if lastissue_label is not None and lastissue_position == int(split_file.index(sf))-1 and sf == 'XCV':
                                logger.info('this should be: %s%s' % (lastissue_label, sf))
                                pi = []
                                for x in possible_issuenumbers:
                                    if (x['number'] == lastissue_label and x['position'] == lastissue_position) or (x['number'] == sf and x['position'] == split_file.index(sf, lastissue_position)):
                                        pass
                                    else:
                                        pi.append({'number':       x['number'],
                                                   'position':     x['position'],
                                                   'mod_position': x['mod_position'],
                                                   'validcountchk': x['validcountchk']})

                                lastissue_label = '%s%s' % (lastissue_label, sf)
                                pi.append({'number':        lastissue_label,
                                           'position':      lastissue_position,
                                           'mod_position':  lastmod_position,
                                           'validcountchk': validcountchk})

                                if len(pi) > 0:
                                    possible_issuenumbers = pi

                            elif sf.lower() == 'of' and lastissue_label is not None and lastissue_position == int(split_file.index(sf))-1:
                                logger.info('MINI-SERIES DETECTED')

                            else:
                                if any([re.sub('[\(\)]', '', sf.lower()).strip() == 'tpb', re.sub('[\(\)]', '', sf.lower()).strip() == 'digital tpb']):
                                    logger.info('TRADE PAPERBACK DETECTED. NOT DETECTING ISSUE NUMBER - ASSUMING VOLUME')
                                    booktype = 'TPB'
                                    try:
                                        if volume_found['volume'] is not None:
                                            possible_issuenumbers.append({'number':       volume_found['volume'],
                                                                          'position':     volume_found['position'],
                                                                          'mod_position': self.char_file_position(modfilename, volume_found['volume'], lastmod_position),
                                                                          'validcountchk': validcountchk})
                                    except:
                                        possible_issuenumbers.append({'number':       '1',
                                                                      'position':      split_file.index(sf, lastissue_position), #modfilename.find(sf)})
                                                                      'mod_position':  self.char_file_position(modfilename, sf, lastmod_position),
                                                                      'validcountchk': validcountchk})

                                elif any([sf.lower() == 'gn', sf.lower() == 'graphic novel']):
                                    logger.info('GRAPHIC NOVEL DETECTED. NOT DETECTING ISSUE NUMBER - ASSUMING VOLUME')
                                    booktype = 'GN'
                                else:
                                    if 'could not convert string to float' not in str(e):
                                        logger.fdebug('[%s] Error detecting issue # - ignoring this result : %s' % (e, sf))

                            volumeprior = False
                            volumeprior_label = None
                            sep_volume = False
                            pass

                #keep track of where in the original modfilename the positions are in order to check against it for decimal places, etc.
                file_length += len(sf) + 1 #1 for space
                if file_length > len(modfilename):
                    file_length = len(modfilename)

                lastmod_position = self.char_file_position(modfilename, sf, lastmod_position)


            highest_series_pos = len(split_file)
            issue2year = False
            issue_year = None
            possible_years = []
            yearmodposition = None
            logger.fdebug('datecheck: %s' % datecheck)
            if len(datecheck) > 0:
                for dc in sorted(datecheck, key=operator.itemgetter('position'), reverse=True):
                    a = self.checkthedate(dc['date'])
                    ab = str(a)
                    sctd = self.checkthedate(str(dt.datetime.now().year))
                    logger.fdebug('sctd: %s' % sctd)
                    # + 1 sctd so that we can allow for issue dates that cross over into the following year when it's nearer to the end of said year.
                    if int(ab) > int(sctd) + 1:
                        logger.fdebug('year is in the future, ignoring and assuming part of series title.')
                        yearposition = None
                        yearmodposition = None
                        continue
                    else:
                        issue_year = dc['date']
                        logger.fdebug('year verified as : %s' % issue_year)
                        if highest_series_pos > dc['position']: highest_series_pos = dc['position']
                        yearposition = dc['position']
                        yearmodposition = dc['mod_position']
                    if len(ab) == 4:
                        issue_year = ab
                        logger.fdebug('year verified as: %s' % issue_year)
                        possible_years.append({'year':            issue_year,
                                               'yearposition':    dc['position'],
                                               'yearmodposition': dc['mod_position']})
                    else:
                        issue_year = ab
                        logger.fdebug('date verified as: %s' % issue_year)

                if len(possible_years) == 1:
                    issueyear = possible_years[0]['year']
                    yearposition = possible_years[0]['yearposition']
                    yearmodposition = possible_years[0]['yearmodposition']
                else:
                    if len(possible_issuenumbers) > 0:
                        for x in possible_years:
                            logger.info('yearposition[%s] -- dc[position][%s]' % (yearposition, x['yearposition']))
                            if yearposition < x['yearposition']:
                                if all([len(possible_issuenumbers) == 1, possible_issuenumbers[0]['number'] == x['year'], x['yearposition'] != possible_issuenumbers[0]['position']]):
                                    issue2year = True
                                    highest_series_pos = x['yearposition']
                                yearposition = x['yearposition']
                                yearmodposition = x['yearmodposition']

                if yearposition is not None and highest_series_pos > yearposition:
                    highest_series_pos = yearposition #dc['position']: highest_series_pos = dc['position']
            else:
                issue_year = None
                yearposition = None
                yearmodposition = None
                logger.fdebug('No year present within title - ignoring as a variable.')


            logger.fdebug('highest_series_position: %s' % highest_series_pos)
            #---2019-11-30 account for scanner Glorith-HD stupid naming conventions
            if len(possible_issuenumbers) == 0 and scangroup == 'Glorith-HD':
                logger.fdebug('Abnormal formatting detected. Time to fix this shiet, yo.')
                if any([yearposition == 0, yearposition is None]):
                    logger.fdebug('Too stupid of a format. Nope. Not gonna happen - just reinvent the wheel you fooker.')
                else:
                    issposs = yearposition + 1
                    #logger.fdebug('split_file: %s' % split_file[issposs])
                    if '(' and ')' in split_file[issposs]:
                        new_issuenumber = split_file[issposs]
                    possible_issuenumbers.append({'number':        re.sub('[/(/)]', '', split_file[issposs]).strip(),
                                                  'position':      split_file.index(new_issuenumber, yearposition),
                                                  'mod_position':  self.char_file_position(modfilename, new_issuenumber, yearmodposition),
                                                  'validcountchk': False})
            #---end 2019-11-30
            issue_number = None
            dash_numbers = []
            issue_number_position = len(split_file)
            if len(possible_issuenumbers) > 0:
                logger.fdebug('possible_issuenumbers: %s' % possible_issuenumbers)
                if len(possible_issuenumbers) >= 1:
                    p = 1
                    if '-' not in split_file[0]:
                        finddash = modfilename.find('-')
                        if finddash != -1:
                            logger.fdebug('hyphen located at position: %s' % finddash)
                            if yearposition:
                                logger.fdebug('yearposition: %s' % yearposition)
                    else:
                        finddash = -1
                        logger.fdebug('dash is in first word, not considering for determing issue number.')

                    for pis in sorted(possible_issuenumbers, key=operator.itemgetter('position'), reverse=True):
                        a = ' '.join(split_file)
                        lenn = pis['mod_position'] + len(pis['number'])
                        if lenn == len(a) and finddash != -1:
                            logger.fdebug('Numeric detected as the last digit after a hyphen. Typically this is the issue number.')
                            if pis['position'] != yearposition:
                                issue_number = pis['number']
                                #logger.info('Issue set to: ' + str(issue_number))
                                issue_number_position = pis['position']
                                if highest_series_pos > pis['position']: highest_series_pos = pis['position']
                            #break
                        elif pis['validcountchk'] == True:
                            issue_number = pis['number']
                            issue_number_position = pis['position']
                            logger.fdebug('Issue verified and detected as part of a numeric count sequnce: %s' % issue_number)
                            if highest_series_pos > pis['position']: highest_series_pos = pis['position']
                            break
                        elif pis['mod_position'] > finddash and finddash != -1:
                            if finddash < yearposition and finddash > (yearmodposition + len(split_file[yearposition])):
                                logger.fdebug('issue number is positioned after a dash - probably not an issue number, but part of an issue title')
                                dash_numbers.append({'mod_position': pis['mod_position'],
                                                     'number':       pis['number'],
                                                     'position':     pis['position']})
                                continue
                            #2019-10-05 fix - if decimal-spaced filename has a series title with a hyphen will include issue # as part of series title
                            elif yearposition == pis['position']:
                                logger.info('Already validated year, ignoring as possible issue number: %s' % pis['number'])
                                continue
                            #end 2019-10-05
                        elif yearposition == pis['position']:
                            logger.fdebug('Already validated year, ignoring as possible issue number: %s' % pis['number'])
                            continue
                        if p == 1:
                            issue_number = pis['number']
                            issue_number_position = pis['position']
                            logger.fdebug('issue number :%s' % issue_number) #(pis)
                            if highest_series_pos > pis['position'] and issue2year is False: highest_series_pos = pis['position']
                        #else:
                            #logger.fdebug('numeric probably belongs to series title: ' + str(pis))
                        p+=1
                else:
                    issue_number = possible_issuenumbers[0]['number']
                    issue_number_position = possible_issuenumbers[0]['position']
                    if highest_series_pos > possible_issuenumbers[0]['position']: highest_series_pos = possible_issuenumbers[0]['position']

            if issue_number:
                issue_number = re.sub('#', '', issue_number).strip()
            else:
                if len(dash_numbers) > 0 and finddash !=-1 :
                    #there are numbers after a dash, which was incorrectly accounted for.
                    fin_num_position = finddash
                    fin_num = None
                    for dn in dash_numbers:
                        if dn['mod_position'] > finddash and dn['mod_position'] > fin_num_position:
                            fin_num_position = dn['mod_position']
                            fin_num = dn['number']
                            fin_pos = dn['position']

                    if fin_num:
                        logger.fdebug('Issue number re-corrected to : %s' % fin_num)
                        issue_number = fin_num
                        if highest_series_pos > fin_pos: highest_series_pos = fin_pos

   #--- this is new - 2016-09-18 /account for unicode in issue number when issue number is not deteted above
            logger.fdebug('issue_position: %s' % issue_number_position)
            if all([issue_number_position == highest_series_pos, 'XCV' in split_file, issue_number is None]):
                for x in list(wrds):
                    if x != '':
                        issue_number = re.sub('XCV', x, split_file[issue_number_position-1])
                        highest_series_pos -=1
                        issue_number_position -=1

            if issue_number is None:
                if any([booktype == 'TPB', booktype == 'GN']):
                    logger.info('%s detected. Volume assumption is number: %s' % (booktype, volume_found))
                else:
                    if len(volume_found) > 0:
                        logger.info('UNKNOWN TPB/GN detected. Volume assumption is number: %s' % (volume_found))
                    else:
                        logger.info('No issue number present in filename.')
            else:
                logger.fdebug('issue verified as : %s' % issue_number)
            issue_volume = None
            if len(volume_found) > 0:
                issue_volume = 'v' + str(volume_found['volume'])
                if all([highest_series_pos + 1 != volume_found['position'], highest_series_pos != volume_found['position'] + 1, sep_volume == False, booktype == 'issue', len(possible_issuenumbers) > 0]):
                    logger.fdebug('Extra item(s) are present between the volume label and the issue number. Checking..')
                    split_file.insert(int(issue_number_position), split_file.pop(volume_found['position'])) #highest_series_pos-1, split_file.pop(volume_found['position']))
                    logger.fdebug('new split: %s' % split_file)
                    highest_series_pos = volume_found['position'] -1
                    #2019-10-02 -  account for volume BEFORE issue number
                    if issue_number_position > highest_series_pos:
                        issue_number_position -=1
                else:
                    if highest_series_pos > volume_found['position']:
                        if sep_volume:
                            highest_series_pos = volume_found['position'] - 1
                        else:
                            highest_series_pos = volume_found['position']
                logger.fdebug('Volume detected as : %s' % issue_volume)
 
            if all([len(volume_found) == 0, booktype != 'issue']) or all([len(volume_found) == 0, issue_number_position == len(split_file)]):
                issue_volume = 'v1'

            #at this point it should be in a SERIES ISSUE VOLUME YEAR kind of format
            #if the position of the issue number is greater than the highest series position, make it the highest series position.
            if issue_number_position != len(split_file) and issue_number_position > highest_series_pos:
                if not volume_found:
                    highest_series_pos = issue_number_position
                else:
                    if sep_volume:
                        highest_series_pos = issue_number_position -2
                    else:
                        if split_file[issue_number_position -1].lower() == 'annual' or split_file[issue_number_position -1].lower() == 'special':
                            highest_series_pos = issue_number_position
                        else:
                            highest_series_pos = issue_number_position - 1
                            #if volume_found['position'] < issue_number_position:
                            #    highest_series_pos = issue_number_position - 1
                            #else:
                            #    highest_series_pos = issue_number_position

            #make sure if we have multiple years detected, that the right one gets picked for the actual year vs. series title
            if len(possible_years) > 1:
                for x in sorted(possible_years, key=operator.itemgetter('yearposition'), reverse=False):
                    if x['yearposition'] <= highest_series_pos:
                        logger.fdebug('year %s is within series title. Ignoring as YEAR value' % x['year'])
                    else:
                        logger.fdebug('year %s is outside of series title range. Accepting of year.' % x['year'])
                        issue_year = x['year']
                        highest_series_pos = x['yearposition']
                        break
            else:
                try:
                    if possible_years[0]['yearposition'] <= highest_series_pos and possible_years[0]['year_position'] != 0:
                       highest_series_pos = possible_years[0]['yearposition']
                    elif possible_years[0]['year_position'] == 0:
                       yearposition = 1
                except:
                    pass

            match_type = None  #folder/file based on how it was matched.

            #logger.fdebug('highest_series_pos is : ' + str(highest_series_pos)
            splitvalue = None
            alt_series = None
            alt_issue = None
            try:
                if yearposition is not None:
                    try:
                        if volume_found['position'] >= issue_number_position:
                            tmpval = highest_series_pos + (issue_number_position - volume_found['position'])
                        else:
                            tmpval = yearposition - issue_number_position
                    except:
                        tmpval = yearposition - issue_number_position
                else:
                    tmpval = 1
            except:
                pass
            else:
                if tmpval > 2:
                    logger.fdebug('There are %s extra words between the issue # and the year position. Deciphering if issue title or part of series title.' % tmpval)
                    tmpval1 = ' '.join(split_file[issue_number_position:yearposition])
                    if split_file[issue_number_position+1] == '-':
                        usevalue = ' '.join(split_file[issue_number_position+2:yearposition])
                        splitv = split_file[issue_number_position+2:yearposition]
                    else:
                        splitv = split_file[issue_number_position:yearposition]
                    splitvalue = ' '.join(splitv)
                else:
                    #store alternate naming of title just in case
                    if '-' not in split_file[0]:
                        c_pos = 1
                        #logger.info('split_file: %s' % split_file)
                        while True:
                            try:
                                fdash = split_file.index("-", c_pos)
                            except:
                                #logger.info('dash not located/finished searching for dashes.')
                                break
                            else:
                                #logger.info('hyphen located at position: ' + str(fdash))
                                c_pos = 2
                                #c_pos = fdash +1
                                break
                        if c_pos > 1:
                            #logger.info('Issue_number_position: %s / fdash: %s' % (issue_number_position, fdash))
                            try:
                                if volume_found['position'] < issue_number_position:
                                    alt_issue = ' '.join(split_file[fdash+1:volume_found['position']])
                                else:
                                    alt_issue = ' '.join(split_file[fdash+1:issue_number_position])
                            except:
                                    alt_issue = ' '.join(split_file[fdash+1:issue_number_position])

                            if alt_issue.endswith('-'): alt_issue = alt_issue[:-1].strip()
                            if len(alt_issue) == 0:
                                alt_issue = None
                            alt_series = ' '.join(split_file[:fdash])
                            logger.fdebug('ALT-SERIES NAME [ISSUE TITLE]: %s [%s]' % (alt_series, alt_issue))

            #logger.info('highest_series_position: ' + str(highest_series_pos))
            #logger.info('issue_number_position: ' + str(issue_number_position))
            #logger.info('volume_found: ' + str(volume_found))

       #2017-10-21
            if highest_series_pos > issue_number_position:
                highest_series_pos = issue_number_position
                #if volume_found['position'] >= issue_number_position:
                #    highest_series_pos = issue_number_position
                #else:
                #    print 'nuhuh'
       #---
            match_type = None  #folder/file based on how it was matched.

            logger.fdebug('sf_highest_series_pos: %s' % split_file[:highest_series_pos])

            #here we should account for some characters that get stripped out due to the regex's
            #namely, unique characters - known so far: +
            #c1 = '+'
            #series_name = ' '.join(split_file[:highest_series_pos])
            if yearposition != 0:
                if yearposition is not None and yearposition < highest_series_pos:
                    if yearposition+1 == highest_series_pos:
                        highest_series_pos = yearposition
                    else:
                        if split_file[yearposition+1] == '-' and yearposition+2 == highest_series_pos:
                            highest_series_pos = yearposition
                series_name = ' '.join(split_file[:highest_series_pos])
            else:
                if highest_series_pos <= issue_number_position and all([len(split_file[0]) == 4, split_file[0].isdigit()]):
                    series_name = ' '.join(split_file[:highest_series_pos])
                else:
                    series_name = ' '.join(split_file[yearposition+1:highest_series_pos])

            for x in list(wrds):
                if x != '':
                    if 'XCV' in series_name:
                        series_name = re.sub('XCV', x, series_name,1)
                    elif 'XCV' in issue_number:
                        issue_number = re.sub('XCV', x, issue_number,1)
                    if alt_series is not None:
                        if 'XCV' in alt_series:
                            alt_series = re.sub('XCV', x, alt_series,1)
                    if alt_issue is not None:
                        if 'XCV' in alt_issue:
                            alt_issue = re.sub('XCV', x, alt_issue,1)

            series_name = re.sub('c11', '+', series_name)
            series_name = re.sub('f11', '&', series_name)
            series_name = re.sub('g11', '\'', series_name)
            series_name = re.sub('h11', '@', series_name)
            if alt_series is not None:
                alt_series = re.sub('c11', '+', alt_series)
                alt_series = re.sub('f11', '&', alt_series)
                alt_series = re.sub('g11', '\'', alt_series)
                alt_series = re.sub('h11', '@', alt_series)

            if series_name.endswith('-'): 
                series_name = series_name[:-1].strip()
            if '\?' in series_name:
                series_name = re.sub('\?', '', series_name).strip()

            logger.fdebug('series title possibly: %s' % series_name)
            if splitvalue is not None:
                logger.fdebug('[SPLITVALUE] possible issue title: %s' % splitvalue)
                alt_series = '%s %s' % (series_name, splitvalue)
                if booktype != 'issue':
                    if alt_issue is not None:
                        alt_issue =  re.sub('tpb', '', splitvalue, flags=re.I).strip()
                    if alt_series is not None:
                        alt_series = re.sub('tpb', '', alt_series, flags=re.I).strip()
            if alt_series is not None:
                if booktype != 'issue':
                    if alt_series is not None:
                        alt_series = re.sub('tpb', '', alt_series, flags=re.I).strip()
                logger.fdebug('Alternate series / issue title: %s [%s]' % (alt_series, alt_issue))

            #if the filename is unicoded, it won't match due to the unicode translation. Keep the unicode as well as the decoded.
            series_name_decoded= unicodedata.normalize('NFKD', helpers.conversion(series_name)).encode('ASCII', 'ignore')
            #check for annual in title(s) here.
            if not self.justparse and all([mylar.CONFIG.ANNUALS_ON, 'annual' not in self.watchcomic.lower(), 'special' not in self.watchcomic.lower()]):
                if 'annual' in series_name.lower():
                    isn = 'Annual'
                    if issue_number is not None:
                        issue_number = '%s %s' % (isn, issue_number)
                    else:
                        issue_number = isn
                    series_name = re.sub('annual', '', series_name, flags=re.I).strip()
                    series_name_decoded = re.sub('annual', '', series_name_decoded, flags=re.I).strip()
                elif 'special' in series_name.lower():
                    isn = 'Special'
                    if issue_number is not None:
                        issue_number = '%s %s' % (isn, issue_number)
                    else:
                        issue_number = isn
                    series_name = re.sub('special', '', series_name, flags=re.I).strip()
                    series_name_decoded = re.sub('special', '', series_name_decoded, flags=re.I).strip()

            if (any([issue_number is None, series_name is None]) and booktype == 'issue'):

                if all([issue_number is None, booktype == 'issue', issue_volume is not None]):
                    logger.info('Possible UKNOWN TPB/GN detected - no issue number present, no clarification in filename, but volume present with series title')
                else:
                    logger.fdebug('Cannot parse the filename properly. I\'m going to make note of this filename so that my evil ruler can make it work.')

                    if series_name is not None:
                        dreplace = self.dynamic_replace(series_name)['mod_seriesname']
                    else:
                        dreplace = None
                    return {'parse_status':        'failure',
                            'sub':                 path_list,
                            'comicfilename':       filename,
                            'comiclocation':       self.dir,
                            'series_name':         series_name,
                            'series_name_decoded': series_name_decoded,
                            'issueid':             issueid,
                            'alt_series':          alt_series,
                            'alt_issue':           alt_issue,
                            'dynamic_name':        dreplace,
                            'issue_number':        issue_number,
                            'justthedigits':       issue_number, #redundant but it's needed atm
                            'series_volume':       issue_volume,
                            'issue_year':          issue_year,
                            'annual_comicid':      None,
                            'scangroup':           scangroup,
                            'booktype':            booktype,
                            'reading_order':       None}

            if self.justparse:
                return {'parse_status':           'success',
                        'type':                   re.sub('\.','', filetype).strip(),
                        'sub':                    path_list,
                        'comicfilename':          filename,
                        'comiclocation':          self.dir,
                        'series_name':            series_name,
                        'series_name_decoded':    series_name_decoded,
                        'issueid':                issueid,
                        'alt_series':             alt_series,
                        'alt_issue':              alt_issue,
                        'dynamic_name':           self.dynamic_replace(series_name)['mod_seriesname'],
                        'series_volume':          issue_volume,
                        'issue_year':             issue_year,
                        'issue_number':           issue_number,
                        'scangroup':              scangroup,
                        'booktype':               booktype,
                        'reading_order':          reading_order}

            series_info = {}
            series_info = {'sub':                    path_list,
                           'type':                   re.sub('\.','', filetype).strip(),
                           'comicfilename':          filename,
                           'comiclocation':          self.dir,
                           'series_name':            series_name,
                           'series_name_decoded':    series_name_decoded,
                           'issueid':                issueid,
                           'alt_series':             alt_series,
                           'alt_issue':              alt_issue,
                           'series_volume':          issue_volume,
                           'issue_year':             issue_year,
                           'issue_number':           issue_number,
                           'scangroup':              scangroup,
                           'booktype':               booktype}

            return self.matchIT(series_info)

    def matchIT(self, series_info):
            series_name = series_info['series_name']
            alt_series = series_info['alt_series']
            filename = series_info['comicfilename']
            #compare here - match comparison against u_watchcomic.
            #logger.fdebug('Series_Name: ' + series_name + ' --- WatchComic: ' + self.watchcomic)
            #check for dynamic handles here.
            mod_dynamicinfo = self.dynamic_replace(series_name)
            mod_seriesname = mod_dynamicinfo['mod_seriesname']
            mod_watchcomic = mod_dynamicinfo['mod_watchcomic']
            mod_altseriesname = None
            mod_altseriesname_decoded = None
            #logger.fdebug('series_info: %s' % series_info)
            if series_info['alt_series'] is not None:
                mod_dynamicalt = self.dynamic_replace(alt_series)
                mod_altseriesname = mod_dynamicalt['mod_seriesname']
                mod_alt_decoded = self.dynamic_replace(alt_series)
                mod_altseriesname_decoded = mod_alt_decoded['mod_seriesname']
            #logger.fdebug('mod_altseriesname: %s' % mod_altseriesname)
            mod_series_decoded = self.dynamic_replace(series_info['series_name_decoded'])
            mod_seriesname_decoded = mod_series_decoded['mod_seriesname']
            mod_watch_decoded = self.dynamic_replace(self.og_watchcomic)
            mod_watchname_decoded = mod_watch_decoded['mod_watchcomic']

            #remove the spaces...
            nspace_seriesname = re.sub(' ', '', mod_seriesname)
            nspace_watchcomic = re.sub(' ', '', mod_watchcomic)
            nspace_altseriesname = None
            if mod_altseriesname is not None:
                nspace_altseriesname = re.sub(' ', '', mod_altseriesname)
                nspace_altseriesname_decoded = re.sub(' ', '', mod_altseriesname_decoded)
            nspace_seriesname_decoded = re.sub(' ', '', mod_seriesname_decoded)
            nspace_watchname_decoded = re.sub(' ', '', mod_watchname_decoded)

            try:
                if self.AS_ALT[0] != '127372873872871091383 abdkhjhskjhkjdhakajhf':
                    logger.fdebug('Possible Alternate Names to match against (if necessary): %s' % self.AS_Alt)
            except:
                pass

            justthedigits = series_info['issue_number']

            if mylar.CONFIG.ANNUALS_ON and 'annual' not in nspace_watchcomic.lower():
                if 'annual' in series_name.lower():
                    justthedigits = 'Annual'
                    if series_info['issue_number'] is not None:
                        justthedigits += ' %s' % series_info['issue_number']
                    nspace_seriesname = re.sub('annual', '', nspace_seriesname.lower()).strip()
                    nspace_seriesname_decoded = re.sub('annual', '', nspace_seriesname_decoded.lower()).strip()
                if alt_series is not None and 'annual' in alt_series.lower():
                    nspace_altseriesname = re.sub('annual', '', nspace_altseriesname.lower()).strip()
                    nspace_altseriesname_decoded = re.sub('annual', '', nspace_altseriesname_decoded.lower()).strip()

            if mylar.CONFIG.ANNUALS_ON and 'special' not in nspace_watchcomic.lower():
                if 'special' in series_name.lower():
                    justthedigits = 'Special'
                    if series_info['issue_number'] is not None:
                        justthedigits += ' %s' % series_info['issue_number']
                    nspace_seriesname = re.sub('special', '', nspace_seriesname.lower()).strip()
                    nspace_seriesname_decoded = re.sub('special', '', nspace_seriesname_decoded.lower()).strip()
                if alt_series is not None and 'special' in alt_series.lower():
                    nspace_altseriesname = re.sub('special', '', nspace_altseriesname.lower()).strip()
                    nspace_altseriesname_decoded = re.sub('special', '', nspace_altseriesname_decoded.lower()).strip()

            seriesalt = False

            if nspace_altseriesname is not None:
                if re.sub('\|','', nspace_altseriesname.lower()).strip() == re.sub('\|', '', nspace_watchcomic.lower()).strip():
                    seriesalt = True

            if any([seriesalt is True, re.sub('\|','', nspace_seriesname.lower()).strip() == re.sub('\|', '', nspace_watchcomic.lower()).strip(), re.sub('\|','', nspace_seriesname_decoded.lower()).strip() == re.sub('\|', '', nspace_watchname_decoded.lower()).strip()]) or any(re.sub('[\|\s]','', x.lower()).strip() == re.sub('[\|\s]','', nspace_seriesname.lower()).strip() for x in self.AS_Alt):
                #logger.fdebug('[MATCH: ' + series_info['series_name'] + '] ' + filename)
                enable_annual = False
                annual_comicid = None
                if any(re.sub('[\|\s]','', x.lower()).strip() == re.sub('[\|\s]','', nspace_seriesname.lower()).strip() for x in self.AS_Alt):
                    #if the alternate search name is almost identical, it won't match up because it will hit the 'normal' first.
                    #not important for series' matches, but for annuals, etc it is very important.
                    #loop through the Alternates picking out the ones that match and then do an overall loop.
                    loopchk = [x for x in self.AS_Alt if re.sub('[\|\s]','', x.lower()).strip() == re.sub('[\|\s]','', nspace_seriesname.lower()).strip()]
                    if len(loopchk) > 0 and loopchk[0] != '':
                        if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
                            logger.fdebug('[FILECHECKER] This should be an alternate: %s' % loopchk)
                        if any(['annual' in series_name.lower(), 'special' in series_name.lower()]):
                            if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
                                logger.fdebug('[FILECHECKER] Annual/Special detected - proceeding')
                            enable_annual = True

                    else:
                        loopchk = []
                    #logger.info('loopchk: ' + str(loopchk))

                    #if the names match up, and enable annuals isn't turned on - keep it all together.
                    if re.sub('\|', '', nspace_watchcomic.lower()).strip() == re.sub('\|', '', nspace_seriesname.lower()).strip() and enable_annual == False:
                        loopchk.append(nspace_watchcomic)
                        if any(['annual' in nspace_seriesname.lower(), 'special' in nspace_seriesname.lower()]):
                            if 'biannual' in nspace_seriesname.lower():
                                if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
                                    logger.fdebug('[FILECHECKER] BiAnnual detected - wouldn\'t Deadpool be proud?')
                                nspace_seriesname = re.sub('biannual', '', nspace_seriesname).strip()
                                enable_annual = True
                            elif 'annual' in nspace_seriesname.lower():
                                if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
                                    logger.fdebug('[FILECHECKER] Annual detected - proceeding cautiously.')
                                nspace_seriesname = re.sub('annual', '', nspace_seriesname).strip()
                                enable_annual = False
                            elif 'special' in nspace_seriesname.lower():
                                if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
                                    logger.fdebug('[FILECHECKER] Special detected - proceeding cautiously.')
                                nspace_seriesname = re.sub('special', '', nspace_seriesname).strip()
                                enable_annual = False

                    if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
                        logger.fdebug('[FILECHECKER] Complete matching list of names to this file [%s] : %s' % (len(loopchk), loopchk))

                    for loopit in loopchk:
                        #now that we have the list of all possible matches for the watchcomic + alternate search names, we go through the list until we find a match.
                        modseries_name = loopit
                        if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
                            logger.fdebug('[FILECHECKER] AS_Tuple : %s' % self.AS_Tuple)
                            for ATS in self.AS_Tuple:
                                if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
                                    logger.fdebug('[FILECHECKER] %s comparing to %s' % (ATS['AS_Alternate'], nspace_seriesname))
                                if re.sub('\|','', ATS['AS_Alternate'].lower()).strip() == re.sub('\|','', nspace_seriesname.lower()).strip():
                                    if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
                                        logger.fdebug('[FILECHECKER] Associating ComiciD : %s' % ATS['ComicID'])
                                    annual_comicid = str(ATS['ComicID'])
                                    modseries_name = ATS['AS_Alternate']
                                    break

                        logger.fdebug('[FILECHECKER] %s - watchlist match on : %s' % (modseries_name, filename))

                if enable_annual:
                    if annual_comicid is not None:
                       if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
                           logger.fdebug('enable annual is on')
                           logger.fdebug('annual comicid is %s' % annual_comicid)
                       if 'biannual' in nspace_watchcomic.lower():
                           if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
                               logger.fdebug('bi annual detected')
                           justthedigits = 'BiAnnual %s' % justthedigits
                       elif 'annual' in nspace_watchcomic.lower():
                           if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
                               logger.fdebug('annual detected')
                           justthedigits = 'Annual %s' % justthedigits
                       elif 'special' in nspace_watchcomic.lower():
                           justthedigits = 'Special %s' % justthedigits

                return {'process_status': 'match',
                        'sub':             series_info['sub'],
                        'volume':          series_info['series_volume'],
                        'match_type':      None,  #match_type - will eventually pass if it wasa folder vs. filename match,
                        'comicfilename':   filename,
                        'comiclocation':   series_info['comiclocation'],
                        'series_name':     series_info['series_name'],
                        'series_volume':   series_info['series_volume'],
                        'alt_series':      series_info['alt_series'],
                        'alt_issue':       series_info['alt_issue'],
                        'issue_year':      series_info['issue_year'],
                        'issueid':         series_info['issueid'],
                        'justthedigits':   justthedigits,
                        'annual_comicid':  annual_comicid,
                        'scangroup':       series_info['scangroup'],
                        'booktype':        series_info['booktype']}

            else:
                #logger.fdebug('[NO MATCH] ' + filename + ' [WATCHLIST:' + self.watchcomic + ']')
                return {'process_status': 'fail',
                        'comicfilename':  filename,
                        'sub':            series_info['sub'],
                        'comiclocation':  series_info['comiclocation'],
                        'series_name':    series_info['series_name'],
                        'alt_series':     series_info['alt_series'],
                        'alt_issue':      series_info['alt_issue'],
                        'issue_number':   series_info['issue_number'],
                        'series_volume':  series_info['series_volume'],
                        'issue_year':     series_info['issue_year'],
                        'issueid':        series_info['issueid'],
                        'scangroup':      series_info['scangroup'],
                        'booktype':       series_info['booktype']}


    def char_file_position(self, file, findchar, lastpos):
        return file.find(findchar, lastpos)

    def traverse_directories(self, dir):
        filelist = []
        comic_ext = ('.cbr','.cbz','.cb7','.pdf')

        dir = dir.encode(mylar.SYS_ENCODING)

        if all([mylar.CONFIG.ENABLE_TORRENTS is True, self.pp_mode is True]):
            import db
            myDB = db.DBConnection()
            pp_crclist =[]
            pp_crc = myDB.select("SELECT a.crc, b.IssueID FROM Snatched as a INNER JOIN issues as b ON a.IssueID=b.IssueID WHERE (a.Status='Post-Processed' or a.status='Snatched' or a.provider='32P' or a.provider='WWT' or a.provider='DEM') and a.crc is not NULL and (b.Status='Downloaded' or b.status='Archived') GROUP BY a.crc ORDER BY a.DateAdded")
            for pp in pp_crc:
                pp_crclist.append({'IssueID':   pp['IssueID'],
                                   'crc':       pp['crc']})

        for (dirname, subs, files) in os.walk(dir):

            for fname in files:
                if dirname == dir:
                    direc = None
                else:
                    direc = dirname
                    if '.AppleDouble' in direc:
                        #Ignoring MAC OS Finder directory of cached files (/.AppleDouble/<name of file(s)>)
                        continue

                if all([mylar.CONFIG.ENABLE_TORRENTS is True, self.pp_mode is True]):
                    tcrc = helpers.crc(os.path.join(dirname, fname).decode(mylar.SYS_ENCODING))
                    crcchk = [x for x in pp_crclist if tcrc == x['crc']]
                    if crcchk:
                        #logger.fdebug('[FILECHECKEER] Already post-processed this item %s - Ignoring' % fname)
                        continue

                if os.path.splitext(fname)[1].lower().endswith(comic_ext):
                    if direc is None:
                        comicsize = os.path.getsize(os.path.join(dir, fname))
                    else:
                        comicsize = os.path.getsize(os.path.join(dir, direc, fname))

                    filelist.append({'directory':  direc,   #subdirectory if it exists
                                     'filename':   fname,
                                     'comicsize':  comicsize})

        logger.info('there are %s files.' % len(filelist))

        return filelist

    def dynamic_replace(self, series_name):
        mod_watchcomic = None

        if self.watchcomic:
            watchdynamic_handlers_match = [x for x in self.dynamic_handlers if x.lower() in self.watchcomic.lower()]
            #logger.fdebug('watch dynamic handlers recognized : ' + str(watchdynamic_handlers_match))
            watchdynamic_replacements_match = [x for x in self.dynamic_replacements if x.lower() in self.watchcomic.lower()]
            #logger.fdebug('watch dynamic replacements recognized : ' + str(watchdynamic_replacements_match))
            mod_watchcomic = re.sub('[\s\s+\_\.]', '%$', self.watchcomic)
            mod_watchcomic = re.sub('[\#]', '', mod_watchcomic)
            mod_find = []
            wdrm_find = []
            if any([watchdynamic_handlers_match, watchdynamic_replacements_match]):
                for wdhm in watchdynamic_handlers_match:
                    #check the watchcomic
                    #first get the position.
                    mod_find.extend([m.start() for m in re.finditer('\\' + wdhm, mod_watchcomic)])
                    if len(mod_find) > 0:
                        for mf in mod_find:
                            spacer = ''
                            for i in range(0, len(wdhm)):
                                spacer+='|'
                            mod_watchcomic = mod_watchcomic[:mf] + spacer + mod_watchcomic[mf+1:]

                for wdrm in watchdynamic_replacements_match:
                    wdrm_find.extend([m.start() for m in re.finditer(wdrm.lower(), mod_watchcomic.lower())])
                    if len(wdrm_find) > 0:
                        for wd in wdrm_find:
                            spacer = ''
                            for i in range(0, len(wdrm)):
                                spacer+='|'
                            mod_watchcomic = mod_watchcomic[:wd] + spacer + mod_watchcomic[wd+len(wdrm):]

        series_name = re.sub(u'\u2014', ' - ', series_name)
        series_name = re.sub(u'\u2013', ' - ', series_name)
        seriesdynamic_handlers_match = [x for x in self.dynamic_handlers if x.lower() in series_name.lower()]
        #logger.fdebug('series dynamic handlers recognized : ' + str(seriesdynamic_handlers_match))
        seriesdynamic_replacements_match = [x for x in self.dynamic_replacements if x.lower() in series_name.lower()]
        #logger.fdebug('series dynamic replacements recognized : ' + str(seriesdynamic_replacements_match))
        mod_seriesname = re.sub('[\s\s+\_\.]', '%$', series_name)
        mod_seriesname = re.sub('[\#]', '', mod_seriesname)
        ser_find = []
        sdrm_find = []
        if any([seriesdynamic_handlers_match, seriesdynamic_replacements_match]):
            for sdhm in seriesdynamic_handlers_match:
                #check the series_name
                ser_find.extend([m.start() for m in re.finditer('\\' + sdhm, mod_seriesname)])
                if len(ser_find) > 0:
                    for sf in ser_find:
                        spacer = ''
                        for i in range(0, len(sdhm)):
                            spacer+='|'
                        mod_seriesname = mod_seriesname[:sf] + spacer + mod_seriesname[sf+1:]

            for sdrm in seriesdynamic_replacements_match:
                sdrm_find.extend([m.start() for m in re.finditer(sdrm.lower(), mod_seriesname.lower())])
                if len(sdrm_find) > 0:
                    for sd in sdrm_find:
                        spacer = ''
                        for i in range(0, len(sdrm)):
                            spacer+='|'
                        mod_seriesname = mod_seriesname[:sd] + spacer + mod_seriesname[sd+len(sdrm):]

        if mod_watchcomic:
            mod_watchcomic = re.sub('\|+', '|', mod_watchcomic)
            if mod_watchcomic.endswith('|'):
                mod_watchcomic = mod_watchcomic[:-1]
            mod_watchcomic = re.sub('[\%\$]+', '', mod_watchcomic)

        mod_seriesname = re.sub('\|+', '|', mod_seriesname)
        if mod_seriesname.endswith('|'):
            mod_seriesname = mod_seriesname[:-1]
        mod_seriesname = re.sub('[\%\$]+', '', mod_seriesname)

        return {'mod_watchcomic':  mod_watchcomic,
                'mod_seriesname':  mod_seriesname}

    def altcheck(self):
       #iniitate the alternate list here so we can add in the different alternate search names (if present)
        AS_Alt = []

        AS_Tuple = []
        if self.AlternateSearch is not None and self.AlternateSearch != 'None':
            chkthealt = self.AlternateSearch.split('##')
            #logger.info('[' + str(len(chkthealt)) + '] chkthealt: ' + str(chkthealt))
            i = 0
            while (i <= len(chkthealt)):
                try:
                    calt = chkthealt[i]
                except IndexError:
                    break
                AS_tupled = False
                AS_Alternate = re.sub('##', '', calt)
                if '!!' in AS_Alternate:
                    # if it's !! present, it's the comicid associated with the series as an added annual.
                    # extract the !!, store it and then remove it so things will continue.
                    as_start = AS_Alternate.find('!!')
                    if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
                        logger.fdebug('as_start: %s --- %s' % (as_start, AS_Alternate[as_start:]))
                    as_end = AS_Alternate.find('##', as_start)
                    if as_end == -1:
                        as_end = len(AS_Alternate)
                    if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
                        logger.fdebug('as_start: %s --- %s' % (as_end, AS_Alternate[as_start:as_end]))
                    AS_ComicID =  AS_Alternate[as_start +2:as_end]
                    if mylar.CONFIG.FOLDER_SCAN_LOG_VERBOSE:
                        logger.fdebug('[FILECHECKER] Extracted comicid for given annual : %s' % AS_ComicID)
                    AS_Alternate = re.sub('!!' + str(AS_ComicID), '', AS_Alternate)
                    AS_tupled = True
                as_dyninfo = self.dynamic_replace(AS_Alternate)
                altsearchcomic = as_dyninfo['mod_seriesname']

                if AS_tupled:
                    AS_Tuple.append({"ComicID":      AS_ComicID,
                                     "AS_Alternate": altsearchcomic})
                AS_Alt.append(altsearchcomic)
                i+=1
        else:
            #create random characters so it will never match.
            altsearchcomic = "127372873872871091383 abdkhjhskjhkjdhakajhf"
            AS_Alt.append(altsearchcomic)

        return {'AS_Alt':   AS_Alt,
                'AS_Tuple': AS_Tuple}
    
    def checkthedate(self, txt, fulldate=False, cnt=0):
    #    txt='''\
    #    Jan 19, 1990
    #    January 19, 1990
    #    Jan 19,1990
    #    01/19/1990
    #    01/19/90
    #    1990
    #    Jan 1990
    #    January1990'''

        fmts = ('%Y','%b %d, %Y','%B %d, %Y','%B %d %Y','%m/%d/%Y','%m/%d/%y','(%m/%d/%Y)','%b %Y','%B%Y','%b %d,%Y','%m-%Y','%B %Y','%Y-%m-%d','%Y-%m','%Y%m')
        mnths = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec')
        parsed=[]

        if fulldate is False:
            for e in txt.splitlines():
                for fmt in fmts:
                    try:
                        t = dt.datetime.strptime(e, fmt)
                        parsed.append((e, fmt, t)) 
                        break
                    except ValueError as err:
                        pass
        else:
            for e in txt.split():
                if cnt == 0:
                    for x in mnths:
                        mnth = re.sub('\.', '', e.lower())
                        if x.lower() in mnth and len(mnth) <= 4:
                            add_date = x + ' '
                            cnt+=1
                            break

                elif cnt == 1:
                    issnumb = re.sub(',', '', e).strip()
                    if issnumb.isdigit() and int(issnumb) < 31:
                        add_date += issnumb + ', '
                        cnt+=1
                elif cnt == 2:
                    possyear = helpers.cleanhtml(re.sub('\.', '', e).strip())
                    if possyear.isdigit() and int(possyear) > 1970 and int(possyear) < 2020:
                        add_date += possyear
                        cnt +=1
                if cnt == 3:
                    return self.checkthedate(add_date, fulldate=False, cnt=-1)


                if cnt <= 0:
                    for fmt in fmts:
                        try:
                            t = dt.datetime.strptime(e, fmt)
                            parsed.append((e, fmt, t))
                            break
                        except ValueError as err:
                            pass

        # check that all the cases are handled        
        success={t[0] for t in parsed}
        for e in txt.splitlines():
            if e not in success:
                pass #print e    

        dateline = None

        #logger.info('parsed: %s' % parsed)

        for t in parsed:
            #logger.fdebug('"{:20}" => "{:20}" => {}'.format(*t))
            if fulldate is False and cnt != -1:
                dateline = t[2].year
            else:
                dateline = t[2].strftime('%Y-%m-%d')
            break

        return dateline

def validateAndCreateDirectory(dir, create=False, module=None, dmode=None):
    if module is None:
        module = ''
    module += '[DIRECTORY-CHECK]'
    if dmode is None:
        dirmode = 'comic'
    else:
        dirmode = dmode

    try:
        if os.path.exists(dir):
            logger.info('%s Found %s directory: %s' % (module, dirmode, dir))
            return True
        else:
            logger.warn('%s Could not find %s directory: %s' % (module, dirmode, dir))
            if create:
                if dir.strip():
                    logger.info('%s Creating %s directory (%s) : %s' % (module, dirmode, mylar.CONFIG.CHMOD_DIR, dir))
                    try:
                        os.umask(0) # this is probably redudant, but it doesn't hurt to clear the umask here.
                        if mylar.CONFIG.ENFORCE_PERMS:
                            permission = int(mylar.CONFIG.CHMOD_DIR, 8)
                            os.makedirs(dir.rstrip(), permission)
                            setperms(dir.rstrip(), True)
                        else:
                            os.makedirs(dir.rstrip())
                    except OSError as e:
                        logger.warn('%s Could not create directory: %s [%s]. Aborting' % (module, dir, e))
                        return False
                    else:
                        return True
                else:
                    logger.warn('%s Provided directory [%s] is blank. Aborting.' % (module, dir))
                    return False
    except OSError as e:
        logger.warn('%s Could not create directory: %s [%s]. Aborting.' % (module, dir, e))
        return False
    return False

def setperms(path, dir=False):

    if 'windows' not in mylar.OS_DETECT.lower():

        try:
            os.umask(0) # this is probably redudant, but it doesn't hurt to clear the umask here.
            if mylar.CONFIG.CHGROUP:
                if mylar.CONFIG.CHOWNER is None or mylar.CONFIG.CHOWNER == 'None' or mylar.CONFIG.CHOWNER == '':
                    import getpass
                    mylar.CONFIG.CHOWNER = getpass.getuser()

                if not mylar.CONFIG.CHOWNER.isdigit():
                    from pwd import getpwnam
                    chowner = getpwnam(mylar.CONFIG.CHOWNER)[2]
                else:
                    chowner = int(mylar.CONFIG.CHOWNER)

                if not mylar.CONFIG.CHGROUP.isdigit():
                    from grp import getgrnam
                    chgroup = getgrnam(mylar.CONFIG.CHGROUP)[2]
                else:
                    chgroup = int(mylar.CONFIG.CHGROUP)

                if dir:
                    permission = int(mylar.CONFIG.CHMOD_DIR, 8)
                    os.chmod(path, permission)
                    os.chown(path, chowner, chgroup)
                elif os.path.isfile(path):
                    permission = int(mylar.CONFIG.CHMOD_FILE, 8)
                    os.chown(path, chowner, chgroup)
                    os.chmod(path, permission)   
                else:
                    for root, dirs, files in os.walk(path):
                        for momo in dirs:
                            permission = int(mylar.CONFIG.CHMOD_DIR, 8)
                            os.chown(os.path.join(root, momo), chowner, chgroup)
                            os.chmod(os.path.join(root, momo), permission)
                        for momo in files:
                            permission = int(mylar.CONFIG.CHMOD_FILE, 8)
                            os.chown(os.path.join(root, momo), chowner, chgroup)
                            os.chmod(os.path.join(root, momo), permission)

                logger.fdebug('Successfully changed ownership and permissions [' + str(mylar.CONFIG.CHOWNER) + ':' + str(mylar.CONFIG.CHGROUP) + '] / [' + str(mylar.CONFIG.CHMOD_DIR) + ' / ' + str(mylar.CONFIG.CHMOD_FILE) + ']')

            elif os.path.isfile(path):
                    permission = int(mylar.CONFIG.CHMOD_FILE, 8)
                    os.chmod(path, permission)
            else:
                for root, dirs, files in os.walk(path):
                    for momo in dirs:
                        permission = int(mylar.CONFIG.CHMOD_DIR, 8)
                        os.chmod(os.path.join(root, momo), permission)
                    for momo in files:
                        permission = int(mylar.CONFIG.CHMOD_FILE, 8)
                        os.chmod(os.path.join(root, momo), permission)

                logger.fdebug('Successfully changed permissions [' + str(mylar.CONFIG.CHMOD_DIR) + ' / ' + str(mylar.CONFIG.CHMOD_FILE) + ']')

        except OSError:
            logger.error('Could not change permissions : %s. Exiting...' % path)

    return


#if __name__ == '__main__':
#    test = FileChecker()
#    test.getlist()