mylar/mylar/updater.py

#  This file is part of Mylar.
#
#  Mylar is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  Mylar is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with Mylar.  If not, see <http://www.gnu.org/licenses/>.

import time
import datetime
from xml.dom.minidom import parseString
import urllib2
import shlex
import re 
import os
import itertools

import mylar
from mylar import db, logger, helpers, filechecker

def dbUpdate(ComicIDList=None):

    myDB = db.DBConnection()
    #print "comicidlist:" + str(ComicIDList)
    if ComicIDList is None:
        comiclist = myDB.select('SELECT ComicID, ComicName from comics WHERE Status="Active" or Status="Loading" order by LastUpdated ASC')
    else:
        comiclist = ComicIDList

    logger.info('Starting update for %i active comics' % len(comiclist))
    
    for comic in comiclist:
        if ComicIDList is None:
            comicid = comic[0]
        else:
            comicid = comic
        #print "comicid:" + str(comicid)
        mismatch = "no"
        if not mylar.CV_ONLY or comicid[:1] == "G":
            CV_EXcomicid = myDB.action("SELECT * from exceptions WHERE ComicID=?", [comicid]).fetchone()
            if CV_EXcomicid is None: pass
            else:
                if CV_EXcomicid['variloop'] == '99':
                    mismatch = "yes"
            if comicid[:1] == "G":
                mylar.importer.GCDimport(comicid)
            else: 
                mylar.importer.addComictoDB(comicid,mismatch)
        else:
            if mylar.CV_ONETIMER == 1:
                logger.fdebug('CV_OneTimer option enabled...')

                #in order to update to JUST CV_ONLY, we need to delete the issues for a given series so it's a clean refresh.
                logger.fdebug('Gathering the status of all issues for the series.')
                issues = myDB.select('SELECT * FROM issues WHERE ComicID=?', [comicid])
                if mylar.ANNUALS_ON:
                    issues += myDB.select('SELECT * FROM annuals WHERE ComicID=?', [comicid])
                #store the issues' status for a given comicid, after deleting and readding, flip the status back to what it is currently.                
                logger.fdebug('Deleting all issue data.')
                myDB.select('DELETE FROM issues WHERE ComicID=?', [comicid])            
                myDB.select('DELETE FROM annuals WHERE ComicID=?', [comicid])
                logger.fdebug('Refreshing the series and pulling in new data using only CV.')             
                mylar.importer.addComictoDB(comicid,mismatch,calledfrom='dbupdate')
                issues_new = myDB.select('SELECT * FROM issues WHERE ComicID=?', [comicid])
                annuals = []
                ann_list = []
                if mylar.ANNUALS_ON:
                    annuals_list = myDB.select('SELECT * FROM annuals WHERE ComicID=?', [comicid])
                    ann_list += annuals_list
                    issues_new += annuals_list

                icount = 0
                logger.fdebug('Attempting to put the Statuses back how they were.')
                for issue in issues:
                    for issuenew in issues_new:
                       if issuenew['IssueID'] == issue['IssueID'] and issuenew['Status'] != issue['Status']:
                            #if the status is now Downloaded, keep status.
                            logger.info('existing status: ' + str(issuenew['Status']))
                            logger.info('new status: ' + str(issue['Status']))
                            if issuenew['Status'] == 'Downloaded' or issue['Status'] == 'Snatched': break
                            #change the status to the previous status
                            ctrlVAL = {'IssueID':  issue['IssueID']}
                            newVAL = {'Status':  issue['Status']}
                            if any(d['IssueID'] == str(issue['IssueID']) for d in ann_list):
                                logger.fdebug('annual detected for ' + str(issue['IssueID']) + ' #: ' + str(issue['Issue_Number']))
                                myDB.upsert("Annuals", newVAL, ctrlVAL)
                            else:
                                myDB.upsert("Issues", newVAL, ctrlVAL)
                            icount+=1
                            break
                logger.info('In converting data to CV only, I changed the status of ' + str(icount) + ' issues.')
                mylar.CV_ONETIMER = 0   
            else:
                mylar.importer.addComictoDB(comicid,mismatch)

        #check global skipped2wanted status here
        #if mylar.GLOBAL_SKIPPED2WANTED:
        #    logger.fdebug('Global change for ' + str(comicid) + ' - Marking all issues not present as Wanted.')
        #    mylar.webserve.skipped2wanted(comicid,True)

        time.sleep(5) #pause for 5 secs so dont hammer CV and get 500 error
    logger.info('Update complete')


def latest_update(ComicID, LatestIssue, LatestDate):
    # here we add to comics.latest
    myDB = db.DBConnection()
    latestCTRLValueDict = {"ComicID":      ComicID}
    newlatestDict = {"LatestIssue":      str(LatestIssue),
                    "LatestDate":       str(LatestDate)}
    myDB.upsert("comics", newlatestDict, latestCTRLValueDict)

def upcoming_update(ComicID, ComicName, IssueNumber, IssueDate, forcecheck=None, futurepull=None):
    # here we add to upcoming table...
    myDB = db.DBConnection()
    dspComicName = ComicName #to make sure that the word 'annual' will be displayed on screen
    if 'annual' in ComicName.lower(): 
        adjComicName = re.sub("\\bannual\\b", "", ComicName.lower()) # for use with comparisons.
        logger.fdebug('annual detected - adjusting name to : ' + adjComicName)
    else:
        adjComicName = ComicName
    controlValue = {"ComicID":      ComicID}
    newValue = {"ComicName":        adjComicName,
                "IssueNumber":      str(IssueNumber),
                "DisplayComicName": dspComicName,
                "IssueDate":        str(IssueDate)}

    #let's refresh the series here just to make sure if an issue is available/not.
    mismatch = "no"
    CV_EXcomicid = myDB.action("SELECT * from exceptions WHERE ComicID=?", [ComicID]).fetchone()
    if CV_EXcomicid is None: pass
    else:
        if CV_EXcomicid['variloop'] == '99':
            mismatch = "yes"
    lastupdatechk = myDB.action("SELECT * FROM comics WHERE ComicID=?", [ComicID]).fetchone()
    if lastupdatechk is None:
        pullupd = "yes"
    else:
        c_date = lastupdatechk['LastUpdated']
        if c_date is None:
            logger.error(lastupdatechk['ComicName'] + ' failed during a previous add /refresh. Please either delete and readd the series, or try a refresh of the series.')
            return
        c_obj_date = datetime.datetime.strptime(c_date, "%Y-%m-%d %H:%M:%S")
        n_date = datetime.datetime.now()
        absdiff = abs(n_date - c_obj_date)
        hours = (absdiff.days * 24 * 60 * 60 + absdiff.seconds) / 3600.0
        # no need to hammer the refresh 
        # let's check it every 5 hours (or more)
        #pullupd = "yes"
    if 'annual' in ComicName.lower():
        if mylar.ANNUALS_ON:
            issuechk = myDB.action("SELECT * FROM annuals WHERE ComicID=? AND Issue_Number=?", [ComicID, IssueNumber]).fetchone()
        else:
            logger.fdebug('Annual detected, but annuals not enabled. Ignoring result.')
            return
    else:
        issuechk = myDB.action("SELECT * FROM issues WHERE ComicID=? AND Issue_Number=?", [ComicID, IssueNumber]).fetchone()

    if issuechk is None:
        if futurepull is None:
            logger.fdebug(adjComicName + ' Issue: ' + str(IssueNumber) + ' not present in listings to mark for download...updating comic and adding to Upcoming Wanted Releases.')
            # we need to either decrease the total issue count, OR indicate that an issue is upcoming.
            upco_results = myDB.action("SELECT COUNT(*) FROM UPCOMING WHERE ComicID=?",[ComicID]).fetchall()
            upco_iss = upco_results[0][0]
            #logger.info("upco_iss: " + str(upco_iss))
            if int(upco_iss) > 0:
                #logger.info("There is " + str(upco_iss) + " of " + str(ComicName) + " that's not accounted for")
                newKey = {"ComicID": ComicID}
                newVal = {"not_updated_db": str(upco_iss)}
                myDB.upsert("comics", newVal, newKey)
            elif int(upco_iss) <=0 and lastupdatechk['not_updated_db']:
                #if not_updated_db has a value, and upco_iss is > 0, let's zero it back out cause it's updated now.
                newKey = {"ComicID": ComicID}
                newVal = {"not_updated_db": ""}
                myDB.upsert("comics", newVal, newKey)

            if hours > 5 or forcecheck == 'yes':
                pullupd = "yes"
                logger.fdebug('Now Refreshing comic ' + ComicName + ' to make sure it is up-to-date')
                if ComicID[:1] == "G": mylar.importer.GCDimport(ComicID,pullupd)
                else: mylar.importer.addComictoDB(ComicID,mismatch,pullupd)
            else:
                logger.fdebug('It has not been longer than 5 hours since we last did this...we will wait so we do not hammer things.')
                return
        else:
            # if futurepull is not None, let's just update the status and ComicID
            nKey = {"ComicID": ComicID}
            nVal = {"Status": "Wanted"}
            myDB.upsert("future", nVal, nKey)

    elif issuechk['Issue_Number'] == IssueNumber:
        logger.fdebug('Comic series already up-to-date ... no need to refresh at this time.')
        logger.fdebug('Available to be marked for download - checking...' + adjComicName + ' Issue: ' + str(issuechk['Issue_Number']))
        logger.fdebug('...Existing status: ' + str(issuechk['Status']))
        control = {"IssueID":   issuechk['IssueID']}
        newValue['IssueID'] = issuechk['IssueID']
        if issuechk['Status'] == "Snatched":
            values = { "Status":   "Snatched"}
            newValue['Status'] = "Snatched"
        elif issuechk['Status'] == "Downloaded":
            values = { "Status":    "Downloaded"}
            newValue['Status'] = "Downloaded"
            #if the status is Downloaded and it's on the pullist - let's mark it so everyone can bask in the glory

        elif issuechk['Status'] == "Wanted":
            values = { "Status":    "Wanted"}
            newValue['Status'] = "Wanted"            
        elif issuechk['Status'] == "Archived":
            values = { "Status":    "Archived"}
            newValue['Status'] = "Archived"
        else:
            values = { "Status":    "Skipped"}
            newValue['Status'] = "Skipped"
        #was in wrong place :(
    else:
        logger.fdebug('Issues do not match for some reason...weekly new issue: ' + str(IssueNumber))
        return

    if mylar.AUTOWANT_UPCOMING:
        #for issues not in db - to be added to Upcoming table.
        if issuechk is None:
            newValue['Status'] = "Wanted"
            logger.fdebug('...Changing Status to Wanted and throwing it in the Upcoming section since it is not published yet.')
        #this works for issues existing in DB...        
        elif issuechk['Status'] == "Skipped":
            newValue['Status'] = "Wanted"
            values = {"Status":  "Wanted"}
            logger.fdebug('...New status of Wanted')
        elif issuechk['Status'] == "Wanted":
            logger.fdebug('...Status already Wanted .. not changing.')
        else:
            logger.fdebug('...Already have issue - keeping existing status of : ' + issuechk['Status'])

    if issuechk is None:
        myDB.upsert("upcoming", newValue, controlValue)
    else:
        logger.fdebug('--attempt to find errant adds to Wanted list')
        logger.fdebug('UpcomingNewValue: ' + str(newValue))
        logger.fdebug('UpcomingcontrolValue: ' + str(controlValue))
        if issuechk['IssueDate'] == '0000-00-00' and newValue['IssueDate'] != '0000-00-00':
            logger.fdebug('Found a 0000-00-00 issue - force updating series to try and get it proper.')
            dateVal = {"IssueDate":        newValue['IssueDate'],
                       "ComicName":        issuechk['ComicName'],
                       "Status":           newValue['Status'],
                       "IssueNumber":      issuechk['Issue_Number']}
            logger.fdebug('updating date in upcoming table to : ' + str(newValue['IssueDate']))
            logger.fdebug('ComicID:' + str(controlValue))
            myDB.upsert("upcoming", dateVal, controlValue)
            logger.fdebug('Temporarily putting the Issue Date for ' + str(issuechk['Issue_Number']) + ' to ' + str(newValue['IssueDate']))
            values = {"IssueDate":  newValue['IssueDate']}
            #if ComicID[:1] == "G": mylar.importer.GCDimport(ComicID,pullupd='yes')
            #else: mylar.importer.addComictoDB(ComicID,mismatch,pullupd='yes')

        if 'annual' in ComicName.lower():
            myDB.upsert("annuals", values, control)
        else:
            myDB.upsert("issues", values, control)

        if issuechk['Status'] == 'Downloaded' or issuechk['Status'] == 'Archived': 
            logger.fdebug('updating Pull-list to reflect status.')
            downstats = {"Status":  issuechk['Status'],
                         "ComicID": issuechk['ComicID']}
            return downstats


def weekly_update(ComicName,IssueNumber,CStatus,CID,futurepull=None):
    # here we update status of weekly table...
    # added Issue to stop false hits on series' that have multiple releases in a week
    # added CStatus to update status flags on Pullist screen
    myDB = db.DBConnection()
    if futurepull is None:
        issuecheck = myDB.action("SELECT * FROM weekly WHERE COMIC=? AND ISSUE=?", [ComicName,IssueNumber]).fetchone()
    else:
        issuecheck = myDB.action("SELECT * FROM future WHERE COMIC=? AND ISSUE=?", [ComicName,IssueNumber]).fetchone()
    if issuecheck is not None:
        controlValue = { "COMIC":         str(ComicName),
                         "ISSUE":         str(IssueNumber)}
        if CStatus:
            newValue = {"STATUS":             CStatus,
                        "ComicID":            CID}
        else:
            if mylar.AUTOWANT_UPCOMING:
                newValue = {"STATUS":             "Wanted"}
            else:
                newValue = {"STATUS":             "Skipped"}

        if futurepull is None:
            myDB.upsert("weekly", newValue, controlValue)
        else:
            if issuecheck['ComicID'] is not None:
                newValue = {"STATUS":       "Wanted",
                            "ComicID":      issuecheck['ComicID']}

            myDB.upsert("future", newValue, controlValue)

def newpullcheck(ComicName, ComicID):
    # When adding a new comic, let's check for new issues on this week's pullist and update.
    mylar.weeklypull.pullitcheck(ComicName, ComicID)
    return

def no_searchresults(ComicID):
    # when there's a mismatch between CV & GCD - let's change the status to
    # something other than 'Loaded'
    myDB = db.DBConnection()
    controlValue = { "ComicID":        ComicID}
    newValue = {"Status":       "Error",
                "LatestDate":   "Error",
                "LatestIssue":  "Error"}    
    myDB.upsert("comics", newValue, controlValue)

def nzblog(IssueID, NZBName, ComicName, SARC=None, IssueArcID=None):
    myDB = db.DBConnection()

    newValue = {"NZBName":  NZBName}

    if IssueID is None or IssueID == 'None':
       #if IssueID is None, it's a one-off download from the pull-list.
       #give it a generic ID above the last one so it doesn't throw an error later.
       print "SARC detected as: " + str(SARC)
       if mylar.HIGHCOUNT == 0:
           IssueID = '900000'
       else: 
           IssueID = int(mylar.HIGHCOUNT) + 1
       
       if SARC:
           IssueID = 'S' + str(IssueArcID)
           newValue['SARC'] = SARC

    controlValue = {"IssueID": IssueID}
    #print controlValue
    #newValue['NZBName'] = NZBName
    #print newValue
    myDB.upsert("nzblog", newValue, controlValue)

def foundsearch(ComicID, IssueID, mode=None, down=None, provider=None, SARC=None, IssueArcID=None):
    # When doing a Force Search (Wanted tab), the resulting search calls this to update.

    # this is all redudant code that forceRescan already does.
    # should be redone at some point so that instead of rescanning entire 
    # series directory, it just scans for the issue it just downloaded and
    # and change the status to Snatched accordingly. It is not to increment the have count
    # at this stage as it's not downloaded - just the .nzb has been snatched and sent to SAB.
    myDB = db.DBConnection()

    logger.info('comicid: ' + str(ComicID))
    logger.info('issueid: ' + str(IssueID))
    if mode != 'story_arc':
        comic = myDB.action('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone()
        ComicName = comic['ComicName']
        if mode == 'want_ann':
            issue = myDB.action('SELECT * FROM annuals WHERE IssueID=?', [IssueID]).fetchone()
        else:
            issue = myDB.action('SELECT * FROM issues WHERE IssueID=?', [IssueID]).fetchone()
        CYear = issue['IssueDate'][:4]

    else:
        issue = myDB.action('SELECT * FROM readinglist WHERE IssueArcID=?', [IssueArcID]).fetchone()
        ComicName = issue['ComicName']
        CYear = issue['IssueYEAR']

    if down is None:
        # update the status to Snatched (so it won't keep on re-downloading!)
        logger.fdebug('updating status to snatched')
        logger.fdebug('provider is ' + provider)
        newValue = {"Status":    "Snatched"}
        if mode == 'story_arc':
            cValue = {"IssueArcID": IssueArcID}
            snatchedupdate = {"IssueArcID": IssueArcID}
            myDB.upsert("readinglist", newValue, cValue)
            # update the snatched DB
            snatchedupdate = {"IssueID":     IssueArcID,
                              "Status":      "Snatched",
                              "Provider":    provider
                              }

        else:
            if mode == 'want_ann':
                controlValue = {"IssueID":   IssueID}
                myDB.upsert("annuals", newValue, controlValue)
            else:
                controlValue = {"IssueID":   IssueID}
                myDB.upsert("issues", newValue, controlValue)

            # update the snatched DB
            snatchedupdate = {"IssueID":     IssueID,
                              "Status":      "Snatched",
                              "Provider":    provider
                              }

        if mode == 'story_arc':
            IssueNum = issue['IssueNumber']
            newsnatchValues = {"ComicName":       ComicName,
                               "ComicID":         'None',
                               "Issue_Number":    IssueNum,
                               "DateAdded":       helpers.now(),
                               "Status":          "Snatched"
                               }
        else:
            if mode == 'want_ann':
                IssueNum = "Annual " + issue['Issue_Number']
            else:
                IssueNum = issue['Issue_Number']

            newsnatchValues = {"ComicName":       ComicName,
                               "ComicID":         ComicID,
                               "Issue_Number":    IssueNum,
                               "DateAdded":       helpers.now(),
                               "Status":          "Snatched"
                               }
        myDB.upsert("snatched", newsnatchValues, snatchedupdate)
        logger.info("updated the snatched.")
    else:
        logger.info("updating the downloaded.")
        if mode == 'want_ann':
            IssueNum = "Annual " + issue['Issue_Number']
        elif mode == 'story_arc':
            IssueNum = issue['IssueNumber']
            IssueID = IssueArcID
        else:
            IssueNum = issue['Issue_Number']

        snatchedupdate = {"IssueID":     IssueID,
                          "Status":      "Downloaded",
                          "Provider":    provider
                          }
        newsnatchValues = {"ComicName":       ComicName,
                           "ComicID":         ComicID,
                           "Issue_Number":    IssueNum,
                           "DateAdded":       helpers.now(),
                           "Status":          "Downloaded"
                           }
        myDB.upsert("snatched", newsnatchValues, snatchedupdate)

        if mode == 'story_arc':
            cValue = {"IssueArcID":   IssueArcID}
            nValue = {"Status":       "Downloaded"}
            myDB.upsert("readinglist", nValue, cValue)

        else:
            controlValue = {"IssueID":   IssueID}
            newValue = {"Status":    "Downloaded"}

            myDB.upsert("issues", newValue, controlValue)

    #print ("finished updating snatched db.")
    logger.info('Updating now complete for ' + ComicName + ' issue: ' + str(IssueNum))
    return

def forceRescan(ComicID,archive=None):
    myDB = db.DBConnection()
    # file check to see if issue exists
    rescan = myDB.action('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone()
    logger.info('Now checking files for ' + rescan['ComicName'] + ' (' + str(rescan['ComicYear']) + ') in ' + rescan['ComicLocation'] )
    if archive is None:
        fc = filechecker.listFiles(dir=rescan['ComicLocation'], watchcomic=rescan['ComicName'], AlternateSearch=rescan['AlternateSearch'])
    else:
        fc = filechecker.listFiles(dir=archive, watchcomic=rescan['ComicName'], AlternateSearch=rescan['AlternateSearch'])
    iscnt = rescan['Total']
    havefiles = 0
    if mylar.ANNUALS_ON:
        an_cnt = myDB.action("SELECT COUNT(*) FROM annuals WHERE ComicID=?", [ComicID]).fetchall()
        anncnt = an_cnt[0][0]
    else:
        anncnt = 0
    fccnt = int(fc['comiccount'])
    issnum = 1
    fcnew = []
    fn = 0
    issuedupechk = []
    annualdupechk = []
    issueexceptdupechk = []
    reissues = myDB.action('SELECT * FROM issues WHERE ComicID=?', [ComicID]).fetchall()
    issID_to_ignore = []
    issID_to_ignore.append(str(ComicID))
    while (fn < fccnt):  
        haveissue = "no"
        issuedupe = "no"
        try:
            tmpfc = fc['comiclist'][fn]
        except IndexError:
            logger.fdebug('Unable to properly retrieve a file listing for the given series.')
            logger.fdebug('Probably because the filenames being scanned are not in a parseable format')
            if fn == 0: 
                return
            else:
                break
        temploc= tmpfc['JusttheDigits'].replace('_', ' ')

#        temploc = tmpfc['ComicFilename'].replace('_', ' ')
        temploc = re.sub('[\#\']', '', temploc)
        logger.fdebug('temploc: ' + str(temploc))
        if 'annual' not in temploc.lower():
            #remove the extension here
            extensions = ('.cbr','.cbz')
            if temploc.lower().endswith(extensions):
                logger.fdebug('removed extension for issue: ' + str(temploc))
                temploc = temploc[:-4]
#            deccnt = str(temploc).count('.')
#            if deccnt > 1:
                #logger.fdebug('decimal counts are :' + str(deccnt))
                #if the file is formatted with '.' in place of spaces we need to adjust.
                #before replacing - check to see if digits on either side of decimal and if yes, DON'T REMOVE
#                occur=1
#                prevstart = 0
#                digitfound = "no"
#                decimalfound = "no"
#                tempreconstruct = ''
#                while (occur <= deccnt):
#                    n = occur
#                    start = temploc.find('.')
#                    while start >=0 and n > 1:
#                        start = temploc.find('.', start+len('.'))
#                        n-=1
#                    #logger.fdebug('occurance ' + str(occur) + ' of . at position: ' + str(start))
#                    if temploc[prevstart:start].isdigit():
#                        if digitfound == "yes":
#                            #logger.fdebug('this is a decimal, assuming decimal issue.')
#                            decimalfound = "yes"
#                            reconst = "." + temploc[prevstart:start] + " "
#                        else:
#                            #logger.fdebug('digit detected.')
#                            digitfound = "yes"
#                            reconst = temploc[prevstart:start]
#                    else:
#                        reconst = temploc[prevstart:start] + " "
#                    #logger.fdebug('word: ' + reconst)
#                    tempreconstruct = tempreconstruct + reconst 
#                    #logger.fdebug('tempreconstruct is : ' + tempreconstruct)
#                    prevstart = (start+1)
#                    occur+=1
#                #logger.fdebug('word: ' + temploc[prevstart:])
#                tempreconstruct = tempreconstruct + " " + temploc[prevstart:]
#                #logger.fdebug('final filename to use is : ' + str(tempreconstruct))
#                temploc = tempreconstruct            
            #logger.fdebug("checking " + str(temploc))
            #fcnew_b4 = shlex.split(str(temploc))            
            fcnew_af = re.findall('[^\()]+', temploc)
            fcnew = shlex.split(fcnew_af[0])

            fcn = len(fcnew)
            n = 0
            while (n <= iscnt):
                som = 0
                try:
                    reiss = reissues[n]
                except IndexError:
                    break
#                int_iss, iss_except = helpers.decimal_issue(reiss['Issue_Number'])
                int_iss = helpers.issuedigits(reiss['Issue_Number'])
                issyear = reiss['IssueDate'][:4]
                old_status = reiss['Status']
                issname = reiss['IssueName']
                #logger.fdebug('integer_issue:' + str(int_iss) + ' ... status: ' + str(old_status))

                #if comic in format of "SomeSeries 5(c2c)(2013).cbr" whatever...it'll die.
                #can't distinguish the 5(c2c) to tell it's the issue #...
                fnd_iss_except = 'None'
                #print ("Issue, int_iss, iss_except: " + str(reiss['Issue_Number']) + "," + str(int_iss) + "," + str(iss_except))


                while (som < fcn):
                    #counts get buggered up when the issue is the last field in the filename - ie. '50.cbr'
                    #logger.fdebug('checking word - ' + str(fcnew[som]))
                    if ".cbr" in fcnew[som].lower():
                        fcnew[som] = fcnew[som].replace(".cbr", "")
                    elif ".cbz" in fcnew[som].lower():
                        fcnew[som] = fcnew[som].replace(".cbz", "")
                    if "(c2c)" in fcnew[som].lower():
                        fcnew[som] = fcnew[som].replace("(c2c)", " ")
                        get_issue = shlex.split(str(fcnew[som]))
                        if fcnew[som] != " ":
                            fcnew[som] = get_issue[0]


                    if som+1 < len(fcnew) and len(fcnew[som+1]) == 2:
                        #print "fcnew[som+1]: " + str(fcnew[som+1])
                        #print "fcnew[som]: " + str(fcnew[som])
                        if 'au' in fcnew[som+1].lower():
                            #if the 'AU' is in 005AU vs 005 AU it will yield different results.
                            fcnew[som] = fcnew[som] + 'AU'
                            fcnew[som+1] = '93939999919190933'
                            logger.info('AU Detected seperate from issue - combining and continuing')
                        elif 'ai' in fcnew[som+1].lower():
                            #if the 'AI' is in 005AI vs 005 AI it will yield different results.
                            fcnew[som] = fcnew[som] + 'AI'
                            fcnew[som+1] = '93939999919190933'
                            logger.info('AI Detected seperate from issue - combining and continuing')

                    #sometimes scanners refuse to use spaces between () and lump the issue right at the start
                    #mylar assumes it's all one word in this case..let's dump the brackets.
                    
                    fcdigit = helpers.issuedigits(fcnew[som])

                    #logger.fdebug("fcdigit: " + str(fcdigit))
                    #logger.fdebug("int_iss: " + str(int_iss))

                    if int(fcdigit) == int_iss:
                        logger.fdebug('issue match - fcdigit: ' + str(fcdigit) + ' ... int_iss: ' + str(int_iss))

                        if '-' in temploc and temploc.find(reiss['Issue_Number']) > temploc.find('-'):
                            logger.fdebug('I have detected a possible Title in the filename')
                            logger.fdebug('the issue # has occured after the -, so I assume that it is part of the Title')
                            break
                        for d in issuedupechk:
                            if int(d['fcdigit']) == int(fcdigit):
                                logger.fdebug('duplicate issue detected - not counting this: ' + str(tmpfc['ComicFilename']))
                                logger.fdebug('is a duplicate of ' + d['filename'])
                                logger.fdebug('fcdigit:' + str(fcdigit) + ' === dupedigit: ' + str(d['fcdigit']))
                                issuedupe = "yes"
                                break
                        if issuedupe == "no":
                            logger.fdebug('matched...issue: ' + rescan['ComicName'] + '#' + str(reiss['Issue_Number']) + ' --- ' + str(int_iss))
                            havefiles+=1
                            haveissue = "yes"
                            isslocation = str(tmpfc['ComicFilename'])
                            issSize = str(tmpfc['ComicSize'])
                            logger.fdebug('.......filename: ' + str(isslocation))
                            logger.fdebug('.......filesize: ' + str(tmpfc['ComicSize'])) 
                            # to avoid duplicate issues which screws up the count...let's store the filename issues then 
                            # compare earlier...
                            issuedupechk.append({'fcdigit': int(fcdigit),
                                                 'filename': tmpfc['ComicFilename']})
                        break
                        #else:
                        # if the issue # matches, but there is no year present - still match.
                        # determine a way to match on year if present, or no year (currently).

                    if issuedupe == "yes":
                        logger.fdebug('I should break out here because of a dupe.')
                        break
                    som+=1
                if haveissue == "yes" or issuedupe == "yes": break
                n+=1
        else:
            # annual inclusion here.
            #logger.fdebug("checking " + str(temploc))
            reannuals = myDB.action('SELECT * FROM annuals WHERE ComicID=?', [ComicID]).fetchall()
            fcnew = shlex.split(str(temploc))
            fcn = len(fcnew)
            n = 0
            reann = None
            while (n < anncnt):
                som = 0
                try:
                    reann = reannuals[n]
                except IndexError:
                    break
                int_iss, iss_except = helpers.decimal_issue(reann['Issue_Number'])
                issyear = reann['IssueDate'][:4]
                old_status = reann['Status']            
                while (som < fcn):
                    #counts get buggered up when the issue is the last field in the filename - ie. '50$
                    #logger.fdebug('checking word - ' + str(fcnew[som]))
                    if ".cbr" in fcnew[som].lower():
                        fcnew[som] = fcnew[som].replace(".cbr", "")
                    elif ".cbz" in fcnew[som].lower():
                        fcnew[som] = fcnew[som].replace(".cbz", "")
                    if "(c2c)" in fcnew[som].lower():
                        fcnew[som] = fcnew[som].replace("(c2c)", " ")
                        get_issue = shlex.split(str(fcnew[som]))
                        if fcnew[som] != " ":
                            fcnew[som] = get_issue[0]
                    if fcnew[som].lower() == 'annual':
                        logger.fdebug('Annual detected.')
                        if fcnew[som+1].isdigit():
                            ann_iss = fcnew[som+1]
                            logger.fdebug('Annual # ' + str(ann_iss) + ' detected.')
                            fcdigit = helpers.issuedigits(ann_iss)
                    logger.fdebug('fcdigit:' + str(fcdigit))
                    logger.fdebug('int_iss:' + str(int_iss))
                    if int(fcdigit) == int_iss:
                        logger.fdebug('annual match - issue : ' + str(int_iss))
                        for d in annualdupechk:
                            if int(d['fcdigit']) == int(fcdigit):
                                logger.fdebug('duplicate annual issue detected - not counting this: ' + str(tmpfc['ComicFilename']))
                                issuedupe = "yes"
                                break
                        if issuedupe == "no":
                            logger.fdebug('matched...annual issue: ' + rescan['ComicName'] + '#' + str(reann['Issue_Number']) + ' --- ' + str(int_iss))
                            havefiles+=1
                            haveissue = "yes"
                            isslocation = str(tmpfc['ComicFilename'])
                            issSize = str(tmpfc['ComicSize'])
                            logger.fdebug('.......filename: ' + str(isslocation))
                            logger.fdebug('.......filesize: ' + str(tmpfc['ComicSize']))
                            # to avoid duplicate issues which screws up the count...let's store the filename issues then
                            # compare earlier...
                            annualdupechk.append({'fcdigit': int(fcdigit)})
                        break
                    som+=1
                if haveissue == "yes": break
                n+=1

        if issuedupe == "yes": pass
        else:
            #we have the # of comics, now let's update the db.
            #even if we couldn't find the physical issue, check the status.
            #-- if annuals aren't enabled, this will bugger out.
            writeit = True
            if mylar.ANNUALS_ON:
                if 'annual' in temploc.lower():
                    if reann is None:
                        logger.fdebug('Annual present in location, but series does not have any annuals attached to it - Ignoring')
                        writeit = False
                    else:
                        iss_id = reann['IssueID']
                else:
                    iss_id = reiss['IssueID']
            else:
                if 'annual' in temploc.lower():
                    logger.fdebug('Annual support not enabled, but annual issue present within directory. Ignoring annual.')
                    writeit = False
                else:
                    iss_id = reiss['IssueID']

            if writeit == True:
                logger.fdebug('issueID to write to db:' + str(iss_id))
                controlValueDict = {"IssueID": iss_id}

                #if Archived, increase the 'Have' count.
                #if archive:
                #    issStatus = "Archived"
  
                if haveissue == "yes":
                    issStatus = "Downloaded"
                    newValueDict = {"Location":           isslocation,
                                    "ComicSize":          issSize,
                                    "Status":             issStatus
                                    }

                    issID_to_ignore.append(str(iss_id))
   
                    if 'annual' in temploc.lower():
                        myDB.upsert("annuals", newValueDict, controlValueDict)
                    else:
                        myDB.upsert("issues", newValueDict, controlValueDict)
        fn+=1

    logger.fdebug('IssueID to ignore: ' + str(issID_to_ignore))

    #here we need to change the status of the ones we DIDN'T FIND above since the loop only hits on FOUND issues.
    update_iss = []
    tmpsql = "SELECT * FROM issues WHERE ComicID=? AND IssueID not in ({seq})".format(seq=','.join(['?']*(len(issID_to_ignore)-1)))
    chkthis = myDB.action(tmpsql, issID_to_ignore).fetchall()
#    chkthis = None
    if chkthis is None: 
        pass
    else:
        for chk in chkthis:
            old_status = chk['Status']
            logger.fdebug('old_status:' + str(old_status))
            if old_status == "Skipped":
                if mylar.AUTOWANT_ALL:
                    issStatus = "Wanted"
                else:
                    issStatus = "Skipped"
            elif old_status == "Archived":
                issStatus = "Archived"
            elif old_status == "Downloaded":
                issStatus = "Archived"
            elif old_status == "Wanted":
                issStatus = "Wanted"
            elif old_status == "Ignored":
                issStatus = "Ignored"
            elif old_status == "Snatched":   #this is needed for torrents, or else it'll keep on queuing..
                issStatus = "Snatched"
            else:
                issStatus = "Skipped"

            logger.fdebug("new status: " + str(issStatus))

            update_iss.append({"IssueID": chk['IssueID'],
                               "Status":  issStatus})
    
    if len(update_iss) > 0:
        i = 0
        #do it like this to avoid DB locks...
        for ui in update_iss:
            controlValueDict = {"IssueID": ui['IssueID']}
            newStatusValue = {"Status": ui['Status']}
            myDB.upsert("issues", newStatusValue, controlValueDict)
            i+=1
        logger.info('Updated the status of ' + str(i) + ' issues for ' + rescan['ComicName'] + ' (' + str(rescan['ComicYear']) + ') that were not found.')

    logger.info('Total files located: ' + str(havefiles))
    foundcount = havefiles
    arcfiles = 0
    # if filechecker returns 0 files (it doesn't find any), but some issues have a status of 'Archived'
    # the loop below won't work...let's adjust :)
    arcissues = myDB.action("SELECT count(*) FROM issues WHERE ComicID=? and Status='Archived'", [ComicID]).fetchall()
    if int(arcissues[0][0]) > 0:
        arcfiles = arcissues[0][0]
        havefiles = havefiles + arcfiles
        logger.fdebug('Adjusting have total to ' + str(havefiles) + ' because of this many archive files:' + str(arcfiles))

    ignorecount = 0
    if mylar.IGNORE_HAVETOTAL:   # if this is enabled, will increase Have total as if in Archived Status
        ignores = myDB.action("SELECT count(*) FROM issues WHERE ComicID=? AND Status='Ignored'", [ComicID]).fetchall()
        if int(ignores[0][0]) > 0:
            ignorecount = ignores[0][0]
            havefiles = havefiles + ignorecount
            logger.fdebug('Adjusting have total to ' + str(havefiles) + ' because of this many Ignored files:' + str(ignorecount))

    #now that we are finished...
    #adjust for issues that have been marked as Downloaded, but aren't found/don't exist.
    #do it here, because above loop only cycles though found comics using filechecker.
    downissues = myDB.select("SELECT * FROM issues WHERE ComicID=? and Status='Downloaded'", [ComicID])
    if downissues is None:
        pass
    else:
        archivedissues = 0 #set this to 0 so it tallies correctly.
        for down in downissues:
            #print "downlocation:" + str(down['Location'])
            #remove special characters from 
            #temploc = rescan['ComicLocation'].replace('_', ' ')
            #temploc = re.sub('[\#\'\/\.]', '', temploc)
            #print ("comiclocation: " + str(rescan['ComicLocation']))
            #print ("downlocation: " + str(down['Location']))
            if down['Location'] is None:
                logger.fdebug('location does not exist which means file was not downloaded successfully, or was moved.')
                controlValue = {"IssueID":  down['IssueID']}
                newValue = {"Status":    "Archived"}
                myDB.upsert("issues", newValue, controlValue)
                archivedissues+=1
                pass
            else:
                comicpath = os.path.join(rescan['ComicLocation'], down['Location'])
                if os.path.exists(comicpath):
                    pass
                    #print "Issue exists - no need to change status."
                else:
                    #print "Changing status from Downloaded to Archived - cannot locate file"
                    controlValue = {"IssueID":   down['IssueID']}
                    newValue = {"Status":    "Archived"}
                    myDB.upsert("issues", newValue, controlValue)
                    archivedissues+=1 
        totalarc = arcfiles + archivedissues
        havefiles = havefiles + archivedissues  #arcfiles already tallied in havefiles in above segment
        logger.fdebug('I have changed the status of ' + str(archivedissues) + ' issues to a status of Archived, as I now cannot locate them in the series directory.')

        
    #let's update the total count of comics that was found.
    controlValueStat = {"ComicID":     rescan['ComicID']}
    newValueStat = {"Have":            havefiles
                   }

    combined_total = rescan['Total'] + anncnt

    myDB.upsert("comics", newValueStat, controlValueStat)
    logger.info('I have physically found ' + str(foundcount) + ' issues, ignored ' + str(ignorecount) + ' issues, and accounted for ' + str(totalarc) + ' in an Archived state. Total Issue Count: ' + str(havefiles) + ' / ' + str(combined_total))

    return