mirror of https://github.com/evilhero/mylar
711 lines
39 KiB
Python
711 lines
39 KiB
Python
# -*- coding: utf-8 -*-
|
|
# This file is part of Mylar.
|
|
#
|
|
# Mylar is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# Mylar is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with Mylar. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
from __future__ import with_statement
|
|
|
|
import os
|
|
import glob
|
|
import re
|
|
import shutil
|
|
import random
|
|
|
|
import mylar
|
|
from mylar import db, logger, helpers, importer, updater, filechecker
|
|
|
|
# You can scan a single directory and append it to the current library by specifying append=True
|
|
def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None, queue=None):
|
|
|
|
if cron and not mylar.LIBRARYSCAN:
|
|
return
|
|
|
|
if not dir:
|
|
dir = mylar.CONFIG.COMIC_DIR
|
|
|
|
# If we're appending a dir, it's coming from the post processor which is
|
|
# already bytestring
|
|
if not append:
|
|
dir = dir.encode(mylar.SYS_ENCODING)
|
|
|
|
if not os.path.isdir(dir):
|
|
logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(mylar.SYS_ENCODING, 'replace'))
|
|
return "Fail"
|
|
|
|
|
|
logger.info('Scanning comic directory: %s' % dir.decode(mylar.SYS_ENCODING, 'replace'))
|
|
|
|
basedir = dir
|
|
|
|
comic_list = []
|
|
failure_list = []
|
|
utter_failure_list = []
|
|
comiccnt = 0
|
|
extensions = ('cbr','cbz')
|
|
cv_location = []
|
|
cbz_retry = 0
|
|
|
|
mylar.IMPORT_STATUS = 'Now attempting to parse files for additional information'
|
|
myDB = db.DBConnection()
|
|
#mylar.IMPORT_PARSED_COUNT #used to count what #/totalfiles the filename parser is currently on
|
|
for r, d, f in os.walk(dir):
|
|
for files in f:
|
|
mylar.IMPORT_FILES +=1
|
|
if any(files.lower().endswith('.' + x.lower()) for x in extensions):
|
|
comicpath = os.path.join(r, files)
|
|
if mylar.CONFIG.IMP_PATHS is True:
|
|
if myDB.select('SELECT * FROM comics JOIN issues WHERE issues.Status="Downloaded" AND ComicLocation=? AND issues.Location=?', [r.decode(mylar.SYS_ENCODING, 'replace'), files.decode(mylar.SYS_ENCODING, 'replace')]):
|
|
logger.info('Skipped known issue path: %s' % comicpath)
|
|
continue
|
|
|
|
comic = files
|
|
comicsize = os.path.getsize(comicpath)
|
|
logger.fdebug('Comic: ' + comic + ' [' + comicpath + '] - ' + str(comicsize) + ' bytes')
|
|
|
|
try:
|
|
t = filechecker.FileChecker(dir=r, file=comic)
|
|
results = t.listFiles()
|
|
|
|
#logger.info(results)
|
|
#'type': re.sub('\.','', filetype).strip(),
|
|
#'sub': path_list,
|
|
#'volume': volume,
|
|
#'match_type': match_type,
|
|
#'comicfilename': filename,
|
|
#'comiclocation': clocation,
|
|
#'series_name': series_name,
|
|
#'series_volume': issue_volume,
|
|
#'series_year': issue_year,
|
|
#'justthedigits': issue_number,
|
|
#'annualcomicid': annual_comicid,
|
|
#'scangroup': scangroup}
|
|
|
|
|
|
if results:
|
|
resultline = '[PARSE-' + results['parse_status'].upper() + ']'
|
|
resultline += '[SERIES: ' + results['series_name'] + ']'
|
|
if results['series_volume'] is not None:
|
|
resultline += '[VOLUME: ' + results['series_volume'] + ']'
|
|
if results['issue_year'] is not None:
|
|
resultline += '[ISSUE YEAR: ' + str(results['issue_year']) + ']'
|
|
if results['issue_number'] is not None:
|
|
resultline += '[ISSUE #: ' + results['issue_number'] + ']'
|
|
logger.fdebug(resultline)
|
|
else:
|
|
logger.fdebug('[PARSED] FAILURE.')
|
|
continue
|
|
|
|
# We need the unicode path to use for logging, inserting into database
|
|
unicode_comic_path = comicpath.decode(mylar.SYS_ENCODING, 'replace')
|
|
|
|
if results['parse_status'] == 'success':
|
|
comic_list.append({'ComicFilename': comic,
|
|
'ComicLocation': comicpath,
|
|
'ComicSize': comicsize,
|
|
'Unicode_ComicLocation': unicode_comic_path,
|
|
'parsedinfo': {'series_name': results['series_name'],
|
|
'series_volume': results['series_volume'],
|
|
'issue_year': results['issue_year'],
|
|
'issue_number': results['issue_number']}
|
|
})
|
|
comiccnt +=1
|
|
mylar.IMPORT_PARSED_COUNT +=1
|
|
else:
|
|
failure_list.append({'ComicFilename': comic,
|
|
'ComicLocation': comicpath,
|
|
'ComicSize': comicsize,
|
|
'Unicode_ComicLocation': unicode_comic_path,
|
|
'parsedinfo': {'series_name': results['series_name'],
|
|
'series_volume': results['series_volume'],
|
|
'issue_year': results['issue_year'],
|
|
'issue_number': results['issue_number']}
|
|
})
|
|
mylar.IMPORT_FAILURE_COUNT +=1
|
|
if comic.endswith('.cbz'):
|
|
cbz_retry +=1
|
|
|
|
except Exception, e:
|
|
logger.info('bang')
|
|
utter_failure_list.append({'ComicFilename': comic,
|
|
'ComicLocation': comicpath,
|
|
'ComicSize': comicsize,
|
|
'Unicode_ComicLocation': unicode_comic_path,
|
|
'parsedinfo': None,
|
|
'error': e
|
|
})
|
|
logger.info('[' + str(e) + '] FAILURE encountered. Logging the error for ' + comic + ' and continuing...')
|
|
mylar.IMPORT_FAILURE_COUNT +=1
|
|
if comic.endswith('.cbz'):
|
|
cbz_retry +=1
|
|
continue
|
|
|
|
if 'cvinfo' in files:
|
|
cv_location.append(r)
|
|
logger.fdebug('CVINFO found: ' + os.path.join(r))
|
|
|
|
mylar.IMPORT_TOTALFILES = comiccnt
|
|
logger.info('I have successfully discovered & parsed a total of ' + str(comiccnt) + ' files....analyzing now')
|
|
logger.info('I have not been able to determine what ' + str(len(failure_list)) + ' files are')
|
|
logger.info('However, ' + str(cbz_retry) + ' out of the ' + str(len(failure_list)) + ' files are in a cbz format, which may contain metadata.')
|
|
logger.info('[ERRORS] I have encountered ' + str(len(utter_failure_list)) + ' file-scanning errors during the scan, but have recorded the necessary information.')
|
|
mylar.IMPORT_STATUS = 'Successfully parsed ' + str(comiccnt) + ' files'
|
|
#return queue.put(valreturn)
|
|
|
|
if len(utter_failure_list) > 0:
|
|
logger.fdebug('Failure list: %s' % utter_failure_list)
|
|
|
|
#let's load in the watchlist to see if we have any matches.
|
|
logger.info("loading in the watchlist to see if a series is being watched already...")
|
|
watchlist = myDB.select("SELECT * from comics")
|
|
ComicName = []
|
|
DisplayName = []
|
|
ComicYear = []
|
|
ComicPublisher = []
|
|
ComicTotal = []
|
|
ComicID = []
|
|
ComicLocation = []
|
|
|
|
AltName = []
|
|
watchcnt = 0
|
|
|
|
watch_kchoice = []
|
|
watchchoice = {}
|
|
import_by_comicids = []
|
|
import_comicids = {}
|
|
|
|
for watch in watchlist:
|
|
#use the comicname_filesafe to start
|
|
watchdisplaycomic = watch['ComicName'].encode('utf-8').strip() #re.sub('[\_\#\,\/\:\;\!\$\%\&\+\'\?\@]', ' ', watch['ComicName']).encode('utf-8').strip()
|
|
# let's clean up the name, just in case for comparison purposes...
|
|
watchcomic = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', '', watch['ComicName_Filesafe']).encode('utf-8').strip()
|
|
#watchcomic = re.sub('\s+', ' ', str(watchcomic)).strip()
|
|
|
|
if ' the ' in watchcomic.lower():
|
|
#drop the 'the' from the watchcomic title for proper comparisons.
|
|
watchcomic = watchcomic[-4:]
|
|
|
|
alt_chk = "no" # alt-checker flag (default to no)
|
|
|
|
# account for alternate names as well
|
|
if watch['AlternateSearch'] is not None and watch['AlternateSearch'] is not 'None':
|
|
altcomic = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', '', watch['AlternateSearch']).encode('utf-8').strip()
|
|
#altcomic = re.sub('\s+', ' ', str(altcomic)).strip()
|
|
AltName.append(altcomic)
|
|
alt_chk = "yes" # alt-checker flag
|
|
|
|
ComicName.append(watchcomic)
|
|
DisplayName.append(watchdisplaycomic)
|
|
ComicYear.append(watch['ComicYear'])
|
|
ComicPublisher.append(watch['ComicPublisher'])
|
|
ComicTotal.append(watch['Total'])
|
|
ComicID.append(watch['ComicID'])
|
|
ComicLocation.append(watch['ComicLocation'])
|
|
watchcnt+=1
|
|
|
|
logger.info("Successfully loaded " + str(watchcnt) + " series from your watchlist.")
|
|
|
|
ripperlist=['digital-',
|
|
'empire',
|
|
'dcp']
|
|
|
|
watchfound = 0
|
|
|
|
datelist = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']
|
|
# datemonth = {'one':1,'two':2,'three':3,'four':4,'five':5,'six':6,'seven':7,'eight':8,'nine':9,'ten':10,'eleven':$
|
|
# #search for number as text, and change to numeric
|
|
# for numbs in basnumbs:
|
|
# #logger.fdebug("numbs:" + str(numbs))
|
|
# if numbs in ComicName.lower():
|
|
# numconv = basnumbs[numbs]
|
|
# #logger.fdebug("numconv: " + str(numconv))
|
|
|
|
issueid_list = []
|
|
cvscanned_loc = None
|
|
cvinfo_CID = None
|
|
cnt = 0
|
|
mylar.IMPORT_STATUS = '[0%] Now parsing individual filenames for metadata if available'
|
|
|
|
for i in comic_list:
|
|
mylar.IMPORT_STATUS = '[' + str(cnt) + '/' + str(comiccnt) + '] Now parsing individual filenames for metadata if available'
|
|
logger.fdebug('Analyzing : ' + i['ComicFilename'])
|
|
comfilename = i['ComicFilename']
|
|
comlocation = i['ComicLocation']
|
|
issueinfo = None
|
|
#probably need to zero these issue-related metadata to None so we can pick the best option
|
|
issuevolume = None
|
|
|
|
#Make sure cvinfo is checked for FIRST (so that CID can be attached to all files properly thereafter as they're scanned in)
|
|
if os.path.dirname(comlocation) in cv_location and os.path.dirname(comlocation) != cvscanned_loc:
|
|
|
|
#if comfilename == 'cvinfo':
|
|
logger.info('comfilename: ' + comfilename)
|
|
logger.info('cvscanned_loc: ' + str(cv_location))
|
|
logger.info('comlocation: ' + os.path.dirname(comlocation))
|
|
#if cvscanned_loc != comlocation:
|
|
try:
|
|
with open(os.path.join(os.path.dirname(comlocation), 'cvinfo')) as f:
|
|
urllink = f.readline()
|
|
|
|
if urllink:
|
|
cid = urllink.strip()
|
|
pattern = re.compile(r"^.*?\b(49|4050)-(?P<num>\d{2,})\b.*$", re.I)
|
|
match = pattern.match(cid)
|
|
if match:
|
|
cvinfo_CID = match.group("num")
|
|
logger.info('CVINFO file located within directory. Attaching everything in directory that is valid to ComicID: ' + str(cvinfo_CID))
|
|
#store the location of the cvinfo so it's applied to the correct directory (since we're scanning multile direcorties usually)
|
|
cvscanned_loc = os.path.dirname(comlocation)
|
|
else:
|
|
logger.error("Could not read cvinfo file properly (or it does not contain any data)")
|
|
except (OSError, IOError):
|
|
logger.error("Could not read cvinfo file properly (or it does not contain any data)")
|
|
#else:
|
|
# don't scan in it again if it's already been done initially
|
|
# continue
|
|
|
|
if mylar.CONFIG.IMP_METADATA:
|
|
#if read tags is enabled during import, check here.
|
|
if i['ComicLocation'].endswith('.cbz'):
|
|
logger.fdebug('[IMPORT-CBZ] Metatagging checking enabled.')
|
|
logger.info('[IMPORT-CBZ} Attempting to read tags present in filename: ' + i['ComicLocation'])
|
|
try:
|
|
issueinfo = helpers.IssueDetails(i['ComicLocation'], justinfo=True)
|
|
except:
|
|
logger.fdebug('[IMPORT-CBZ] Unable to retrieve metadata - possibly doesn\'t exist. Ignoring meta-retrieval')
|
|
pass
|
|
else:
|
|
logger.info('issueinfo: ' + str(issueinfo))
|
|
|
|
if issueinfo is None:
|
|
logger.fdebug('[IMPORT-CBZ] No valid metadata contained within filename. Dropping down to parsing the filename itself.')
|
|
pass
|
|
else:
|
|
issuenotes_id = None
|
|
logger.info('[IMPORT-CBZ] Successfully retrieved some tags. Lets see what I can figure out.')
|
|
comicname = issueinfo[0]['series']
|
|
if comicname is not None:
|
|
logger.fdebug('[IMPORT-CBZ] Series Name: ' + comicname)
|
|
as_d = filechecker.FileChecker()
|
|
as_dyninfo = as_d.dynamic_replace(comicname)
|
|
logger.fdebug('Dynamic-ComicName: ' + as_dyninfo['mod_seriesname'])
|
|
else:
|
|
logger.fdebug('[IMPORT-CBZ] No series name found within metadata. This is bunk - dropping down to file parsing for usable information.')
|
|
issueinfo = None
|
|
issue_number = None
|
|
|
|
if issueinfo is not None:
|
|
try:
|
|
issueyear = issueinfo[0]['year']
|
|
except:
|
|
issueyear = None
|
|
|
|
#if the issue number is a non-numeric unicode string, this will screw up along with impID
|
|
issue_number = issueinfo[0]['issue_number']
|
|
if issue_number is not None:
|
|
logger.fdebug('[IMPORT-CBZ] Issue Number: ' + issue_number)
|
|
else:
|
|
issue_number = i['parsed']['issue_number']
|
|
|
|
if 'annual' in comicname.lower() or 'annual' in comfilename.lower():
|
|
if issue_number is None or issue_number == 'None':
|
|
logger.info('Annual detected with no issue number present within metadata. Assuming year as issue.')
|
|
try:
|
|
issue_number = 'Annual ' + str(issueyear)
|
|
except:
|
|
issue_number = 'Annual ' + i['parsed']['issue_year']
|
|
else:
|
|
logger.info('Annual detected with issue number present within metadata.')
|
|
if 'annual' not in issue_number.lower():
|
|
issue_number = 'Annual ' + issue_number
|
|
mod_series = re.sub('annual', '', comicname, flags=re.I).strip()
|
|
else:
|
|
mod_series = comicname
|
|
|
|
logger.fdebug('issue number SHOULD Be: ' + issue_number)
|
|
|
|
try:
|
|
issuetitle = issueinfo[0]['title']
|
|
except:
|
|
issuetitle = None
|
|
try:
|
|
issueyear = issueinfo[0]['year']
|
|
except:
|
|
issueyear = None
|
|
try:
|
|
issuevolume = str(issueinfo[0]['volume'])
|
|
if all([issuevolume is not None, issuevolume != 'None', not issuevolume.lower().startswith('v')]):
|
|
issuevolume = 'v' + str(issuevolume)
|
|
if any([issuevolume is None, issuevolume == 'None']):
|
|
logger.info('EXCEPT] issue volume is NONE')
|
|
issuevolume = None
|
|
else:
|
|
logger.fdebug('[TRY]issue volume is: ' + str(issuevolume))
|
|
except:
|
|
logger.fdebug('[EXCEPT]issue volume is: ' + str(issuevolume))
|
|
issuevolume = None
|
|
|
|
if any([comicname is None, comicname == 'None', issue_number is None, issue_number == 'None']):
|
|
logger.fdebug('[IMPORT-CBZ] Improperly tagged file as the metatagging is invalid. Ignoring meta and just parsing the filename.')
|
|
issueinfo = None
|
|
pass
|
|
else:
|
|
# if used by ComicTagger, Notes field will have the IssueID.
|
|
issuenotes = issueinfo[0]['notes']
|
|
logger.fdebug('[IMPORT-CBZ] Notes: ' + issuenotes)
|
|
if issuenotes is not None and issuenotes != 'None':
|
|
if 'Issue ID' in issuenotes:
|
|
st_find = issuenotes.find('Issue ID')
|
|
tmp_issuenotes_id = re.sub("[^0-9]", " ", issuenotes[st_find:]).strip()
|
|
if tmp_issuenotes_id.isdigit():
|
|
issuenotes_id = tmp_issuenotes_id
|
|
logger.fdebug('[IMPORT-CBZ] Successfully retrieved CV IssueID for ' + comicname + ' #' + issue_number + ' [' + str(issuenotes_id) + ']')
|
|
elif 'CVDB' in issuenotes:
|
|
st_find = issuenotes.find('CVDB')
|
|
tmp_issuenotes_id = re.sub("[^0-9]", " ", issuenotes[st_find:]).strip()
|
|
if tmp_issuenotes_id.isdigit():
|
|
issuenotes_id = tmp_issuenotes_id
|
|
logger.fdebug('[IMPORT-CBZ] Successfully retrieved CV IssueID for ' + comicname + ' #' + issue_number + ' [' + str(issuenotes_id) + ']')
|
|
else:
|
|
logger.fdebug('[IMPORT-CBZ] Unable to retrieve IssueID from meta-tagging. If there is other metadata present I will use that.')
|
|
|
|
logger.fdebug('[IMPORT-CBZ] Adding ' + comicname + ' to the import-queue!')
|
|
#impid = comicname + '-' + str(issueyear) + '-' + str(issue_number) #com_NAME + "-" + str(result_comyear) + "-" + str(comiss)
|
|
impid = str(random.randint(1000000,99999999))
|
|
logger.fdebug('[IMPORT-CBZ] impid: ' + str(impid))
|
|
#make sure we only add in those issueid's which don't already have a comicid attached via the cvinfo scan above (this is for reverse-lookup of issueids)
|
|
issuepopulated = False
|
|
if cvinfo_CID is None:
|
|
if issuenotes_id is None:
|
|
logger.info('[IMPORT-CBZ] No ComicID detected where it should be. Bypassing this metadata entry and going the parsing route [' + comfilename + ']')
|
|
else:
|
|
#we need to store the impid here as well so we can look it up.
|
|
issueid_list.append({'issueid': issuenotes_id,
|
|
'importinfo': {'impid': impid,
|
|
'comicid': None,
|
|
'comicname': comicname,
|
|
'dynamicname': as_dyninfo['mod_seriesname'],
|
|
'comicyear': issueyear,
|
|
'issuenumber': issue_number,
|
|
'volume': issuevolume,
|
|
'comfilename': comfilename,
|
|
'comlocation': comlocation.decode(mylar.SYS_ENCODING)}
|
|
})
|
|
mylar.IMPORT_CID_COUNT +=1
|
|
issuepopulated = True
|
|
|
|
if issuepopulated == False:
|
|
if cvscanned_loc == os.path.dirname(comlocation):
|
|
cv_cid = cvinfo_CID
|
|
logger.fdebug('[IMPORT-CBZ] CVINFO_COMICID attached : ' + str(cv_cid))
|
|
else:
|
|
cv_cid = None
|
|
import_by_comicids.append({
|
|
"impid": impid,
|
|
"comicid": cv_cid,
|
|
"watchmatch": None,
|
|
"displayname": mod_series,
|
|
"comicname": comicname,
|
|
"dynamicname": as_dyninfo['mod_seriesname'],
|
|
"comicyear": issueyear,
|
|
"issuenumber": issue_number,
|
|
"volume": issuevolume,
|
|
"issueid": issuenotes_id,
|
|
"comfilename": comfilename,
|
|
"comlocation": comlocation.decode(mylar.SYS_ENCODING)
|
|
})
|
|
|
|
mylar.IMPORT_CID_COUNT +=1
|
|
else:
|
|
pass
|
|
#logger.fdebug(i['ComicFilename'] + ' is not in a metatagged format (cbz). Bypassing reading of the metatags')
|
|
|
|
if issueinfo is None:
|
|
if i['parsedinfo']['issue_number'] is None:
|
|
if 'annual' in i['parsedinfo']['series_name'].lower():
|
|
logger.fdebug('Annual detected with no issue number present. Assuming year as issue.')##1 issue')
|
|
if i['parsedinfo']['issue_year'] is not None:
|
|
issuenumber = 'Annual ' + str(i['parsedinfo']['issue_year'])
|
|
else:
|
|
issuenumber = 'Annual 1'
|
|
else:
|
|
issuenumber = i['parsedinfo']['issue_number']
|
|
|
|
if 'annual' in i['parsedinfo']['series_name'].lower():
|
|
mod_series = re.sub('annual', '', i['parsedinfo']['series_name'], flags=re.I).strip()
|
|
logger.fdebug('Annual detected with no issue number present. Assuming year as issue.')##1 issue')
|
|
if i['parsedinfo']['issue_number'] is not None:
|
|
issuenumber = 'Annual ' + str(i['parsedinfo']['issue_number'])
|
|
else:
|
|
if i['parsedinfo']['issue_year'] is not None:
|
|
issuenumber = 'Annual ' + str(i['parsedinfo']['issue_year'])
|
|
else:
|
|
issuenumber = 'Annual 1'
|
|
else:
|
|
mod_series = i['parsedinfo']['series_name']
|
|
issuenumber = i['parsedinfo']['issue_number']
|
|
|
|
|
|
logger.fdebug('[' + mod_series + '] Adding to the import-queue!')
|
|
isd = filechecker.FileChecker()
|
|
is_dyninfo = isd.dynamic_replace(helpers.conversion(mod_series))
|
|
logger.fdebug('Dynamic-ComicName: ' + is_dyninfo['mod_seriesname'])
|
|
|
|
#impid = dispname + '-' + str(result_comyear) + '-' + str(comiss) #com_NAME + "-" + str(result_comyear) + "-" + str(comiss)
|
|
impid = str(random.randint(1000000,99999999))
|
|
logger.fdebug("impid: " + str(impid))
|
|
if cvscanned_loc == os.path.dirname(comlocation):
|
|
cv_cid = cvinfo_CID
|
|
logger.fdebug('CVINFO_COMICID attached : ' + str(cv_cid))
|
|
else:
|
|
cv_cid = None
|
|
|
|
if issuevolume is None:
|
|
logger.fdebug('issue volume is : ' + str(issuevolume))
|
|
if i['parsedinfo']['series_volume'] is None:
|
|
issuevolume = None
|
|
else:
|
|
if str(i['parsedinfo']['series_volume'].lower()).startswith('v'):
|
|
issuevolume = i['parsedinfo']['series_volume']
|
|
else:
|
|
issuevolume = 'v' + str(i['parsedinfo']['series_volume'])
|
|
else:
|
|
logger.fdebug('issue volume not none : ' + str(issuevolume))
|
|
if issuevolume.lower().startswith('v'):
|
|
issuevolume = issuevolume
|
|
else:
|
|
issuevolume = 'v' + str(issuevolume)
|
|
|
|
logger.fdebug('IssueVolume is : ' + str(issuevolume))
|
|
|
|
import_by_comicids.append({
|
|
"impid": impid,
|
|
"comicid": cv_cid,
|
|
"issueid": None,
|
|
"watchmatch": None, #watchmatch (should be true/false if it already exists on watchlist)
|
|
"displayname": mod_series,
|
|
"comicname": i['parsedinfo']['series_name'],
|
|
"dynamicname": is_dyninfo['mod_seriesname'].lower(),
|
|
"comicyear": i['parsedinfo']['issue_year'],
|
|
"issuenumber": issuenumber, #issuenumber,
|
|
"volume": issuevolume,
|
|
"comfilename": comfilename,
|
|
"comlocation": helpers.conversion(comlocation)
|
|
})
|
|
cnt+=1
|
|
#logger.fdebug('import_by_ids: ' + str(import_by_comicids))
|
|
|
|
#reverse lookup all of the gathered IssueID's in order to get the related ComicID
|
|
reverse_issueids = []
|
|
for x in issueid_list:
|
|
reverse_issueids.append(x['issueid'])
|
|
|
|
vals = []
|
|
if len(reverse_issueids) > 0:
|
|
mylar.IMPORT_STATUS = 'Now Reverse looking up ' + str(len(reverse_issueids)) + ' IssueIDs to get the ComicIDs'
|
|
vals = mylar.cv.getComic(None, 'import', comicidlist=reverse_issueids)
|
|
#logger.fdebug('vals returned:' + str(vals))
|
|
|
|
if len(watch_kchoice) > 0:
|
|
watchchoice['watchlist'] = watch_kchoice
|
|
#logger.fdebug("watchchoice: " + str(watchchoice))
|
|
|
|
logger.info("I have found " + str(watchfound) + " out of " + str(comiccnt) + " comics for series that are being watched.")
|
|
wat = 0
|
|
comicids = []
|
|
|
|
if watchfound > 0:
|
|
if mylar.CONFIG.IMP_MOVE:
|
|
logger.info('You checked off Move Files...so that\'s what I am going to do')
|
|
#check to see if Move Files is enabled.
|
|
#if not being moved, set the archive bit.
|
|
logger.fdebug('Moving files into appropriate directory')
|
|
while (wat < watchfound):
|
|
watch_the_list = watchchoice['watchlist'][wat]
|
|
watch_comlocation = watch_the_list['ComicLocation']
|
|
watch_comicid = watch_the_list['ComicID']
|
|
watch_comicname = watch_the_list['ComicName']
|
|
watch_comicyear = watch_the_list['ComicYear']
|
|
watch_comiciss = watch_the_list['ComicIssue']
|
|
logger.fdebug('ComicLocation: ' + watch_comlocation)
|
|
orig_comlocation = watch_the_list['OriginalLocation']
|
|
orig_filename = watch_the_list['OriginalFilename']
|
|
logger.fdebug('Orig. Location: ' + orig_comlocation)
|
|
logger.fdebug('Orig. Filename: ' + orig_filename)
|
|
#before moving check to see if Rename to Mylar structure is enabled.
|
|
if mylar.CONFIG.IMP_RENAME:
|
|
logger.fdebug('Renaming files according to configuration details : ' + str(mylar.CONFIG.FILE_FORMAT))
|
|
renameit = helpers.rename_param(watch_comicid, watch_comicname, watch_comicyear, watch_comiciss)
|
|
nfilename = renameit['nfilename']
|
|
|
|
dst_path = os.path.join(watch_comlocation, nfilename)
|
|
if str(watch_comicid) not in comicids:
|
|
comicids.append(watch_comicid)
|
|
else:
|
|
logger.fdebug('Renaming files not enabled, keeping original filename(s)')
|
|
dst_path = os.path.join(watch_comlocation, orig_filename)
|
|
|
|
#os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder,str(nfilename + ext)))
|
|
#src = os.path.join(, str(nfilename + ext))
|
|
logger.fdebug('I am going to move ' + orig_comlocation + ' to ' + dst_path)
|
|
try:
|
|
shutil.move(orig_comlocation, dst_path)
|
|
except (OSError, IOError):
|
|
logger.info("Failed to move directory - check directories and manually re-run.")
|
|
wat+=1
|
|
else:
|
|
# if move files isn't enabled, let's set all found comics to Archive status :)
|
|
while (wat < watchfound):
|
|
watch_the_list = watchchoice['watchlist'][wat]
|
|
watch_comicid = watch_the_list['ComicID']
|
|
watch_issue = watch_the_list['ComicIssue']
|
|
logger.fdebug('ComicID: ' + str(watch_comicid))
|
|
logger.fdebug('Issue#: ' + str(watch_issue))
|
|
issuechk = myDB.selectone("SELECT * from issues where ComicID=? AND INT_IssueNumber=?", [watch_comicid, watch_issue]).fetchone()
|
|
if issuechk is None:
|
|
logger.fdebug('No matching issues for this comic#')
|
|
else:
|
|
logger.fdebug('...Existing status: ' + str(issuechk['Status']))
|
|
control = {"IssueID": issuechk['IssueID']}
|
|
values = {"Status": "Archived"}
|
|
logger.fdebug('...changing status of ' + str(issuechk['Issue_Number']) + ' to Archived ')
|
|
myDB.upsert("issues", values, control)
|
|
if str(watch_comicid) not in comicids:
|
|
comicids.append(watch_comicid)
|
|
wat+=1
|
|
if comicids is None: pass
|
|
else:
|
|
c_upd = len(comicids)
|
|
c = 0
|
|
while (c < c_upd ):
|
|
logger.fdebug('Rescanning.. ' + str(c))
|
|
updater.forceRescan(c)
|
|
if not len(import_by_comicids):
|
|
return "Completed"
|
|
|
|
if len(import_by_comicids) > 0 or len(vals) > 0:
|
|
#import_comicids['comic_info'] = import_by_comicids
|
|
#if vals:
|
|
# import_comicids['issueid_info'] = vals
|
|
#else:
|
|
# import_comicids['issueid_info'] = None
|
|
if vals:
|
|
cvimport_comicids = vals
|
|
import_cv_ids = len(vals)
|
|
else:
|
|
cvimport_comicids = None
|
|
import_cv_ids = 0
|
|
else:
|
|
import_cv_ids = 0
|
|
cvimport_comicids = None
|
|
|
|
return {'import_by_comicids': import_by_comicids,
|
|
'import_count': len(import_by_comicids),
|
|
'CV_import_comicids': cvimport_comicids,
|
|
'import_cv_ids': import_cv_ids,
|
|
'issueid_list': issueid_list,
|
|
'failure_list': failure_list,
|
|
'utter_failure_list': utter_failure_list}
|
|
|
|
|
|
def scanLibrary(scan=None, queue=None):
|
|
mylar.IMPORT_FILES = 0
|
|
mylar.IMPORT_PARSED_COUNT = 0
|
|
valreturn = []
|
|
if scan:
|
|
try:
|
|
soma = libraryScan(queue=queue)
|
|
except Exception, e:
|
|
logger.error('[IMPORT] Unable to complete the scan: %s' % e)
|
|
mylar.IMPORT_STATUS = None
|
|
valreturn.append({"somevalue": 'self.ie',
|
|
"result": 'error'})
|
|
return queue.put(valreturn)
|
|
if soma == "Completed":
|
|
logger.info('[IMPORT] Sucessfully completed import.')
|
|
elif soma == "Fail":
|
|
mylar.IMPORT_STATUS = 'Failure'
|
|
valreturn.append({"somevalue": 'self.ie',
|
|
"result": 'error'})
|
|
return queue.put(valreturn)
|
|
else:
|
|
mylar.IMPORT_STATUS = 'Now adding the completed results to the DB.'
|
|
logger.info('[IMPORT] Parsing/Reading of files completed!')
|
|
logger.info('[IMPORT] Attempting to import ' + str(int(soma['import_cv_ids'] + soma['import_count'])) + ' files into your watchlist.')
|
|
logger.info('[IMPORT-BREAKDOWN] Files with ComicIDs successfully extracted: ' + str(soma['import_cv_ids']))
|
|
logger.info('[IMPORT-BREAKDOWN] Files that had to be parsed: ' + str(soma['import_count']))
|
|
logger.info('[IMPORT-BREAKDOWN] Files that were unable to be parsed: ' + str(len(soma['failure_list'])))
|
|
logger.info('[IMPORT-BREAKDOWN] Files that caused errors during the import: ' + str(len(soma['utter_failure_list'])))
|
|
#logger.info('[IMPORT-BREAKDOWN] Failure Files: ' + str(soma['failure_list']))
|
|
|
|
myDB = db.DBConnection()
|
|
|
|
#first we do the CV ones.
|
|
if int(soma['import_cv_ids']) > 0:
|
|
for i in soma['CV_import_comicids']:
|
|
#we need to find the impid in the issueid_list as that holds the impid + other info
|
|
abc = [x for x in soma['issueid_list'] if x['issueid'] == i['IssueID']]
|
|
ghi = abc[0]['importinfo']
|
|
|
|
nspace_dynamicname = re.sub('[\|\s]', '', ghi['dynamicname'].lower()).strip()
|
|
#these all have related ComicID/IssueID's...just add them as is.
|
|
controlValue = {"impID": ghi['impid']}
|
|
newValue = {"Status": "Not Imported",
|
|
"ComicName": helpers.conversion(i['ComicName']),
|
|
"DisplayName": helpers.conversion(i['ComicName']),
|
|
"DynamicName": helpers.conversion(nspace_dynamicname),
|
|
"ComicID": i['ComicID'],
|
|
"IssueID": i['IssueID'],
|
|
"IssueNumber": helpers.conversion(i['Issue_Number']),
|
|
"Volume": ghi['volume'],
|
|
"ComicYear": ghi['comicyear'],
|
|
"ComicFilename": helpers.conversion(ghi['comfilename']),
|
|
"ComicLocation": helpers.conversion(ghi['comlocation']),
|
|
"ImportDate": helpers.today(),
|
|
"WatchMatch": None} #i['watchmatch']}
|
|
myDB.upsert("importresults", newValue, controlValue)
|
|
|
|
if int(soma['import_count']) > 0:
|
|
for ss in soma['import_by_comicids']:
|
|
|
|
nspace_dynamicname = re.sub('[\|\s]', '', ss['dynamicname'].lower()).strip()
|
|
|
|
controlValue = {"impID": ss['impid']}
|
|
newValue = {"ComicYear": ss['comicyear'],
|
|
"Status": "Not Imported",
|
|
"ComicName": helpers.conversion(ss['comicname']),
|
|
"DisplayName": helpers.conversion(ss['displayname']),
|
|
"DynamicName": helpers.conversion(nspace_dynamicname),
|
|
"ComicID": ss['comicid'], #if it's been scanned in for cvinfo, this will be the CID - otherwise it's None
|
|
"IssueID": None,
|
|
"Volume": ss['volume'],
|
|
"IssueNumber": helpers.conversion(ss['issuenumber']),
|
|
"ComicFilename": helpers.conversion(ss['comfilename']),
|
|
"ComicLocation": helpers.conversion(ss['comlocation']),
|
|
"ImportDate": helpers.today(),
|
|
"WatchMatch": ss['watchmatch']}
|
|
myDB.upsert("importresults", newValue, controlValue)
|
|
|
|
# because we could be adding volumes/series that span years, we need to account for this
|
|
# add the year to the db under the term, valid-years
|
|
# add the issue to the db under the term, min-issue
|
|
|
|
#locate metadata here.
|
|
# unzip -z filename.cbz will show the comment field of the zip which contains the metadata.
|
|
|
|
#self.importResults()
|
|
mylar.IMPORT_STATUS = 'Import completed.'
|
|
valreturn.append({"somevalue": 'self.ie',
|
|
"result": 'success'})
|
|
return queue.put(valreturn)
|