From 606114c743ddb4245b8f0dd652295199ae459d7a Mon Sep 17 00:00:00 2001 From: evilhero Date: Wed, 26 Feb 2014 14:48:50 -0500 Subject: [PATCH] FIX:(#627) Rebooted #1's showing incorrectly on pull-list/matching, FIX:(#630) identify cbr/cbz files only in filechecker, FIX:(#632) NZBGet Download directory option added, IMP: Manual Annual add now supported via comicid directly (on bottom of comicdetails screen), IMP: Publisher now accounted for when searching, IMP: / added for file format values, IMP: Store/Publication dates now used to help better matching during et all, FIX:(#620) dognzb/nzb.su should be working (again), IMP: Very generic API implemented (taken from headphones), lots of other fixes... --- API_REFERENCE | 61 ++++ data/interfaces/default/comicdetails.html | 37 ++- data/interfaces/default/config.html | 155 +++++++--- data/interfaces/default/images/x.png | Bin 0 -> 18142 bytes data/interfaces/default/searchresults.html | 2 +- mylar/PostProcessor.py | 47 ++- mylar/__init__.py | 78 +++-- mylar/api.py | 342 +++++++++++++++++++++ mylar/cv.py | 123 ++++++-- mylar/filechecker.py | 332 ++++++++++++-------- mylar/findcomicfeed.py | 20 +- mylar/helpers.py | 35 +++ mylar/importer.py | 111 +++++-- mylar/librarysync.py | 57 +++- mylar/mb.py | 20 +- mylar/rsscheck.py | 46 +-- mylar/search.py | 198 +++++++++--- mylar/solicit.py | 5 + mylar/updater.py | 8 +- mylar/webserve.py | 214 ++++++++++--- mylar/weeklypull.py | 100 +++++- 21 files changed, 1591 insertions(+), 400 deletions(-) create mode 100644 API_REFERENCE create mode 100644 data/interfaces/default/images/x.png create mode 100644 mylar/api.py diff --git a/API_REFERENCE b/API_REFERENCE new file mode 100644 index 00000000..b57627e5 --- /dev/null +++ b/API_REFERENCE @@ -0,0 +1,61 @@ +Because of how Mylar is based upon headphones, logistically the API that rembo10 has created for headphones +will work reasonably well for Mylar - with some obvious changes. With that said, this was all taken from headphones +as a starting base and will be added to as required. +The API is really new and probably needs alot of cleaning up and to be tested out properly. +There are no error codes yet, but will be added soon. + + +General structure: +http://localhost:8090 + HTTP_ROOT + /api?apikey=$apikey&cmd=$command + +Data returned in json format. +If executing a command like "delComic" or "addComic" you'll get back an "OK", else, you'll get the data you requested + +$commands¶meters[&optionalparameters]: + + +getIndex (fetch data from index page. Returns: ArtistName, ArtistSortName, ArtistID, Status, DateAdded, + [LatestAlbum, ReleaseDate, AlbumID], HaveTracks, TotalTracks, + IncludeExtras, LastUpdated, [ArtworkURL, ThumbURL]: a remote url to the artwork/thumbnail. To get the cached image path, see getArtistArt command. + ThumbURL is added/updated when an artist is added/updated. If your using the database method to get the artwork, + it's more reliable to use the ThumbURL than the ArtworkURL) + +getComic&id=$comicid (fetch artist data. returns the artist object (see above) and album info: Status, AlbumASIN, DateAdded, AlbumTitle, ArtistName, ReleaseDate, AlbumID, ArtistID, Type, ArtworkURL: hosted image path. For cached image, see getAlbumArt command) + +getIssue&id=$comicid (fetch data from album page. Returns the album object, a description object and a tracks object. Tracks contain: AlbumASIN, AlbumTitle, TrackID, Format, TrackDuration (ms), ArtistName, TrackTitle, AlbumID, ArtistID, Location, TrackNumber, CleanName (stripped of punctuation /styling), BitRate) + +getUpcoming (Returns: Status, AlbumASIN, DateAdded, AlbumTitle, ArtistName, ReleaseDate, AlbumID, ArtistID, Type) + +getWanted (Returns: Status, AlbumASIN, DateAdded, AlbumTitle, ArtistName, ReleaseDate, AlbumID, ArtistID, Type) + +getHistory (Returns: Status, DateAdded, Title, URL (nzb), FolderName, AlbumID, Size (bytes)) + +getLogs (not working yet) + +findArtist&name=$artistname[&limit=$limit] (perform artist query on musicbrainz. Returns: url, score, name, uniquename (contains disambiguation info), id) + +findAlbum&name=$albumname[&limit=$limit] (perform album query on musicbrainz. Returns: title, url (artist), id (artist), albumurl, albumid, score, uniquename (artist - with disambiguation) + +addComic&id=$comicid (add an comic to the db by comicid) +addAlbum&id=$releaseid (add an album to the db by album release id) + +delComic&id=$comicid (delete Comic from db by comicid) + +pauseComic&id=$artistid (pause an comic in db) +resumeComic&id=$artistid (resume an comic in db) + +refreshComic&id=$comicid (refresh info for comic in db) + +queueIssue&id=$issueid (Mark an issue as wanted and start the search. +unqueueIssue&id=$issueid (Unmark issue as wanted / i.e. mark as skipped) + +forceSearch (force search for wanted issues - not launched in a separate thread so it may take a bit to complete) +forceProcess (force post process issues in download directory - also not launched in a separate thread) + +getVersion (Returns some version information: git_path, install_type, current_version, installed_version, commits_behind +checkGithub (updates the version information above and returns getVersion data) + +shutdown (shut down mylar) +restart (restart mylar) +update (update mylar - you may want to check the install type in get version and not allow this if type==exe) + diff --git a/data/interfaces/default/comicdetails.html b/data/interfaces/default/comicdetails.html index 96f0ecdb..0ff8470f 100644 --- a/data/interfaces/default/comicdetails.html +++ b/data/interfaces/default/comicdetails.html @@ -401,8 +401,12 @@ + %if annuals: -

Annuals

+

Annuals

+ %for aninfo in annualinfo: + ${aninfo['annualComicName']} + %endfor
Mark selected annuals as @@ -445,6 +449,7 @@ %for annual in annuals: + <% if annual['Status'] == 'Skipped': grade = 'Z' @@ -483,17 +488,29 @@ - + %endfor - + %endif - +
+
+ + + +
+
+
+
+ +
+
+ <%def name="headIncludes()"> @@ -653,7 +670,17 @@ { 'bVisible': false, 'aTargets': [1] }, { 'sType': 'numeric', 'aTargets': [1] }, { 'iDataSort': [1], 'aTargets': [2] } - ] + ], + "oLanguage": { + "sLengthMenu":"", + "sEmptyTable": "No issue information available", + "sInfo":"Showing _TOTAL_ issues", + "sInfoEmpty":"Showing 0 to 0 of 0 issues", + "sInfoFiltered":"", + "sSearch": ""}, + "bStateSave": true, + "bFilter": false, + "iDisplayLength": 10 }); resetFilters("issue"); diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 7fbbd649..025c6ae9 100755 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -137,6 +137,20 @@ +
+ API +
+ +
+
+
+ + + + Current API key: ${config['api_key']} +
+
+
Interval
@@ -195,11 +209,11 @@ - - + @@ -624,7 +630,7 @@ <% - file_options = "$Series = SeriesName\n$Year = SeriesYear\n$Annual = Annual (word)\n$Issue = IssueNumber\n$VolumeY = V{SeriesYear}\n$VolumeN = V{Volume#}" + file_options = "$Series = SeriesName\n$Year = SeriesYear\n$Annual = Annual (word)\n$Issue = IssueNumber\n$VolumeY = V{SeriesYear}\n$VolumeN = V{Volume#}\n$month = publication month number\n$monthname = publication month name" %> Use: $Series, $Year, $Issue
@@ -849,6 +855,26 @@ { + if ($("#api_enabled").is(":checked")) + { + $("#apioptions").show(); + } + else + { + $("#apioptions").hide(); + } + + $("#api_enabled").click(function(){ + if ($("#api_enabled").is(":checked")) + { + $("#apioptions").slideDown(); + } + else + { + $("#apioptions").slideUp(); + } + }); + if ($("#prowl").is(":checked")) { $("#prowloptions").show(); @@ -927,8 +953,38 @@ { $("#boxcaroptions").slideUp(); } + + if ($("#nzb_downloader_sabnzbd").is(":checked")) + { + $("#sabnzbd_options").show(); + $("#nzbget_options,#blackhole_options").hide(); + } + if ($("#nzb_downloader_nzbget").is(":checked")) + { + $("#sabnzbd_options,#blackhole_options").hide(); + $("#nzbget_options").show(); + } + if ($("#nzb_downloader_blackhole").is(":checked")) + { + $("#sabnzbd_options,#nzbget_options").hide(); + $("#blackhole_options").show(); + } }); + $('input[type=radio]').change(function(){ + if ($("#nzb_downloader_sabnzbd").is(":checked")) + { + $("#nzbget_options,#blackhole_options").fadeOut("fast", function() { $("#sabnzbd_options").fadeIn() }); + } + if ($("#nzb_downloader_nzbget").is(":checked")) + { + $("#sabnzbd_options,#blackhole_options").fadeOut("fast", function() { $("#nzbget_options").fadeIn() }); + } + if ($("#nzb_downloader_blackhole").is(":checked")) + { + $("#sabnzbd_options,#nzbget_options").fadeOut("fast", function() { $("#blackhole_options").fadeIn() }); + } + }); var deletedNewznabs = 0; @@ -937,6 +993,17 @@ deletedNewznabs = deletedNewznabs + 1; }); + $('#api_key').click(function(){ $('#api_key').select() }); + $("#generate_api").click(function(){ + $.get('generateAPI', + function(data){ + if (data.error != undefined) { + alert(data.error); + return; + } + $('#api_key').val(data); + }); + }); $("#add_newznab").click(function() { @@ -958,9 +1025,7 @@ }); initActions(); initConfigCheckbox("#launch_browser"); - initConfigCheckbox("#use_sabnzbd"); - initConfigCheckbox("#use_nzbget"); - initConfigCheckbox("#useblackhole"); + initConfigCheckbox("#enable_api"); initConfigCheckbox("#usenewznab"); initConfigCheckbox("#usenzbsu"); initConfigCheckbox("#usedognzb"); diff --git a/data/interfaces/default/images/x.png b/data/interfaces/default/images/x.png new file mode 100644 index 0000000000000000000000000000000000000000..ea247a9d930a1545d6b466144608b46a9b31d026 GIT binary patch literal 18142 zcmX|p2{@E%*#9h+N`^sH*2W0gl093?SSDMg>|4r6b}@Fw(a9jm*!NQ@$=;A9G-WJV zYsy#>(?k-&gzWy$^nKs|Tvyk*yz|cU-p~Eq>+k-(lX%h6g!jPF0}u#=7i)^S41qvd z5D3g_9~Zcipk(tM{Ey@IIqa2v;J>JSH;CZx{Q;)-w;|9@JN7@QrnJu(xOgPk$S&C0 z9~T^YEyx2B8XBtX<>!6-`n3QLW&a@0{9k%UArL7D7IXGWSi$_4@RrKkUpkiurvv)Z z%C3|=dl0UeJCoa&YOijZHiOl-u)2NX=#|bsHJ$He2a(9cbI7!Z8LDqp#b>s*11E#U zB8yw;6W*1J`c8|(<*@^!k(CPb(Up^nySz#J`uh5wU!e6-YzdWwVptc3`mOBY35;@( znp*h@toP-NKPc<%<=>RlI28=l>`2pds*xWKmnrRqU`V;CoN135O`)tI11J9rx|)_e z-q&}+h!z>g6Zh}wb(Uf^H3tGROSD#x+v|s6 zfDB&fey##8pRxArR#7_@)UU;_-Zoim5iXlXh;}@B`DAN}75iB3x9--`SSGV9L`_Xp z8{2E`BROZ12yfwC&n%hVT-nV0iqcXe$B|afGHfZ^(J|%Wajq8}DhC(Sd)OO=OV2_W zif&oZ)@e~fvDsMInMJLD7-R5Bw5-=@hAd?Qi*$kE6Nzf1iP!#m?6ulR>iPk&a2do& zc);_LR5b1fk4RH)?y5!j*jO%QVANjj^r9)Y_#J{~R^-6GU#?nGpL;|~x0fGrk2vU= z@PqU3{RJhX@@vHEqC$_O2yO!;GMyZM?_c)*A}-LJC{HL!qxPo*PSiMJ%zF^09M}go zm&+>n-Aj-mlY_sWbNfsTGtlk6*~r$fELsWT1Z`2!?Ui%l^$|u|d2-MG)m5iC4Mmxz zd?u6cy|JL=A(wsd5B$Qma&vQ25=;JE7He{t5#Y9qW4_OG2u#b*9etLWUF>ABH)tW5 z@kQq9wZX_Py?UUG$0)C>3{=wZzWk{pZX3i#1zl6wzsjh>I`FOQ^eY4sMhR-@{v7cV z!M~Cwl_(K-4v+KVj$}dGvc(?>{#C_&?4}9>?cB8t zZOZ*os43ER9+drt`y*e8P65^(09U;7M7&j+%Pae z`hjr!mzeW+m#jPm6@`&>W`f@vM4N{|fcjDG+3|Kt3PbZat@Q$&YgbTH{(g!93#jhnVxebXbd=l(wSD2x91u z+MmFoYOolmOmKJ-X~SI&v#yEfRfl%fu#s!a`h68BbgN(^SN8fv98@`@E&~6+G7RGa zC($8QH%>~nz}8JMV)}$Xr^p;6;nJ|!oa)%bBaVM<}n$}WyBO`<~w;JE&Q%hwy=jz-ik zqi3H^A;aqztbXx^z+%u|yVQd?O=FTKT=}hxfsGmDb!J^S{(*c`x59J1Cg19LF^U+4 zk9tNzT~bLdfe$j5wdZ($81bl@lYY3jP(+z^fTXUnWc6I@I;)VlJ~_{i)AS{+ zwWm_KW;xx8iP}#K{9F-s36C0WIdMn7ZW$gzCfr-=cb+RGThT_RP=c(G{y_-q;EJJT^Bz|T=3n&i_~1fv^)Un8R%Kn z?fNFru(yuIwEWUrv)SQBd0`o15w6UO6`F?DG%;tKK*u|C3%Ni-D7=k4?sKrSCZ;RT zR}wpCzLQXSB+bkL(kKz;T|D`7o;cF{QAN2pR($fcw!F)KhchQL!6W=48o?K4p>FSp z>wgqt-zV)|>=EryezFJ3hs=^_-Ho?FGC#jY52Rl+PO>iTtKcEwOG4*DD{_T7Vzj3v z)nhO&zh1vOXy;4t-})Q$49uVA%K0gb-6rIoniodO;u21l!?Amu1(@3sWcyfOVWh~p z_T-lFZ_!*e7H>auckDM$fH%N-APQM~pEDwy@1Jr=y*%#dvbr_Ye>qdv%kqtkU*VQ) z4F;n(v4S$>_;EAu)!=ccxOmLunJ#0@M^A}9<4Qdl))glvy;V)xeXCQrsRT|~#Lb+9 z*-4|S-XYv=G)OO>$SuE&dEZd``C*!oi=}np-Bh)TBwaZ_(@0faqWx9VvAfJa8D`_1 z%-zn)dy0ROB$LfrE(pUB-H2;8he$6wW&B!~^NmxwW%~DT3)`V}lUoOV3-PG^0?TJ% zX*NR10nSW1&u2{>#-->1UO;*Id0ry!SFYAvP6;0j7Cx|CLdz18UOw`IY|>koSCO*Z52n&EsRq( zIOk%tFIRz4te;ceTHZcI3FM#iJ0`UzUj?eAL5g7gBk8S(&1VWU?5?jU$RIS7gRol= zUC)`COO68T#lYluq`U;(V9J4Wt!C0p4PC-59U$o?2F~sKyrSlD!MH!h)MyZ|BAz%! zMQV10wBcG;I69oC_$)RO>P6GCKi0~A74ZMZp3wf{e9mf8NlHMX<6czMPl1G^!a7Sv zZvW2n)73;XMND#r@=AE)ZiA0w3xtuL2UkVv^Bl74PyylwhswNmp^Bgs^Wat$<;99Z zW-b@F-K2ZRmgZdE8@?E8pN|Z08k)yAF)h>$+k)t~F3>iCR$^fz*Bile4(W6pLnY%9 zYT7nxN;E)j3OP@UpoFvU0Kq5QVvgq)Tl>C@pePNmk2{=pCs0Mb2zN~Z>>O<7+rJHT zL)R2y^}Q9NOvyB}Jh-+*c9^IlEJ{iD&ZsH6GTv-_bBZ#nj&ClKEX~Az zUYfG?;I4^tJh1MA-$$0mI!H9`%hAwH)|iO@5)~-^l#f!u2 zvt3E-M+q%*8>ET99W$f-LpemrCzI{wqUt`X1gAL;MVryijUKCrMz`iZqKfnlxL2+i zW51o=c-t(r;SZ4%pjnb(CP1zDG}CtYZV=Q^OuJHNgoN^3=dxCgC=3-ZfH{y1ohV zd^Wzyy!aTU;#?q3q4o1csesSp(NGikXAhv<2m#PI`APOBf5P1oKSR@vi+pk0PNqT> zX?r7~CL(?1den~x(_wkcKO-sEqaQk?bW1Z`7%%6=Z1x*ws+-~Wm8)PUK`7BF;?(?Q zlz6^BsU_GC4=KMB3Yi3fW)(e~yna`|cUi2)9^>@6T%=;?(`f=I?()gtDdt=@iA2#C z@pjRspppkwJN<$R#1TUsjNEusuMh;m{(<@r4qVGzGI47PvW*{A$8MesxPnJ`KglLk zN;Cg)c^x2LN`I(b_vT>Ib#=pH@wd;dX!j||QJsoR7cTs~N#glvlPPbgEpDc!Ck$y9 zQ{kvx(S-KhzkM*rKsUL5;D%c^QM=yN|Kc!8(j-t|U>uy{DL;-0Np`1YS8qB7(Uv^S zJ~dea=wlmW+OSHc^xpC>%6&kp7Xme|VE@9{a`+5Gpg~M>F*NbKMRLoO?@i5m|BM8# z++$uwNqpOF#cxi{OCsU+>6d!TLI|vr4@{tv2i&cjQ7C#p;#w10$tdtT(}`_ByIQC= zwP#x83q(ECaoFo_@D%d>5`FS{I|=L}RJk4Kx$RUpaD9>RcOU08Gva0X!KS&aXy-r! z7uAHVc+w`0bQNF8r?+;il#CR6YCh440|9Ep4I9A|z`692Z8nCW!}MqgtOTWr1b0kk zVfzU$Gwf$@+HW4Ht8|!-CyOh!e?`{D(({L#yUmJR8oq90NX6-z1tmK2^j^g*wh`O8 z7571ugS=m8K_>?h*LI}6G#LVJx$NajA(Ho}7F_GU#AwQ3U&FI{!6xd@V!{&6x0~i2 zX*_JFrju0v(Vl{shI*9cokS%ttI6qw|wG;esGg-v2GHd={$`c+)bi*gVjv zVD#byA_~+TtH$BOZ_6mgY*i_9zDy?e1Y1fcfo(vQd%#m(rk3!5ZPccrLBBJ{mcVG1lRK)YeDJOhSGa6YpUCXQ63C=Ih+bKBn_(&iJJ-Bln(v!9Y7{$O9X%n1c zpae8Fp}nDW0cTM70`U<;4&u0vT(}M%b{r}T9Ia=ogdKA2LebHjAe{iqZGeaxQ$LQL zz59{4;932k?(kA6@RIOY&a5P&=ZpA}ZglifZ8_b9dYUiAVOsuC=1>k&=L}{X>QoGl zU{SpB@U647iS^0M=|(}jhMzJBUIi8{a1>f0Y;fGGb)H{S5Bmz104&*Nlc&bAN%}eK zs75_PYL=H`2&uMwnWnc-5sBOzK)6?5bqTwD`lK@3z;`G=k;`X5JEq|WLg!@z1>J<9 zw=i}iGvbXM%VAPPvKx8FCJFU(SD5c9D3$PtFQ@AhN-ThC&e9x&OsWSSfa9Ji7Wz2* z2gf~Y^X)19>(IyI-G3I3zZ}?xhxuc+L7$(GjgSmD!nnQc`(A>RevjzzE8^A^v8Cnm zz~i8~PurAK^7*>l5)XMqNKoSE|8{uns;j{E#M9uWpq5L$8)H}7HgWZA?2qF^F0Hl zvVl4@*BM3m3O+tZ`z&{1TM3v+$Z=S{F-97}?>*0sJ4fV&PqnJ*CJ*%>kb%Z{;_-BK z$pGG^U%*yQt3dLNs2-zdBc6PaX0ZlFtU-iY5IZqzF?uZTxnAAuWsqXRRA7!;Wz>gX zja*nO%^}`?jA!%w7HF^I44B&}dy#ZG+-Kz;1kF1|g+**UvVI`j~|YJRc-H`hLcFWFTmwW4z3O11$Sc zD@1Q!&PYx9@`vyRoXi}@QY}$iRoB$8{!w@D)(zPkKr>?(X)dG3ewx!};&y=K3JW60 z6xT)Cj|ER_Qr*#|a$t*QR&X<8>dxrdpT;zkIBB+Ni}NG7JrfS!vANd`MKO;u*4Vd^ z;awSn;g#n}FF6swdn;F-yv)$9d)93xab)CXC49b6{E;%c(xg`NiSO&{ij!K>OwrL} zTxA;8b4x$Z3h_b7;P@OOgzZkFKrMOMgJi>1MKSc$3^Pn5THz87*V#lnfHVm&1!9lo z9n!YETpr`se9SA5;m%fZDQpXu(EYyC)wE5hz0+O8?e#qWRWxYmRR~R>s+@B=I4|Lw z=U#NK{3+e_*lYwN8if0oMewW?qM;{uSrSVi!#ge%q&E77MokRHybnQok|G_n^lZEn zhhv0czrBdR@d%wqx_v`9q5DQ8ywK!Rz+R9eG^3IsS&XN16%s_xg5-=&R}r5&n3h;`BZ0EafQM$ab-+g}bTqQdi&ifG~p%L{yL7 z?B7>#apLFD32Uc+UKrtpy$faKjhPc%4!3;4I$wk$o%=i8!x_Ivj|qq!oG%i8>oB9^ zMkY3D=$dBMz3et~S;4g~8DV>GBpepP7Q=USLEsob13<4)Q`dB;E@*31p-2A17GzS# z_j-szjXOqtv)KtN`7lh;L(>#Ge-tSTbn(b0wOiU{~I^$v7Th zGt&3qmRahK_y;beI)MNLaKe^$XQs4wbXvBa^UYS4C4c~qMR+8cT|Bz_m3r&zL53i( z%l}r{)=@O_ieSWU4SWQVf-lbBGYa3|Q&0KKKGy95_xw>aV4D=_p#NG_XNVxC%9&sz z%<#ky0PN_nz_HzZ65{xZsyl-W$I`|rlZ0IMT27XM_ij|iboZEDjM=Z4&33R<5ca;l zI@!!XhRdTQN*m2PDs1VT2ASfIMA;_`I*&K6Qkn79W-VqCY$$+5u>eMN$>l$7x$iIM zrKIIm&gK#Zts34$a@f`Do&MZBh1?KAF++v2Kjp9X+@r{oaD4!iYUAh-*byChFJlm) z%o_IYqaQ23Cllun9O}*4bpavKmKk}Q(n*qd$$(zHK_+W^^)+*-neKQtaFq!H&C}zv zv2k$Vt8?a$>#)C zGCfrBUDn5MR>07%TUe=fObmPZVae9!rLTay{(@-Y6aQo^F5%zVZ_)*< znr^mL6)wd-dej#Sv3 z5q0uKd|g~|TIdWQyjz|S{6l{8lIRZ`e|R#w+8pMhwNQv2 z#0Q)*OBq8YQ&fRXA6irC4acyJ9_F0N?C<=MLj)Zd@#SwdmNM31Un6(yz8AY&x&WTh zQymvEb;w};7z*dg>Zu6OtZ~M)6o{8BI2k)95x3+ldrr+YG{8&{f3iVL1C90BR{|p+ zWL$WaSs#{vONnOqq8Bt*B`@S9fTrqj6xzEfpP(eo{C8f;X6e(W9JZOuP6Gt7w;zGM zE!&2z3S_u4e}D8{+C5lt6q)M5UAwnLj9O^}MViBPe3B!rb3;JLY(=VLV^(=NX_*+6 z(F1q);}#wS9n!Lf8}`2$k&2uYhN@dVu~AKz*jkhv1#rW&f&FE<_SX9F3UTD?euVq^ z+i9tBCCijnq4MPZzP#^7Zr_NFC|x4&;9z%`6v~AoCa18yuY&KmwU6rUMB4(Bsq1J^ zQNbx*$r3`nyih{RY3s1Q%8i9Y5b?0}pSs)t%kW3iG9p*;icIL!lEP;U&CIxj^R2dk9m z$$j0P7X#yy0Lhv%vz8*Ksn+T1j{OyWyPQ5kNIp;Q+7E$ge=cI2xVKO1Cgj>DxNsQ! zcx*e@Z!{4pk(qBV&QW86aViw2c6Nd*B9Ksk{i%t*5kWCOftnGo%#uiU7p1%z)#>Yd z`n41p=!V}n$Lnu67tcDC`5NZ`jxAEASzJHL{W6|^jyJV?i;`WIV{f!0F+!J_P?Sjf zgALE^iKmFZAm`*>+BS%A#pq>-QzLq%Sq94?7u~g%S^Z^v)*hO;~{F^8E|Q4>F6=V)Ka>jX*0|Y zWaQkTi89K-SJ$b^?PfH(YHBI;$EOylTf*>U2a zF47PQulg-FV5F%g`Yr|;couY?5+dP`f_l5P$UxZXXcZI!bdy6TA87AAomv#;34 zAXCWcGt-(`L73PI#Bu+In(98oAHXUjidDi)4g z>j{P?I(C7j`2jrsiqPQcsY(|5pTu-+m5B`hH>2Tjv_4SYll5A&apQJrR+7AJ&E$T? z0KowqSvrAkHmYAi*eQeBb;Jfrbw(V#`11l*iif(IAH6=(e$(6C%je z8{D;<>`zck4~7D;^NQ-C{~B$_W&Ey#rx{@dmbl&Cv2VYv0s&E9WAtprE?E2-%V%1J zt@0usx6tgy)uiN>u>nrM0mL<&{%R9ZTOXh#3CS&QdoFG40dFQ3>6}8frHjAK>lp0z zK11^-WU?j8aXRSZqKwm+JCfQx_tk6^+UH~L$N3Uyz!0cQkQBpNkp%)w^V+_e%b4+E zacbT>Y1ZZvIf%R&j$Ho_;AiiQTm+GUhXOo+mCW^+pqR}7IP_6wMF1NiZ)(e-ECGz7 zTyf3d|F{7CGoDi#(z60??cgXiEkt0B>nI^n4w3MFQwQj}FIYyDzCbyFTn@(bdvjY= z4uio2M*-+D`|hqBuE{kU=K>-eCDo}1@cakVvi4jgE+qPHGjE>Bl!g#87IORs*zNGh zXBodEKuz-4WO&0@zU#QFOFKk1Zc4XQj>}6T8nV0KlWNLPHk`aLjSNr0I`B90iZ0o1IrJjc8i{&DN}DpaP|(+l2%>R^ zKxEVCM!Urm8&ByHM=$E3{2#~}WbOdKlYDn{SjO+_ca%RE9K+U)*jKdGQ3wEeXet#s z_)B4XVXxbV3-VGrrYXkLn}XyXV2N=krZ8g#h^Db7;-;J3I230x#Vg}=je&5x3pT9r z`(TGA*$~IAI(G+ra0vMaVghtoo(sW;Bsvy`VJ6&26Btz^ZAZK?Id44zvkhP-vLE|8 za{1)a4twFiV@uOO5nsU5zM}gr1N&!NK*BmPKS;UI<~aTnR1tTRiKTJ=;4I5)V&RSe z6RraLf?~hP8;h~My*%$fxn-Ug@i{43CR0lMNEMhXfKY7Cy5bWZzkon2Kza)IVIb^7V%R}Rf$IxF3Nl$er5l_*@VD zoeV_SocD0!&TiDmO>XHxF%=oZ>}x;Nkb&XsC4jKKScm=Vyll(Y$6vtA7l4ONf`>H` z8IJ5s@&@BJO>EQ$wkQ>L>x;`{J-znOEXTrrSb>b!+PoAl zo`Gt37?tO~T5a##%B_8xEp21>LI{d!8RgkDGzn(Nx$JFi-y6R>rQ2J1%!R`ym5sCJ ztR|aGX`(0X{diR+w5@65z&SQMq=Q7WfPh?093$v$p24Ht(n0xIT5vUw1hBspf(j@o z*edolC4GlgaSHbixYMQ6<5s1aNkEp?DOAI7nz~wm4#;!DWC_qPX`&>05bkt^B8P_I z2WW;mm|~g#hI*Rj8L&;DvD^1nG4v&f8g!F#7&i_qe#olwb2+*av+NYb_7C{d8=|%r z*pyvUH!p$Z^wW{>FE|3;Zfi;q=v3H7xAdvGl^FmQR=$1FsfwwTW#g?Jbe`5lCz_*J$DtsT1l60W&Gp z^8Xgl(Wk&nR0|aOr%3l926=_;6>tB)+C$z-vxL)vAaGOVKt^?iN-KZ;3V#m_{!ERAwngpfxZ^V>m^Lb=}Jt~0bp}fQa z)XP37UOZ3D1&kiZY6ro-50fT%%qcQ{bpKwMVGrWk+W!RaeEBN0^GWcy#~8Z+v32yBuAEeia z`VABS&U%l^kyw9jWvYyyjS&w_Hak>2hxcA`%W-r+)ahW0yl!$XulIS7FphzH>j9Gm z2T|sv1K-9;=N+t&1j7;TT096y68GPN6+}KlOD_PV0_0nf@D9L0s)_uY&7&wsNKb)n z_0OQHPhxeg$CdnWtK<_)3m}a~0huoWf=mqb9%yNB*IEB#^cva8pvGzC1|#n4cwseB zaw%b2npp(81@j_@>ciOImwnW3C+2IkiwR|y_klt@4-^o`NI0$mZgyDgi@0@!(FLT= zb(%VKsfa5?V5tpkoh}lI9n%06O=@wP?VXZ%f)%^+Y0KMJA;xBb)P4_0vn78me zY0ivuR`9dwVYN8mL8q{y-#k5WzAQh+4f(}4BL5?NIjaky#z|)TC)fZuC zp9!Ln;w~S$r3$p%h{_A}JmEC8AKi~|;!(`}TQcrb1oavyZZ}zv>L7`Y5mcjN!limHoFC4p9EHi|U`C=_8bE zu$@N=7^4RLJcWf^9t6#_QjD>IF$u@Xwq4xLsxYIE#brdcZR+m&uO(tBIEhmphue91 zbP0?MYyfMjR)=FmUH49IeNP(+58QI-tLM~Xv{x)(+JI6y^EhlVmFNhDT?Qf>a;BS7 z0m4NePLlQhz!Wgl*Pzo<7P*v-uH-2AXKysuzA6+}YRX!Cl=V9n;w(3)-v-Ii(Mrrc z0uhC}g2K58Xzv*IHViO{fN`khpJUJRI6=z;>S(qH#ULu!X5h2W3aVhtW?=a&fp6qK z4ou9q59x;%I7?jgQ2K_Fj4M%rzXAm#Xk!Hch;fX4n*lEg548kiHyr)ny~KURfge!H ztZSwY>CC3}zdiVu9F7VRB$)ixz6nUE&h5cW5e z9m&a?d3-ri@6?i%7^vcZN^<29QQ;=QcERq}b_3Gjq%~*^iH#;$NpNI=Iu>YekZlz)n z)*n*Ene}%&Zi5g76KXM7-M`N7mNRXIV$WWr5CntxcX0;P62MEGM|pwTo!)9}S()v4!V>hpRyh-trQc{6?V8>BFo9Ua^m^e@%>-BwJ%GYKCmA!J+ zZH6s`X2~IfAX;A6)bVOFFi2>WJcv!X79U?y0Ps?G)kEYJ)`6pOe~vWRa&E}fYZP;3 zp5NE{9nb_1=v4rlI49S>tB4!S_COg~hrL=#Pi2Q68!a{H%moK2Fao?%)h!b(rs7Bb z87TqEraShWt-0=wV--N^N0+VVrkBfK!#aW8h`wo~T&I+Q%JaleC%5uvm*1&h@i3ku zedinisdOk5i`mvjS+JQ-2KF!vlYZ3$vm4^XwBd_vxy*zhRgohSVah5b`zJeBXxuN(tJ_DUnT?69100Hs z0MAe^+T@dF;@$S{=;1<6N@$zVKtw;P5CUB4tx!T+U#{@m3bh6}h*ZDZqvJ}{K0ofFCAH|?*o$zR&4AzvBPHzQQGWS!5o{j3ku$98%;nsn!43)zsj8!E zntsEwQZmkL{O4D)bf)$hj2`Daq~+6|DXzXwH6Q>_e2SF^Mh?gKhxwD0oduAFKUb2a znZ^vB=)UVVp<8CGJd(uG^-s+unFQ~pn85Nj6T%Y|^Q&K5ScrdGCQHI zAc-v)_w@}>)dU!lI|9!!Wd1d=B(_nUsc3 ziLQ!T92cPaV(<@6i*^IyEaHAcS9kjRI=^2644(B!6&E&t`h?q%OG}acd3_GxMZbCx zmrbHog0PDvVamVGyH@->4j08xr(2^wu>8JH!f&s#Li&$>_e(kztFCMMWrckd<`fS0 zk-9E~B!UT4JVl~4#+iIKTqK~ZWRx8L$>Mc!nj}K;s*v1x(E1 z+k~2cuXsaapTBq>x!+C+<#(70#@iwRY9~KSO$=6a#&FFGAq`kz9Qwr$+3H#W!T?H0 zu;P181@lusx5cxq#r7$_JmyR4Tip>3Q_~MsN+JG>({p*N?{dEsJf2WM2A&Fd``_en z#a+$EBc29k_y;HEW`X{yVSB@55%sxZL+f#sq9CmofE3;EH677^I*_*}aq*O-!ypLD zos@sq-;0#DB~p5jfswdP1H!GnMa91D%e1J`v(Iq=X<@z3cD0`0Q!8(NjAA|>J?3)BU~5ai&Q&=o()( z!Slu~WS~AUY6Lm2@wD!lp?eYF41HV4^A`eX1-ZNn4%zE-A z;z7iRL}DZ7dIWgXfmbJV&m2m@VRmCF#;u^0ebP+%`#r9Nr%>Af<4wG2QTVL7H5c;+&Zm%1M6R^ZSqCNFJ$S- z=`7%~Rj3u!TE7^15}1cd`J03A-}r>x|8SgkgFQ*KpOb^668fr$`B8THtNjF{}LNT z!0Z`pXI-}AiM-|=FOWYy;@Y_~I(#bDQi;Zu>-UM$1Xlp*nBnV+Q~CTyuFjDr_H83T zyb;Vz87M>6TH}c^4^}jjUiGp|pDediOZSOkAUa>=)j_H7MeBg3a;UwSka{EGM>cU? z6!4b-4kOY;E%j|ITUCMOA3Q}ImD7iV>1>LN5Ma8(WvHD%!THAp0Z_nU@?lGjJhW((59Wv8lYX$!M=h@S`4_;6CvQ$n=LS*m*;C41^gEKV(~oj|CFDm1B5dS z^x24L1(fmtB25XU42@y;(Y1 zfmKNWl>Rta zWT|qVKjaDV49!VMq#-im%X4NpV+Xms1(rPKr56CNF$b7{GDi)*X`>qdU zRQsz@G?|G)R>`YEIRlQ|p$=JR=5@7cWToemCnqB-1yOGmY)y`#b~&Onfi4r;4kHcT z_}+WkqVD0BG`l)$EolGw3s;4cGULUi2k4}7q86YmYVvZMP(f9d#~4C}WAqSCnIawc zO5=Q;nAZh6N`LDUy4d*5x0~5vi{{abjkeW}Px*|f&fp7Ds9k6jp!$#U4^Xp%S3i9+ zia|^y-_k)O?yLUz;~`O7R2zsfDx}PbzfXKOdNyxr$aAWCH!hf~#@SNLyKF8_1I2tY zFJ-|dpz%?3*)-1`fe~uzK!pEKqz)yNyXfuMyedFp0U}LRqYQ*S?GF+^;KLADOdUQH zj@d>m$~U~pIS68UY%hX#14OT-em0wH5FUx4%5Gn4jlFv`E#72W^RR0&(e*~c&BSWB z7T|{3g!6;saDNa%3$Bhd-u>xn`kFgS!bi+I%@raV{-Kv)8X*Ek=4>iV=c}Wjs*~tP z(74)Dx}k`pb_cxkUtcmWdqw4zL;d1f1I1%X=)RgNv#5Zl8mMB4b)KcQxZ^LLMkc!d zySWMqs0*}k%6Bl#4J?8i0A_~=w^#l(lPRR?{`EHyvYoCNL%^y*WGMh(BNfsmzRG-r4mKrK5pi8&WT@8Q!%lpM_+8P+N>@n@&PVfrnqJuXzq`kyCppxaDp0!v0KtMlJ z2zss?&WCgeCAv#()}mw5lo!b5i=c(l$Nq-D=Kc=4LHR1E!C_=fhJ)Z2J?YpEYEfy; z@vvNXqjIyB?+Ws0FMw(gI($*lCsGr1Q`Mi58_%P$*Fx}y zHQ-%<@SX)@U1*FGEvh&qaoD!Os4d*}vYm6QY2c4!J^`<1#4wJLICtJV8pF&{sLx8X z603ic^!DeiJ>n9c3dRfG02-dDX&aN$C8W4$c~(|mSQhIz$fzTZsOsL~y^Ds5K=N;R z*0zPga)wJAhOD)WcFo-S+n&}oH}cpSpo-oh+~cJIc$pj^43brJZ4hQ0z>&?vAJ~Zo z%m%(1CG9Ykm$T%D&-o-Qs`x)&CBsL41aIEk&X2aZ7Q$*`A-ugG5*wlG=6J*$kD+XJ z8M2fvD zptTYvp*-O1ipEZ%1^}h|%Mq_z(Vignj9+RkHyd|BBCRgzsl*=BOmYdWb;KEzoi52Siwkl|QgXxz7NJBuQD2DmV^w z+2Tz{?PO6=$1M9>pMtkNEL5KuxH_Q9|A~})`l+?}Q!~ozHbboW^@FngMVz|_6|==l z;IZ5`OKQuwoXvy<1@O+%FBvzJ?8X#hOo5fI|AnnX9XdR{@O;>x<1@KgU2&_<%>LzD zpzk;`BBPpowLLp3@ShYTOq8g$jBXbsn$J!R^@|h46~Yra`q8TkLR3K7L@}*E&mZbi z2wsH{3<&ZQGsEk2$gV!$Vb$iPsS9?fjby$cuZ((8EZcYbOdUkD@7@|e{0G*7?6t*l z*G}XG-TK0FWo+S6(#OwWIF3Uu>($6CUBqV@G;@tycQ6D4ADtR>z&zzL?*e-u6sBgO zgNhaG;ByugDLd>VfE4jesKKSx!8(3_1M{7g5ogL)Z@IHB%8Q3_)2*oO$ti`3H89qrf-$OKxfGjfbt0NifP~aZ2cd9k4wQQSf zM6BRy;1Ra^SnDY7VL zMbVuXV~4A{MWBA%II%F4=$$PBs9?$fu4X{8fmcCr-2=3TJaT^f=H_yzNE60^Eei)8 zudk&OtU_Kv26tb#Zt;zlSA5gn-L*XXkNPpxvuB2D35UMYDqo_m?OYTzgC&@eDpq?> zJ-*7bSEH`rV3&3DIM>m`WS!$Luf}s(^bS#mYB0m8X_iVB2^Fd@zq39{4u?zpqLXKu zinrPZng*H%XI5v{^oCcV9*aQ)Zh61-?|f~!Sxi0NYm44_8N6>!3u{=i<7&2D zuyS|v;Zx0w;Ksbu>qfX(zc!Keg8_LZN-1~9^t|O;(VHsL6Ffe*leikhYt-_Jt8J}xRM9gBt*11JQ-b)NmZ0PSt>d@~m8+J#uz zX}AonFE;O45w1h_Jr#g!!a>veE|M{DF2W$%OK_0dACyb3ZZ7dO9dxgF<%dT$ynLSY z{laCdHIkn!FCqxpV43?HW3UsC%#Zu?Nwn)0FvR|=WJxkTiJ1Lh;@h*wA4i@1pa#?4 z-gA#Ogjlwusu8O)jkM|DeRh{a9-Q)Sy>;Ll-tAia=LxBI;!2g|yAt*et?2rvor+nL zw?qQxNZ@lFis8|bU;Il*@OSe=tErPacV|`Nrb9Si`25NWzBbE`drTyLjN2kudyg?y zCGXW2AmJK}gYLdO-# z&X!V6xQ#cw`vddDHhXPf!7ZFxEMJ{wifmSU^NjBoch;7!UA{W9_Gn`a)pZi$ z^!*`9v&>jOO?vhiW9**2kKSNf)hB1FFVl1rv6K?NJ626-xMDl2%+#zkHel0dVKk?& z(@n6^ZX44vyVG~<3sFoT8q|GM(Qu7F;7LibMtV<0MePCbPCyGRjQ@|j%B&5xSPpbN z^2k7mfJ0^fS$$Fb#o2an{OkT$y6kn4AOaDfln1FJ$)i#FFh##0+zWdExWA+16KtV# zGCQvV0NmptKLr6rZO!Rkcg>1+%4MzklWKF-X|DMEH<^@ydvl}X^hRQS?yBQUK^^nZ z=+B%51-_Sbi~31pCtr{D^&RL5M-~6v{GMI)^G_(F{mFTN1BTo4unU{G&q3ARA6+2I zX?4}X{Ay|nDv`&koz6z&kFgvTyd1${;^+wb;drg1$>a&8Qw&K*#JC*Epyi=R2=+Fmsa69bLO0RECeMaE-4(?x_ON@$0 zC#FIh=q>%^lEslwSSqX=VUymuDJES`-whbxJ*@Wo^Y@5WaIKUk|NSNe z4-!5T0e(GT;ZyK2LujLDj=#x^cFJjnF_!OsKYUE?Gnz zi#3^r+-|%;ga5UhP;kZn_wVIOO32^88!YD4SU)$`V;CJz$bF0>j9{$QUzLB+td(Uw zKR=#F)4@qGF0(h$d(E9~c8&z^3p3iCyrh>IfT@K`zvf;|=LX<7Mb>Nvf`|5^#pT7R zeSOEzyB^qpx#@d_U6ALF>mco3IT;TC@0`RohvLinDz7<4Y)*C_QU>FSEJqc~DQgr{ zh~dJxcelYsWp;;m(F=#ue_zf%Q5Ei&a30Yy)+I&?X-%uH82pHbsY(@{8yj&{TO_PG zZs9)3p{5ylkiAlbZ8sFX0G;67F8^Lxl~wonyFSoXS43#NfgEZA;RCgNer>~-^6I(B z0{Fp|TOa1w>x;lb)BfPyC2LG+=|_gu)zr2OTU#6?(O%L4;om)Mo7gK0M!0q#i&Pk1 z#ExZnDXFQE?N@hiMPIyYQxyxzx^TKco|KzATH(~qeQ&g9mA?iuzuqBU3{=Cu wL}u@ku3p^jdGegXa0=-H;s!XcqD_^zxxYGl8jP=k-=cwFjVv)$=iL7Je^Sd{WB>pF literal 0 HcmV?d00001 diff --git a/data/interfaces/default/searchresults.html b/data/interfaces/default/searchresults.html index 0cce296f..7498ac3f 100755 --- a/data/interfaces/default/searchresults.html +++ b/data/interfaces/default/searchresults.html @@ -32,7 +32,7 @@ - + %endfor %endif diff --git a/mylar/PostProcessor.py b/mylar/PostProcessor.py index 3e518dbf..57a0fc02 100755 --- a/mylar/PostProcessor.py +++ b/mylar/PostProcessor.py @@ -148,7 +148,8 @@ class PostProcessor(object): # if the SAB Directory option is enabled, let's use that folder name and append the jobname. if mylar.SAB_DIRECTORY is not None and mylar.SAB_DIRECTORY is not 'None' and len(mylar.SAB_DIRECTORY) > 4: self.nzb_folder = os.path.join(mylar.SAB_DIRECTORY, self.nzb_name).encode(mylar.SYS_ENCODING) - + logger.fdebug('SABnzbd Download folder option enabled. Directory set to : ' + self.nzb_folder) + #lookup nzb_name in nzblog table to get issueid #query SAB to find out if Replace Spaces enabled / not as well as Replace Decimals @@ -172,6 +173,10 @@ class PostProcessor(object): if mylar.USE_NZBGET==1: logger.fdebug("Using NZBGET") logger.fdebug("NZB name as passed from NZBGet: " + self.nzb_name) + # if the NZBGet Directory option is enabled, let's use that folder name and append the jobname. + if mylar.NZBGET_DIRECTORY is not None and mylar.NZBGET_DIRECTORY is not 'None' and len(mylar.NZBGET_DIRECTORY) > 4: + self.nzb_folder = os.path.join(mylar.NZBGET_DIRECTORY, self.nzb_name).encode(mylar.SYS_ENCODING) + logger.fdebug('NZBGET Download folder option enabled. Directory set to : ' + self.nzb_folder) myDB = db.DBConnection() if self.nzb_name == 'Manual Run': @@ -195,12 +200,14 @@ class PostProcessor(object): watchvals = {"SeriesYear": cs['ComicYear'], "LatestDate": cs['LatestDate'], "ComicVersion": cs['ComicVersion'], + "Publisher": cs['ComicPublisher'], "Total": cs['Total']} - watchmatch = filechecker.listFiles(self.nzb_folder,cs['ComicName'],cs['AlternateSearch'], manual=watchvals) + watchmatch = filechecker.listFiles(self.nzb_folder,cs['ComicName'],cs['ComicPublisher'],cs['AlternateSearch'], manual=watchvals) if watchmatch['comiccount'] == 0: # is None: nm+=1 continue else: + print 'i made it here...' fn = 0 fccnt = int(watchmatch['comiccount']) if len(watchmatch) == 1: continue @@ -260,12 +267,32 @@ class PostProcessor(object): if issuechk is None: logger.info("No corresponding issue # found for " + str(cs['ComicID'])) else: - logger.info("Found matching issue # " + str(fcdigit) + " for ComicID: " + str(cs['ComicID']) + " / IssueID: " + str(issuechk['IssueID'])) - manual_list.append({"ComicLocation": tmpfc['ComicLocation'], - "ComicID": cs['ComicID'], - "IssueID": issuechk['IssueID'], - "IssueNumber": issuechk['Issue_Number'], - "ComicName": cs['ComicName']}) + datematch = "True" + if len(watchmatch) > 1: + #if the # of matches is more than 1, we need to make sure we get the right series + #compare the ReleaseDate for the issue, to the found issue date in the filename. + #if ReleaseDate doesn't exist, use IssueDate + #if no issue date was found, then ignore. + if issuechk['ReleaseDate'] is not None: + if int(issuechk['ReleaseDate'][:4]) < int(tmpfc['ComicYear']): + logger.fdebug(str(issuechk['ReleaseDate']) + ' is before the issue year of ' + str(tmpfc['ComicYear']) + ' that was discovered in the filename') + datematch = "False" + else: + if int(issuechk['IssueDate'][:4]) < int(tmpfc['ComicYear']): + logger.fdebug(str(issuechk['IssueDate']) + ' is before the issue year ' + str(tmpfc['ComicYear']) + ' that was discovered in the filename') + datematch = "False" + + else: + logger.info("Found matching issue # " + str(fcdigit) + " for ComicID: " + str(cs['ComicID']) + " / IssueID: " + str(issuechk['IssueID'])) + + if datematch == "True": + manual_list.append({"ComicLocation": tmpfc['ComicLocation'], + "ComicID": cs['ComicID'], + "IssueID": issuechk['IssueID'], + "IssueNumber": issuechk['Issue_Number'], + "ComicName": cs['ComicName']}) + else: + logger.fdebug('Incorrect series - not populating..continuing post-processing') ccnt+=1 #print manual_list wdc+=1 @@ -563,6 +590,8 @@ class PostProcessor(object): issueyear = issuenzb['IssueDate'][:4] self._log("Issue Year: " + str(issueyear), logger.DEBUG) logger.fdebug("Issue Year : " + str(issueyear)) + month = issuenzb['IssueDate'][5:7].replace('-','').strip() + month_name = helpers.fullmonth(month) # comicnzb= myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone() publisher = comicnzb['ComicPublisher'] self._log("Publisher: " + publisher, logger.DEBUG) @@ -675,6 +704,8 @@ class PostProcessor(object): '$publisher': publisher.lower(), '$VolumeY': 'V' + str(seriesyear), '$VolumeN': comversion, + '$monthname': monthname, + '$month': month, '$Annual': 'Annual' } diff --git a/mylar/__init__.py b/mylar/__init__.py index 9a312c59..c7cac072 100755 --- a/mylar/__init__.py +++ b/mylar/__init__.py @@ -54,6 +54,7 @@ __INITIALIZED__ = False started = False DATA_DIR = None +DBLOCK = False CONFIG_FILE = None CFG = None @@ -74,6 +75,8 @@ HTTP_HOST = None HTTP_USERNAME = None HTTP_PASSWORD = None HTTP_ROOT = None +API_ENABLED = False +API_KEY = None LAUNCH_BROWSER = False LOGVERBOSE = 1 GIT_PATH = None @@ -111,8 +114,6 @@ PREFERRED_QUALITY = 0 CORRECT_METADATA = False MOVE_FILES = False RENAME_FILES = False -BLACKHOLE = False -BLACKHOLE_DIR = None FOLDER_FORMAT = None FILE_FORMAT = None REPLACE_SPACES = False @@ -151,7 +152,9 @@ CVINFO = False LOG_LEVEL = None POST_PROCESSING = 1 -USE_SABNZBD = True +NZB_DOWNLOADER = None #0 = sabnzbd, #1 = nzbget, #2 = blackhole + +USE_SABNZBD = False SAB_HOST = None SAB_USERNAME = None SAB_PASSWORD = None @@ -167,6 +170,10 @@ NZBGET_USERNAME = None NZBGET_PASSWORD = None NZBGET_PRIORITY = None NZBGET_CATEGORY = None +NZBGET_DIRECTORY = None + +USE_BLACKHOLE = False +BLACKHOLE_DIR = None PROVIDER_ORDER = None @@ -309,11 +316,11 @@ def initialize(): with INIT_LOCK: global __INITIALIZED__, FULL_PATH, PROG_DIR, VERBOSE, DAEMON, COMICSORT, DATA_DIR, CONFIG_FILE, CFG, CONFIG_VERSION, LOG_DIR, CACHE_DIR, LOGVERBOSE, OLDCONFIG_VERSION, OS_DETECT, OS_LANG, OS_ENCODING, \ - HTTP_PORT, HTTP_HOST, HTTP_USERNAME, HTTP_PASSWORD, HTTP_ROOT, LAUNCH_BROWSER, GIT_PATH, \ + HTTP_PORT, HTTP_HOST, HTTP_USERNAME, HTTP_PASSWORD, HTTP_ROOT, API_ENABLED, API_KEY, LAUNCH_BROWSER, GIT_PATH, \ CURRENT_VERSION, LATEST_VERSION, CHECK_GITHUB, CHECK_GITHUB_ON_STARTUP, CHECK_GITHUB_INTERVAL, USER_AGENT, DESTINATION_DIR, \ DOWNLOAD_DIR, USENET_RETENTION, SEARCH_INTERVAL, NZB_STARTUP_SEARCH, INTERFACE, AUTOWANT_ALL, AUTOWANT_UPCOMING, ZERO_LEVEL, ZERO_LEVEL_N, COMIC_COVER_LOCAL, HIGHCOUNT, \ - LIBRARYSCAN, LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, USE_SABNZBD, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, SAB_PRIORITY, SAB_DIRECTORY, BLACKHOLE, BLACKHOLE_DIR, ADD_COMICS, COMIC_DIR, IMP_MOVE, IMP_RENAME, IMP_METADATA, \ - USE_NZBGET, NZBGET_HOST, NZBGET_PORT, NZBGET_USERNAME, NZBGET_PASSWORD, NZBGET_CATEGORY, NZBGET_PRIORITY, NZBSU, NZBSU_UID, NZBSU_APIKEY, DOGNZB, DOGNZB_UID, DOGNZB_APIKEY, NZBX,\ + LIBRARYSCAN, LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, NZB_DOWNLOADER, USE_SABNZBD, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, SAB_PRIORITY, SAB_DIRECTORY, USE_BLACKHOLE, BLACKHOLE_DIR, ADD_COMICS, COMIC_DIR, IMP_MOVE, IMP_RENAME, IMP_METADATA, \ + USE_NZBGET, NZBGET_HOST, NZBGET_PORT, NZBGET_USERNAME, NZBGET_PASSWORD, NZBGET_CATEGORY, NZBGET_PRIORITY, NZBGET_DIRECTORY, NZBSU, NZBSU_UID, NZBSU_APIKEY, DOGNZB, DOGNZB_UID, DOGNZB_APIKEY, NZBX,\ NEWZNAB, NEWZNAB_NAME, NEWZNAB_HOST, NEWZNAB_APIKEY, NEWZNAB_UID, NEWZNAB_ENABLED, EXTRA_NEWZNABS, NEWZNAB_EXTRA, \ RAW, RAW_PROVIDER, RAW_USERNAME, RAW_PASSWORD, RAW_GROUPS, EXPERIMENTAL, ALTEXPERIMENTAL, \ ENABLE_META, CMTAGGER_PATH, INDIE_PUB, BIGGIE_PUB, IGNORE_HAVETOTAL, PROVIDER_ORDER, \ @@ -350,6 +357,8 @@ def initialize(): HTTP_USERNAME = check_setting_str(CFG, 'General', 'http_username', '') HTTP_PASSWORD = check_setting_str(CFG, 'General', 'http_password', '') HTTP_ROOT = check_setting_str(CFG, 'General', 'http_root', '/') + API_ENABLED = bool(check_setting_int(CFG, 'General', 'api_enabled', 0)) + API_KEY = check_setting_str(CFG, 'General', 'api_key', '') LAUNCH_BROWSER = bool(check_setting_int(CFG, 'General', 'launch_browser', 1)) LOGVERBOSE = bool(check_setting_int(CFG, 'General', 'logverbose', 1)) GIT_PATH = check_setting_str(CFG, 'General', 'git_path', '') @@ -387,7 +396,7 @@ def initialize(): RENAME_FILES = bool(check_setting_int(CFG, 'General', 'rename_files', 0)) FOLDER_FORMAT = check_setting_str(CFG, 'General', 'folder_format', '$Series ($Year)') FILE_FORMAT = check_setting_str(CFG, 'General', 'file_format', '$Series $Issue ($Year)') - BLACKHOLE = bool(check_setting_int(CFG, 'General', 'blackhole', 0)) + USE_BLACKHOLE = bool(check_setting_int(CFG, 'General', 'use_blackhole', 0)) BLACKHOLE_DIR = check_setting_str(CFG, 'General', 'blackhole_dir', '') REPLACE_SPACES = bool(check_setting_int(CFG, 'General', 'replace_spaces', 0)) REPLACE_CHAR = check_setting_str(CFG, 'General', 'replace_char', '') @@ -488,7 +497,13 @@ def initialize(): ENABLE_CBT = bool(check_setting_int(CFG, 'Torrents', 'enable_cbt', 0)) CBT_PASSKEY = check_setting_str(CFG, 'Torrents', 'cbt_passkey', '') - USE_SABNZBD = bool(check_setting_int(CFG, 'SABnzbd', 'use_sabnzbd', 0)) + #this needs to have it's own category - for now General will do. + NZB_DOWNLOADER = check_setting_int(CFG, 'General', 'nzb_downloader', 0) + #legacy support of older config - reload into old values for consistency. + if NZB_DOWNLOADER == 0: USE_SABNZBD = True + elif NZB_DOWNLOADER == 1: USE_NZBGET = True + elif NZB_DOWNLOADER == 2: USE_BLACKHOLE = True + #USE_SABNZBD = bool(check_setting_int(CFG, 'SABnzbd', 'use_sabnzbd', 0)) SAB_HOST = check_setting_str(CFG, 'SABnzbd', 'sab_host', '') SAB_USERNAME = check_setting_str(CFG, 'SABnzbd', 'sab_username', '') SAB_PASSWORD = check_setting_str(CFG, 'SABnzbd', 'sab_password', '') @@ -504,13 +519,17 @@ def initialize(): elif SAB_PRIORITY == "4": SAB_PRIORITY = "Paused" else: SAB_PRIORITY = "Default" - USE_NZBGET = bool(check_setting_int(CFG, 'NZBGet', 'use_nzbget', 0)) + #USE_NZBGET = bool(check_setting_int(CFG, 'NZBGet', 'use_nzbget', 0)) NZBGET_HOST = check_setting_str(CFG, 'NZBGet', 'nzbget_host', '') NZBGET_PORT = check_setting_str(CFG, 'NZBGet', 'nzbget_port', '') NZBGET_USERNAME = check_setting_str(CFG, 'NZBGet', 'nzbget_username', '') NZBGET_PASSWORD = check_setting_str(CFG, 'NZBGet', 'nzbget_password', '') NZBGET_CATEGORY = check_setting_str(CFG, 'NZBGet', 'nzbget_category', '') NZBGET_PRIORITY = check_setting_str(CFG, 'NZBGet', 'nzbget_priority', '') + NZBGET_DIRECTORY = check_setting_str(CFG, 'NZBGet', 'nzbget_directory', '') + + #USE_BLACKHOLE = bool(check_setting_int(CFG, 'General', 'use_blackhole', 0)) + BLACKHOLE_DIR = check_setting_str(CFG, 'General', 'blackhole_dir', '') PR_NUM = 0 # provider counter here (used for provider orders) PR = [] @@ -716,9 +735,15 @@ def initialize(): # With the addition of NZBGet, it's possible that both SAB and NZBget are unchecked initially. # let's force default SAB. - if USE_NZBGET == 0 and USE_SABNZBD == 0 : - logger.info('No Download Server option given - defaulting to SABnzbd.') - USE_SABNZBD = 1 + #if NZB_DOWNLOADER == None: + # logger.info('No Download Option selected - default to SABnzbd.') + # NZB_DOWNLOADER = 0 + # USE_SABNZBD = 1 + #else: + # logger.info('nzb_downloader is set to : ' + str(NZB_DOWNLOADER)) + #if USE_NZBGET == 0 and USE_SABNZBD == 0 : + # logger.info('No Download Server option given - defaulting to SABnzbd.') + # USE_SABNZBD = 1 # Get the currently installed version - returns None, 'win32' or the git hash # Also sets INSTALL_TYPE variable to 'win', 'git' or 'source' @@ -851,6 +876,8 @@ def config_write(): new_config['General']['http_username'] = HTTP_USERNAME new_config['General']['http_password'] = HTTP_PASSWORD new_config['General']['http_root'] = HTTP_ROOT + new_config['General']['api_enabled'] = int(API_ENABLED) + new_config['General']['api_key'] = API_KEY new_config['General']['launch_browser'] = int(LAUNCH_BROWSER) new_config['General']['log_dir'] = LOG_DIR new_config['General']['logverbose'] = int(LOGVERBOSE) @@ -890,7 +917,7 @@ def config_write(): new_config['General']['rename_files'] = int(RENAME_FILES) new_config['General']['folder_format'] = FOLDER_FORMAT new_config['General']['file_format'] = FILE_FORMAT - new_config['General']['blackhole'] = int(BLACKHOLE) + #new_config['General']['use_blackhole'] = int(USE_BLACKHOLE) new_config['General']['blackhole_dir'] = BLACKHOLE_DIR new_config['General']['replace_spaces'] = int(REPLACE_SPACES) new_config['General']['replace_char'] = REPLACE_CHAR @@ -939,6 +966,7 @@ def config_write(): flattened_providers.append(item) new_config['General']['provider_order'] = flattened_providers + new_config['General']['nzb_downloader'] = NZB_DOWNLOADER new_config['Torrents'] = {} new_config['Torrents']['enable_torrents'] = int(ENABLE_TORRENTS) @@ -957,9 +985,8 @@ def config_write(): new_config['Torrents']['enable_cbt'] = int(ENABLE_CBT) new_config['Torrents']['cbt_passkey'] = CBT_PASSKEY - new_config['SABnzbd'] = {} - new_config['SABnzbd']['use_sabnzbd'] = int(USE_SABNZBD) + #new_config['SABnzbd']['use_sabnzbd'] = int(USE_SABNZBD) new_config['SABnzbd']['sab_host'] = SAB_HOST new_config['SABnzbd']['sab_username'] = SAB_USERNAME new_config['SABnzbd']['sab_password'] = SAB_PASSWORD @@ -969,14 +996,14 @@ def config_write(): new_config['SABnzbd']['sab_directory'] = SAB_DIRECTORY new_config['NZBGet'] = {} - new_config['NZBGet']['use_nzbget'] = int(USE_NZBGET) + #new_config['NZBGet']['use_nzbget'] = int(USE_NZBGET) new_config['NZBGet']['nzbget_host'] = NZBGET_HOST new_config['NZBGet']['nzbget_port'] = NZBGET_PORT new_config['NZBGet']['nzbget_username'] = NZBGET_USERNAME new_config['NZBGet']['nzbget_password'] = NZBGET_PASSWORD new_config['NZBGet']['nzbget_category'] = NZBGET_CATEGORY new_config['NZBGet']['nzbget_priority'] = NZBGET_PRIORITY - + new_config['NZBGet']['nzbget_directory'] = NZBGET_DIRECTORY new_config['NZBsu'] = {} new_config['NZBsu']['nzbsu'] = int(NZBSU) @@ -1104,7 +1131,7 @@ def dbcheck(): c.execute('CREATE TABLE IF NOT EXISTS importresults (impID TEXT, ComicName TEXT, ComicYear TEXT, Status TEXT, ImportDate TEXT, ComicFilename TEXT, ComicLocation TEXT, WatchMatch TEXT)') c.execute('CREATE TABLE IF NOT EXISTS readlist (IssueID TEXT, ComicName TEXT, Issue_Number TEXT, Status TEXT, DateAdded TEXT, Location TEXT, inCacheDir TEXT, SeriesYear TEXT, ComicID TEXT)') c.execute('CREATE TABLE IF NOT EXISTS readinglist(StoryArcID TEXT, ComicName TEXT, IssueNumber TEXT, SeriesYear TEXT, IssueYEAR TEXT, StoryArc TEXT, TotalIssues TEXT, Status TEXT, inCacheDir TEXT, Location TEXT, IssueArcID TEXT, ReadingOrder INT, IssueID TEXT)') - c.execute('CREATE TABLE IF NOT EXISTS annuals (IssueID TEXT, Issue_Number TEXT, IssueName TEXT, IssueDate TEXT, Status TEXT, ComicID TEXT, GCDComicID TEXT, Location TEXT, ComicSize TEXT, Int_IssueNumber INT, ComicName TEXT)') + c.execute('CREATE TABLE IF NOT EXISTS annuals (IssueID TEXT, Issue_Number TEXT, IssueName TEXT, IssueDate TEXT, Status TEXT, ComicID TEXT, GCDComicID TEXT, Location TEXT, ComicSize TEXT, Int_IssueNumber INT, ComicName TEXT, ReleaseDate TEXT, ReleaseComicID TEXT, ReleaseComicName TEXT)') c.execute('CREATE TABLE IF NOT EXISTS rssdb (Title TEXT UNIQUE, Link TEXT, Pubdate TEXT, Site TEXT, Size TEXT)') c.execute('CREATE TABLE IF NOT EXISTS futureupcoming (ComicName TEXT, IssueNumber TEXT, ComicID TEXT, IssueID TEXT, IssueDate TEXT, Publisher TEXT, Status TEXT, DisplayComicName TEXT)') conn.commit @@ -1283,6 +1310,20 @@ def dbcheck(): except: c.execute('ALTER TABLE issues ADD COLUMN AltIssueNumber TEXT') + try: + c.execute('SELECT ReleaseDate from annuals') + except: + c.execute('ALTER TABLE annuals ADD COLUMN ReleaseDate TEXT') + + try: + c.execute('SELECT ReleaseComicID from annuals') + except: + c.execute('ALTER TABLE annuals ADD COLUMN ReleaseComicID TEXT') + + try: + c.execute('SELECT ReleaseComicName from annuals') + except: + c.execute('ALTER TABLE annuals ADD COLUMN ReleaseComicName TEXT') #if it's prior to Wednesday, the issue counts will be inflated by one as the online db's everywhere #prepare for the next 'new' release of a series. It's caught in updater.py, so let's just store the @@ -1309,6 +1350,7 @@ def dbcheck(): #let's delete errant comics that are stranded (ie. Comicname = Comic ID: ) c.execute("DELETE from COMICS WHERE ComicName='None' OR ComicName LIKE 'Comic ID%' OR ComicName is NULL") + c.execute("DELETE from ISSUES WHERE ComicName='None' OR ComicName LIKE 'Comic ID%' OR ComicName is NULL") logger.info('Ensuring DB integrity - Removing all Erroneous Comics (ie. named None)') logger.info('Correcting Null entries that make the main page break on startup.') diff --git a/mylar/api.py b/mylar/api.py new file mode 100644 index 00000000..6249d1a3 --- /dev/null +++ b/mylar/api.py @@ -0,0 +1,342 @@ +# This file is part of Mylar. +# +# Mylar is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Mylar is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Mylar. If not, see . + +import mylar + +from mylar import db, mb, importer, search, PostProcessor, versioncheck, logger + +import lib.simplejson as simplejson +from xml.dom.minidom import Document +import copy + +cmd_list = [ 'getIndex', 'getComic', 'getComic', 'getUpcoming', 'getWanted', 'getHistory', 'getLogs', + 'findComic', 'findIssue', 'addComic', 'delComic', 'pauseComic', 'resumeComic', 'refreshComic', + 'addIssue', 'queueIssue', 'unqueueIssue', 'forceSearch', 'forceProcess', 'getVersion', 'checkGithub', + 'shutdown', 'restart', 'update', 'getComicInfo', 'getIssueInfo'] + +class Api(object): + + def __init__(self): + + self.apikey = None + self.cmd = None + self.id = None + + self.kwargs = None + + self.data = None + + self.callback = None + + + def checkParams(self,*args,**kwargs): + + if not mylar.API_ENABLED: + self.data = 'API not enabled' + return + if not mylar.API_KEY: + self.data = 'API key not generated' + return + if len(mylar.API_KEY) != 32: + self.data = 'API key not generated correctly' + return + + if 'apikey' not in kwargs: + self.data = 'Missing api key' + return + + if kwargs['apikey'] != mylar.API_KEY: + self.data = 'Incorrect API key' + return + else: + self.apikey = kwargs.pop('apikey') + + if 'cmd' not in kwargs: + self.data = 'Missing parameter: cmd' + return + + if kwargs['cmd'] not in cmd_list: + self.data = 'Unknown command: %s' % kwargs['cmd'] + return + else: + self.cmd = kwargs.pop('cmd') + + self.kwargs = kwargs + self.data = 'OK' + + def fetchData(self): + + if self.data == 'OK': + logger.info('Recieved API command: ' + self.cmd) + methodToCall = getattr(self, "_" + self.cmd) + result = methodToCall(**self.kwargs) + if 'callback' not in self.kwargs: + if type(self.data) == type(''): + return self.data + else: + return simplejson.dumps(self.data) + else: + self.callback = self.kwargs['callback'] + self.data = simplejson.dumps(self.data) + self.data = self.callback + '(' + self.data + ');' + return self.data + else: + return self.data + + def _dic_from_query(self,query): + + myDB = db.DBConnection() + rows = myDB.select(query) + + rows_as_dic = [] + + for row in rows: + row_as_dic = dict(zip(row.keys(), row)) + rows_as_dic.append(row_as_dic) + + return rows_as_dic + + def _getIndex(self, **kwargs): + + self.data = self._dic_from_query('SELECT * from comics order by ComicSortName COLLATE NOCASE') + return + + def _getComic(self, **kwargs): + + if 'id' not in kwargs: + self.data = 'Missing parameter: id' + return + else: + self.id = kwargs['id'] + + comic = self._dic_from_query('SELECT * from comics WHERE ComicID="' + self.id + '"') + issues = self._dic_from_query('SELECT * from issues WHERE ComicID="' + self.id + '"order by Int_IssueNumber DESC') + if mylar.ANNUALS_ON: + annuals = self._dic_from_query('SELECT * FROM annuals WHERE ComicID="' + self.id + '"') + else: annuals = None + + self.data = { 'comic': comic, 'issues': issues, 'annuals': annuals } + return + + def _getHistory(self, **kwargs): + self.data = self._dic_from_query('SELECT * from snatched order by DateAdded DESC') + return + + def _getUpcoming(self, **kwargs): + self.data = self._dic_from_query("SELECT * from upcoming WHERE IssueID is NULL order by IssueDate DESC") + return + + def _getWanted(self, **kwargs): + self.data = self._dic_from_query("SELECT * from issues WHERE Status='Wanted'") + return + + def _getLogs(self, **kwargs): + pass + + def _findArtist(self, **kwargs): + if 'name' not in kwargs: + self.data = 'Missing parameter: name' + return + if 'limit' in kwargs: + limit = kwargs['limit'] + else: + limit=50 + + self.data = mb.findArtist(kwargs['name'], limit) + + def _findAlbum(self, **kwargs): + if 'name' not in kwargs: + self.data = 'Missing parameter: name' + return + if 'limit' in kwargs: + limit = kwargs['limit'] + else: + limit=50 + + self.data = mb.findRelease(kwargs['name'], limit) + + def _addArtist(self, **kwargs): + if 'id' not in kwargs: + self.data = 'Missing parameter: id' + return + else: + self.id = kwargs['id'] + + try: + importer.addComictoDB(self.id) + except Exception, e: + self.data = e + + return + + def _delComic(self, **kwargs): + if 'id' not in kwargs: + self.data = 'Missing parameter: id' + return + else: + self.id = kwargs['id'] + + myDB = db.DBConnection() + myDB.action('DELETE from comics WHERE ComicID="' + self.id + '"') + myDB.action('DELETE from issues WHERE ComicID="' + self.id + '"') + myDB.action('DELETE from upcoming WHERE ComicID="' + self.id + '"') + + def _pauseComic(self, **kwargs): + if 'id' not in kwargs: + self.data = 'Missing parameter: id' + return + else: + self.id = kwargs['id'] + + myDB = db.DBConnection() + controlValueDict = {'ComicID': self.id} + newValueDict = {'Status': 'Paused'} + myDB.upsert("comics", newValueDict, controlValueDict) + + def _resumeComic(self, **kwargs): + if 'id' not in kwargs: + self.data = 'Missing parameter: id' + return + else: + self.id = kwargs['id'] + + myDB = db.DBConnection() + controlValueDict = {'ComicID': self.id} + newValueDict = {'Status': 'Active'} + myDB.upsert("comics", newValueDict, controlValueDict) + + def _refreshComic(self, **kwargs): + if 'id' not in kwargs: + self.data = 'Missing parameter: id' + return + else: + self.id = kwargs['id'] + + try: + importer.addComictoDB(self.id) + except Exception, e: + self.data = e + + return + + def _addComic(self, **kwargs): + if 'id' not in kwargs: + self.data = 'Missing parameter: id' + return + else: + self.id = kwargs['id'] + + try: + importer.addReleaseById(self.id) + except Exception, e: + self.data = e + + return + + def _queueIssue(self, **kwargs): + + if 'id' not in kwargs: + self.data = 'Missing parameter: id' + return + else: + self.id = kwargs['id'] + + myDB = db.DBConnection() + controlValueDict = {'IssueID': self.id} + newValueDict = {'Status': 'Wanted'} + myDB.upsert("issues", newValueDict, controlValueDict) + search.searchforissue(self.id) + + def _unqueueIssue(self, **kwargs): + + if 'id' not in kwargs: + self.data = 'Missing parameter: id' + return + else: + self.id = kwargs['id'] + + myDB = db.DBConnection() + controlValueDict = {'IssueID': self.id} + newValueDict = {'Status': 'Skipped'} + myDB.upsert("issues", newValueDict, controlValueDict) + + def _forceSearch(self, **kwargs): + search.searchforissue() + + def _forceProcess(self, **kwargs): + PostProcessor.forcePostProcess() + + def _getVersion(self, **kwargs): + self.data = { + 'git_path' : mylar.GIT_PATH, + 'install_type' : mylar.INSTALL_TYPE, + 'current_version' : mylar.CURRENT_VERSION, + 'latest_version' : mylar.LATEST_VERSION, + 'commits_behind' : mylar.COMMITS_BEHIND, + } + + def _checkGithub(self, **kwargs): + versioncheck.checkGithub() + self._getVersion() + + def _shutdown(self, **kwargs): + mylar.SIGNAL = 'shutdown' + + def _restart(self, **kwargs): + mylar.SIGNAL = 'restart' + + def _update(self, **kwargs): + mylar.SIGNAL = 'update' + + def _getArtistArt(self, **kwargs): + + if 'id' not in kwargs: + self.data = 'Missing parameter: id' + return + else: + self.id = kwargs['id'] + + self.data = cache.getArtwork(ComicID=self.id) + + def _getIssueArt(self, **kwargs): + + if 'id' not in kwargs: + self.data = 'Missing parameter: id' + return + else: + self.id = kwargs['id'] + + self.data = cache.getArtwork(IssueID=self.id) + + def _getComicInfo(self, **kwargs): + + if 'id' not in kwargs: + self.data = 'Missing parameter: id' + return + else: + self.id = kwargs['id'] + + self.data = cache.getInfo(ComicID=self.id) + + def _getIssueInfo(self, **kwargs): + + if 'id' not in kwargs: + self.data = 'Missing parameter: id' + return + else: + self.id = kwargs['id'] + + self.data = cache.getInfo(IssueID=self.id) + diff --git a/mylar/cv.py b/mylar/cv.py index 7b12625f..53002282 100755 --- a/mylar/cv.py +++ b/mylar/cv.py @@ -31,14 +31,15 @@ def pulldetails(comicid,type,issueid=None,offset=1): comicapi='583939a3df0a25fc4e8b7a29934a13078002dc27' if type == 'comic': - PULLURL= mylar.CVURL + 'volume/4050-' + str(comicid) + '/?api_key=' + str(comicapi) + '&format=xml&field_list=name,count_of_issues,issues,start_year,site_detail_url,image,publisher,description,first_issue' + if not comicid.startswith('4050-'): comicid = '4050-' + comicid + PULLURL= mylar.CVURL + 'volume/' + str(comicid) + '/?api_key=' + str(comicapi) + '&format=xml&field_list=name,count_of_issues,issues,start_year,site_detail_url,image,publisher,description,first_issue,deck' elif type == 'issue': if mylar.CV_ONLY: cv_type = 'issues' searchset = 'filter=volume:' + str(comicid) + '&field_list=cover_date,description,id,image,issue_number,name,date_last_updated,store_date' else: cv_type = 'volume/' + str(comicid) - searchset = 'name,count_of_issues,issues,start_year,site_detail_url,image,publisher,description' + searchset = 'name,count_of_issues,issues,start_year,site_detail_url,image,publisher,description,store_date' PULLURL = mylar.CVURL + str(cv_type) + '/?api_key=' + str(comicapi) + '&format=xml&' + str(searchset) + '&offset=' + str(offset) elif type == 'firstissue': #this is used ONLY for CV_ONLY @@ -131,39 +132,93 @@ def GetComicInfo(comicid,dom): except: comic['ComicYear'] = '0000' comic['ComicURL'] = dom.getElementsByTagName('site_detail_url')[trackcnt].firstChild.wholeText + + desdeck = 0 #the description field actually holds the Volume# - so let's grab it try: descchunk = dom.getElementsByTagName('description')[0].firstChild.wholeText - comic['ComicDescription'] = drophtml(descchunk) + comic_desc = drophtml(descchunk) + desdeck +=1 except: - comic['ComicDescription'] = 'None' - #extract the first 60 characters - comicDes = comic['ComicDescription'][:60] - if 'volume' in comicDes.lower(): - #found volume - let's grab it. - v_find = comicDes.lower().find('volume') - #arbitrarily grab the next 10 chars (6 for volume + 1 for space + 3 for the actual vol #) - #increased to 10 to allow for text numbering (+5 max) - vfind = comicDes[v_find:v_find+15] - volconv = '' - basenums = {'zero':'0','one':'1','two':'2','three':'3','four':'4','five':'5','six':'6','seven':'7','eight':'8'} - for nums in basenums: - if nums in vfind.lower(): - sconv = basenums[nums] - volconv = re.sub(nums, sconv, vfind.lower()) - break - if volconv != '': - vfind = volconv - if '(' in vfind: - #bracket detected in versioning' - vfindit = re.findall('[^()]+', vfind) - vfind = vfindit[0] - vf = re.findall('[^<>]+', vfind) - comic['ComicVersion'] = re.sub("[^0-9]", "", vf[0]) - - logger.info("Volume information found! Adding to series record : volume " + comic['ComicVersion']) - else: - comic['ComicVersion'] = "noversion" + comic_desc = 'None' + + #sometimes the deck has volume labels + try: + deckchunk = dom.getElementsByTagName('deck')[0].firstChild.wholeText + comic_deck = deckchunk + desdeck +=1 + except: + comic_deck = 'None' + + comic['ComicVersion'] = 'noversion' + #logger.info('comic_desc:' + comic_desc) + #logger.info('comic_deck:' + comic_deck) + #logger.info('desdeck: ' + str(desdeck)) + while (desdeck > 0): + if desdeck == 1: + if comic_desc == 'None': + comicDes = comic_deck[:30] + else: + #extract the first 60 characters + comicDes = comic_desc[:60].replace('New 52', '') + elif desdeck == 2: + #extract the characters from the deck + comicDes = comic_deck[:30].replace('New 52', '') + else: + break + + i = 0 + while (i < 2): + if 'volume' in comicDes.lower(): + #found volume - let's grab it. + v_find = comicDes.lower().find('volume') + #arbitrarily grab the next 10 chars (6 for volume + 1 for space + 3 for the actual vol #) + #increased to 10 to allow for text numbering (+5 max) + #sometimes it's volume 5 and ocassionally it's fifth volume. + if i == 0: + vfind = comicDes[v_find:v_find+15] #if it's volume 5 format + basenums = {'zero':'0','one':'1','two':'2','three':'3','four':'4','five':'5','six':'6','seven':'7','eight':'8','nine':'9','ten':'10'} + #logger.fdebug(str(i) + ': ' + str(vfind)) + else: + vfind = comicDes[:v_find] # if it's fifth volume format + basenums = {'zero':'0','first':'1','second':'2','third':'3','fourth':'4','fifth':'5','sixth':'6','seventh':'7','eighth':'8','nineth':'9','tenth':'10'} + #logger.fdebug(str(i) + ': ' + str(vfind)) + volconv = '' + for nums in basenums: + if nums in vfind.lower(): + sconv = basenums[nums] + vfind = re.sub(nums, sconv, vfind.lower()) + break + #logger.fdebug('volconv: ' + str(volconv)) + + #now we attempt to find the character position after the word 'volume' + if i == 0: + volthis = vfind.lower().find('volume') + volthis = volthis + 6 # add on the actual word to the position so that we can grab the subsequent digit + vfind = vfind[volthis:volthis+4] #grab the next 4 characters ;) + elif i == 1: + volthis = vfind.lower().find('volume') + vfind = vfind[volthis-4:volthis] #grab the next 4 characters ;) + + if '(' in vfind: + #bracket detected in versioning' + vfindit = re.findall('[^()]+', vfind) + vfind = vfindit[0] + vf = re.findall('[^<>]+', vfind) + ledigit = re.sub("[^0-9]", "", vf[0]) + if ledigit != '': + comic['ComicVersion'] = ledigit + logger.info("Volume information found! Adding to series record : volume " + comic['ComicVersion']) + break + i+=1 + else: + i+=1 + + if comic['ComicVersion'] == 'noversion': + logger.info('comic[ComicVersion]:' + str(comic['ComicVersion'])) + desdeck -=1 + else: + break if vari == "yes": comic['ComicIssues'] = str(cntit) @@ -235,11 +290,17 @@ def GetIssuesInfo(comicid,dom): tempissue['CoverDate'] = subtrack.getElementsByTagName('cover_date')[0].firstChild.wholeText except: tempissue['CoverDate'] = '0000-00-00' + try: + tempissue['StoreDate'] = subtrack.getElementsByTagName('store_date')[0].firstChild.wholeText + except: + tempissue['StoreDate'] = '0000-00-00' tempissue['Issue_Number'] = subtrack.getElementsByTagName('issue_number')[0].firstChild.wholeText issuech.append({ + 'Comic_ID': comicid, 'Issue_ID': tempissue['Issue_ID'], 'Issue_Number': tempissue['Issue_Number'], 'Issue_Date': tempissue['CoverDate'], + 'Store_Date': tempissue['StoreDate'], 'Issue_Name': tempissue['Issue_Name'] }) diff --git a/mylar/filechecker.py b/mylar/filechecker.py index 526500ff..7d1c51f2 100755 --- a/mylar/filechecker.py +++ b/mylar/filechecker.py @@ -27,7 +27,7 @@ def file2comicmatch(watchmatch): #print ("match: " + str(watchmatch)) pass -def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): +def listFiles(dir,watchcomic,Publisher,AlternateSearch=None,manual=None,sarc=None): # use AlternateSearch to check for filenames that follow that naming pattern # ie. Star Trek TNG Doctor Who Assimilation won't get hits as the @@ -35,9 +35,9 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): # we need to convert to ascii, as watchcomic is utf-8 and special chars f'it up u_watchcomic = watchcomic.encode('ascii', 'ignore').strip() - logger.fdebug('comic: ' + watchcomic) + logger.fdebug('[FILECHECKER] comic: ' + watchcomic) basedir = dir - logger.fdebug('Looking in: ' + dir) + logger.fdebug('[FILECHECKER] Looking in: ' + dir) watchmatch = {} comiclist = [] comiccnt = 0 @@ -64,8 +64,14 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): 'B', 'C'] + extensions = ('.cbr', '.cbz') + for item in os.listdir(basedir): if item == 'cover.jpg' or item == 'cvinfo': continue + if not item.endswith(extensions): + logger.fdebug('[FILECHECKER] filename not a valid cbr/cbz - ignoring: ' + item) + continue + #print item #subname = os.path.join(basedir, item) subname = item @@ -86,7 +92,7 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): elif subit.lower()[:3] == 'vol': #if in format vol.2013 etc #because the '.' in Vol. gets removed, let's loop thru again after the Vol hit to remove it entirely - logger.fdebug('volume indicator detected as version #:' + str(subit)) + logger.fdebug('[FILECHECKER] volume indicator detected as version #:' + str(subit)) subname = re.sub(subit, '', subname) volrem = subit @@ -104,42 +110,73 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): find19 = i.find('19') if find19: stf = i[find19:4].strip() - logger.fdebug('stf is : ' + str(stf)) + logger.fdebug('[FILECHECKER] stf is : ' + str(stf)) if stf.isdigit(): numberinseries = 'True' - logger.fdebug('numberinseries: ' + numberinseries) + logger.fdebug('[FILECHECKER] numberinseries: ' + numberinseries) #remove the brackets.. subnm = re.findall('[^()]+', subname) - logger.fdebug('subnm len : ' + str(len(subnm))) + logger.fdebug('[FILECHECKER] subnm len : ' + str(len(subnm))) if len(subnm) == 1: - logger.fdebug(str(len(subnm)) + ': detected invalid filename - attempting to detect year to continue') + logger.fdebug('[FILECHECKER] ' + str(len(subnm)) + ': detected invalid filename - attempting to detect year to continue') #if the series has digits this f's it up. if numberinseries == 'True': #we need to remove the series from the subname and then search the remainder. watchname = re.sub('[-\:\;\!\'\/\?\+\=\_\%\.]', '', watchcomic) #remove spec chars for watchcomic match. - logger.fdebug('watch-cleaned: ' + str(watchname)) + logger.fdebug('[FILECHECKER] watch-cleaned: ' + str(watchname)) subthis = re.sub('.cbr', '', subname) subthis = re.sub('.cbz', '', subthis) subthis = re.sub('[-\:\;\!\'\/\?\+\=\_\%\.]', '', subthis) - logger.fdebug('sub-cleaned: ' + str(subthis)) + logger.fdebug('[FILECHECKER] sub-cleaned: ' + str(subthis)) subthis = subthis[len(watchname):] #remove watchcomic #we need to now check the remainder of the string for digits assuming it's a possible year - logger.fdebug('new subname: ' + str(subthis)) + logger.fdebug('[FILECHECKER] new subname: ' + str(subthis)) subname = re.sub('(.*)\s+(19\d{2}|20\d{2})(.*)', '\\1 (\\2) \\3', subthis) subname = watchcomic + subname subnm = re.findall('[^()]+', subname) else: - subname = re.sub('(.*)\s+(19\d{2}|20\d{2})(.*)', '\\1 (\\2) \\3', subname) + subit = re.sub('(.*)\s+(19\d{2}|20\d{2})(.*)', '\\1 (\\2) \\3', subname) + subthis2 = re.sub('.cbr', '', subit) + subthis1 = re.sub('.cbz', '', subthis2) + subname = re.sub('[-\:\;\!\'\/\?\+\=\_\%\.]', '', subthis1) subnm = re.findall('[^()]+', subname) + if Publisher.lower() in subname.lower(): + #if the Publisher is given within the title or filename even (for some reason, some people + #have this to distinguish different titles), let's remove it entirely. + lenm = len(subnm) + cnt = 0 + pub_removed = None + + while (cnt < lenm): + if subnm[cnt] is None: break + if subnm[cnt] == ' ': + pass + else: + logger.fdebug(str(cnt) + ". Bracket Word: " + str(subnm[cnt])) + + if Publisher.lower() in subnm[cnt].lower() and cnt >= 1: + logger.fdebug('Publisher detected within title : ' + str(subnm[cnt])) + logger.fdebug('cnt is : ' + str(cnt) + ' --- Publisher is: ' + Publisher) + pub_removed = subnm[cnt] + #-strip publisher if exists here- + logger.fdebug('removing publisher from title') + subname_pubremoved = re.sub(pub_removed, '', subname) + logger.fdebug('pubremoved : ' + str(subname_pubremoved)) + subname_pubremoved = re.sub('\(\)', '', subname_pubremoved) #remove empty brackets + subname_pubremoved = re.sub('\s+', ' ', subname_pubremoved) #remove spaces > 1 + logger.fdebug('blank brackets removed: ' + str(subname_pubremoved)) + subnm = re.findall('[^()]+', subname_pubremoved) + break + cnt+=1 subname = subnm[0] if len(subnm): # if it still has no year (brackets), check setting and either assume no year needed. subname = subname - logger.fdebug('subname no brackets: ' + str(subname)) + logger.fdebug('[FILECHECKER] subname no brackets: ' + str(subname)) subname = re.sub('\_', ' ', subname) nonocount = 0 charpos = 0 @@ -162,22 +199,22 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): else: sublimit = subname[j+1:j+2] if sublimit.isdigit(): - logger.fdebug('possible negative issue detected.') + logger.fdebug('[FILECHECKER] possible negative issue detected.') nonocount = nonocount + subcnt - 1 detneg = "yes" elif '-' in watchcomic and i < len(watchcomic): - logger.fdebug('- appears in series title.') - logger.fdebug('up to - :' + subname[:j+1].replace('-', ' ')) - logger.fdebug('after - :' + subname[j+1:]) + logger.fdebug('[FILECHECKER] - appears in series title.') + logger.fdebug('[FILECHECKER] up to - :' + subname[:j+1].replace('-', ' ')) + logger.fdebug('[FILECHECKER] after - :' + subname[j+1:]) subname = subname[:j+1].replace('-', ' ') + subname[j+1:] - logger.fdebug('new subname is : ' + str(subname)) + logger.fdebug('[FILECHECKER] new subname is : ' + str(subname)) should_restart = True leavehyphen = True i+=1 if detneg == "no" or leavehyphen == False: subname = re.sub(str(nono), ' ', subname) nonocount = nonocount + subcnt - #logger.fdebug(str(nono) + " detected " + str(subcnt) + " times.") + #logger.fdebug('[FILECHECKER] (str(nono) + " detected " + str(subcnt) + " times.") # segment '.' having a . by itself will denote the entire string which we don't want elif nono == '.': x = 0 @@ -186,7 +223,7 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): while x < subcnt: fndit = subname.find(nono, fndit) if subname[fndit-1:fndit].isdigit() and subname[fndit+1:fndit+2].isdigit(): - logger.fdebug('decimal issue detected.') + logger.fdebug('[FILECHECKER] decimal issue detected.') dcspace+=1 x+=1 if dcspace == 1: @@ -204,7 +241,7 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): #print ("space before check: " + str(subname[fndit-1:fndit])) #print ("space after check: " + str(subname[fndit+1:fndit+2])) if subname[fndit-1:fndit] == ' ' and subname[fndit+1:fndit+2] == ' ': - logger.fdebug('blankspace detected before and after ' + str(nono)) + logger.fdebug('[FILECHECKER] blankspace detected before and after ' + str(nono)) blspc+=1 x+=1 subname = re.sub(str(nono), ' ', subname) @@ -213,7 +250,7 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): modwatchcomic = re.sub('[\_\#\,\/\:\;\.\!\$\%\'\?\@\-]', ' ', u_watchcomic) #if leavehyphen == False: - # logger.fdebug('removing hyphen for comparisons') + # logger.fdebug('[FILECHECKER] ('removing hyphen for comparisons') # modwatchcomic = re.sub('-', ' ', modwatchcomic) # subname = re.sub('-', ' ', subname) detectand = False @@ -221,7 +258,7 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): modwatchcomic = re.sub('\&', ' and ', modwatchcomic) if ' the ' in modwatchcomic.lower(): modwatchcomic = re.sub("\\bthe\\b", "", modwatchcomic.lower()) - logger.fdebug('new modwatchcomic: ' + str(modwatchcomic)) + logger.fdebug('[FILECHECKER] new modwatchcomic: ' + str(modwatchcomic)) detectthe = True modwatchcomic = re.sub('\s+', ' ', str(modwatchcomic)).strip() if '&' in subname: @@ -251,17 +288,17 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): AS_Alt.append(altsearchcomic) #if '_' in subname: # subname = subname.replace('_', ' ') - logger.fdebug('watchcomic:' + str(modwatchcomic) + ' ..comparing to found file: ' + str(subname)) + logger.fdebug('[FILECHECKER] watchcomic:' + str(modwatchcomic) + ' ..comparing to found file: ' + str(subname)) if modwatchcomic.lower() in subname.lower() or any(x.lower() in subname.lower() for x in AS_Alt):#altsearchcomic.lower() in subname.lower(): comicpath = os.path.join(basedir, item) - logger.fdebug( modwatchcomic + ' - watchlist match on : ' + comicpath) + logger.fdebug('[FILECHECKER] ' + modwatchcomic + ' - watchlist match on : ' + comicpath) comicsize = os.path.getsize(comicpath) #print ("Comicsize:" + str(comicsize)) comiccnt+=1 stann = 0 if 'annual' in subname.lower(): - logger.fdebug('Annual detected - proceeding') + logger.fdebug('[FILECHECKER] Annual detected - proceeding') jtd_len = subname.lower().find('annual') cchk = modwatchcomic else: @@ -272,11 +309,11 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): cchk = cchk_ls[0] #print "something: " + str(cchk) - logger.fdebug('we should remove ' + str(nonocount) + ' characters') + logger.fdebug('[FILECHECKER] we should remove ' + str(nonocount) + ' characters') findtitlepos = subname.find('-') if charpos != 0: - logger.fdebug('detected ' + str(len(charpos)) + ' special characters') + logger.fdebug('[FILECHECKER] detected ' + str(len(charpos)) + ' special characters') i=0 while (i < len(charpos)): for i,j in enumerate(charpos): @@ -284,22 +321,22 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): #print subname #print "digitchk: " + str(subname[j:]) if j >= len(subname): - logger.fdebug('end reached. ignoring remainder.') + logger.fdebug('[FILECHECKER] end reached. ignoring remainder.') break elif subname[j:] == '-': if i <= len(subname) and subname[i+1].isdigit(): - logger.fdebug('negative issue detected.') + logger.fdebug('[FILECHECKER] negative issue detected.') #detneg = "yes" elif j > findtitlepos: if subname[j:] == '#': if subname[i+1].isdigit(): - logger.fdebug('# detected denoting issue#, ignoring.') + logger.fdebug('[FILECHECKER] # detected denoting issue#, ignoring.') else: nonocount-=1 elif '-' in watchcomic and i < len(watchcomic): - logger.fdebug('- appears in series title, ignoring.') + logger.fdebug('[FILECHECKER] - appears in series title, ignoring.') else: - logger.fdebug('special character appears outside of title - ignoring @ position: ' + str(charpos[i])) + logger.fdebug('[FILECHECKER] special character appears outside of title - ignoring @ position: ' + str(charpos[i])) nonocount-=1 i+=1 @@ -313,9 +350,9 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): removest = subname.find(' ') # the - gets removed above so we test for the first blank space... if subname[:removest].isdigit(): jtd_len += removest + 1 # +1 to account for space in place of - - logger.fdebug('adjusted jtd_len to : ' + str(removest) + ' because of story-arc reading order tags') + logger.fdebug('[FILECHECKER] adjusted jtd_len to : ' + str(removest) + ' because of story-arc reading order tags') - logger.fdebug('nonocount [' + str(nonocount) + '] cchk [' + cchk + '] length [' + str(len(cchk)) + ']') + logger.fdebug('[FILECHECKER] nonocount [' + str(nonocount) + '] cchk [' + cchk + '] length [' + str(len(cchk)) + ']') #if detectand: # jtd_len = jtd_len - 2 # char substitution diff between & and 'and' = 2 chars @@ -324,54 +361,54 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): #justthedigits = item[jtd_len:] - logger.fdebug('final jtd_len to prune [' + str(jtd_len) + ']') - logger.fdebug('before title removed from FILENAME [' + str(item) + ']') - logger.fdebug('after title removed from FILENAME [' + str(item[jtd_len:]) + ']') - logger.fdebug('creating just the digits using SUBNAME, pruning first [' + str(jtd_len) + '] chars from [' + subname + ']') + logger.fdebug('[FILECHECKER] final jtd_len to prune [' + str(jtd_len) + ']') + logger.fdebug('[FILECHECKER] before title removed from FILENAME [' + str(item) + ']') + logger.fdebug('[FILECHECKER] after title removed from FILENAME [' + str(item[jtd_len:]) + ']') + logger.fdebug('[FILECHECKER] creating just the digits using SUBNAME, pruning first [' + str(jtd_len) + '] chars from [' + subname + ']') justthedigits_1 = subname[jtd_len:].strip() - logger.fdebug('after title removed from SUBNAME [' + justthedigits_1 + ']') + logger.fdebug('[FILECHECKER] after title removed from SUBNAME [' + justthedigits_1 + ']') #remove the title if it appears #findtitle = justthedigits.find('-') #if findtitle > 0 and detneg == "no": # justthedigits = justthedigits[:findtitle] - # logger.fdebug("removed title from name - is now : " + str(justthedigits)) + # logger.fdebug('[FILECHECKER] ("removed title from name - is now : " + str(justthedigits)) justthedigits = justthedigits_1.split(' ', 1)[0] digitsvalid = "false" for jdc in list(justthedigits): - #logger.fdebug('jdc:' + str(jdc)) + #logger.fdebug('[FILECHECKER] ('jdc:' + str(jdc)) if not jdc.isdigit(): - #logger.fdebug('alpha') + #logger.fdebug('[FILECHECKER] ('alpha') jdc_start = justthedigits.find(jdc) alpha_isschk = justthedigits[jdc_start:] - #logger.fdebug('alpha_isschk:' + str(alpha_isschk)) + #logger.fdebug('[FILECHECKER] ('alpha_isschk:' + str(alpha_isschk)) for issexcept in issue_exceptions: if issexcept.lower() in alpha_isschk.lower() and len(alpha_isschk) <= len(issexcept): - logger.fdebug('ALPHANUMERIC EXCEPTION : [' + justthedigits + ']') + logger.fdebug('[FILECHECKER] ALPHANUMERIC EXCEPTION : [' + justthedigits + ']') digitsvalid = "true" break if digitsvalid == "true": break try: tmpthedigits = justthedigits_1.split(' ', 1)[1] - logger.fdebug('If the series has a decimal, this should be a number [' + tmpthedigits + ']') + logger.fdebug('[FILECHECKER] If the series has a decimal, this should be a number [' + tmpthedigits + ']') if 'cbr' in tmpthedigits.lower() or 'cbz' in tmpthedigits.lower(): tmpthedigits = tmpthedigits[:-3].strip() - logger.fdebug('Removed extension - now we should just have a number [' + tmpthedigits + ']') + logger.fdebug('[FILECHECKER] Removed extension - now we should just have a number [' + tmpthedigits + ']') poss_alpha = tmpthedigits if poss_alpha.isdigit(): digitsvalid = "true" if justthedigits.lower() == 'annual': - logger.fdebug('ANNUAL DETECTED [' + poss_alpha + ']') + logger.fdebug('[FILECHECKER] ANNUAL DETECTED [' + poss_alpha + ']') justthedigits += ' ' + poss_alpha else: justthedigits += '.' + poss_alpha - logger.fdebug('DECIMAL ISSUE DETECTED [' + justthedigits + ']') + logger.fdebug('[FILECHECKER] DECIMAL ISSUE DETECTED [' + justthedigits + ']') else: for issexcept in issue_exceptions: decimalexcept = False @@ -382,7 +419,7 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): if decimalexcept: issexcept = '.' + issexcept justthedigits += issexcept #poss_alpha - logger.fdebug('ALPHANUMERIC EXCEPTION. COMBINING : [' + justthedigits + ']') + logger.fdebug('[FILECHECKER] ALPHANUMERIC EXCEPTION. COMBINING : [' + justthedigits + ']') digitsvalid = "true" break except: @@ -391,7 +428,7 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): # justthedigits = justthedigits.split(' ', 1)[0] #if the issue has an alphanumeric (issue_exceptions, join it and push it through) - logger.fdebug('JUSTTHEDIGITS [' + justthedigits + ']' ) + logger.fdebug('[FILECHECKER] JUSTTHEDIGITS [' + justthedigits + ']' ) if digitsvalid == "true": pass else: @@ -403,7 +440,7 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): b4dec = justthedigits[:tmpdec] a4dec = justthedigits[tmpdec+1:] if a4dec.isdigit() and b4dec.isdigit(): - logger.fdebug('DECIMAL ISSUE DETECTED') + logger.fdebug('[FILECHECKER] DECIMAL ISSUE DETECTED') digitsvalid = "true" else: try: @@ -418,11 +455,11 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): # else: -# logger.fdebug('NO DECIMALS DETECTED') +# logger.fdebug('[FILECHECKER] NO DECIMALS DETECTED') # digitsvalid = "false" # if justthedigits.lower() == 'annual': -# logger.fdebug('ANNUAL [' + tmpthedigits.split(' ', 1)[1] + ']') +# logger.fdebug('[FILECHECKER] ANNUAL [' + tmpthedigits.split(' ', 1)[1] + ']') # justthedigits += ' ' + tmpthedigits.split(' ', 1)[1] # digitsvalid = "true" # else: @@ -432,19 +469,19 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): # if poss_alpha.isdigit(): # digitsvalid = "true" # justthedigits += '.' + poss_alpha -# logger.fdebug('DECIMAL ISSUE DETECTED [' + justthedigits + ']') +# logger.fdebug('[FILECHECKER] DECIMAL ISSUE DETECTED [' + justthedigits + ']') # for issexcept in issue_exceptions: # if issexcept.lower() in poss_alpha.lower() and len(poss_alpha) <= len(issexcept): # justthedigits += poss_alpha -# logger.fdebug('ALPHANUMERIC EXCEPTION. COMBINING : [' + justthedigits + ']') +# logger.fdebug('[FILECHECKER] ALPHANUMERIC EXCEPTION. COMBINING : [' + justthedigits + ']') # digitsvalid = "true" # break # except: # pass - logger.fdebug('final justthedigits [' + justthedigits + ']') + logger.fdebug('[FILECHECKER] final justthedigits [' + justthedigits + ']') if digitsvalid == "false": - logger.fdebug('Issue number not properly detected...ignoring.') + logger.fdebug('[FILECHECKER] Issue number not properly detected...ignoring.') comiccnt -=1 # remove the entry from the list count as it was incorrrectly tallied. continue @@ -455,21 +492,35 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): # in case it matches on an Alternate Search pattern, set modwatchcomic to the cchk value modwatchcomic = cchk - logger.fdebug('cchk = ' + cchk.lower()) - logger.fdebug('modwatchcomic = ' + modwatchcomic.lower()) - logger.fdebug('subname = ' + subname.lower()) + logger.fdebug('[FILECHECKER] cchk = ' + cchk.lower()) + logger.fdebug('[FILECHECKER] modwatchcomic = ' + modwatchcomic.lower()) + logger.fdebug('[FILECHECKER] subname = ' + subname.lower()) comyear = manual['SeriesYear'] issuetotal = manual['Total'] comicvolume = manual['ComicVersion'] - logger.fdebug('SeriesYear: ' + str(comyear)) - logger.fdebug('IssueTotal: ' + str(issuetotal)) - logger.fdebug('Comic Volume: ' + str(comicvolume)) - logger.fdebug('volume detected: ' + str(volrem)) + logger.fdebug('[FILECHECKER] SeriesYear: ' + str(comyear)) + logger.fdebug('[FILECHECKER] IssueTotal: ' + str(issuetotal)) + logger.fdebug('[FILECHECKER] Comic Volume: ' + str(comicvolume)) + logger.fdebug('[FILECHECKER] volume detected: ' + str(volrem)) + + if comicvolume: + ComVersChk = re.sub("[^0-9]", "", comicvolume) + if ComVersChk == '' or ComVersChk == '1': + ComVersChk = 0 + else: + ComVersChk = 0 + + # even if it's a V1, we need to pull the date for the given issue ID and get the publication year + # for the issue. Because even if it's a V1, if there are additional Volumes then it's possible that + # it will take the incorrect series. (ie. Detective Comics (1937) & Detective Comics (2011). + # If issue #28 (2013) is found, it exists in both series, and because DC 1937 is a V1, it will bypass + # the year check which will result in the incorrect series being picked (1937) + #set the issue/year threshold here. # 2013 - (24issues/12) = 2011. #minyear = int(comyear) - (int(issuetotal) / 12) - + maxyear = manual['LatestDate'][:4] # yyyy-mm-dd #subnm defined at being of module. @@ -477,28 +528,28 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): #print ("there are " + str(lenm) + " words.") cnt = 0 - yearmatch = "false" + yearmatch = "none" vers4year = "no" vers4vol = "no" for ct in subsplit: if ct.lower().startswith('v') and ct[1:].isdigit(): - logger.fdebug("possible versioning..checking") + logger.fdebug('[FILECHECKER] possible versioning..checking') #we hit a versioning # - account for it if ct[1:].isdigit(): if len(ct[1:]) == 4: #v2013 - logger.fdebug("Version detected as " + str(ct)) + logger.fdebug('[FILECHECKER] Version detected as ' + str(ct)) vers4year = "yes" #re.sub("[^0-9]", " ", str(ct)) #remove the v break else: if len(ct) < 4: - logger.fdebug("Version detected as " + str(ct)) + logger.fdebug('[FILECHECKER] Version detected as ' + str(ct)) vers4vol = str(ct) break - logger.fdebug("false version detection..ignoring.") + logger.fdebug('[FILECHECKER] false version detection..ignoring.') + versionmatch = "false" if vers4year is not "no" or vers4vol is not "no": - yearmatch = "false" if comicvolume: #is not "None" and comicvolume is not None: D_ComicVersion = re.sub("[^0-9]", "", comicvolume) @@ -509,42 +560,62 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): F_ComicVersion = re.sub("[^0-9]", "", volrem) S_ComicVersion = str(comyear) - logger.fdebug("FCVersion: " + str(F_ComicVersion)) - logger.fdebug("DCVersion: " + str(D_ComicVersion)) - logger.fdebug("SCVersion: " + str(S_ComicVersion)) + logger.fdebug('[FILECHECKER] FCVersion: ' + str(F_ComicVersion)) + logger.fdebug('[FILECHECKER] DCVersion: ' + str(D_ComicVersion)) + logger.fdebug('[FILECHECKER] SCVersion: ' + str(S_ComicVersion)) #if annualize == "true" and int(ComicYear) == int(F_ComicVersion): - # logger.fdebug("We matched on versions for annuals " + str(volrem)) + # logger.fdebug('[FILECHECKER] ("We matched on versions for annuals " + str(volrem)) if int(F_ComicVersion) == int(D_ComicVersion) or int(F_ComicVersion) == int(S_ComicVersion): - logger.fdebug("We matched on versions..." + str(volrem)) + logger.fdebug('[FILECHECKER] We matched on versions...' + str(volrem)) + versionmatch = "true" + else: + logger.fdebug('[FILECHECKER] Versions wrong. Ignoring possible match.') + + #else: + while (cnt < len_sm): + if subnm[cnt] is None: break + if subnm[cnt] == ' ': + pass + else: + logger.fdebug('[FILECHECKER] ' + str(cnt) + ' Bracket Word: ' + str(subnm[cnt])) + + #if ComVersChk == 0: + # logger.fdebug('[FILECHECKER] Series version detected as V1 (only series in existance with that title). Bypassing year check') + # yearmatch = "true" + # break + if subnm[cnt][:-2] == '19' or subnm[cnt][:-2] == '20': + logger.fdebug('[FILECHECKER] year detected: ' + str(subnm[cnt])) + result_comyear = subnm[cnt] + if int(result_comyear) <= int(maxyear): + logger.fdebug('[FILECHECKER] ' + str(result_comyear) + ' is within the series range of ' + str(comyear) + '-' + str(maxyear)) + #still possible for incorrect match if multiple reboots of series end/start in same year + yearmatch = "true" + break + else: + logger.fdebug('[FILECHECKER] ' + str(result_comyear) + ' - not right - year not within series range of ' + str(comyear) + '-' + str(maxyear)) + yearmatch = "false" + break + cnt+=1 + if versionmatch == "false": + if yearmatch == "false": + logger.fdebug('[FILECHECKER] Failed to match on both version and issue year.') + continue + else: + logger.fdebug('[FILECHECKER] Matched on versions, not on year - continuing.') + else: + if yearmatch == "false": + logger.fdebug('[FILECHECKER] Matched on version, but not on year - continuing.') + else: + logger.fdebug('[FILECHECKER] Matched on both version, and issue year - continuing.') + + if yearmatch == "none": + if ComVersChk == 0: + logger.fdebug('[FILECHECKER] Series version detected as V1 (only series in existance with that title). Bypassing year check.') yearmatch = "true" else: - logger.fdebug("Versions wrong. Ignoring possible match.") - - else: - while (cnt < len_sm): - if subnm[cnt] is None: break - if subnm[cnt] == ' ': - pass - else: - logger.fdebug(str(cnt) + ". Bracket Word: " + str(subnm[cnt])) - - if subnm[cnt][:-2] == '19' or subnm[cnt][:-2] == '20': - logger.fdebug("year detected: " + str(subnm[cnt])) - result_comyear = subnm[cnt] - if int(result_comyear) <= int(maxyear): - logger.fdebug(str(result_comyear) + ' is within the series range of ' + str(comyear) + '-' + str(maxyear)) - #still possible for incorrect match if multiple reboots of series end/start in same year - yearmatch = "true" - break - else: - logger.fdebug(str(result_comyear) + ' - not right - year not within series range of ' + str(comyear) + '-' + str(maxyear)) - yearmatch = "false" - break - cnt+=1 - - if yearmatch == "false": continue + continue if 'annual' in subname.lower(): subname = re.sub('annual', '', subname.lower()) @@ -554,66 +625,81 @@ def listFiles(dir,watchcomic,AlternateSearch=None,manual=None,sarc=None): # if it's an alphanumeric with a space, rejoin, so we can remove it cleanly just below this. substring_removal = None poss_alpha = subname.split(' ')[-1:] - logger.fdebug('poss_alpha: ' + str(poss_alpha)) - logger.fdebug('lenalpha: ' + str(len(''.join(poss_alpha)))) + logger.fdebug('[FILECHECKER] poss_alpha: ' + str(poss_alpha)) + logger.fdebug('[FILECHECKER] lenalpha: ' + str(len(''.join(poss_alpha)))) for issexcept in issue_exceptions: if issexcept.lower()in str(poss_alpha).lower() and len(''.join(poss_alpha)) <= len(issexcept): #get the last 2 words so that we can remove them cleanly substring_removal = ' '.join(subname.split(' ')[-2:]) substring_join = ''.join(subname.split(' ')[-2:]) - logger.fdebug('substring_removal: ' + str(substring_removal)) - logger.fdebug('substring_join: ' + str(substring_join)) + logger.fdebug('[FILECHECKER] substring_removal: ' + str(substring_removal)) + logger.fdebug('[FILECHECKER] substring_join: ' + str(substring_join)) break if substring_removal is not None: sub_removed = subname.replace('_', ' ').replace(substring_removal, substring_join) else: sub_removed = subname.replace('_', ' ') - logger.fdebug('sub_removed: ' + str(sub_removed)) + logger.fdebug('[FILECHECKER] sub_removed: ' + str(sub_removed)) split_sub = sub_removed.rsplit(' ',1)[0].split(' ') #removes last word (assuming it's the issue#) split_mod = modwatchcomic.replace('_', ' ').split() #batman - logger.fdebug('split_sub: ' + str(split_sub)) - logger.fdebug('split_mod: ' + str(split_mod)) + logger.fdebug('[FILECHECKER] split_sub: ' + str(split_sub)) + logger.fdebug('[FILECHECKER] split_mod: ' + str(split_mod)) x = len(split_sub)-1 scnt = 0 if x > len(split_mod)-1: - logger.fdebug('number of words do not match...aborting.') + logger.fdebug('[FILECHECKER] number of words do not match...aborting.') else: while ( x > -1 ): print str(split_sub[x]) + ' comparing to ' + str(split_mod[x]) if str(split_sub[x]).lower() == str(split_mod[x]).lower(): scnt+=1 - logger.fdebug('word match exact. ' + str(scnt) + '/' + str(len(split_mod))) + logger.fdebug('[FILECHECKER] word match exact. ' + str(scnt) + '/' + str(len(split_mod))) x-=1 wordcnt = int(scnt) - logger.fdebug('scnt:' + str(scnt)) + logger.fdebug('[FILECHECKER] scnt:' + str(scnt)) totalcnt = int(len(split_mod)) - logger.fdebug('split_mod length:' + str(totalcnt)) + logger.fdebug('[FILECHECKER] split_mod length:' + str(totalcnt)) try: spercent = (wordcnt/totalcnt) * 100 except ZeroDivisionError: spercent = 0 - logger.fdebug('we got ' + str(spercent) + ' percent.') + logger.fdebug('[FILECHECKER] we got ' + str(spercent) + ' percent.') if int(spercent) >= 80: - logger.fdebug("this should be considered an exact match.") + logger.fdebug('[FILECHECKER] this should be considered an exact match.Justthedigits:' + justthedigits) else: - logger.fdebug('failure - not an exact match.') + logger.fdebug('[FILECHECKER] failure - not an exact match.') continue - comiclist.append({ - 'ComicFilename': item, - 'ComicLocation': comicpath, - 'ComicSize': comicsize, - 'JusttheDigits': justthedigits - }) + if manual: + print item + print comicpath + print comicsize + print result_comyear + print justthedigits + comiclist.append({ + 'ComicFilename': item, + 'ComicLocation': comicpath, + 'ComicSize': comicsize, + 'ComicYear': result_comyear, + 'JusttheDigits': justthedigits + }) + print('appended.') + else: + comiclist.append({ + 'ComicFilename': item, + 'ComicLocation': comicpath, + 'ComicSize': comicsize, + 'JusttheDigits': justthedigits + }) watchmatch['comiclist'] = comiclist else: pass #print ("directory found - ignoring") - logger.fdebug('you have a total of ' + str(comiccnt) + ' ' + watchcomic + ' comics') + logger.fdebug('[FILECHECKER] you have a total of ' + str(comiccnt) + ' ' + watchcomic + ' comics') watchmatch['comiccount'] = comiccnt return watchmatch diff --git a/mylar/findcomicfeed.py b/mylar/findcomicfeed.py index fadd952e..7fb4a9bc 100755 --- a/mylar/findcomicfeed.py +++ b/mylar/findcomicfeed.py @@ -59,7 +59,8 @@ def Startit(searchName, searchIssue, searchYear, ComicVersion, IssDateFix): totNum = len(feed.entries) tallycount += len(feed.entries) - keyPair = {} + #keyPair = {} + keyPair = [] regList = [] countUp = 0 @@ -68,7 +69,11 @@ def Startit(searchName, searchIssue, searchYear, ComicVersion, IssDateFix): while countUp < totNum: urlParse = feed.entries[countUp].enclosures[0] #keyPair[feed.entries[countUp].title] = feed.entries[countUp].link - keyPair[feed.entries[countUp].title] = urlParse["href"] + #keyPair[feed.entries[countUp].title] = urlParse["href"] + keyPair.append({"title": feed.entries[countUp].title, + "link": urlParse["href"], + "length": urlParse["length"], + "pubdate": feed.entries[countUp].updated}) countUp=countUp+1 @@ -90,13 +95,14 @@ def Startit(searchName, searchIssue, searchYear, ComicVersion, IssDateFix): except_list=['releases', 'gold line', 'distribution', '0-day', '0 day'] - for title, link in keyPair.items(): + for entry in keyPair: + title = entry['title'] #logger.fdebug("titlesplit: " + str(title.split("\""))) splitTitle = title.split("\"") noYear = 'False' for subs in splitTitle: - logger.fdebug(subs) + #logger.fdebug('sub:' + subs) regExCount = 0 if len(subs) > 10 and not any(d in subs.lower() for d in except_list): #Looping through dictionary to run each regEx - length + regex is determined by regexList up top. @@ -128,8 +134,10 @@ def Startit(searchName, searchIssue, searchYear, ComicVersion, IssDateFix): if noYear == 'False': entries.append({ - 'title': subs, - 'link': str(link) + 'title': subs, + 'link': entry['link'], + 'pubdate': entry['pubdate'], + 'length': entry['length'] }) break # break out so we don't write more shit. diff --git a/mylar/helpers.py b/mylar/helpers.py index a805434f..21957dfe 100755 --- a/mylar/helpers.py +++ b/mylar/helpers.py @@ -382,6 +382,8 @@ def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=N logger.fdebug('Pretty Comic Issue is : ' + str(prettycomiss)) issueyear = issuenzb['IssueDate'][:4] + month = issuenzb['IssueDate'][5:7].replace('-','').strip() + month_name = fullmonth(month) logger.fdebug('Issue Year : ' + str(issueyear)) comicnzb= myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone() publisher = comicnzb['ComicPublisher'] @@ -428,6 +430,8 @@ def rename_param(comicid, comicname, issue, ofilename, comicyear=None, issueid=N '$publisher': publisher.lower(), '$VolumeY': 'V' + str(seriesyear), '$VolumeN': comversion, + '$monthname': monthname, + '$month': month, '$Annual': 'Annual' } @@ -886,3 +890,34 @@ def checkFolder(): result = PostProcess.Process() logger.info('Finished checking for newly snatched downloads') +def LoadAlternateSearchNames(seriesname_alt, comicid): + import logger + #seriesname_alt = db.comics['AlternateSearch'] + AS_Alt = [] + Alternate_Names = {} + alt_count = 0 + + logger.fdebug('seriesname_alt:' + str(seriesname_alt)) + if seriesname_alt is None or seriesname_alt == 'None': + logger.fdebug('no Alternate name given. Aborting search.') + return "no results" + else: + chkthealt = seriesname_alt.split('##') + if chkthealt == 0: + AS_Alternate = seriesname_alt + AS_Alt.append(seriesname_alt) + for calt in chkthealt: + AS_Alter = re.sub('##','',calt) + u_altsearchcomic = AS_Alter.encode('ascii', 'ignore').strip() + AS_formatrem_seriesname = re.sub('\s+', ' ', u_altsearchcomic) + if AS_formatrem_seriesname[:1] == ' ': AS_formatrem_seriesname = AS_formatrem_seriesname[1:] + + AS_Alt.append({"AlternateName": AS_formatrem_seriesname}) + alt_count+=1 + + Alternate_Names['AlternateName'] = AS_Alt + Alternate_Names['ComicID'] = comicid + Alternate_Names['Count'] = alt_count + logger.info('AlternateNames returned:' + str(Alternate_Names)) + + return Alternate_Names diff --git a/mylar/importer.py b/mylar/importer.py index 6183164e..ce2ff0b7 100755 --- a/mylar/importer.py +++ b/mylar/importer.py @@ -43,7 +43,7 @@ def is_exists(comicid): return False -def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None,calledfrom=None): +def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None,calledfrom=None,annload=None): # Putting this here to get around the circular import. Will try to use this to update images at later date. # from mylar import cache @@ -143,6 +143,19 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None,c #let's do the Annual check here. if mylar.ANNUALS_ON: + #we need to check first to see if there are pre-existing annuals that have been manually added, or else they'll get + #wiped out. + annualids = [] #to be used to make sure an ID isn't double-loaded + + if annload is None: + pass + else: + for manchk in annload: + if manchk['ReleaseComicID'] is not None or manchk['ReleaseComicID'] is not None: #if it exists, then it's a pre-existing add. + #print str(manchk['ReleaseComicID']), comic['ComicName'], str(SeriesYear), str(comicid) + manualAnnual(manchk['ReleaseComicID'], comic['ComicName'], SeriesYear, comicid) + annualids.append(manchk['ReleaseComicID']) + annualcomicname = re.sub('[\,\:]', '', comic['ComicName']) #----- CBDB (outdated) @@ -177,7 +190,6 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None,c #print "annualyear: " + str(annualval['AnnualYear']) logger.fdebug('annualyear:' + str(annualyear)) sresults = mb.findComic(annComicName, mode, issue=None) - logger.fdebug('sresults : ' + str(sresults)) type='comic' @@ -188,7 +200,7 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None,c num_res = 0 while (num_res < len(sresults)): sr = sresults[num_res] - #logger.fdebug("description:" + sr['description']) + logger.fdebug("description:" + sr['description']) if 'paperback' in sr['description'] or 'collecting' in sr['description'] or 'reprints' in sr['description'] or 'collected' in sr['description']: logger.fdebug('tradeback/collected edition detected - skipping ' + str(sr['comicid'])) else: @@ -196,6 +208,10 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None,c logger.fdebug(str(comicid) + ' found. Assuming it is part of the greater collection.') issueid = sr['comicid'] logger.fdebug(str(issueid) + ' added to series list as an Annual') + if issueid in annualids: + logger.fdebug(str(issueid) + ' already exists & was refreshed.') + num_res+=1 # need to manually increment since not a for-next loop + continue issued = cv.getComic(issueid,'issue') if len(issued) is None or len(issued) == 0: logger.fdebug('Could not find any annual information...') @@ -213,14 +229,18 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None,c issnum = str(firstval['Issue_Number']) issname = cleanname issdate = str(firstval['Issue_Date']) + stdate = str(firstval['Store_Date']) newCtrl = {"IssueID": issid} - newVals = {"Issue_Number": issnum, - "Int_IssueNumber": helpers.issuedigits(issnum), - "IssueDate": issdate, - "IssueName": issname, - "ComicID": comicid, - "ComicName": comic['ComicName'], - "Status": "Skipped"} + newVals = {"Issue_Number": issnum, + "Int_IssueNumber": helpers.issuedigits(issnum), + "IssueDate": issdate, + "ReleaseDate": stdate, + "IssueName": issname, + "ComicID": comicid, + "ComicName": comic['ComicName'], + "ReleaseComicID": re.sub('4050-','',firstval['Comic_ID']).strip(), + "ReleaseComicName": sr['name'], + "Status": "Skipped"} myDB.upsert("annuals", newVals, newCtrl) n+=1 num_res+=1 @@ -613,6 +633,7 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None,c #print ("issnum: " + str(issnum)) issname = cleanname issdate = str(firstval['Issue_Date']) + storedate = str(firstval['Store_Date']) if issnum.isdigit(): int_issnum = int( issnum ) * 1000 else: @@ -741,6 +762,7 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None,c "IssueName": issname, "Issue_Number": issnum, "IssueDate": issdate, + "ReleaseDate": storedate, "Int_IssueNumber": int_issnum}) #logger.info('issuedata: ' + str(issuedata)) @@ -836,10 +858,10 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None,c #check for existing files... statbefore = myDB.action("SELECT * FROM issues WHERE ComicID=? AND Issue_Number=?", [comicid,str(latestiss)]).fetchone() - logger.fdebug('issue: ' + str(latestiss) + ' status before chk :' + statbefore['Status']) + logger.fdebug('issue: ' + str(latestiss) + ' status before chk :' + str(statbefore['Status'])) updater.forceRescan(comicid) statafter = myDB.action("SELECT * FROM issues WHERE ComicID=? AND Issue_Number=?", [comicid,str(latestiss)]).fetchone() - logger.fdebug('issue: ' + str(latestiss) + ' status after chk :' + statafter['Status']) + logger.fdebug('issue: ' + str(latestiss) + ' status after chk :' + str(statafter['Status'])) if pullupd is None: # lets' check the pullist for anything at this time as well since we're here. @@ -848,7 +870,7 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None,c logger.fdebug('latestissue: #' + str(latestiss)) chkstats = myDB.action("SELECT * FROM issues WHERE ComicID=? AND Issue_Number=?", [comicid,str(latestiss)]).fetchone() logger.fdebug('latestissue status: ' + chkstats['Status']) - if chkstats['Status'] == 'Skipped' or chkstats['Status'] == 'Wanted': # or chkstats['Status'] == 'Snatched': + if chkstats['Status'] == 'Skipped' or chkstats['Status'] == 'Wanted' or chkstats['Status'] == 'Snatched': logger.info('Checking this week pullist for new issues of ' + comic['ComicName']) updater.newpullcheck(comic['ComicName'], comicid) @@ -869,7 +891,9 @@ def addComictoDB(comicid,mismatch=None,pullupd=None,imported=None,ogcname=None,c logger.info('Already have the latest issue : #' + str(latestiss)) if calledfrom == 'addbyid': - return comic['ComicName'], SeriesYear + logger.info('Sucessfully added ' + comic['ComicName'] + ' (' + str(SeriesYear) + ') by directly using the ComicVine ID') + return + def GCDimport(gcomicid, pullupd=None,imported=None,ogcname=None): # this is for importing via GCD only and not using CV. @@ -1224,6 +1248,7 @@ def issue_collection(issuedata,nostatus): "IssueName": issue['IssueName'], "Issue_Number": issue['Issue_Number'], "IssueDate": issue['IssueDate'], + "ReleaseDate": issue['ReleaseDate'], "Int_IssueNumber": issue['Int_IssueNumber'] } @@ -1235,7 +1260,7 @@ def issue_collection(issuedata,nostatus): # Only change the status & add DateAdded if the issue is already in the database if iss_exists is None: newValueDict['DateAdded'] = helpers.today() - #print "issue doesn't exist in db." + print 'issue #' + str(issue['Issue_Number']) + 'does not exist in db.' if mylar.AUTOWANT_ALL: newValueDict['Status'] = "Wanted" elif issue['IssueDate'] > helpers.today() and mylar.AUTOWANT_UPCOMING: @@ -1244,12 +1269,12 @@ def issue_collection(issuedata,nostatus): newValueDict['Status'] = "Skipped" else: - #print ("Existing status : " + str(iss_exists['Status'])) + #logger.info('Existing status for issue #' + str(issue['Issue_Number']) + ' : ' + str(iss_exists['Status'])) newValueDict['Status'] = iss_exists['Status'] else: - #print ("Not changing the status at this time - reverting to previous module after to re-append existing status") - newValueDict['Status'] = "Skipped" + #logger.info("Not changing the status at this time - reverting to previous module after to re-append existing status") + pass #newValueDict['Status'] = "Skipped" try: myDB.upsert("issues", newValueDict, controlValueDict) @@ -1259,3 +1284,53 @@ def issue_collection(issuedata,nostatus): myDB.action("DELETE FROM comics WHERE ComicID=?", [issue['ComicID']]) return +def manualAnnual(manual_comicid, comicname, comicyear, comicid): + #called when importing/refreshing an annual that was manually added. + myDB = db.DBConnection() + issueid = manual_comicid + logger.fdebug(str(issueid) + ' added to series list as an Annual') + sr = cv.getComic(manual_comicid, 'comic') + logger.info('Attempting to integrate ' + sr['ComicName'] + ' (' + str(issueid) + ') to the existing series of ' + comicname + '(' + str(comicyear) + ')') + if len(sr) is None or len(sr) == 0: + logger.fdebug('Could not find any information on the series indicated : ' + str(manual_comicid)) + pass + else: + n = 0 + noissues = sr['ComicIssues'] + logger.fdebug('there are ' + str(noissues) + ' annuals within this series.') + issued = cv.getComic(re.sub('4050-','',manual_comicid).strip(),'issue') + while (n < int(noissues)): + try: + firstval = issued['issuechoice'][n] + except IndexError: + break + cleanname = helpers.cleanName(firstval['Issue_Name']) + issid = str(firstval['Issue_ID']) + issnum = str(firstval['Issue_Number']) + issname = cleanname + issdate = str(firstval['Issue_Date']) + stdate = str(firstval['Store_Date']) + logger.fdebug('comicname:' + str(comicname)) + logger.fdebug('comicid:' + str(comicid)) + logger.fdebug('issid:' + str(issid)) + logger.fdebug('cleanname:' + str(cleanname)) + logger.fdebug('issnum:' + str(issnum)) + logger.fdebug('issdate:' + str(issdate)) + logger.fdebug('stdate:' + str(stdate)) + newCtrl = {"IssueID": issid} + newVals = {"Issue_Number": issnum, + "Int_IssueNumber": helpers.issuedigits(issnum), + "IssueDate": issdate, + "ReleaseDate": stdate, + "IssueName": issname, + "ComicID": comicid, #this is the series ID + "ReleaseComicID": re.sub('4050-','',manual_comicid).strip(), #this is the series ID for the annual(s) + "ComicName": comicname, #series ComicName + "ReleaseComicName" :sr['ComicName'], #series ComicName for the manual_comicid + "Status": "Skipped"} + #need to add in the values for the new series to be added. + #"M_ComicName": sr['ComicName'], + #"M_ComicID": manual_comicid} + myDB.upsert("annuals", newVals, newCtrl) + n+=1 + return diff --git a/mylar/librarysync.py b/mylar/librarysync.py index 7b14f952..7d199a82 100755 --- a/mylar/librarysync.py +++ b/mylar/librarysync.py @@ -128,6 +128,16 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None) watchfound = 0 + datelist = ['jan','feb','mar','apr','may','jun','jul','aug','sep','oct','nov','dec'] +# datemonth = {'one':1,'two':2,'three':3,'four':4,'five':5,'six':6,'seven':7,'eight':8,'nine':9,'ten':10,'eleven':$ +# #search for number as text, and change to numeric +# for numbs in basnumbs: +# #print ("numbs:" + str(numbs)) +# if numbs in ComicName.lower(): +# numconv = basnumbs[numbs] +# #print ("numconv: " + str(numconv)) + + for i in comic_list: print i['ComicFilename'] @@ -157,15 +167,16 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None) #because the '.' in Vol. gets removed, let's loop thru again after the Vol hit to remove it entirely logger.fdebug('volume indicator detected as version #:' + str(subit)) cfilename = re.sub(subit, '', cfilename) - volyr = re.sub("[^0-9]", " ", subit) - + cfilename = " ".join(cfilename.split()) + volyr = re.sub("[^0-9]", " ", subit).strip() + logger.fdebug('volume year set as : ' + str(volyr)) cm_cn = 0 #we need to track the counter to make sure we are comparing the right array parts #this takes care of the brackets :) m = re.findall('[^()]+', cfilename) lenm = len(m) - print ("there are " + str(lenm) + " words.") + logger.fdebug("there are " + str(lenm) + " words.") cnt = 0 yearmatch = "false" foundonwatch = "False" @@ -187,7 +198,7 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None) extensions = ('cbr', 'cbz') if comic_andiss.lower().endswith(extensions): comic_andiss = comic_andiss[:-4] - print ("removed extension from filename.") + logger.fdebug("removed extension from filename.") #now we have to break up the string regardless of formatting. #let's force the spaces. comic_andiss = re.sub('_', ' ', comic_andiss) @@ -199,42 +210,41 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None) decimaldetect = 'no' for i in reversed(xrange(len(cs))): #start at the end. - print ("word: " + str(cs[i])) + logger.fdebug("word: " + str(cs[i])) #assume once we find issue - everything prior is the actual title #idetected = no will ignore everything so it will assume all title if cs[i][:-2] == '19' or cs[i][:-2] == '20' and idetected == 'no': - print ("year detected: " + str(cs[i])) + logger.fdebug("year detected: " + str(cs[i])) ydetected = 'yes' result_comyear = cs[i] elif cs[i].isdigit() and idetected == 'no' or '.' in cs[i]: issue = cs[i] - print ("issue detected : " + str(issue)) + logger.fdebug("issue detected : " + str(issue)) idetected = 'yes' if '.' in cs[i]: #make sure it's a number on either side of decimal and assume decimal issue. decst = cs[i].find('.') dec_st = cs[i][:decst] dec_en = cs[i][decst+1:] - print ("st: " + str(dec_st)) - print ("en: " + str(dec_en)) + logger.fdebug("st: " + str(dec_st)) + logger.fdebug("en: " + str(dec_en)) if dec_st.isdigit() and dec_en.isdigit(): - print ("decimal issue detected...adjusting.") + logger.fdebug("decimal issue detected...adjusting.") issue = dec_st + "." + dec_en - print ("issue detected: " + str(issue)) + logger.fdebug("issue detected: " + str(issue)) idetected = 'yes' else: - print ("false decimal represent. Chunking to extra word.") + logger.fdebug("false decimal represent. Chunking to extra word.") cn = cn + cs[i] + " " break elif '\#' in cs[i] or decimaldetect == 'yes': - print ("issue detected: " + str(cs[i])) + logger.fdebug("issue detected: " + str(cs[i])) idetected = 'yes' - else: cn = cn + cs[i] + " " if ydetected == 'no': #assume no year given in filename... result_comyear = "0000" - print ("cm?: " + str(cn)) + logger.fdebug("cm?: " + str(cn)) if issue is not '999999': comiss = issue else: @@ -252,11 +262,20 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None) print ("com_NAME : " + com_NAME) yearmatch = "True" else: + logger.fdebug('checking ' + m[cnt]) # we're assuming that the year is in brackets (and it should be damnit) if m[cnt][:-2] == '19' or m[cnt][:-2] == '20': print ("year detected: " + str(m[cnt])) ydetected = 'yes' result_comyear = m[cnt] + elif m[cnt][:3].lower() in datelist: + logger.fdebug('possible issue date format given - verifying') + #if the date of the issue is given as (Jan 2010) or (January 2010) let's adjust. + #keeping in mind that ',' and '.' are already stripped from the string + if m[cnt][-4:].isdigit(): + ydetected = 'yes' + result_comyear = m[cnt][-4:] + logger.fdebug('Valid Issue year of ' + str(result_comyear) + 'detected in format of ' + str(m[cnt])) cnt+=1 splitit = [] @@ -447,6 +466,13 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None) else: if result_comyear is None: result_comyear = volyr + if volno is None: + if volyr is None: + vol_label = None + else: + vol_label = volyr + else: + vol_label = volno print ("adding " + com_NAME + " to the import-queue!") impid = com_NAME + "-" + str(result_comyear) + "-" + str(comiss) @@ -456,6 +482,7 @@ def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None) "watchmatch": watchmatch, "comicname" : com_NAME, "comicyear" : result_comyear, + "volume" : vol_label, "comfilename" : comfilename, "comlocation" : comlocation.decode(mylar.SYS_ENCODING) }) diff --git a/mylar/mb.py b/mylar/mb.py index 6a3327a0..0409ff84 100755 --- a/mylar/mb.py +++ b/mylar/mb.py @@ -29,7 +29,14 @@ mb_lock = threading.Lock() def pullsearch(comicapi,comicquery,offset): u_comicquery = urllib.quote(comicquery.encode('utf-8').strip()) - PULLURL = mylar.CVURL + 'search?api_key=' + str(comicapi) + '&resources=volume&query=' + u_comicquery + '&field_list=id,name,start_year,site_detail_url,count_of_issues,image,publisher,description&format=xml&page=' + str(offset) + u_comicquery = u_comicquery.replace(" ", "%20") + + # as of 02/15/2014 this is buggered up. + #PULLURL = mylar.CVURL + 'search?api_key=' + str(comicapi) + '&resources=volume&query=' + u_comicquery + '&field_list=id,name,start_year,site_detail_url,count_of_issues,image,publisher,description&format=xml&page=' + str(offset) + + # 02/22/2014 use the volume filter label to get the right results. + PULLURL = mylar.CVURL + 'volumes?api_key=' + str(comicapi) + '&filter=name:' + u_comicquery + '&field_list=id,name,start_year,site_detail_url,count_of_issues,image,publisher,description&format=xml&page=' + str(offset) #offset=' + str(offset) # 2012/22/02 - CVAPI flipped back to offset instead of page + #all these imports are standard on most modern python implementations #download the file: try: @@ -59,8 +66,9 @@ def findComic(name, mode, issue, limityear=None): #print ("limityear: " + str(limityear)) if limityear is None: limityear = 'None' + comicquery = name #comicquery=name.replace(" ", "%20") - comicquery=name.replace(" ", " AND ") + #comicquery=name.replace(" ", " AND ") comicapi='583939a3df0a25fc4e8b7a29934a13078002dc27' offset = 1 @@ -68,15 +76,19 @@ def findComic(name, mode, issue, limityear=None): searched = pullsearch(comicapi,comicquery,1) if searched is None: return False totalResults = searched.getElementsByTagName('number_of_total_results')[0].firstChild.wholeText - #print ("there are " + str(totalResults) + " search results...") + logger.fdebug("there are " + str(totalResults) + " search results...") if not totalResults: return False countResults = 0 while (countResults < int(totalResults)): - #print ("querying " + str(countResults)) + #logger.fdebug("querying " + str(countResults)) if countResults > 0: #new api - have to change to page # instead of offset count offsetcount = (countResults/100) + 1 + + #2012/22/02 - CV API flipped back to offset usage instead of page :( + #if countResults == 1: offsetcount = 0 + #else: offsetcount = countResults searched = pullsearch(comicapi,comicquery,offsetcount) comicResults = searched.getElementsByTagName('volume') body = '' diff --git a/mylar/rsscheck.py b/mylar/rsscheck.py index 7d690e46..91c7372d 100755 --- a/mylar/rsscheck.py +++ b/mylar/rsscheck.py @@ -36,7 +36,7 @@ def tehMain(forcerss=None): #function for looping through nzbs/torrent feeds if mylar.ENABLE_TORRENTS: - logger.fdebug("[RSS] Initiating Torrent RSS Check.") + logger.fdebug('[RSS] Initiating Torrent RSS Check.') if mylar.ENABLE_KAT: logger.fdebug('[RSS] Initiating Torrent RSS Feed Check on KAT.') torrents(pickfeed='3') @@ -44,7 +44,7 @@ def tehMain(forcerss=None): logger.fdebug('[RSS] Initiating Torrent RSS Feed Check on CBT.') torrents(pickfeed='1') torrents(pickfeed='4') - logger.fdebug('RSS] Initiating RSS Feed Check for NZB Providers.') + logger.fdebug('[RSS] Initiating RSS Feed Check for NZB Providers.') nzbs() logger.fdebug('[RSS] RSS Feed Check/Update Complete') logger.fdebug('[RSS] Watchlist Check for new Releases') @@ -285,32 +285,37 @@ def nzbs(provider=None): if nonexp == "yes": #print str(ft) + " sites checked. There are " + str(totNum) + " entries to be updated." #print feedme - #i = 0 for ft in feedthis: + sitei = 0 site = ft['site'] logger.fdebug(str(site) + " now being updated...") + logger.fdebug('feedthis:' + str(ft)) for entry in ft['feed'].entries: if site == 'dognzb': #because the rss of dog doesn't carry the enclosure item, we'll use the newznab size value - if entry.attrib.get('name') == 'size': - tmpsz = entry.attrib.get('value') + tmpsz = 0 + #for attr in entry['newznab:attrib']: + # if attr('@name') == 'size': + # tmpsz = attr['@value'] + # logger.fdebug('size retrieved as ' + str(tmpsz)) + # break feeddata.append({ 'Site': site, - 'Title': ft['feed'].entries[i].title, - 'Link': ft['feed'].entries[i].link, - 'Pubdate': ft['feed'].entries[i].updated, + 'Title': entry.title, #ft['feed'].entries[i].title, + 'Link': entry.link, #ft['feed'].entries[i].link, + 'Pubdate': entry.updated, #ft['feed'].entries[i].updated, 'Size': tmpsz }) else: #this should work for all newznabs (nzb.su included) #only difference is the size of the file between this and above (which is probably the same) - tmpsz = ft['feed'].entries[i].enclosures[0] + tmpsz = entry.enclosures[0] #ft['feed'].entries[i].enclosures[0] feeddata.append({ 'Site': site, - 'Title': ft['feed'].entries[i].title, - 'Link': ft['feed'].entries[i].link, - 'Pubdate': ft['feed'].entries[i].updated, + 'Title': entry.title, #ft['feed'].entries[i].title, + 'Link': entry.link, #ft['feed'].entries[i].link, + 'Pubdate': entry.updated, #ft['feed'].entries[i].updated, 'Size': tmpsz['length'] }) @@ -319,9 +324,10 @@ def nzbs(provider=None): #logger.fdebug("Link: " + str(feeddata[i]['Link'])) #logger.fdebug("pubdate: " + str(feeddata[i]['Pubdate'])) #logger.fdebug("size: " + str(feeddata[i]['Size'])) - i+=1 - logger.info(str(site) + ' : ' + str(i) + ' entries indexed.') - + sitei+=1 + logger.info(str(site) + ' : ' + str(sitei) + ' entries indexed.') + i+=sitei + logger.info('[RSS] ' + str(i) + ' entries have been indexed and are now going to be stored for caching.') rssdbupdate(feeddata,i,'usenet') return @@ -446,7 +452,7 @@ def torrentdbsearch(seriesname,issue,comicid=None,nzbprov=None): titletemp = re.sub('cbr', '', str(titletemp)) titletemp = re.sub('cbz', '', str(titletemp)) titletemp = re.sub('none', '', str(titletemp)) - + if i == 0: rebuiltline = str(titletemp) else: @@ -465,13 +471,13 @@ def torrentdbsearch(seriesname,issue,comicid=None,nzbprov=None): seriesname_mod = re.sub('[\&]', ' ', seriesname_mod) foundname_mod = re.sub('[\&]', ' ', foundname_mod) - formatrem_seriesname = re.sub('[\'\!\@\#\$\%\:\;\=\?\.\-]', '',seriesname_mod) - formatrem_seriesname = re.sub('[\/]', '-', formatrem_seriesname) + formatrem_seriesname = re.sub('[\'\!\@\#\$\%\:\;\=\?\.\-\/]', '',seriesname_mod) + #formatrem_seriesname = re.sub('[\/]', '-', formatrem_seriesname) #not necessary since seriesname in a torrent file won't have / formatrem_seriesname = re.sub('\s+', ' ', formatrem_seriesname) if formatrem_seriesname[:1] == ' ': formatrem_seriesname = formatrem_seriesname[1:] - formatrem_torsplit = re.sub('[\'\!\@\#\$\%\:\;\\=\?\.\-]', '',foundname_mod) - formatrem_torsplit = re.sub('[\/]', '-', formatrem_torsplit) + formatrem_torsplit = re.sub('[\'\!\@\#\$\%\:\;\\=\?\.\-\/]', '',foundname_mod) + #formatrem_torsplit = re.sub('[\/]', '-', formatrem_torsplit) #not necessary since if has a /, should be removed in above line formatrem_torsplit = re.sub('\s+', ' ', formatrem_torsplit) logger.fdebug(str(len(formatrem_torsplit)) + ' - formatrem_torsplit : ' + formatrem_torsplit.lower()) logger.fdebug(str(len(formatrem_seriesname)) + ' - formatrem_seriesname :' + formatrem_seriesname.lower()) diff --git a/mylar/search.py b/mylar/search.py index 0d4a3088..52bb4492 100755 --- a/mylar/search.py +++ b/mylar/search.py @@ -33,12 +33,14 @@ import time import urlparse from xml.dom.minidom import parseString import urllib2 -from datetime import datetime +import email.utils +import datetime -def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, IssueDate, IssueID, AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=None, IssueArcID=None, mode=None, rsscheck=None, ComicID=None): +def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, IssueID, AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=None, IssueArcID=None, mode=None, rsscheck=None, ComicID=None): if ComicYear == None: ComicYear = '2014' else: ComicYear = str(ComicYear)[:4] - + if Publisher == 'IDW Publishing': Publisher = 'IDW' + logger.info('Publisher is : ' + str(Publisher)) if mode == 'want_ann': logger.info("Annual issue search detected. Appending to issue #") #anything for mode other than None indicates an annual. @@ -179,7 +181,7 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, IssueDate, IssueI torprov = 'KAT' if searchmode == 'rss': - findit = NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, torprov, torpr, IssDateFix, IssueID, UseFuzzy, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="yes", ComicID=ComicID) + findit = NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, torprov, torpr, IssDateFix, IssueID, UseFuzzy, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="yes", ComicID=ComicID) if findit == 'yes': logger.fdebug("findit = found!") break @@ -192,12 +194,12 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, IssueDate, IssueI for calt in chkthealt: AS_Alternate = re.sub('##','',calt) logger.info(u"Alternate Search pattern detected...re-adjusting to : " + str(AS_Alternate) + " " + str(ComicYear)) - findit = NZB_SEARCH(AS_Alternate, IssueNumber, ComicYear, SeriesYear, torprov, torp, IssDateFix, IssueID, UseFuzzy, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="yes", ComicID=ComicID) + findit = NZB_SEARCH(AS_Alternate, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, torprov, torp, IssDateFix, IssueID, UseFuzzy, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="yes", ComicID=ComicID) if findit == 'yes': break else: - findit = NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, torprov, torpr, IssDateFix, IssueID, UseFuzzy, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, ComicID=ComicID) + findit = NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, torprov, torpr, IssDateFix, IssueID, UseFuzzy, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, ComicID=ComicID) if findit == 'yes': logger.fdebug("findit = found!") break @@ -210,7 +212,7 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, IssueDate, IssueI for calt in chkthealt: AS_Alternate = re.sub('##','',calt) logger.info(u"Alternate Search pattern detected...re-adjusting to : " + str(AS_Alternate) + " " + str(ComicYear)) - findit = NZB_SEARCH(AS_Alternate, IssueNumber, ComicYear, SeriesYear, torprov, torp, IssDateFix, IssueID, UseFuzzy, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, ComicID=ComicID) + findit = NZB_SEARCH(AS_Alternate, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, torprov, torp, IssDateFix, IssueID, UseFuzzy, ComicVersion=ComicVersion, SARC=SARC, IssueArcID=IssueArcID, ComicID=ComicID) if findit == 'yes': break @@ -260,7 +262,7 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, IssueDate, IssueI #if it's rss - search both seriesname/alternates via rss then return. if searchmode == 'rss': if mylar.ENABLE_RSS: - findit = NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="yes", ComicID=ComicID) + findit = NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, nzbprov, nzbpr, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="yes", ComicID=ComicID) if findit == 'yes': logger.fdebug("Found via RSS.") break @@ -273,7 +275,7 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, IssueDate, IssueI for calt in chkthealt: AS_Alternate = re.sub('##','',calt) logger.info(u"Alternate Search pattern detected...re-adjusting to : " + str(AS_Alternate) + " " + str(ComicYear)) - findit = NZB_SEARCH(AS_Alternate, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="yes", ComicID=ComicID) + findit = NZB_SEARCH(AS_Alternate, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, nzbprov, nzbpr, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="yes", ComicID=ComicID) if findit == 'yes': break if findit == 'yes': @@ -284,7 +286,7 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, IssueDate, IssueI break else: #normal api-search here. - findit = NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion, SARC=SARC, IssueArcID=IssueArcID, ComicID=ComicID) + findit = NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, nzbprov, nzbpr, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion, SARC=SARC, IssueArcID=IssueArcID, ComicID=ComicID) if findit == 'yes': logger.fdebug("Found via API.") break @@ -296,7 +298,7 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, IssueDate, IssueI for calt in chkthealt: AS_Alternate = re.sub('##','',calt) logger.info(u"Alternate Search pattern detected...re-adjusting to : " + str(AS_Alternate) + " " + str(ComicYear)) - findit = NZB_SEARCH(AS_Alternate, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion, SARC=SARC, IssueArcID=IssueArcID, ComicID=ComicID) + findit = NZB_SEARCH(AS_Alternate, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, nzbprov, nzbpr, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion, SARC=SARC, IssueArcID=IssueArcID, ComicID=ComicID) if findit == 'yes': break if findit == 'yes': @@ -312,7 +314,7 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, IssueDate, IssueI nzbprov = nzbprovider[nzbpr] if searchmode == 'rss': if mylar.ENABLE_RSS: - findit = NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS='yes', ComicID=ComicID) + findit = NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, nzbprov, nzbpr, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS='yes', ComicID=ComicID) if findit == 'yes': logger.fdebug("Found via RSS on " + nzbprov) break @@ -324,7 +326,7 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, IssueDate, IssueI for calt in chkthealt: AS_Alternate = re.sub('##','',calt) logger.info(u"Alternate Search pattern detected...re-adjusting to : " + str(AS_Alternate) + " " + str(ComicYear)) - findit = NZB_SEARCH(AS_Alternate, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="yes", ComicID=ComicID) + findit = NZB_SEARCH(AS_Alternate, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, nzbprov, nzbpr, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion, SARC=SARC, IssueArcID=IssueArcID, RSS="yes", ComicID=ComicID) if findit == 'yes': logger.fdebug("Found via RSS Alternate Naming on " + nzbprov) break @@ -333,7 +335,7 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, IssueDate, IssueI break else: #normal api-search here. - findit = NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion, SARC=SARC, IssueArcID=IssueArcID, ComicID=ComicID) + findit = NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, nzbprov, nzbpr, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion, SARC=SARC, IssueArcID=IssueArcID, ComicID=ComicID) if findit == 'yes': logger.fdebug("Found via API on " + nzbprov) break @@ -344,7 +346,7 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, IssueDate, IssueI for calt in chkthealt: AS_Alternate = re.sub('##','',calt) logger.info(u"Alternate Search pattern detected...re-adjusting to : " + str(AS_Alternate)) - findit = NZB_SEARCH(AS_Alternate, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion, SARC=SARC, IssueArcID=IssueArcID, ComicID=ComicID) + findit = NZB_SEARCH(AS_Alternate, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, nzbprov, nzbpr, IssDateFix, IssueID, UseFuzzy, newznab_host, ComicVersion, SARC=SARC, IssueArcID=IssueArcID, ComicID=ComicID) if findit == 'yes': break if findit == 'yes': @@ -362,7 +364,7 @@ def search_init(ComicName, IssueNumber, ComicYear, SeriesYear, IssueDate, IssueI return findit, nzbprov -def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, IssDateFix, IssueID, UseFuzzy, newznab_host=None, ComicVersion=None, SARC=None, IssueArcID=None, RSS=None, ComicID=None): +def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, Publisher, IssueDate, StoreDate, nzbprov, nzbpr, IssDateFix, IssueID, UseFuzzy, newznab_host=None, ComicVersion=None, SARC=None, IssueArcID=None, RSS=None, ComicID=None): if nzbprov == 'nzb.su': apikey = mylar.NZBSU_APIKEY @@ -586,9 +588,9 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is #if bb is not None: logger.fdebug("results: " + str(bb)) elif nzbprov != 'experimental': if nzbprov == 'dognzb': - findurl = "http://dognzb.cr/api?t=search&q=" + str(comsearch) + "&o=xml&cat=7030&apikey=" + str(mylar.DOGNZB_APIKEY) + findurl = "https://dognzb.cr/api?t=search&q=" + str(comsearch) + "&o=xml&cat=7030" elif nzbprov == 'nzb.su': - findurl = "https://nzb.su/api?t=search&q=" + str(comsearch) + "&o=xml&cat=7030&apikey=" + str(mylar.NZBSU_APIKEY) + findurl = "https://nzb.su/api?t=search&q=" + str(comsearch) + "&o=xml&cat=7030" elif nzbprov == 'newznab': #let's make sure the host has a '/' at the end, if not add it. if host_newznab[len(host_newznab)-1:len(host_newznab)] != '/': @@ -601,13 +603,14 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is # helper function to replace apikey here so we avoid logging it ;) findurl = findurl + "&apikey=" + str(apikey) logsearch = helpers.apiremove(str(findurl),'nzb') - logger.fdebug("search-url: " + str(logsearch)) ### IF USENET_RETENTION is set, honour it ### For newznab sites, that means appending "&maxage=" on the URL if mylar.USENET_RETENTION != None: findurl = findurl + "&maxage=" + str(mylar.USENET_RETENTION) + logger.fdebug("search-url: " + str(findurl)) + # Add a user-agent #print ("user-agent:" + str(mylar.USER_AGENT)) request = urllib2.Request(findurl) @@ -646,12 +649,19 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is except Exception, e: logger.warn('Error fetching data from %s: %s' % (nzbprov, e)) data = False - + #logger.info('data: ' + data) if data: bb = feedparser.parse(data) else: bb = "no results" - + #logger.info('Search results:' + str(bb)) + try: + if bb['feed']['error']: + logger.error('[ERROR CODE: ' + str(bb['feed']['error']['code']) + '] ' + str(bb['feed']['error']['description'])) + bb = "no results" + except: + #logger.info('no errors on data retrieval...proceeding') + pass elif nzbprov == 'experimental': #bb = parseit.MysterBinScrape(comsearch[findloop], comyear) bb = findcomicfeed.Startit(u_ComicName, isssearch, comyear, ComicVersion, IssDateFix) @@ -667,7 +677,7 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is else: for entry in bb['entries']: logger.fdebug("checking search result: " + entry['title']) - if nzbprov != "experimental" and nzbprov != "CBT": + if nzbprov != "experimental" and nzbprov != "CBT" and nzbprov != "dognzb": if RSS == "yes": comsize_b = entry['length'] else: @@ -698,11 +708,67 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is logger.fdebug("Failure to meet the Maximium size threshold - skipping") continue +#---- date constaints. + # if the posting date is prior to the publication date, dump it and save the time. + #logger.info('entry' + str(entry)) + if nzbprov == 'experimental' or nzbprov =='CBT': + pubdate = entry['pubdate'] + else: + try: + pubdate = entry['updated'] + except: + try: + pubdate = entry['pubdate'] + except: + logger.fdebug('invalid date found. Unable to continue - skipping result.') + continue + #use store date instead of publication date for comparisons since publication date is usually +2 months + if StoreDate is None or StoreDate is '0000-00-00': + stdate = IssueDate + else: + stdate = StoreDate + #logger.fdebug('Posting date of : ' + str(pubdate)) + # convert it to a tuple + dateconv = email.utils.parsedate_tz(pubdate) + # convert it to a numeric time, then subtract the timezone difference (+/- GMT) + postdate_int = time.mktime(dateconv[:len(dateconv)-1]) - dateconv[-1] + #logger.fdebug('Issue date of : ' + str(stdate)) + #convert it to a Thu, 06 Feb 2014 00:00:00 format + issue_convert = datetime.datetime.strptime(stdate.rstrip(), '%Y-%m-%d') + #logger.fdebug('issue_convert:' + str(issue_convert)) + issconv = issue_convert.strftime('%a, %d %b %Y %H:%M:%S') + #logger.fdebug('issue date is :' + str(issconv)) + #convert it to a tuple + econv = email.utils.parsedate_tz(issconv) + #logger.fdebug('econv:' + str(econv)) + #convert it to a numeric + issuedate_int = time.mktime(econv[:len(econv)-1]) + if postdate_int < issuedate_int: + logger.fdebug(str(pubdate) + ' is before store date of ' + str(stdate) + '. Ignoring search result as this is not the right issue.') + continue + else: + logger.fdebug(str(pubdate) + ' is after store date of ' + str(stdate)) + # -- end size constaints. + thisentry = entry['title'] logger.fdebug("Entry: " + thisentry) - cleantitle = re.sub('[\_\.]', ' ', entry['title']) + cleantitle = thisentry + + #remove the extension. + extensions = ('.cbr', '.cbz') + if cleantitle.lower().endswith(extensions): + fd, ext = os.path.splitext(cleantitle) + logger.fdebug("Removed extension from filename: " + ext) + #name = re.sub(str(ext), '', str(subname)) + cleantitle = fd + + if 'mixed format' in cleantitle.lower(): + cleantitle = re.sub('mixed format', '', cleantitle).strip() + logger.fdebug('removed extra information after issue # that is not necessary: ' + str(cleantitle)) + + cleantitle = re.sub('[\_\.]', ' ', cleantitle) cleantitle = helpers.cleanName(cleantitle) # this is new - if title contains a '&' in the title it will assume the filename has ended at that point # which causes false positives (ie. wolverine & the x-men becomes the x-men, which matches on x-men. @@ -733,29 +799,30 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is ComVersChk = 0 else: ComVersChk = 0 + + ctchk = cleantitle.split() + for ct in ctchk: + if ct.lower().startswith('v') and ct[1:].isdigit(): + logger.fdebug("possible versioning..checking") + #we hit a versioning # - account for it + if ct[1:].isdigit(): + if len(ct[1:]) == 4: #v2013 + logger.fdebug("Version detected as " + str(ct)) + vers4year = "yes" #re.sub("[^0-9]", " ", str(ct)) #remove the v + #cleantitle = re.sub(ct, "(" + str(vers4year) + ")", cleantitle) + #logger.fdebug("volumized cleantitle : " + cleantitle) + break + else: + if len(ct) < 4: + logger.fdebug("Version detected as " + str(ct)) + vers4vol = str(ct) + break + logger.fdebug("false version detection..ignoring.") + if len(re.findall('[^()]+', cleantitle)) == 1 or 'cover only' in cleantitle.lower(): #some sites don't have (2013) or whatever..just v2 / v2013. Let's adjust: #this handles when there is NO YEAR present in the title, otherwise versioning is way below. - ctchk = cleantitle.split() - for ct in ctchk: - if ct.lower().startswith('v') and ct[1:].isdigit(): - logger.fdebug("possible versioning..checking") - #we hit a versioning # - account for it - if ct[1:].isdigit(): - if len(ct[1:]) == 4: #v2013 - logger.fdebug("Version detected as " + str(ct)) - vers4year = "yes" #re.sub("[^0-9]", " ", str(ct)) #remove the v - #cleantitle = re.sub(ct, "(" + str(vers4year) + ")", cleantitle) - #logger.fdebug("volumized cleantitle : " + cleantitle) - break - else: - if len(ct) < 4: - logger.fdebug("Version detected as " + str(ct)) - vers4vol = str(ct) - break - logger.fdebug("false version detection..ignoring.") - if vers4year == "no" and vers4vol == "no": # if the series is a v1, let's remove the requirements for year and volume label # even if it's a v1, the nzbname might not contain a valid year format (20xx) or v3, @@ -767,7 +834,6 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is if len(re.findall('[^()]+', cleantitle)): logger.fdebug("detected invalid nzb filename - attempting to detect year to continue") cleantitle = re.sub('(.*)\s+(19\d{2}|20\d{2})(.*)', '\\1 (\\2) \\3', cleantitle) - continue else: logger.fdebug("invalid nzb and/or cover only - skipping.") cleantitle = "abcdefghijk 0 (1901).cbz" @@ -793,6 +859,7 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is #print ("there are " + str(lenm) + " words.") cnt = 0 yearmatch = "false" + pub_removed = None while (cnt < lenm): if m[cnt] is None: break @@ -802,6 +869,9 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is logger.fdebug(str(cnt) + ". Bracket Word: " + str(m[cnt])) if cnt == 0: comic_andiss = m[cnt] + if 'mixed format' in comic_andiss.lower(): + comic_andiss = re.sub('mixed format', '', comic_andiss).strip() + logger.fdebug('removed extra information after issue # that is not necessary: ' + str(comic_andiss)) logger.fdebug("Comic: " + str(comic_andiss)) logger.fdebug("UseFuzzy is : " + str(UseFuzzy)) logger.fdebug('ComVersChk : ' + str(ComVersChk)) @@ -844,6 +914,25 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is elif UseFuzzy == "1": yearmatch = "true" + if Publisher.lower() in m[cnt].lower() and cnt >= 1: + #if the Publisher is given within the title or filename even (for some reason, some people + #have this to distinguish different titles), let's remove it entirely. + logger.fdebug('Publisher detected within title : ' + str(m[cnt])) + logger.fdebug('cnt is : ' + str(cnt) + ' --- Publisher is: ' + Publisher) + pub_removed = m[cnt] + #-strip publisher if exists here- + logger.fdebug('removing publisher from title') + cleantitle_pubremoved = re.sub(pub_removed, '', cleantitle) + logger.fdebug('pubremoved : ' + str(cleantitle_pubremoved)) + cleantitle_pubremoved = re.sub('\(\)', '', cleantitle_pubremoved) #remove empty brackets + cleantitle_pubremoved = re.sub('\s+', ' ', cleantitle_pubremoved) #remove spaces > 1 + logger.fdebug('blank brackets removed: ' + str(cleantitle_pubremoved)) + #reset the values to initial without the publisher in the title + m = re.findall('[^()]+', cleantitle_pubremoved) + lenm = len(m) + cnt = 0 + yearmatch = "false" + continue if 'digital' in m[cnt] and len(m[cnt]) == 7: logger.fdebug("digital edition detected") pass @@ -1161,7 +1250,7 @@ def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr, Is #blackhole functinality--- #let's download the file to a temporary cache. sent_to = None - if mylar.BLACKHOLE and nzbprov != 'CBT' and nzbprov != 'KAT': + if mylar.USE_BLACKHOLE and nzbprov != 'CBT' and nzbprov != 'KAT': logger.fdebug("using blackhole directory at : " + str(mylar.BLACKHOLE_DIR)) if os.path.exists(mylar.BLACKHOLE_DIR): #pretty this biatch up. @@ -1378,6 +1467,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None): 'IssueID': iss['IssueID'], 'Issue_Number': iss['Issue_Number'], 'IssueDate': iss['IssueDate'], + 'StoreDate': iss['ReleaseDate'], 'mode': 'want' }) elif stloop == 2: @@ -1387,6 +1477,7 @@ def searchforissue(issueid=None, new=False, rsscheck=None): 'IssueID': iss['IssueID'], 'Issue_Number': iss['Issue_Number'], 'IssueDate': iss['IssueDate'], + 'StoreDate': iss['ReleaseDate'], #need to replace with Store date 'mode': 'want_ann' }) stloop-=1 @@ -1397,8 +1488,10 @@ def searchforissue(issueid=None, new=False, rsscheck=None): comic = myDB.action("SELECT * from comics WHERE ComicID=? AND ComicName != 'None'", [result['ComicID']]).fetchone() foundNZB = "none" SeriesYear = comic['ComicYear'] + Publisher = comic['ComicPublisher'] AlternateSearch = comic['AlternateSearch'] IssueDate = result['IssueDate'] + StoreDate = result['StoreDate'] UseFuzzy = comic['UseFuzzy'] ComicVersion = comic['ComicVersion'] if result['IssueDate'] == None: @@ -1406,8 +1499,8 @@ def searchforissue(issueid=None, new=False, rsscheck=None): else: ComicYear = str(result['IssueDate'])[:4] mode = result['mode'] - if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL or mylar.NEWZNAB or mylar.NZBX or mylar.ENABLE_KAT or mylar.ENABLE_CBT) and (mylar.USE_SABNZBD or mylar.USE_NZBGET or mylar.ENABLE_TORRENTS): - foundNZB, prov = search_init(comic['ComicName'], result['Issue_Number'], str(ComicYear), comic['ComicYear'], IssueDate, result['IssueID'], AlternateSearch, UseFuzzy, ComicVersion, SARC=None, IssueArcID=None, mode=mode, rsscheck=rsscheck, ComicID=result['ComicID']) + if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL or mylar.NEWZNAB or mylar.NZBX or mylar.ENABLE_KAT or mylar.ENABLE_CBT) and (mylar.USE_SABNZBD or mylar.USE_NZBGET or mylar.ENABLE_TORRENTS or mylar.USE_BLACKHOLE): + foundNZB, prov = search_init(comic['ComicName'], result['Issue_Number'], str(ComicYear), comic['ComicYear'], Publisher, IssueDate, StoreDate, result['IssueID'], AlternateSearch, UseFuzzy, ComicVersion, SARC=None, IssueArcID=None, mode=mode, rsscheck=rsscheck, ComicID=result['ComicID']) if foundNZB == "yes": #print ("found!") updater.foundsearch(result['ComicID'], result['IssueID'], mode=mode, provider=prov) @@ -1426,8 +1519,10 @@ def searchforissue(issueid=None, new=False, rsscheck=None): ComicID = result['ComicID'] comic = myDB.action('SELECT * FROM comics where ComicID=?', [ComicID]).fetchone() SeriesYear = comic['ComicYear'] + Publisher = comic['ComicPublisher'] AlternateSearch = comic['AlternateSearch'] IssueDate = result['IssueDate'] + StoreDate = result['ReleaseDate'] UseFuzzy = comic['UseFuzzy'] ComicVersion = comic['ComicVersion'] if result['IssueDate'] == None: @@ -1436,10 +1531,10 @@ def searchforissue(issueid=None, new=False, rsscheck=None): IssueYear = str(result['IssueDate'])[:4] foundNZB = "none" - if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL or mylar.NEWZNAB or mylar.NZBX) and (mylar.USE_SABNZBD or mylar.USE_NZBGET): - foundNZB, prov = search_init(result['ComicName'], result['Issue_Number'], str(IssueYear), comic['ComicYear'], IssueDate, result['IssueID'], AlternateSearch, UseFuzzy, ComicVersion, mode=mode, ComicID=ComicID) + if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL or mylar.NEWZNAB or mylar.NZBX or mylar.ENABLE_KAT or mylar.ENABLE_CBT) and (mylar.USE_SABNZBD or mylar.USE_NZBGET or mylar.ENABLE_TORRENTS or mylar.USE_BLACKHOLE): + foundNZB, prov = search_init(comic['ComicName'], result['Issue_Number'], str(IssueYear), comic['ComicYear'], Publisher, IssueDate, StoreDate, result['IssueID'], AlternateSearch, UseFuzzy, ComicVersion, SARC=None, IssueArcID=None, mode=mode, rsscheck=rsscheck, ComicID=result['ComicID']) if foundNZB == "yes": - logger.fdebug("I found " + result['ComicName'] + ' #:' + str(result['Issue_Number'])) + logger.fdebug("I found " + comic['ComicName'] + ' #:' + str(result['Issue_Number'])) updater.foundsearch(ComicID=result['ComicID'], IssueID=result['IssueID'], mode=mode, provider=prov) else: pass @@ -1462,14 +1557,15 @@ def searchIssueIDList(issuelist): foundNZB = "none" SeriesYear = comic['ComicYear'] AlternateSearch = comic['AlternateSearch'] + Publisher = comic['ComicPublisher'] UseFuzzy = comic['UseFuzzy'] ComicVersion = comic['ComicVersion'] if issue['IssueDate'] == None: IssueYear = comic['ComicYear'] else: IssueYear = str(issue['IssueDate'])[:4] - if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL or mylar.NEWZNAB or mylar.NZBX or mylar.ENABLE_CBT or mylar.ENABLE_KAT) and (mylar.USE_SABNZBD or mylar.USE_NZBGET or mylar.ENABLE_TORRENTS): - foundNZB, prov = search_init(comic['ComicName'], issue['Issue_Number'], str(IssueYear), comic['ComicYear'], issue['IssueDate'], issue['IssueID'], AlternateSearch, UseFuzzy, ComicVersion, SARC=None, IssueArcID=None, mode=mode, ComicID=issue['ComicID']) + if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL or mylar.NEWZNAB or mylar.NZBX or mylar.ENABLE_CBT or mylar.ENABLE_KAT) and (mylar.USE_SABNZBD or mylar.USE_NZBGET or mylar.ENABLE_TORRENTS or mylar.USE_BLACKHOLE): + foundNZB, prov = search_init(comic['ComicName'], issue['Issue_Number'], str(IssueYear), comic['ComicYear'], Publisher, issue['IssueDate'], issue['ReleaseDate'], issue['IssueID'], AlternateSearch, UseFuzzy, ComicVersion, SARC=None, IssueArcID=None, mode=mode, ComicID=issue['ComicID']) if foundNZB == "yes": #print ("found!") updater.foundsearch(ComicID=issue['ComicID'], IssueID=issue['IssueID'], mode=mode, provider=prov) diff --git a/mylar/solicit.py b/mylar/solicit.py index fb9f485a..d96cd411 100755 --- a/mylar/solicit.py +++ b/mylar/solicit.py @@ -117,6 +117,11 @@ def solicit(month, year): connection = sqlite3.connect(str(mylardb)) cursor = connection.cursor() + # we should extract the issues that are being watched, but no data is available yet ('Watch For' status) + # once we get the data, store it, wipe the existing table, retrieve the new data, populate the data into + # the table, recheck the series against the current watchlist and then restore the Watch For data. + + cursor.executescript('drop table if exists future;') cursor.execute("CREATE TABLE IF NOT EXISTS future (SHIPDATE, PUBLISHER text, ISSUE text, COMIC VARCHAR(150), EXTRA text, STATUS text, FutureID text, ComicID text);") diff --git a/mylar/updater.py b/mylar/updater.py index b7040153..94fb1fcd 100755 --- a/mylar/updater.py +++ b/mylar/updater.py @@ -236,7 +236,7 @@ def upcoming_update(ComicID, ComicName, IssueNumber, IssueDate, forcecheck=None, elif issuechk['Status'] == "Wanted": logger.fdebug('...Status already Wanted .. not changing.') else: - logger.fdebug('...Already have issue - keeping existing status of : ' + issuechk['Status']) + logger.fdebug('...Already have issue - keeping existing status of : ' + str(issuechk['Status'])) if issuechk is None: myDB.upsert("upcoming", newValue, controlValue) @@ -271,6 +271,8 @@ def upcoming_update(ComicID, ComicName, IssueNumber, IssueDate, forcecheck=None, def weekly_update(ComicName,IssueNumber,CStatus,CID,futurepull=None): + logger.fdebug('weekly_update of table : ' + str(ComicName) + ' #:' + str(IssueNumber)) + logger.fdebug('weekly_update of table : ' + str(CStatus)) # here we update status of weekly table... # added Issue to stop false hits on series' that have multiple releases in a week # added CStatus to update status flags on Pullist screen @@ -459,9 +461,9 @@ def forceRescan(ComicID,archive=None): rescan = myDB.action('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone() logger.info('Now checking files for ' + rescan['ComicName'] + ' (' + str(rescan['ComicYear']) + ') in ' + rescan['ComicLocation'] ) if archive is None: - fc = filechecker.listFiles(dir=rescan['ComicLocation'], watchcomic=rescan['ComicName'], AlternateSearch=rescan['AlternateSearch']) + fc = filechecker.listFiles(dir=rescan['ComicLocation'], watchcomic=rescan['ComicName'], Publisher=rescan['ComicPublisher'], AlternateSearch=rescan['AlternateSearch']) else: - fc = filechecker.listFiles(dir=archive, watchcomic=rescan['ComicName'], AlternateSearch=rescan['AlternateSearch']) + fc = filechecker.listFiles(dir=archive, watchcomic=rescan['ComicName'], Publisher=rescan['ComicPublisher'], AlternateSearch=rescan['AlternateSearch']) iscnt = rescan['Total'] havefiles = 0 if mylar.ANNUALS_ON: diff --git a/mylar/webserve.py b/mylar/webserve.py index bf24c4ff..956d44a0 100755 --- a/mylar/webserve.py +++ b/mylar/webserve.py @@ -119,10 +119,13 @@ class WebInterface(object): for curResult in issues: baseissues = {'skipped':1,'wanted':2,'archived':3,'downloaded':4,'ignored':5} for seas in baseissues: - if seas in curResult['Status'].lower(): - sconv = baseissues[seas] - isCounts[sconv]+=1 - continue + if curResult['Status'] is None: + continue + else: + if seas in curResult['Status'].lower(): + sconv = baseissues[seas] + isCounts[sconv]+=1 + continue isCounts = { "Skipped" : str(isCounts[1]), "Wanted" : str(isCounts[2]), @@ -145,13 +148,27 @@ class WebInterface(object): } if mylar.ANNUALS_ON: annuals = myDB.select("SELECT * FROM annuals WHERE ComicID=?", [ComicID]) + #we need to load in the annual['ReleaseComicName'] and annual['ReleaseComicID'] + #then group by ReleaseComicID, in an attempt to create seperate tables for each different annual series. + #this should allow for annuals, specials, one-shots, etc all to be included if desired. + acnt = 0 + aName = [] + annualinfo = {} + for ann in annuals: + if not any(d.get('annualComicID', None) == str(ann['ReleaseComicID']) for d in aName): + aName.append({"annualComicName": ann['ReleaseComicName'], + "annualComicID" : ann['ReleaseComicID']}) + #logger.info('added : ' + str(ann['ReleaseComicID'])) + acnt+=1 + annualinfo = aName + #annualinfo['count'] = acnt else: annuals = None - return serve_template(templatename="comicdetails.html", title=comic['ComicName'], comic=comic, issues=issues, comicConfig=comicConfig, isCounts=isCounts, series=series, annuals=annuals) + return serve_template(templatename="comicdetails.html", title=comic['ComicName'], comic=comic, issues=issues, comicConfig=comicConfig, isCounts=isCounts, series=series, annuals=annuals, annualinfo=aName) comicDetails.exposed = True def searchit(self, name, issue=None, mode=None, type=None): if type is None: type = 'comic' # let's default this to comic search only for the time being (will add story arc, characters, etc later) - else: print (str(type) + " mode enabled.") + else: logger.fdebug(str(type) + " mode enabled.") #mode dictates type of search: # --series ... search for comicname displaying all results # --pullseries ... search for comicname displaying a limited # of results based on issue @@ -162,6 +179,12 @@ class WebInterface(object): if type == 'comic' and mode == 'pullseries': searchresults = mb.findComic(name, mode, issue=issue) elif type == 'comic' and mode == 'series': + if name.startswith('4050-'): + mismatch = "no" + comicid = re.sub('4050-','', name) + logger.info('Attempting to add directly by ComicVineID: ' + str(comicid) + '. I sure hope you know what you are doing.') + threading.Thread(target=importer.addComictoDB, args=[comicid,mismatch,None]).start() + raise cherrypy.HTTPRedirect("comicDetails?ComicID=%s" % comicid) searchresults = mb.findComic(name, mode, issue=None) elif type == 'comic' and mode == 'want': searchresults = mb.findComic(name, mode, issue) @@ -279,12 +302,8 @@ class WebInterface(object): mismatch = "no" logger.info('Attempting to add directly by ComicVineID: ' + str(comicid)) if comicid.startswith('4050-'): comicid = re.sub('4050-','', comicid) - comicname, year = importer.addComictoDB(comicid,mismatch) - if comicname is None: - logger.error('There was an error during the add, check the mylar.log file for futher details.') - else: - logger.info('Sucessfully added ' + comicname + ' (' + str(year) + ') to your watchlist') - raise cherrypy.HTTPRedirect("home") + threading.Thread(target=importer.addComictoDB, args=[comicid,mismatch,None]).start() + raise cherrypy.HTTPRedirect("comicDetails?ComicID=%s" % comicid) addbyid.exposed = True def wanted_Export(self): @@ -456,14 +475,33 @@ class WebInterface(object): #in order to update to JUST CV_ONLY, we need to delete the issues for a given series so it's a clea$ logger.fdebug("Gathering the status of all issues for the series.") issues = myDB.select('SELECT * FROM issues WHERE ComicID=?', [ComicID]) + + annload = [] #initiate the list here so we don't error out below. + if mylar.ANNUALS_ON: - issues += myDB.select('SELECT * FROM annuals WHERE ComicID=?', [ComicID]) + #now we load the annuals into memory to pass through to importer when refreshing so that it can + #refresh even the manually added annuals. + annual_load = myDB.select('SELECT * FROM annuals WHERE ComicID=?', [ComicID]) + logger.fdebug('checking annual db') + for annthis in annual_load: + if not any(d['ReleaseComicID'] == annthis['ReleaseComicID'] for d in annload): + #print 'matched on annual' + annload.append({ + 'ReleaseComicID': annthis['ReleaseComicID'], + 'ReleaseComicName': annthis['ReleaseComicName'], + 'ComicID': annthis['ComicID'], + 'ComicName': annthis['ComicName'] + }) + #print 'added annual' + issues += annual_load #myDB.select('SELECT * FROM annuals WHERE ComicID=?', [ComicID]) #store the issues' status for a given comicid, after deleting and readding, flip the status back to$ logger.fdebug("Deleting all issue data.") myDB.select('DELETE FROM issues WHERE ComicID=?', [ComicID]) myDB.select('DELETE FROM annuals WHERE ComicID=?', [ComicID]) logger.fdebug("Refreshing the series and pulling in new data using only CV.") - mylar.importer.addComictoDB(ComicID,mismatch,calledfrom='dbupdate') + mylar.importer.addComictoDB(ComicID,mismatch,calledfrom='dbupdate',annload=annload) + #reload the annuals here. + issues_new = myDB.select('SELECT * FROM issues WHERE ComicID=?', [ComicID]) annuals = [] ann_list = [] @@ -476,17 +514,37 @@ class WebInterface(object): icount = 0 for issue in issues: for issuenew in issues_new: + #logger.info('issuenew:' + str(issuenew['IssueID']) + ' : ' + str(issuenew['Status'])) + #logger.info('issuenew:' + str(issue['IssueID']) + ' : ' + str(issue['Status'])) if issuenew['IssueID'] == issue['IssueID'] and issuenew['Status'] != issue['Status']: - #if the status is now Downloaded/Snatched, keep status. - if issuenew['Status'] == 'Downloaded' or issue['Status'] == 'Snatched': - break - #change the status to the previous status - ctrlVAL = {'IssueID': issue['IssueID']} - newVAL = {'Status': issue['Status']} + ctrlVAL = {"IssueID": issue['IssueID']} + #if the status is None and the original status is either Downloaded / Archived, keep status & stats + if issuenew['Status'] == None and (issue['Status'] == 'Downloaded' or issue['Status'] == 'Archived'): + newVAL = {"Location": issue['Location'], + "ComicSize": issue['ComicSize'], + "Status": issue['Status']} + #if the status is now Downloaded/Snatched, keep status & stats (downloaded only) + elif issuenew['Status'] == 'Downloaded' or issue['Status'] == 'Snatched': + newVAL = {"Location": issue['Location'], + "ComicSize": issue['ComicSize']} + if issuenew['Status'] == 'Downloaded': + newVAL['Status'] = issuenew['Status'] + else: + newVAL['Status'] = issue['Status'] + + elif issue['Status'] == 'Archived': + newVAL = {"Status": issue['Status'], + "Location": issue['Location'], + "ComicSize": issue['ComicSize']} + else: + #change the status to the previous status + newVAL = {"Status": issue['Status']} + if any(d['IssueID'] == str(issue['IssueID']) for d in ann_list): logger.fdebug("annual detected for " + str(issue['IssueID']) + " #: " + str(issue['Issue_Number'])) myDB.upsert("Annuals", newVAL, ctrlVAL) else: + #logger.info('writing issuedata: ' + str(newVAL)) myDB.upsert("Issues", newVAL, ctrlVAL) icount+=1 break @@ -613,7 +671,7 @@ class WebInterface(object): controlValueDict = {"IssueArcID": IssueArcID} newStatus = {"Status": "Wanted"} myDB.upsert("readinglist", newStatus, controlValueDict) - foundcom, prov = search.search_init(ComicName=ComicName, IssueNumber=ComicIssue, ComicYear=ComicYear, SeriesYear=None, IssueDate=None, IssueID=None, AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=SARC, IssueArcID=IssueArcID) + foundcom, prov = search.search_init(ComicName=ComicName, IssueNumber=ComicIssue, ComicYear=ComicYear, SeriesYear=None, Publisher=None, IssueDate=None, StoreDate=None, IssueID=None, AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=SARC, IssueArcID=IssueArcID) if foundcom == "yes": logger.info(u"Downloaded " + ComicName + " #" + ComicIssue + " (" + str(ComicYear) + ")") #raise cherrypy.HTTPRedirect("readlist") @@ -627,7 +685,7 @@ class WebInterface(object): ComicYear = str(cyear['SHIPDATE'])[:4] if ComicYear == '': ComicYear = now.year logger.info(u"Marking " + ComicName + " " + ComicIssue + " as wanted...") - foundcom, prov = search.search_init(ComicName=ComicName, IssueNumber=ComicIssue, ComicYear=ComicYear, SeriesYear=None, IssueDate=cyear['SHIPDATE'], IssueID=None, AlternateSearch=None, UseFuzzy=None, ComicVersion=None) + foundcom, prov = search.search_init(ComicName=ComicName, IssueNumber=ComicIssue, ComicYear=ComicYear, SeriesYear=None, Publisher=None, IssueDate=cyear['SHIPDATE'], StoreDate=cyear['SHIPDATE'], IssueID=None, AlternateSearch=None, UseFuzzy=None, ComicVersion=None) if foundcom == "yes": logger.info(u"Downloaded " + ComicName + " " + ComicIssue ) raise cherrypy.HTTPRedirect("pullist") @@ -640,6 +698,7 @@ class WebInterface(object): if mode == 'want': logger.info(u"Marking " + ComicName + " issue: " + ComicIssue + " as wanted...") myDB.upsert("issues", newStatus, controlValueDict) + logger.info('Written to db.') else: logger.info(u"Marking " + ComicName + " Annual: " + ComicIssue + " as wanted...") myDB.upsert("annuals", newStatus, controlValueDict) @@ -650,18 +709,31 @@ class WebInterface(object): # newStatus = {"Status": "Wanted"} # myDB.upsert("issues", newStatus, controlValueDict) #for future reference, the year should default to current year (.datetime) + print 'before db' if mode == 'want': - issues = myDB.action("SELECT IssueDate FROM issues WHERE IssueID=?", [IssueID]).fetchone() + issues = myDB.action("SELECT IssueDate, ReleaseDate FROM issues WHERE IssueID=?", [IssueID]).fetchone() elif mode == 'want_ann': - issues = myDB.action("SELECT IssueDate FROM annuals WHERE IssueID=?", [IssueID]).fetchone() + issues = myDB.action("SELECT IssueDate, ReleaseDate FROM annuals WHERE IssueID=?", [IssueID]).fetchone() + print 'after db' if ComicYear == None: ComicYear = str(issues['IssueDate'])[:4] + print 'after year' + if issues['ReleaseDate'] is None: + logger.info('No Store Date found for given issue. This is probably due to not Refreshing the Series beforehand.') + logger.info('I Will assume IssueDate as Store Date, but you should probably Refresh the Series and try again if required.') + storedate = issues['IssueDate'] + else: + storedate = issues['ReleaseDate'] + print 'there' miy = myDB.action("SELECT * FROM comics WHERE ComicID=?", [ComicID]).fetchone() + print 'miy' SeriesYear = miy['ComicYear'] AlternateSearch = miy['AlternateSearch'] + Publisher = miy['ComicPublisher'] UseAFuzzy = miy['UseFuzzy'] ComicVersion = miy['ComicVersion'] - foundcom, prov = search.search_init(ComicName, ComicIssue, ComicYear, SeriesYear, issues['IssueDate'], IssueID, AlternateSearch, UseAFuzzy, ComicVersion, mode=mode, ComicID=ComicID) + print 'here' + foundcom, prov = search.search_init(ComicName, ComicIssue, ComicYear, SeriesYear, Publisher, issues['IssueDate'], storedate, IssueID, AlternateSearch, UseAFuzzy, ComicVersion, mode=mode, ComicID=ComicID) if foundcom == "yes": # file check to see if issue exists and update 'have' count if IssueID is not None: @@ -925,7 +997,7 @@ class WebInterface(object): timenow = datetime.datetime.now().strftime('%Y%m%d') #convert to yyyymmdd tmpdate = re.sub("[^0-9]", "", upc['IssueDate']) #convert date to numerics only (should be in yyyymmdd) - logger.fdebug('comparing pubdate of: ' + str(tmpdate) + ' to now date of: ' + str(timenow)) + #logger.fdebug('comparing pubdate of: ' + str(tmpdate) + ' to now date of: ' + str(timenow)) if int(tmpdate) >= int(timenow): if upc['Status'] == 'Wanted': @@ -1009,10 +1081,14 @@ class WebInterface(object): raise cherrypy.HTTPRedirect("comicDetails?ComicID=%s" % [comicid]) skipped2wanted.exposed = True - def annualDelete(self, comicid): + def annualDelete(self, comicid, ReleaseComicID=None): myDB = db.DBConnection() - myDB.action("DELETE FROM annuals WHERE ComicID=?", [comicid]) - logger.fdebug("Deleted all annuals from DB for ComicID of " + str(comicid)) + if ReleaseComicID is None: + myDB.action("DELETE FROM annuals WHERE ComicID=?", [comicid]) + logger.fdebug("Deleted all annuals from DB for ComicID of " + str(comicid)) + else: + myDB.action("DELETE FROM annuals WHERE ReleaseComicID=?", [ReleaseComicID]) + logger.fdebug("Deleted selected annual from DB with a ComicID of " + str(ReleaseComicID)) raise cherrypy.HTTPRedirect("comicDetails?ComicID=%s" % [comicid]) annualDelete.exposed = True @@ -1392,7 +1468,7 @@ class WebInterface(object): dstloc = os.path.join(mylar.DESTINATION_DIR, 'StoryArcs', arc['storyarc']) logger.fdebug('destination location set to : ' + dstloc) - filechk = filechecker.listFiles(dstloc, arc['ComicName'], sarc='true') + filechk = filechecker.listFiles(dstloc, arc['ComicName'], Publisher=None, sarc='true') fn = 0 fccnt = filechk['comiccount'] while (fn < fccnt): @@ -1505,14 +1581,14 @@ class WebInterface(object): logger.fdebug(want['ComicName'] + " -- #" + str(want['IssueNumber'])) logger.info(u"Story Arc : " + str(SARC) + " queueing selected issue...") logger.info(u"IssueArcID : " + str(IssueArcID)) - foundcom, prov = search.search_init(ComicName=want['ComicName'], IssueNumber=want['IssueNumber'], ComicYear=want['IssueYear'], SeriesYear=want['SeriesYear'], IssueDate=None, IssueID=None, AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=SARC, IssueArcID=IssueArcID, mode=None, rsscheck=None, ComicID=None) + foundcom, prov = search.search_init(ComicName=want['ComicName'], IssueNumber=want['IssueNumber'], ComicYear=want['IssueYear'], SeriesYear=want['SeriesYear'], Publisher=None, IssueDate=None, StoreDate=None, IssueID=None, AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=SARC, IssueArcID=IssueArcID, mode=None, rsscheck=None, ComicID=None) else: # it's a watched series s_comicid = issuechk['ComicID'] s_issueid = issuechk['IssueID'] logger.fdebug("-- watched series queue.") logger.fdebug(issuechk['ComicName'] + " -- #" + str(issuechk['Issue_Number'])) - foundcom, prov = search.search_init(ComicName=issuechk['ComicName'], IssueNumber=issuechk['Issue_Number'], ComicYear=issuechk['IssueYear'], SeriesYear=issuechk['SeriesYear'], IssueDate=None, IssueID=issuechk['IssueID'], AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=SARC, IssueArcID=IssueArcID) + foundcom, prov = search.search_init(ComicName=issuechk['ComicName'], IssueNumber=issuechk['Issue_Number'], ComicYear=issuechk['IssueYear'], SeriesYear=issuechk['SeriesYear'], Publisher=None, IssueDate=None, StoreDate=issuechk['ReleaseDate'], IssueID=issuechk['IssueID'], AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=SARC, IssueArcID=IssueArcID) if foundcom == "yes": print "sucessfully found." @@ -1539,14 +1615,14 @@ class WebInterface(object): logger.fdebug(watchchk['ComicName'] + " -- #" + str(watchchk['IssueNumber'])) logger.info(u"Story Arc : " + str(SARC) + " queueing selected issue...") logger.info(u"IssueArcID : " + str(IssueArcID)) - foundcom, prov = search.search_init(ComicName=watchchk['ComicName'], IssueNumber=watchchk['IssueNumber'], ComicYear=watchchk['IssueYEAR'], SeriesYear=watchchk['SeriesYear'], IssueDate=None, IssueID=None, AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=SARC, IssueArcID=IssueArcID, mode=None, rsscheck=None, ComicID=None) + foundcom, prov = search.search_init(ComicName=watchchk['ComicName'], IssueNumber=watchchk['IssueNumber'], ComicYear=watchchk['IssueYEAR'], SeriesYear=watchchk['SeriesYear'], Publisher=watchchk['ComicPublisher'], IssueDate=None, StoreDate=None, IssueID=None, AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=SARC, IssueArcID=IssueArcID, mode=None, rsscheck=None, ComicID=None) else: # it's a watched series s_comicid = issuechk['ComicID'] s_issueid = issuechk['IssueID'] logger.fdebug("-- watched series queue.") logger.fdebug(issuechk['ComicName'] + " -- #" + str(issuechk['Issue_Number'])) - foundcom,prov = search.search_init(ComicName=issuechk['ComicName'], IssueNumber=issuechk['Issue_Number'], ComicYear=issuechk['IssueYear'], SeriesYear=issuechk['SeriesYear'], IssueDate=None, IssueID=issuechk['IssueID'], AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=SARC, IssueArcID=IssueArcID, mode=None, rsscheck=None, ComicID=None) + foundcom,prov = search.search_init(ComicName=issuechk['ComicName'], IssueNumber=issuechk['Issue_Number'], ComicYear=issuechk['IssueYear'], SeriesYear=issuechk['SeriesYear'], Publisher=None, IssueDate=None, StoreDate=issuechk['ReleaseDate'], IssueID=issuechk['IssueID'], AlternateSearch=None, UseFuzzy=None, ComicVersion=None, SARC=SARC, IssueArcID=IssueArcID, mode=None, rsscheck=None, ComicID=None) if foundcom == "yes": print "sucessfully found." updater.foundsearch(s_comicid, s_issueid, mode=mode, provider=prov, SARC=SARC, IssueArcID=IssueArcID) @@ -2070,6 +2146,8 @@ class WebInterface(object): "http_user" : mylar.HTTP_USERNAME, "http_port" : mylar.HTTP_PORT, "http_pass" : mylar.HTTP_PASSWORD, + "api_enabled" : helpers.checked(mylar.API_ENABLED), + "api_key" : mylar.API_KEY, "launch_browser" : helpers.checked(mylar.LAUNCH_BROWSER), "logverbose" : helpers.checked(mylar.LOGVERBOSE), "download_scan_interval" : mylar.DOWNLOAD_SCAN_INTERVAL, @@ -2077,7 +2155,9 @@ class WebInterface(object): "nzb_startup_search" : helpers.checked(mylar.NZB_STARTUP_SEARCH), "libraryscan_interval" : mylar.LIBRARYSCAN_INTERVAL, "search_delay" : mylar.SEARCH_DELAY, - "use_sabnzbd" : helpers.checked(mylar.USE_SABNZBD), + "nzb_downloader_sabnzbd" : helpers.radio(int(mylar.NZB_DOWNLOADER), 0), + "nzb_downloader_nzbget" : helpers.radio(int(mylar.NZB_DOWNLOADER), 1), + "nzb_downloader_blackhole" : helpers.radio(int(mylar.NZB_DOWNLOADER), 2), "sab_host" : mylar.SAB_HOST, "sab_user" : mylar.SAB_USERNAME, "sab_api" : mylar.SAB_APIKEY, @@ -2085,14 +2165,13 @@ class WebInterface(object): "sab_cat" : mylar.SAB_CATEGORY, "sab_priority" : mylar.SAB_PRIORITY, "sab_directory" : mylar.SAB_DIRECTORY, - "use_nzbget" : helpers.checked(mylar.USE_NZBGET), "nzbget_host" : mylar.NZBGET_HOST, "nzbget_port" : mylar.NZBGET_PORT, "nzbget_user" : mylar.NZBGET_USERNAME, "nzbget_pass" : mylar.NZBGET_PASSWORD, "nzbget_cat" : mylar.NZBGET_CATEGORY, "nzbget_priority" : mylar.NZBGET_PRIORITY, - "use_blackhole" : helpers.checked(mylar.BLACKHOLE), + "nzbget_directory" : mylar.NZBGET_DIRECTORY, "blackhole_dir" : mylar.BLACKHOLE_DIR, "usenet_retention" : mylar.USENET_RETENTION, "use_nzbsu" : helpers.checked(mylar.NZBSU), @@ -2217,6 +2296,22 @@ class WebInterface(object): error_change.exposed = True + def manual_annual_add(self, manual_comicid, comicname, comicyear, comicid, x=None, y=None): + import urllib + b = urllib.unquote_plus(comicname) + cname = b.encode('utf-8') + + print ('comicid to be attached : ' + str(manual_comicid)) + print ('comicname : ' + str(cname)) + print ('comicyear : ' + str(comicyear)) + print ('comicid : ' + str(comicid)) + issueid = manual_comicid + logger.fdebug(str(issueid) + ' added to series list as an Annual') + threading.Thread(target=importer.manualAnnual, args=[manual_comicid, comicname, comicyear, comicid]).start() + + raise cherrypy.HTTPRedirect("comicDetails?ComicID=%s" % comicid) + manual_annual_add.exposed = True + def comic_config(self, com_location, ComicID, alt_search=None, fuzzy_year=None, comic_version=None, force_continuing=None): myDB = db.DBConnection() #--- this is for multipe search terms............ @@ -2334,9 +2429,9 @@ class WebInterface(object): readOptions.exposed = True - def configUpdate(self, http_host='0.0.0.0', http_username=None, http_port=8090, http_password=None, launch_browser=0, logverbose=0, download_scan_interval=None, nzb_search_interval=None, nzb_startup_search=0, libraryscan_interval=None, - use_sabnzbd=0, sab_host=None, sab_username=None, sab_apikey=None, sab_password=None, sab_category=None, sab_priority=None, sab_directory=None, log_dir=None, log_level=0, blackhole=0, blackhole_dir=None, - use_nzbget=0, nzbget_host=None, nzbget_port=None, nzbget_username=None, nzbget_password=None, nzbget_category=None, nzbget_priority=None, + def configUpdate(self, http_host='0.0.0.0', http_username=None, http_port=8090, http_password=None, api_enabled=0, api_key=None, launch_browser=0, logverbose=0, download_scan_interval=None, nzb_search_interval=None, nzb_startup_search=0, libraryscan_interval=None, + nzb_downloader=0, sab_host=None, sab_username=None, sab_apikey=None, sab_password=None, sab_category=None, sab_priority=None, sab_directory=None, log_dir=None, log_level=0, blackhole_dir=None, + nzbget_host=None, nzbget_port=None, nzbget_username=None, nzbget_password=None, nzbget_category=None, nzbget_priority=None, nzbget_directory=None, usenet_retention=None, nzbsu=0, nzbsu_uid=None, nzbsu_apikey=None, dognzb=0, dognzb_uid=None, dognzb_apikey=None, nzbx=0, newznab=0, newznab_host=None, newznab_name=None, newznab_apikey=None, newznab_uid=None, newznab_enabled=0, raw=0, raw_provider=None, raw_username=None, raw_password=None, raw_groups=None, experimental=0, enable_meta=0, cmtagger_path=None, enable_rss=0, rss_checkinterval=None, enable_torrent_search=0, enable_kat=0, enable_cbt=0, cbt_passkey=None, @@ -2348,6 +2443,8 @@ class WebInterface(object): mylar.HTTP_PORT = http_port mylar.HTTP_USERNAME = http_username mylar.HTTP_PASSWORD = http_password + mylar.API_ENABLED = api_enabled + mylar.API_KEY = api_key mylar.LAUNCH_BROWSER = launch_browser mylar.LOGVERBOSE = logverbose mylar.DOWNLOAD_SCAN_INTERVAL = download_scan_interval @@ -2355,7 +2452,7 @@ class WebInterface(object): mylar.NZB_STARTUP_SEARCH = nzb_startup_search mylar.LIBRARYSCAN_INTERVAL = libraryscan_interval mylar.SEARCH_DELAY = search_delay - mylar.USE_SABNZBD = use_sabnzbd + mylar.NZB_DOWNLOADER = int(nzb_downloader) mylar.SAB_HOST = sab_host mylar.SAB_USERNAME = sab_username mylar.SAB_PASSWORD = sab_password @@ -2363,14 +2460,13 @@ class WebInterface(object): mylar.SAB_CATEGORY = sab_category mylar.SAB_PRIORITY = sab_priority mylar.SAB_DIRECTORY = sab_directory - mylar.USE_NZBGET = use_nzbget mylar.NZBGET_HOST = nzbget_host mylar.NZBGET_USERNAME = nzbget_username mylar.NZBGET_PASSWORD = nzbget_password mylar.NZBGET_PORT = nzbget_port mylar.NZBGET_CATEGORY = nzbget_category mylar.NZBGET_PRIORITY = nzbget_priority - mylar.BLACKHOLE = blackhole + mylar.NZBGET_DIRECTORY = nzbget_directory mylar.BLACKHOLE_DIR = blackhole_dir mylar.USENET_RETENTION = usenet_retention mylar.NZBSU = nzbsu @@ -2510,7 +2606,14 @@ class WebInterface(object): mylar.CMTAGGER_PATH = re.sub(os.path.basename(mylar.CMTAGGER_PATH), '', mylar.CMTAGGER_PATH) logger.fdebug("Removed application name from ComicTagger path") + logger.info('nzb_downloader') + #legacy support of older config - reload into old values for consistency. + if mylar.NZB_DOWNLOADER == 0: mylar.USE_SABNZBD = True + elif mylar.NZB_DOWNLOADER == 1: mylar.USE_NZBGET = True + elif mylar.NZB_DOWNLOADER == 2: mylar.USE_BLACKHOLE = True + # Write the config + logger.info('sending to config..') mylar.config_write() raise cherrypy.HTTPRedirect("config") @@ -2600,4 +2703,29 @@ class WebInterface(object): return cache.getArtwork(ComicID, imageURL) getComicArtwork.exposed = True + + def generateAPI(self): + + import hashlib, random + + apikey = hashlib.sha224( str(random.getrandbits(256)) ).hexdigest()[0:32] + logger.info("New API generated") + mylar.API_KEY = apikey + return apikey + + generateAPI.exposed = True + def api(self, *args, **kwargs): + + from mylar.api import Api + + a = Api() + + a.checkParams(*args, **kwargs) + + data = a.fetchData() + + return data + + api.exposed = True + diff --git a/mylar/weeklypull.py b/mylar/weeklypull.py index 1be8d450..5636f538 100755 --- a/mylar/weeklypull.py +++ b/mylar/weeklypull.py @@ -405,11 +405,13 @@ def pullitcheck(comic1off_name=None,comic1off_id=None,forcecheck=None, futurepul ccname = [] pubdate = [] w = 0 + wc = 0 tot = 0 chkout = [] watchfnd = [] watchfndiss = [] watchfndextra = [] + alternate = [] #print ("----------WATCHLIST--------") a_list = [] @@ -432,7 +434,7 @@ def pullitcheck(comic1off_name=None,comic1off_id=None,forcecheck=None, futurepul w = 1 else: #let's read in the comic.watchlist from the db here - cur.execute("SELECT ComicID, ComicName, ComicYear, ComicPublisher, ComicPublished, LatestDate, ForceContinuing from comics") + cur.execute("SELECT ComicID, ComicName, ComicYear, ComicPublisher, ComicPublished, LatestDate, ForceContinuing, AlternateSearch from comics") while True: watchd = cur.fetchone() #print ("watchd: " + str(watchd)) @@ -467,16 +469,48 @@ def pullitcheck(comic1off_name=None,comic1off_id=None,forcecheck=None, futurepul b_list.append(watchd[2]) comicid.append(watchd[0]) pubdate.append(watchd[4]) - #print ( "Comic:" + str(a_list[w]) + " Year: " + str(b_list[w]) ) - #if "WOLVERINE AND THE X-MEN" in str(a_list[w]): a_list[w] = "WOLVERINE AND X-MEN" lines.append(a_list[w].strip()) unlines.append(a_list[w].strip()) - llen.append(a_list[w].splitlines()) - ccname.append(a_list[w].strip()) - tmpwords = a_list[w].split(None) - ltmpwords = len(tmpwords) - ltmp = 1 - w+=1 + w+=1 # we need to increment the count here, so we don't count the same comics twice (albeit with alternate names) + + #here we load in the alternate search names for a series and assign them the comicid and + #alternate names + Altload = helpers.LoadAlternateSearchNames(watchd[7], watchd[0]) + if Altload == 'no results': + pass + else: + wc = 0 + alt_cid = Altload['ComicID'] + n = 0 + iscnt = Altload['Count'] + while (n <= iscnt): + try: + altval = Altload['AlternateName'][n] + except IndexError: + break + cleanedname = altval['AlternateName'] + a_list.append(altval['AlternateName']) + b_list.append(watchd[2]) + comicid.append(alt_cid) + pubdate.append(watchd[4]) + lines.append(a_list[w+wc].strip()) + unlines.append(a_list[w+wc].strip()) + logger.info('loading in Alternate name for ' + str(cleanedname)) + n+=1 + wc+=1 + w+=wc + + #-- to be removed - + #print ( "Comic:" + str(a_list[w]) + " Year: " + str(b_list[w]) ) + #if "WOLVERINE AND THE X-MEN" in str(a_list[w]): a_list[w] = "WOLVERINE AND X-MEN" + #lines.append(a_list[w].strip()) + #unlines.append(a_list[w].strip()) + #llen.append(a_list[w].splitlines()) + #ccname.append(a_list[w].strip()) + #tmpwords = a_list[w].split(None) + #ltmpwords = len(tmpwords) + #ltmp = 1 + #-- end to be removed else: logger.fdebug("Determined to not be a Continuing series at this time.") cnt = int(w-1) @@ -561,6 +595,7 @@ def pullitcheck(comic1off_name=None,comic1off_id=None,forcecheck=None, futurepul modcomicnm = re.sub(r'\s', '', modcomicnm) logger.fdebug("watchcomic : " + str(watchcomic) + " / mod :" + str(modwatchcomic)) logger.fdebug("comicnm : " + str(comicnm) + " / mod :" + str(modcomicnm)) + if comicnm == watchcomic.upper() or modcomicnm == modwatchcomic.upper(): logger.fdebug("matched on:" + comicnm + "..." + watchcomic.upper()) pass @@ -569,6 +604,37 @@ def pullitcheck(comic1off_name=None,comic1off_id=None,forcecheck=None, futurepul # print ( row[3] + " matched on ANNUAL") else: break + +#this all needs to get redone, so the ability to compare issue dates can be done systematically. +#Everything below should be in it's own function - at least the callable sections - in doing so, we can +#then do comparisons when two titles of the same name exist and are by definition 'current'. Issue date comparisons +#would identify the difference between two #1 titles within the same series year, but have different publishing dates. +#Wolverine (2013) & Wolverine (2014) are good examples of this situation. +#of course initially, the issue data for the newer series wouldn't have any issue data associated with it so it would be +#a null value, but given that the 2013 series (as an exmaple) would be from 2013-05-01, it obviously wouldn't be a match to +#the current date & year (2014). Throwing out that, we could just assume that the 2014 would match the #1. + + #get the issue number of the 'weeklypull' series. + #load in the actual series issue number's store-date (not publishing date) + #---use a function to check db, then return the results in a tuple/list to avoid db locks. + #if the store-date is >= weeklypull-list date then continue processing below. + #if the store-date is <= weeklypull-list date then break. + ### week['ISSUE'] #issue # from pullist + ### week['SHIPDATE'] #weeklypull-list date + ### comicid[cnt] #comicid of matched series + datecheck = loaditup(watchcomic, comicid[cnt], week['ISSUE']) + logger.fdebug('Now checking date comparison using an issue store date of ' + str(datecheck)) + + if datecheck == 'no results': + pass + elif datecheck >= week['SHIPDATE']: + #logger.info('The issue date of issue #' + str(week['ISSUE']) + ' was on ' + str(datecheck) + ' which is on/ahead of ' + str(week['SHIPDATE'])) + logger.fdebug('Store Date falls within acceptable range - series MATCH') + pass + elif datecheck < week['SHIPDATE']: + logger.fdebug('The issue date of issue #' + str(week['ISSUE']) + ' was on ' + str(datecheck) + ' which is prior to ' + str(week['SHIPDATE'])) + break + if ("NA" not in week['ISSUE']) and ("HC" not in week['ISSUE']): if ("COMBO PACK" not in week['EXTRA']) and ("2ND PTG" not in week['EXTRA']) and ("3RD PTG" not in week['EXTRA']): otot+=1 @@ -635,3 +701,19 @@ def pullitcheck(comic1off_name=None,comic1off_id=None,forcecheck=None, futurepul def check(fname, txt): with open(fname) as dataf: return any(txt in line for line in dataf) + + +def loaditup(comicname, comicid, issue): + myDB = db.DBConnection() + issue_number = helpers.issuedigits(issue) + logger.fdebug('[' + comicname + '] trying to locate issue ' + str(issue) + ' to do comparitive issue analysis for pull-list') + issueload = myDB.action('SELECT * FROM issues WHERE ComicID=? AND Int_IssueNumber=?', [comicid, issue_number]).fetchone() + if issueload is None: + logger.fdebug('No results matched for Issue number - either this is a NEW series with no data yet, or something is wrong') + return 'no results' + if issueload['ReleaseDate'] is not None or issueload['ReleaseDate'] is not 'None': + logger.fdebug('Returning Release Date for issue # ' + str(issue) + ' of ' + str(issueload['ReleaseDate'])) + return issueload['ReleaseDate'] + else: + logger.fdebug('Returning Publication Date for issue # ' + str(issue) + ' of ' + str(issueload['PublicationDate'])) + return issueload['PublicationDate']
-
-
- -
-
+
+ Usenet + Sabnzbd NZBget Black Hole +
+
@@ -208,27 +222,27 @@
-
+
-
+
-
-
+
+
Where your SAB downloads go... (optional) -
+ -
+
- -
-
+ +
+
-
-
- Test SABnzbd -
+
+
+ Test SABnzbd +
- -
- -
-
+
+
@@ -268,6 +279,11 @@
+
+ + + Where your NZBGet downloads go... (optional) +
@@ -287,32 +303,23 @@
- - -
- 		- Usenet -
-
- -
-
+
+
Folder your Download program watches for NZBs
- -
-
+
+
-
+ 		Torrents
@@ -354,10 +361,9 @@
Folder path your torrent seedbox client watches
- +
-
${result['comicyear']} ${result['issues']} Add this Comic Add this Comic