From 6bc011f678dbdc7a587b93cd1c1350030d3882d2 Mon Sep 17 00:00:00 2001 From: JigSawFr Date: Sun, 3 Jan 2016 19:39:53 +0100 Subject: [PATCH] FNT Provider --- README.md | 1 + src/Jackett/Content/logos/fnt.png | Bin 0 -> 23049 bytes src/Jackett/Indexers/Fnt.cs | 811 ++++++++++++++++++ src/Jackett/Jackett.csproj | 7 +- .../Bespoke/ConfigurationDataFnt.cs | 63 ++ 5 files changed, 881 insertions(+), 1 deletion(-) create mode 100644 src/Jackett/Content/logos/fnt.png create mode 100644 src/Jackett/Indexers/Fnt.cs create mode 100644 src/Jackett/Models/IndexerConfig/Bespoke/ConfigurationDataFnt.cs diff --git a/README.md b/README.md index c8daae89a..98494179d 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ Developer note: The software implements the [Torznab](https://github.com/Sonarr/ * Demonoid * EuTorrents * FileList + * Fnt * Freshon * HD-Space * HD-Torrents diff --git a/src/Jackett/Content/logos/fnt.png b/src/Jackett/Content/logos/fnt.png new file mode 100644 index 0000000000000000000000000000000000000000..e552d2645a0916a9df1c9e38980d434f2d9bbeda GIT binary patch literal 23049 zcmeI4dpuO@`}gf?hY+Dij^mh&nPJ9^F~}*0BIgL%V`eZ;!x#oRq@qIP9EBp}SdK-Z zoF#{nRA`cfRFoVF`7Iso+3k7u_u0Sa_q?9xWnM34?scu}y4QN&*L~fed;PIa85`+x zY!cYS#Kgp*ucvJad|w9sFf1E^|L*NI2;Xyihlcm z)HVUXFl4+V+Ejbbx9)(JD#)2a@kB!)R4P@Tswht)J3(N|%E}O^0z^Rp44eV>_V=J* z{lFgH;$MS&kE2cS#*vAh6e7t(bR{mA3mNwWVbJuhQ~n&ZUlFN2gMukgZ3*5ms zBER(XHuv`=KuihVBp))4pyLZ%iTJNB-kV}d_zCwPgaS_gWgt&y5{2aLO!^yuKg@rN zkwSF*n+sX7{H+_okN7ukE0(WrUvlc}P-6hQ5 zFa#zM9!tUgDT*T090&nLpcSC2qx|Vy`Zy#!(b4}OMfrK=su*}2+L1(d$5PaY?pP-R z#M8qG1NnJy)tO(i7p?E%O~HEL2>RM;zyWz85sy|>P=Z4Vcsy7M22%nnD#0DV4h|?q zFhU8Z07oHUcofv}3l2lW)$V@@txdxDtV|#v^w(O0C*gqbKN1OzR6@ZV72!CrG66{d z!(api7>htCf>8tn5|40DKsrK|zjw4M*k3x@PbLDT5bL(;oRxdQ1Dz3o5C}!Q5*P`? zDuW&Ha1+gejlN>2jESaF;1Z2oRnZsYB{@IN_JO;9|Jh&16 z4-AAi*7x5H1D1e95U>siumcwA07hWpSTI%*2c#nb2gfTxQ8MDO=68(Sdp8xgCM4(Vu z1qT!YSoD>EnW+q{cgje-GMIou!r)3UC>*7T_@2LOGxO()^R?Fgi^c6W0i$|MYiM1qxJ$}lhr3v+}bk#Gk^N8GQa*w>?f?s4_)FuGpM$}SQOh;%?Oep!pv zU>L|>j;+G*#b&i4w%vTbh-st(6!<`3EC0W_t~S%VS~GTCZKikq>1p9Xq^K!;>y7U4 zDzBBT?$@r%G5sZ-eykb)oj-ow(SK)Dx>#$S&;eoG zT*Jkf51n-l7ab7B%{5$%`OsO{aM1x_++4%Om=B$G4Hq2{#?3WcjQP-6*KpARVccB9 z#h4GBbqyCC5XQ|lT#Wh9S=VsU0b$%+!^M~noplWt9T3LNHC&AO&{@}T(E(xHT*Jkf z51n-l7ab7B%{5$%`OsO{aM1x_++4%Om=B$G4Hq2{#?3WcjQP-6*KpARVccB9#h4GB zbqyCC5XQ|lT#Wh9S=VsU0b$%+!^M~noplWt9T3LNHC&AO&{@}T(E(xHT*Jkf51n-l z7ab7B%{5$%`OsO{aM1x_++4%Om=B$G4Hq2{#!ZH}HvN2S1Hl7$RRa}xCqqFAk_Nob zK@_KFYRJSCAj!mZvA z)}FSbDd`zyRHT4S);*@*Jma-pDkKgU>OE-INzjXak(y#|$Zg*$n`huFkGZDrAr-%` zBg$_0&Lh67hWHE|Q&VorQ(I1h-!4z9JV>M8WlRUClmX2VO7Rl!ewZjp4P> zv9Q2f?}@u48FHL!Xj&modkQ1UB&NMcGVGB_ZjWN;C?mGNJU6>uF4^G4=*w=OCGi6B z5Ii+=Zr>xa*iq@&SS}U`{ZEaq^1<>?kX_SU^}0!kF|#~VVYlXj*c<-vbLJ0y7RoGq z-G>#euHKcsH`&uJ*-UzW9MQD_$`15wVd0XapJ%%#mSD=kQBF``5zXjg?vvS+KFKq4 zOHN7}#wsactr<&@u->D2^h{y;WV})K1=%RS6zB#EsGC^O)fa6 zhI{b)6-N%E9=ISdUE+xb*nXwxVNreV{QTzu!s8TyLFgHA@~%y#nHOY5s&ZR7o>ZQm zs4m&j?(>5GdSKY}XxH?MoYNEvWguZg$LRFkkhO@=1 z+efI+$HXF$N7~3(ZrkH>VvnJIXIqreo(c>5-Q1!H%1*J9PBai}C*RDwSD|qR@_aN1 zm_9)X(8}MtebQX2>c`D;^(444v-879A?4iM+?@wx&#N3xYHr>`t`h8ZL9n>vm4nr9 zpB{5i*<3k%L3UFu7guv^tfZabEA6*7fiFJ13yYc3T(|e)4Nm?3)HcuTWd*)~!&P(X zhpozd-xTS0zkBDUkixE|)i7{)vgT3cX+p23Dzn&z>f)??m&LIFVGw1h*JW#;Pl@}E zu9-9Voko{I_wPhRftZuTOOw(^vHiYN0mYG{{)GoCA`~a<_LE2u&fsAU%R(!!o~E=! zgRr5CVt2B+?dVgEKC#X`5oT z)O*u=GZCPk?(SQm>zPx|pMOCMUg*D;a~G}#yFE1pU0w|N42d_3=dTVWrlLG{8zG`Z zg1VSBUB?!@e0Lt-ZPEbUeMDlf71q{;e`9LC*e-?q%;z74izhZd+~_IDY}ljOE~b~e z<7`@*P9H~lAbHqO$Ea(XUqr-duGsGN(+alsN|S~uU6TZ39q!QM5^b%e<&9f)Pl?CE z5{}NjW)q=JZ>i_J5-ImWw~tx4B~lWzbDt8y;^OtIyXE_Ci^pKd6I&>5Zc!qOGc?a} zQ<0B>>EYqwyJur1KtA{FFAar_BjdCeLs{&fEeIEnhC)Oa(;OlRLDjZ>B3NOkoX z_x{jU;qfGq{s&F(MqA2S_c!QX*?Kkt@~Ny;F5W2jipY8+!y!N6Y=?VahsI;~8prPN~FWusT3;d{1UY z#mt;XL-5jseHKw*;^7r13WZmdeM@Hd`rp{t*b)pR9~Rn0PA(K$m%WUCfjOL%vGgGq zvrt6xnt9c$Q<8R6(7+K~R3xbG8`YD$fBJnpWZL4mfsj2*P*4!N?e=H;mpd{_$-Viz zc{$TECG=b4w9UN5D^hjTGIULNKP@a&EzS1h+T;S#hFfB?#?j}lxeYz}fB=mS1qctS zstT$EBpe8t*%dt1AQDWNTgVX`3uEIu+umhToB*tCUhc{fiA?)?*atEoTpJ$OaINd} zIwP^YkFC^Sto~7gXfVfzYKb;Hw}l067vZ6c1UD=9sSWjf&f%TUZZQE}UChfH=W!Jj z5XhXY*YUWqs8`~hpVmEY%gv?ZZ)O71TQK6;lsl}R+b zskrI_M*;N56;K@HcJXdv zd*L)4_~!E!lOoG4nrHS&Df_?Y-61@9WGd*X%93y;0SKKwaX+bp{JxQOqO^EiCPMIrtzqv~ zw;lR%-3TGWK8FIEJac2)Om z2uDXpV+(L=Qr0<%yKZ4z*V7}pE=&k$vRD?5JwL~(+PE}FiCQ^`{>nmCaH7F^RPcH4(aw^fmd)%9EEEU*OtG})X^dRErs&w) zw^)b3sD=}pr;+N5x25k~wD&(MjP~O)?6azJ**Z8l7cbk35faa+PYhV!uSYwBx>8P{4#wpL1$x zVoK_H{L&|ts=3;Qb&=Z>Y@S!JY|!K?wrd=*D03lK)n#0~T>!sUenCd|jk~qw?1v4T zj*JfZ9qN7|)^=ofFT4_%?S-nU!LRv&a$o6|qz&3~#$=1vsRJybIYd$hZ8sVy-C$>B_3S~OS$Spo{odZD#l=9m!N8lDTj0K^l0$q5eY zt`jqG6@h9Z$I%yOw>RC5lYl8HHMh31P-)%}>VTJS+J#83`EIdIVe6=iw}tR!Z_YaU zoN^rZ*Z93*F37g;G}7NIOftVrx5zhrG%Q9fUm)RgW_0dY%w*)*o1>j+S;jI~aV^m{ zpy>+tb4v?z@%tp($_XXzDl#AMkh>uQ%_GGV_W_+YKPCfc70hzl?Hu(`Fc%Efax-bNIJ+p!Ig$=rz95Jb~G zZfAXvcOqOMf07N;r=tf|Qm8kxxkesMqXv7;1uvp{`ik!j_J-xl+`W5Oj=0%`7m;-H z{`f{9si!a@kB1t;d%MY1cTWvVJG-_6L$uYYDv1ClvPQ}g9)$7Q?Wj1b(<`E5a@0zm zos-x4H|+szofSK?v#aayhw&cvLp67zg;bn9JZj&fmq#JNp*iC#+Th}l7~HVQN7c6F zcBOl&&B2wGq&6~Wz<1$Ip-o-1n^OCO2fQO>W#0{ht6vw9U%JG(JZ0ABKig0SXqPG7kVRbz#n&O#v0K-rANWNcdB62g$Q#7x z!E1G=vHXjRi3hA|0yw(V=HxoSV0Mnp8nbT3Xc_4KR0?{`NMw9 z;#5P1wbYW_>&In$T1e*({8g>3;vmNlH-LM}%R2=qQK@EW|M6_TyZxr()=JGYgJ7^y}&P&`~!!vz{vaT9GyVoMDXKo`n2=S#9PtCeNBePDrXjnXutSNAPScs+71|{s3Okdyy z&rW;3yt^G(8i?*4XY}~ZWfLVk+MzevPU7s)|&6bZxELC&*k`z|5=bFF!A-FR0B8cq1XT^PuICuI2Ni_Vb%*^T$ZQ7>%IX z%3QW>vmA&tu=)c?J66LUb%gL|%*~p^R4!jQvWaX;!TF{0&2^%qwerjz0$fr7!FJ9! z;2AZiRTz*SuE2B|xPLT1|HZUFus;A+$EUY0m$QQI))O`|Gs9e^&LL@M_1ljzZwBcr z2nh+DwNUhpjUMiPFmU%-Zvi`J%vJY=O3AD`oAiJPtrMHQX??ZLwB@sXI^hz{EUi3 zBF*HJt9;)G^jEqsJa}7lrSNxEzcDT*p(?>W^(S5H?w!b;2yipZy2AGoc+`+8k=$M~ zbWgYL1LEbhzuNrNda}@)!m+nG#xl>sj~z}Thy`ZsI zY@5MYQhXIuRSiqbZGDTDlPWIMPL_o%2MY+?$3J;;Ii&){8ewb4d#(-G{eF}%+Zi6d zQOmVL_42ilHk9vc{t?rZS=;Dl3hun;<$MF#L{LVT_slD%v|W}{t>F{Ygk2H!`$J|G z6^e>(zE?kV(O8bjIL};WTO^In;Q*{(#8@&W&6AcD1 zSrZx#N&9m2`1f?5KqF%^W!mL)pO$V=^?xrB&%;w*4)1yNK)cA&CKFh#Q{DxIj#j~u z2p9}@%A(Bg<@gTo77^t@eu=SGpFdJ9_VqNv4I`RBdqB6lpK)c0i@VWkRfUDBgyn6o z-`Y_Zbj%=opN$kxJSH{8Ah!n)yHZlk)-}EybAwb`>BAe;E@264(BPx%5>MID$#Qo_ z6cPA#l$QZj^+V zFQw84=3gY&HoH{ZxiMM0HwU5BbtmzxBf=3GpZ*#7sFB!2^htSmG+9cA_(k@46$|gogD-o=lE8n0@O6AzZWB+gqjJg+VqU7;} zGI*#Xgm@n1b=2YFZu#Uapt2q8h|r48$&msk5>SgMKGbPI=?1=Zt@-%!oakO48DtYT zB@vn?OKjzb*;clDh<1FtnN8B@Q`(a->wj%G<-YZ5z|84NIt} zAPyA*Mm9Z)b3WLeYFT9IDR{!A!l1}fe6ai0*c8i3+WH*3_rQaE|u7PpLy6g#kT$(CwNS`W(U&4)|dK`@Ai04Zc0bj{f7_t?A^!!pM-PxScy zii6pvd1u;3&Q|%JuBuY59K66Wc#R|i446Af7g)WMyai8C&>fE+p^j-Ca~rIlQFNjL z)qCuX>V7jbo~r20fnAZ{T$UCsK)*k!APU^)XfGD7nR#gAcj#(z5&_tb>`c6q(7ttF z*|wI9*sW<98Pa`EJ9#pAR6<5<7dw$u5Lcniwe*J%EoBpB;{+=Ut+z!_tY{)C@H$OI zCQ+cFq2Y3VK*PFEX)4tFz`fg1X{Y7dB)IkQ%Bbt2K&=F{=DwcAfw$9xH+;p8|MABC z+P$O>Ktu%Ar)SE*m6Q}zRN5OGH=IMdOU1VZ4plsnB7=m4Y!VFKgD + /// Provider for Fnt Private French Tracker + /// + public class Fnt : BaseIndexer, IIndexer + { + private string LoginUrl { get { return SiteLink + "Login/"; } } + private string LoginCheckUrl { get { return SiteLink + "account-login.php"; } } + private string SearchUrl { get { return SiteLink + "torrents-recherche.php"; } } + private string TorrentCommentUrl { get { return SiteLink + "FnT/comm/torrent/"; } } + private string TorrentDescriptionUrl { get { return SiteLink + "FnT/fiche_film/"; } } + private string TorrentDownloadUrl { get { return SiteLink + "download.php?id={id}&dl=oui"; } } + private bool Latency { get { return ConfigData.Latency.Value; } } + private bool DevMode { get { return ConfigData.DevMode.Value; } } + private bool CacheMode { get { return ConfigData.HardDriveCache.Value; } } + private string directory { get { return System.IO.Path.GetTempPath() + "Jackett\\" + MethodBase.GetCurrentMethod().DeclaringType.Name + "\\"; } } + + private Dictionary emulatedBrowserHeaders = new Dictionary(); + private CQ fDom = null; + + private ConfigurationDataFnt ConfigData + { + get { return (ConfigurationDataFnt)configData; } + set { base.configData = value; } + } + + public Fnt(IIndexerManagerService i, IWebClient w, Logger l, IProtectionService ps) + : base( + name: "Fnt", + description: "Your French Movies & Series Provider", + link: "http://fnt.nu/", + caps: new TorznabCapabilities(), + manager: i, + client: w, + logger: l, + p: ps, + downloadBase: "http://fnt.nu/download.php?id={id}&dl=oui", + configData: new ConfigurationDataFnt()) + { + // Clean capabilities + TorznabCaps.Categories.Clear(); + + // Movies + AddCategoryMapping("101", TorznabCatType.MoviesSD); // BDRIP + AddCategoryMapping("100", TorznabCatType.MoviesSD); // DVDRIP + AddCategoryMapping("104", TorznabCatType.MoviesDVD); // DVD FULL + AddCategoryMapping("103", TorznabCatType.MoviesDVD); // DVD NTSC + AddCategoryMapping("102", TorznabCatType.MoviesDVD); // DVD PAL + AddCategoryMapping("106", TorznabCatType.MoviesHD); // M-HD 1080P + AddCategoryMapping("105", TorznabCatType.MoviesHD); // M-HD 720P + AddCategoryMapping("108", TorznabCatType.MoviesHD); // 1080P + AddCategoryMapping("107", TorznabCatType.MoviesHD); // 720P + AddCategoryMapping("130", TorznabCatType.MoviesBluRay); // BLURAY + AddCategoryMapping("127", TorznabCatType.MoviesSD); // VOSTFR + + // Series + AddCategoryMapping("121", TorznabCatType.TVSD); // DVDRIP + AddCategoryMapping("126", TorznabCatType.TVSD); // BDRIP + AddCategoryMapping("146", TorznabCatType.TVHD); // DVD R + AddCategoryMapping("120", TorznabCatType.TVSD); // HDTV + AddCategoryMapping("119", TorznabCatType.TVHD); // HDTV 720P + AddCategoryMapping("137", TorznabCatType.TVHD); // M-HD + AddCategoryMapping("138", TorznabCatType.TVHD); // BLURAY + AddCategoryMapping("135", TorznabCatType.TV); // PACK + AddCategoryMapping("153", TorznabCatType.TVWEBDL); // WEB-DL + AddCategoryMapping("150", TorznabCatType.TVWEBDL); // WEB-DL 1080P + AddCategoryMapping("149", TorznabCatType.TVWEBDL); // WEB-DL 720P + AddCategoryMapping("154", TorznabCatType.TVSD); // WEB-RIP + AddCategoryMapping("156", TorznabCatType.TVHD); // WEB-RIP 1080P + AddCategoryMapping("155", TorznabCatType.TVHD); // WEB-RIP 720P + AddCategoryMapping("110", TorznabCatType.TVHD); // VOSTFR HD + AddCategoryMapping("109", TorznabCatType.TV); // VOSTFR PACK + AddCategoryMapping("122", TorznabCatType.TVSD); // VOSTFR SD + + // TV + AddCategoryMapping("118", TorznabCatType.TV); // HDTV + AddCategoryMapping("129", TorznabCatType.TV); // HDTV 720P + AddCategoryMapping("115", TorznabCatType.TV); // SHOWS + + // Sport + AddCategoryMapping("133", TorznabCatType.TVSport); // SD + AddCategoryMapping("134", TorznabCatType.TVSport); // HD + + // Anime + AddCategoryMapping("151", TorznabCatType.TVAnime); // HD + AddCategoryMapping("140", TorznabCatType.TVAnime); // M-HD + AddCategoryMapping("116", TorznabCatType.TVAnime); // ALL + AddCategoryMapping("148", TorznabCatType.TVAnime); // SERIES + + // Manga + AddCategoryMapping("152", TorznabCatType.TVAnime); // HD + AddCategoryMapping("141", TorznabCatType.TVAnime); // M-HD + AddCategoryMapping("117", TorznabCatType.TVAnime); // ALL + + // Documentaries + AddCategoryMapping("128", TorznabCatType.TVDocumentary); // SD + AddCategoryMapping("131", TorznabCatType.TVDocumentary); // HD + + // Music + AddCategoryMapping("125", TorznabCatType.AudioVideo); // CLIP + AddCategoryMapping("123", TorznabCatType.AudioVideo); // DIVX + AddCategoryMapping("124", TorznabCatType.AudioVideo); // DVD + AddCategoryMapping("136", TorznabCatType.AudioLossless); // FLAC + AddCategoryMapping("132", TorznabCatType.Audio); // HD + + // Other + AddCategoryMapping("114", TorznabCatType.Other); // SUBPACK + AddCategoryMapping("147", TorznabCatType.AudioOther); // AUDIO TRACKS OF MOVIES/SERIES + } + + /// + /// Configure our WiHD Provider + /// + /// Our params in Json + /// Configuration state + public async Task ApplyConfiguration(JToken configJson) + { + // Retrieve config values set by Jackett's user + ConfigData.LoadValuesFromJson(configJson); + + // Check & Validate Config + validateConfig(); + + // Setting our data for a better emulated browser (maximum security) + // TODO: Encoded Content not supported by Jackett at this time + // emulatedBrowserHeaders.Add("Accept-Encoding", "gzip, deflate"); + + // If we want to simulate a browser + if (ConfigData.Browser.Value) { + + // Clean headers + emulatedBrowserHeaders.Clear(); + + // Inject headers + emulatedBrowserHeaders.Add("Accept", ConfigData.HeaderAccept.Value); + emulatedBrowserHeaders.Add("Accept-Language", ConfigData.HeaderAcceptLang.Value); + emulatedBrowserHeaders.Add("DNT", Convert.ToInt32(ConfigData.HeaderDNT.Value).ToString()); + emulatedBrowserHeaders.Add("Upgrade-Insecure-Requests", Convert.ToInt32(ConfigData.HeaderUpgradeInsecure.Value).ToString()); + emulatedBrowserHeaders.Add("User-Agent", ConfigData.HeaderUserAgent.Value); + } + + + // Getting login form to retrieve CSRF token + var myRequest = new Utils.Clients.WebRequest() + { + Url = LoginUrl + }; + + // Add our headers to request + myRequest.Headers = emulatedBrowserHeaders; + + // Building login form data + var pairs = new Dictionary { + { "username", ConfigData.Username.Value }, + { "password", ConfigData.Password.Value }, + { "submit", "Se loguer" }, + { "returnto", "/" } + }; + + // Do the login + var request = new Utils.Clients.WebRequest(){ + PostData = pairs, + Referer = LoginUrl, + Type = RequestType.POST, + Url = LoginUrl, + Headers = emulatedBrowserHeaders + }; + + // Perform loggin + latencyNow(); + output("\nPerform loggin.. with " + LoginCheckUrl); + var response = await RequestLoginAndFollowRedirect(LoginCheckUrl, pairs, string.Empty, true, SiteLink + "accueil"); + + // Test if we are logged in + await ConfigureIfOK(response.Cookies, !response.Cookies.Contains("deleted"), () => + { + // Parse error page + CQ dom = response.Content; + string message = dom[".NB-fm > p > b"].Text().Trim(); + + // Oops, unable to login + output("-> Login failed: " + message, "error"); + throw new ExceptionWithConfigData("Login failed: " + message, configData); + }); + + output("-> Login Success"); + + return IndexerConfigurationStatus.RequiresTesting; + } + + /// + /// Execute our search query + /// + /// Query + /// Releases + public async Task> PerformQuery(TorznabQuery query) + { + var releases = new List(); + var torrentRowList = new List(); + var searchTerm = query.GetQueryString(); + var searchUrl = SearchUrl; + int nbResults = 0; + int pageLinkCount = 0; + + // Check cache first so we don't query the server (if search term used or not in dev mode) + if (!DevMode && !string.IsNullOrEmpty(searchTerm)) + { + lock (cache) + { + // Remove old cache items + CleanCache(); + + // Search in cache + var cachedResult = cache.Where(i => i.Query == searchTerm).FirstOrDefault(); + if (cachedResult != null) + return cachedResult.Results.Select(s => (ReleaseInfo)s.Clone()).ToArray(); + } + } + + // Build our query + var request = buildQuery(searchTerm, query, searchUrl); + + // Getting results & Store content + WebClientStringResult results = await queryExec(request); + fDom = results.Content; + + try + { + // Find torrent rows + var firstPageRows = findTorrentRows(); + + // Add them to torrents list + torrentRowList.AddRange(firstPageRows.Select(fRow => fRow.Cq())); + + // Check if there are pagination links at bottom + Boolean pagination = (fDom[".NB-fm > p > a"].Length != 0); + + // If pagination available + if (pagination) { + // Retrieve total count from last page + nbResults = ParseUtil.CoerceInt(Regex.Match(fDom[".NB-fm > p > a:eq(-2)"].Text().Split('-')[1].Trim(), @"\d+").Value); + + // Calculate numbers of pages available for this search query (Based on number results and number of torrents on first page) + pageLinkCount = (int)Math.Ceiling((double)nbResults / firstPageRows.Length); + } + else { + // Check if we have a minimum of one result + if (firstPageRows.Length >= 1) + { + // Retrieve total count on our alone page + nbResults = ParseUtil.CoerceInt(Regex.Match(fDom[".NB-fm > p > b"].Text().Split('-')[1].Trim(), @"\d+").Value); + pageLinkCount = 1; + } + else + { + output("\nNo result found for your query, please try another search term ...", "info"); + // No result found for this query + return releases; + } + } + output("\nFound " + nbResults + " result(s) in " + pageLinkCount + " page(s) for this query !"); + output("\nThere are " + firstPageRows.Length + " results on the first page !"); + + // If we have a term used for search and pagination result superior to one + if (!string.IsNullOrWhiteSpace(query.GetQueryString()) && pageLinkCount > 1) + { + // Starting with page #2 + for (int i = 2; i <= Math.Min(Int32.Parse(ConfigData.Pages.Value), pageLinkCount); i++) + { + output("\nProcessing page #" + i); + + // Request our page + latencyNow(); + + // Build our query -- Minus 1 to page due to strange pagination number on tracker side, starting with page 0... + var pageRequest = buildQuery(searchTerm, query, searchUrl, (i - 1)); + + // Getting results & Store content + WebClientStringResult pageResults = await queryExec(pageRequest); + + // Assign response + fDom = pageResults.Content; + + // Process page results + var additionalPageRows = findTorrentRows(); + + // Add them to torrents list + torrentRowList.AddRange(additionalPageRows.Select(fRow => fRow.Cq())); + } + } + + // Loop on results + foreach (CQ tRow in torrentRowList) + { + output("\n=>> Torrent #" + (releases.Count + 1)); + + // ID + int id = ParseUtil.CoerceInt(Regex.Match(tRow.Find("td:eq(1) > div:last > a").Attr("href").ToString(), @"\d+").Value); + output("ID: " + id); + + // Release Name + string name = tRow.Find("td:eq(1) > div:last > a > b").Text().ToString(); + output("Release: " + name); + + // Category + int categoryID = ParseUtil.CoerceInt(Regex.Match(tRow.Find("td:eq(0) > a").Attr("href").ToString(), @"\d+").Value); + //string categoryName = tRow.Find(".category > img").Attr("title").ToString(); + output("Category: " + MapTrackerCatToNewznab(categoryID.ToString()) + " (" + categoryID + ")"); + + // Uploader & Size & Publish Date & Seeders & Leechers & Completed + string infosData = tRow.Find("td:eq(1) > div:last > a").Attr("mtcontent").ToString(); + IList infosList = Regex.Split(infosData, "
").Select(s => Regex.Replace(s, "<.*?>", String.Empty)).Where(s => s != String.Empty).ToList(); + IList infosTorrent = infosList.Select(s => s.Split(new[] { ':' }, 2)[1].Trim()).ToList(); + + // --> Uploader + string uploader = infosTorrent[2]; + output("Uploader: " + uploader); + + // --> Seeders + int seeders = ParseUtil.CoerceInt(infosTorrent[4]); + output("Seeders: " + seeders); + + // --> Leechers ~~ Some torrents are bugged with 4294967294 leechers (so, arbitrary set to 0) + int leechers = infosTorrent[5] == "4294967294" ? 0 : ParseUtil.CoerceInt(infosTorrent[5]); + output("Leechers: " + leechers); + + // --> Completed + int completed = ParseUtil.CoerceInt(infosTorrent[6]); + output("Completed: " + completed); + + // Comments + int nbComments = 0; + string comments = tRow.Find("td:last > div > span:nth-child(5)").Attr("mttitle").ToString(); + if(comments != "Ajouter un commentaire") { + nbComments = ParseUtil.CoerceInt(Regex.Match(comments, @"\d+").Value); + } + output("Comments: " + nbComments); + + // --> Size + var size = ReleaseInfo.GetBytes(infosTorrent[0].ToLowerInvariant()); + output("Size: " + infosTorrent[0] + " (" + size + " bytes)"); + + // --> Publish Date + DateTime date = DateTime.ParseExact(infosTorrent[1], "dd-MM-yyyy HH:mm:ss", CultureInfo.GetCultureInfo("fr-FR"), DateTimeStyles.AssumeLocal).ToUniversalTime(); + output("Released on: " + date.ToLocalTime().ToString()); + + // Torrent Details URL + Uri detailsLink = new Uri(TorrentDescriptionUrl + id + "/"); + output("Details: " + detailsLink.AbsoluteUri); + + // Torrent Comments URL + Uri commentsLink = new Uri(TorrentCommentUrl + id + "/"); + output("Comments Link: " + commentsLink.AbsoluteUri); + + // Torrent Download URL + Uri downloadLink = new Uri(TorrentDownloadUrl.Replace("{id}", id.ToString())); + output("Download Link: " + downloadLink.AbsoluteUri); + + // Building release infos + var release = new ReleaseInfo(); + release.Category = MapTrackerCatToNewznab(categoryID.ToString()); + release.Title = name; + release.Seeders = seeders; + release.Peers = seeders + leechers; + release.MinimumRatio = 1; + release.MinimumSeedTime = 172800; + release.PublishDate = date; + release.Size = size; + release.Guid = detailsLink; + release.Comments = commentsLink; + release.Link = downloadLink; + releases.Add(release); + } + + } + catch (Exception ex) + { + OnParseError("Error, unable to parse result \n" + ex.StackTrace, ex); + } + + // Return found releases + return releases; + } + + /// + /// Build query to process + /// + /// Term to search + /// Torznab Query for categories mapping + /// Search url for provider + /// Page number to request + /// URL to query for parsing and processing results + private string buildQuery(string term, TorznabQuery query, string url, int page = 0) + { + var parameters = new NameValueCollection(); + List categoriesList = MapTorznabCapsToTrackers(query); + + // Building our tracker query + parameters.Add("afficher", "1"); + + // If search term provider + if (!string.IsNullOrWhiteSpace(term)) + { + // Add search term + parameters.Add("recherche", term); + } + else + { + // Showing all torrents (just for output function) + term = "all"; + } + + // Building our tracker query + parameters.Add("visible", (ConfigData.Dead.Value == true) ? "0" : "1"); + parameters.Add("freeleech", (ConfigData.Freeleech.Value == true) ? "2" : "0"); + parameters.Add("nuke", (ConfigData.Nuke.Value == true) ? "0" : "1"); + parameters.Add("3D", (ConfigData.ThreeD.Value == true) ? "2" : "0"); + parameters.Add("langue", (ConfigData.Language.Value == true) ? "2" : "0"); + + // Loop on Categories needed + foreach (string category in categoriesList) + { + parameters.Add("c" + category, Convert.ToString(1)); + } + + // Check if we are processing a new page + if (page > 0) + { + // Adding page number to query + parameters.Add("page", page.ToString()); + } + + // Building our query -- Cannot use GetQueryString due to UrlEncode (generating wrong subcat[] param) + url += "?" + parameters.GetQueryString(); + + output("\nBuilded query for \"" + term + "\"... " + url); + + // Return our search url + return url; + } + + /// + /// Switch Method for Querying + /// + /// URL created by Query Builder + /// Results from query + private async Task queryExec(string request) + { + WebClientStringResult results = null; + + // Switch in we are in DEV mode with Hard Drive Cache or not + if (DevMode && CacheMode) + { + // Check Cache before querying and load previous results if available + results = await queryCache(request); + } + else + { + // Querying tracker directly + results = await queryTracker(request); + } + return results; + } + + /// + /// Get Torrents Page from Cache by Query Provided + /// + /// URL created by Query Builder + /// Results from query + private async Task queryCache(string request) + { + WebClientStringResult results = null; + + // Create Directory if not exist + System.IO.Directory.CreateDirectory(directory); + + // Clean Storage Provider Directory from outdated cached queries + cleanCacheStorage(); + + // Create fingerprint for request + string file = directory + request.GetHashCode() + ".json"; + + // Checking modes states + if (System.IO.File.Exists(file)) + { + // File exist... loading it right now ! + output("Loading results from hard drive cache ..." + request.GetHashCode() + ".json"); + results = JsonConvert.DeserializeObject(System.IO.File.ReadAllText(file)); + } + else + { + // No cached file found, querying tracker directly + results = await queryTracker(request); + + // Cached file didn't exist for our query, writing it right now ! + output("Writing results to hard drive cache ..." + request.GetHashCode() + ".json"); + System.IO.File.WriteAllText(file, JsonConvert.SerializeObject(results)); + } + return results; + } + + /// + /// Get Torrents Page from Tracker by Query Provided + /// + /// URL created by Query Builder + /// Results from query + private async Task queryTracker(string request) + { + WebClientStringResult results = null; + + // Cache mode not enabled or cached file didn't exist for our query + output("\nQuerying tracker for results...."); + + // Request our first page + latencyNow(); + results = await RequestStringWithCookiesAndRetry(request, null, null, emulatedBrowserHeaders); + + // Return results from tracker + return results; + } + + /// + /// Clean Hard Drive Cache Storage + /// + /// Force Provider Folder deletion + private void cleanCacheStorage(Boolean force = false) + { + // Check cleaning method + if(force) + { + // Deleting Provider Storage folder and all files recursively + output("\nDeleting Provider Storage folder and all files recursively ..."); + + // Check if directory exist + if(System.IO.Directory.Exists(directory)) + { + // Delete storage directory of provider + System.IO.Directory.Delete(directory, true); + output("-> Storage folder deleted successfully."); + } + else + { + // No directory, so nothing to do + output("-> No Storage folder found for this provider !"); + } + } + else + { + int i = 0; + // Check if there is file older than ... and delete them + output("\nCleaning Provider Storage folder... in progress."); + System.IO.Directory.GetFiles(directory) + .Select(f => new System.IO.FileInfo(f)) + .Where(f => f.LastAccessTime < DateTime.Now.AddMilliseconds(-Convert.ToInt32(ConfigData.HardDriveCacheKeepTime.Value))) + .ToList() + .ForEach(f => { + output("Deleting cached file << " + f.Name + " >> ... done."); + f.Delete(); + i++; + }); + + // Inform on what was cleaned during process + if(i > 0) { + output("-> Deleted " + i + " cached files during cleaning."); + } + else { + output("-> Nothing deleted during cleaning."); + } + } + } + + /// + /// Generate a random fake latency to avoid detection on tracker side + /// + private void latencyNow() + { + // Need latency ? + if(Latency) + { + // Generate a random value in our range + var random = new Random(DateTime.Now.Millisecond); + int waiting = random.Next(Convert.ToInt32(ConfigData.LatencyStart.Value), Convert.ToInt32(ConfigData.LatencyEnd.Value)); + output("\nLatency Faker => Sleeping for " + waiting + " ms..."); + + // Sleep now... + System.Threading.Thread.Sleep(waiting); + } + } + + /// + /// Find torrent rows in search pages + /// + /// JQuery Object + private CQ findTorrentRows() + { + // Return all occurencis of torrents found + return fDom[".ligntorrent"]; + } + + /// + /// Output message for logging or developpment (console) + /// + /// Message to output + /// Level for Logger + private void output(string message, string level = "debug") + { + // Check if we are in dev mode + if(DevMode) + { + // Output message to console + Console.WriteLine(message); + } + else + { + // Send message to logger with level + switch (level) + { + default: + goto case "debug"; + case "debug": + // Only if Debug Level Enabled on Jackett + if (Engine.Logger.IsDebugEnabled) + { + logger.Debug(message); + } + break; + case "info": + logger.Info(message); + break; + case "error": + logger.Error(message); + break; + } + } + } + + /// + /// Validate Config entered by user on Jackett + /// + private void validateConfig() + { + output("\nValidating Settings ... \n"); + + // Check Username Setting + if (string.IsNullOrEmpty(ConfigData.Username.Value)) + { + throw new ExceptionWithConfigData("You must provide a username for this tracker to login !", ConfigData); + } + else + { + output("Validated Setting -- Username (auth) => " + ConfigData.Username.Value.ToString()); + } + + // Check Password Setting + if (string.IsNullOrEmpty(ConfigData.Password.Value)) + { + throw new ExceptionWithConfigData("You must provide a password with your username for this tracker to login !", ConfigData); + } + else + { + output("Validated Setting -- Password (auth) => " + ConfigData.Password.Value.ToString()); + } + + // Check Max Page Setting + if (!string.IsNullOrEmpty(ConfigData.Pages.Value)) + { + try + { + output("Validated Setting -- Max Pages => " + Convert.ToInt32(ConfigData.Pages.Value)); + } + catch (Exception) + { + throw new ExceptionWithConfigData("Please enter a numeric maximum number of pages to crawl !", ConfigData); + } + } + else + { + throw new ExceptionWithConfigData("Please enter a maximum number of pages to crawl !", ConfigData); + } + + // Check Latency Setting + if (ConfigData.Latency.Value) + { + output("\nValidated Setting -- Latency Simulation enabled"); + + // Check Latency Start Setting + if (!string.IsNullOrEmpty(ConfigData.LatencyStart.Value)) + { + try + { + output("Validated Setting -- Latency Start => " + Convert.ToInt32(ConfigData.LatencyStart.Value)); + } + catch (Exception) + { + throw new ExceptionWithConfigData("Please enter a numeric latency start in ms !", ConfigData); + } + } + else + { + throw new ExceptionWithConfigData("Latency Simulation enabled, Please enter a start latency !", ConfigData); + } + + // Check Latency End Setting + if (!string.IsNullOrEmpty(ConfigData.LatencyEnd.Value)) + { + try + { + output("Validated Setting -- Latency End => " + Convert.ToInt32(ConfigData.LatencyEnd.Value)); + } + catch (Exception) + { + throw new ExceptionWithConfigData("Please enter a numeric latency end in ms !", ConfigData); + } + } + else + { + throw new ExceptionWithConfigData("Latency Simulation enabled, Please enter a end latency !", ConfigData); + } + } + + // Check Browser Setting + if (ConfigData.Browser.Value) + { + output("\nValidated Setting -- Browser Simulation enabled"); + + // Check ACCEPT header Setting + if (string.IsNullOrEmpty(ConfigData.HeaderAccept.Value)) + { + throw new ExceptionWithConfigData("Browser Simulation enabled, Please enter an ACCEPT header !", ConfigData); + } + else + { + output("Validated Setting -- ACCEPT (header) => " + ConfigData.HeaderAccept.Value.ToString()); + } + + // Check ACCEPT-LANG header Setting + if (string.IsNullOrEmpty(ConfigData.HeaderAcceptLang.Value)) + { + throw new ExceptionWithConfigData("Browser Simulation enabled, Please enter an ACCEPT-LANG header !", ConfigData); + } + else + { + output("Validated Setting -- ACCEPT-LANG (header) => " + ConfigData.HeaderAcceptLang.Value.ToString()); + } + + // Check USER-AGENT header Setting + if (string.IsNullOrEmpty(ConfigData.HeaderUserAgent.Value)) + { + throw new ExceptionWithConfigData("Browser Simulation enabled, Please enter an USER-AGENT header !", ConfigData); + } + else + { + output("Validated Setting -- USER-AGENT (header) => " + ConfigData.HeaderUserAgent.Value.ToString()); + } + } + + // Check Dev Cache Settings + if (ConfigData.HardDriveCache.Value == true) + { + output("\nValidated Setting -- DEV Hard Drive Cache enabled"); + + // Check if Dev Mode enabled ! + if (!ConfigData.DevMode.Value) + { + throw new ExceptionWithConfigData("Hard Drive is enabled but not in DEV MODE, Please enable DEV MODE !", ConfigData); + } + + // Check Cache Keep Time Setting + if (!string.IsNullOrEmpty(ConfigData.HardDriveCacheKeepTime.Value)) + { + try + { + output("Validated Setting -- Cache Keep Time (ms) => " + Convert.ToInt32(ConfigData.HardDriveCacheKeepTime.Value)); + } + catch (Exception) + { + throw new ExceptionWithConfigData("Please enter a numeric hard drive keep time in ms !", ConfigData); + } + } + else + { + throw new ExceptionWithConfigData("Hard Drive Cache enabled, Please enter a maximum keep time for cache !", ConfigData); + } + } + else + { + // Delete cache if previously existed + cleanCacheStorage(true); + } + } + } +} \ No newline at end of file diff --git a/src/Jackett/Jackett.csproj b/src/Jackett/Jackett.csproj index 628b554dd..02614bf3f 100644 --- a/src/Jackett/Jackett.csproj +++ b/src/Jackett/Jackett.csproj @@ -195,6 +195,7 @@ + @@ -211,6 +212,7 @@ + @@ -457,6 +459,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest @@ -626,4 +631,4 @@ - + \ No newline at end of file diff --git a/src/Jackett/Models/IndexerConfig/Bespoke/ConfigurationDataFnt.cs b/src/Jackett/Models/IndexerConfig/Bespoke/ConfigurationDataFnt.cs new file mode 100644 index 000000000..b03efe3cc --- /dev/null +++ b/src/Jackett/Models/IndexerConfig/Bespoke/ConfigurationDataFnt.cs @@ -0,0 +1,63 @@ +namespace Jackett.Models.IndexerConfig.Bespoke +{ + class ConfigurationDataFnt : ConfigurationData + { + public DisplayItem CredentialsWarning { get; private set; } + public StringItem Username { get; private set; } + public StringItem Password { get; private set; } + public DisplayItem PagesWarning { get; private set; } + public StringItem Pages { get; private set; } + public BoolItem Dead { get; private set; } + public BoolItem Freeleech { get; private set; } + public BoolItem Nuke { get; private set; } + public BoolItem ThreeD { get; private set; } + public BoolItem Language { get; private set; } + public DisplayItem SecurityWarning { get; private set; } + public BoolItem Latency { get; private set; } + public BoolItem Browser { get; private set; } + public DisplayItem LatencyWarning { get; private set; } + public StringItem LatencyStart { get; private set; } + public StringItem LatencyEnd { get; private set; } + public DisplayItem HeadersWarning { get; private set; } + public StringItem HeaderAccept { get; private set; } + public StringItem HeaderAcceptLang { get; private set; } + public BoolItem HeaderDNT { get; private set; } + public BoolItem HeaderUpgradeInsecure { get; private set; } + public StringItem HeaderUserAgent { get; private set; } + public DisplayItem DevWarning { get; private set; } + public BoolItem DevMode { get; private set; } + public BoolItem HardDriveCache { get; private set; } + public StringItem HardDriveCacheKeepTime { get; private set; } + + public ConfigurationDataFnt() + : base() + { + CredentialsWarning = new DisplayItem("Credentials Configuration (Private Tracker),

  • Username is your account name on this tracker.
  • Password is your password associated to your account name.
") { Name = "Credentials" }; + Username = new StringItem { Name = "Username (Required)", Value = "" }; + Password = new StringItem { Name = "Password (Required)", Value = "" }; + PagesWarning = new DisplayItem("Preferences Configuration (Tweak your search settings),

  • Max Pages to Process let you specify how many page (max) Jackett can process when doing a search. Setting a value higher than 4 is dangerous for you account ! (Result of too many requests to tracker...that will be suspect).
  • With Dead Torrents let you search with torrents which are marked Dead.
  • Freeleech Only let you search only for torrents which are marked Freeleech.
  • With Nuked Releases let you search with torrents which are marked Nuked.
  • 3D Releases Only let you search only for torrents which are marked 3D.
  • French Only let you search only for torrents which are marked FRENCH or TRUEFRENCH (MULTI not included !).
") { Name = "Preferences" }; + Pages = new StringItem { Name = "Max Pages to Process (Required)", Value = "4" }; + Dead = new BoolItem() { Name = "With Dead Torrents", Value = false }; + Freeleech = new BoolItem() { Name = "Freeleech Only", Value = false }; + Nuke = new BoolItem() { Name = "With Nuked Releases", Value = false }; + ThreeD = new BoolItem() { Name = "3D Releases only", Value = false }; + Language = new BoolItem() { Name = "French Only (No MULTI or FOREIGN)", Value = false }; + SecurityWarning = new DisplayItem("Security Configuration (Read this area carefully !),

  • Latency Simulation will simulate human browsing with Jacket by pausing Jacket for an random time between each request, to fake a real content browsing.
  • Browser Simulation will simulate a real human browser by injecting additionals headers when doing requests to tracker.
") { Name = "Security" }; + Latency = new BoolItem() { Name = "Latency Simulation (Optional)", Value = true }; + Browser = new BoolItem() { Name = "Browser Simulation (Optional)", Value = true }; + LatencyWarning = new DisplayItem("Latency Configuration (Required if latency simulation enabled),

  • By filling this range, Jackett will make a random timed pause between requests to tracker to simulate a real browser.
  • MilliSeconds only
") { Name = "Simulate Latency" }; + LatencyStart = new StringItem { Name = "Minimum Latency (ms)", Value = "1589" }; + LatencyEnd = new StringItem { Name = "Maximum Latency (ms)", Value = "3674" }; + HeadersWarning = new DisplayItem("Browser Headers Configuration (Required if browser simulation enabled),

  • By filling these fields, Jackett will inject headers with your values to simulate a real browser.
  • You can get your browser values here: www.whatismybrowser.com

Note that some headers are not necessary because they are injected automatically by this provider such as Accept_Encoding, Connection, Host or X-Requested-With") { Name = "Injecting headers" }; + HeaderAccept = new StringItem { Name = "Accept", Value = "" }; + HeaderAcceptLang = new StringItem { Name = "Accept-Language", Value = "" }; + HeaderDNT = new BoolItem { Name = "DNT", Value = false }; + HeaderUpgradeInsecure = new BoolItem { Name = "Upgrade-Insecure-Requests", Value = false }; + HeaderUserAgent = new StringItem { Name = "User-Agent", Value = "" }; + DevWarning = new DisplayItem("Development Facility (For Developers ONLY),

  • By enabling development mode, Jackett will bypass his cache and will output debug messages to console instead of his log file.
  • By enabling Hard Drive Cache, This provider will save each query answers from tracker in temp directory, in fact this reduce drastically HTTP requests when building a provider at parsing step for example. So, Jackett will search for a cached query answer on hard drive before executing query on tracker side ! DEV MODE must be enabled to use it !
") { Name = "Development" }; + DevMode = new BoolItem { Name = "Enable DEV MODE (Developers ONLY)", Value = false }; + HardDriveCache = new BoolItem { Name = "Enable HARD DRIVE CACHE (Developers ONLY)", Value = false }; + HardDriveCacheKeepTime = new StringItem { Name = "Keep Cached files for (ms)", Value = "300000" }; + } + } +}