mirror
/
Jackett
mirror of https://github.com/Jackett/Jackett
1
0
Fork 0
Code Issues Releases Wiki Activity

Implement server backhole downloading

This commit is contained in:
KZ 2015-08-07 22:40:45 +01:00
parent 6d0aa05761
commit 6ea759aeab
15 changed files with 284 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/Jackett/Content/custom.css
View File

@ -231,4 +231,16 @@ hr {
.jackettlogError {
background-color: #FF6060 !important;
}
.jackettdownloaded {
color: blueviolet;
}
.jacketdownloadlocal {
padding-left: 10px;
}
.downloadcolumn {
text-align:center;
}

66
src/Jackett/Content/custom.js
View File

@ -2,14 +2,36 @@
$.ajaxSetup({ cache: false });
reloadIndexers();
loadJackettSettings();
$('body').on('click', '.downloadlink', function (e, b) {
$(e.target).addClass('jackettdownloaded');
});
$('body').on('click', '.jacketdownloadserver', function (event) {
var href = $(event.target).parent().attr('href');
var jqxhr = $.get(href, function (data) {
if (data.result == "error") {
doNotify("Error: " + data.error, "danger", "glyphicon glyphicon-alert");
return;
} else {
doNotify("Downloaded sent to the blackhole successfully.", "success", "glyphicon glyphicon-ok");
}
}).fail(function () {
doNotify("Request to Jackett server failed", "danger", "glyphicon glyphicon-alert");
});
event.preventDefault();
return false;
});
});
function loadJackettSettings() {
getJackettConfig(function (data) {
$("#api-key-input").val(data.config.api_key);
$("#app-version").html(data.app_version);
$("#jackett-port").val(data.config.port);
$("#jackett-savedir").val(data.config.blackholedir);
$("#jackett-allowext").attr('checked', data.config.external);
var password = data.config.password;
$("#jackett-adminpwd").val(password);
@ -33,12 +55,14 @@ $("#jackett-show-releases").click(function () {
{
"targets": 0,
"visible": false,
"searchable": false
"searchable": false,
"type": 'date'
},
{
"targets": 1,
"visible": false,
"searchable": false
"searchable": false,
"type": 'date'
},
{
"targets": 2,
@ -52,7 +76,31 @@ $("#jackett-show-releases").click(function () {
"searchable": false,
"iDataSort": 1
}
]
],
initComplete: function () {
var count = 0;
this.api().columns().every(function () {
count++;
if (count === 5 || count ===7) {
var column = this;
var select = $('<select><option value=""></option></select>')
.appendTo($(column.footer()).empty())
.on('change', function () {
var val = $.fn.dataTable.util.escapeRegex(
$(this).val()
);
column
.search(val ? '^' + val + '$' : '', true, false)
.draw();
});
column.data().unique().sort().each(function (d, j) {
select.append('<option value="' + d + '">' + d + '</option>')
});
}
});
}
});
$("#modals").append(releaseDialog);
releaseDialog.modal("show");
@ -80,13 +128,17 @@ $("#view-jackett-logs").click(function () {
$("#change-jackett-port").click(function () {
var jackett_port = $("#jackett-port").val();
var jackett_external = $("#jackett-allowext").is(':checked');
var jsonObject = { port: jackett_port, external: jackett_external };
var jqxhr = $.post("/admin/set_port", JSON.stringify(jsonObject), function (data) {
var jsonObject = {
port: jackett_port,
external: jackett_external,
blackholedir: $("#jackett-savedir").val()
};
var jqxhr = $.post("/admin/set_config", JSON.stringify(jsonObject), function (data) {
if (data.result == "error") {
doNotify("Error: " + data.error, "danger", "glyphicon glyphicon-alert");
return;
} else {
doNotify("The port has been changed. Redirecting you to the new port.", "success", "glyphicon glyphicon-ok");
doNotify("Redirecting you to complete configuration update..", "success", "glyphicon glyphicon-ok");
window.setTimeout(function () {
url = window.location.href;
if (data.external) {

31
src/Jackett/Content/index.html
View File

@ -50,17 +50,36 @@
<tr>
<td>{{PublishDate}}</td>
<td>{{FirstSeen}}</td>
<td>{{dateFormatRel PublishDate}}</td>
<td>{{dateFormatRel FirstSeen}}</td>
<td>{{jacketTimespan PublishDate}}</td>
<td>{{jacketTimespan FirstSeen}}</td>
<td>{{Tracker}}</td>
<td><a href="{{Comments}}">{{Title}}</a></td>
<td>{{CategoryDesc}}</td>
<td>{{Seeders}}</td>
<td>{{Peers}}</td>
<td><a href="{{Link}}"><i class="fa fa-download"></i></a></td>
<td class="downloadcolumn">
<a class="downloadlink" title="Download locally" href="{{Link}}"><i class="fa fa-download"></i></a>
{{#if BlackholeLink}}
<a class="downloadlink jacketdownloadserver" title="Save to server blackhole directory" href="{{BlackholeLink}}"><i class="fa fa-upload"></i></a>
{{/if}}
</td>
</tr>
{{/each}}
</tbody>
<tfoot>
<tr>
<td></td>
<td></td>
<td></td>
<td></td>
<td></td>
<td></td>
<td></td>
<td></td>
<td></td>
<td></td>
</tr>
</tfoot>
</table>
</div>
<div class="modal-footer">
@ -93,7 +112,7 @@
<tr class="jackettlog{{Level}}">
<td>{{dateFormat When}}</td>
<td>{{Level}}</td>
<td>{{Message}}</td>
<td><pre>{{Message}}</pre></td>
</tr>
{{/each}}
</tbody>
@ -196,6 +215,10 @@
View logs <span class="glyphicon glyphicon-ok-wrench" aria-hidden="true"></span>
</button>
</div>
<div class="input-area">
<span class="input-header">Manual download blackhole directory: </span>
<input id="jackett-savedir" class="form-control input-right" type="text" value="" placeholder="c:\torrents\">
</div>
<div class="input-area">
<span class="input-header">External access: </span>
<input id="jackett-allowext" class="form-control input-right" type="checkbox" />

31
src/Jackett/Content/libs/handlebarsmoment.js
View File

@ -8,10 +8,27 @@ Handlebars.registerHelper('dateFormat', function (context, block) {
};
});
Handlebars.registerHelper('dateFormatRel', function (context, block) {
if (window.moment) {
return moment(context).fromNow(true);
} else {
return context;
};
});
Handlebars.registerHelper('jacketTimespan', function (context, block) {
var now = moment();
var from = moment(context);
var timeSpan = moment.duration(now.diff(from));
var minutes = timeSpan.asMinutes();
if (minutes < 120) {
return Math.round(minutes) + 'm ago';
}
var hours = timeSpan.asHours();
if (hours < 48) {
return Math.round(hours) + 'h ago';
}
var days = timeSpan.asDays();
if (days < 365) {
return Math.round(days) + 'd ago';
}
var years = timeSpan.asYears();
return Math.round(years) + 'y ago';
});

19
src/Jackett/Controllers/AdminController.cs
View File

@ -286,6 +286,7 @@ namespace Jackett.Controllers
cfg["port"] = serverService.Config.Port;
cfg["external"] = serverService.Config.AllowExternal;
cfg["api_key"] = serverService.Config.APIKey;
cfg["blackholedir"] = serverService.Config.BlackholeDir;
cfg["password"] = string.IsNullOrEmpty(serverService.Config.AdminPassword )? string.Empty:serverService.Config.AdminPassword.Substring(0,10);
jsonReply["config"] = cfg;
@ -300,7 +301,7 @@ namespace Jackett.Controllers
return Json(jsonReply);
}
[Route("set_port")]
[Route("set_config")]
[HttpPost]
public async Task<IHttpActionResult> SetConfig()
{
@ -312,6 +313,7 @@ namespace Jackett.Controllers
var postData = await ReadPostDataJson();
int port = (int)postData["port"];
bool external = (bool)postData["external"];
string saveDir = (string)postData["blackholedir"];
if (port != Engine.Server.Config.Port || external != Engine.Server.Config.AllowExternal)
{
@ -361,6 +363,21 @@ namespace Jackett.Controllers
})).Start();
}
if(saveDir != Engine.Server.Config.BlackholeDir)
{
if (!string.IsNullOrEmpty(saveDir))
{
if (!Directory.Exists(saveDir))
{
throw new Exception("Blackhole directory does not exist");
}
}
Engine.Server.Config.BlackholeDir = saveDir;
Engine.Server.SaveConfig();
}
jsonReply["result"] = "success";
jsonReply["port"] = port;
jsonReply["external"] = external;

71
src/Jackett/Controllers/BlackholeController.cs Normal file
View File

@ -0,0 +1,71 @@
using Jackett.Services;
using Newtonsoft.Json.Linq;
using NLog;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading.Tasks;
using System.Web;
using System.Web.Http;
namespace Jackett.Controllers
{
[AllowAnonymous]
public class BlackholeController : ApiController
{
private Logger logger;
private IIndexerManagerService indexerService;
public BlackholeController(IIndexerManagerService i, Logger l)
{
logger = l;
indexerService = i;
}
[HttpGet]
public async Task<IHttpActionResult> Blackhole(string indexerID, string path)
{
var jsonReply = new JObject();
try
{
var indexer = indexerService.GetIndexer(indexerID);
if (!indexer.IsConfigured)
{
logger.Warn(string.Format("Rejected a request to {0} which is unconfigured.", indexer.DisplayName));
throw new Exception("This indexer is not configured.");
}
var remoteFile = Encoding.UTF8.GetString(HttpServerUtility.UrlTokenDecode(path));
var downloadBytes = await indexer.Download(new Uri(remoteFile, UriKind.RelativeOrAbsolute));
if (string.IsNullOrWhiteSpace(Engine.Server.Config.BlackholeDir))
{
throw new Exception("Blackhole directory not set!");
}
if (!Directory.Exists(Engine.Server.Config.BlackholeDir))
{
throw new Exception("Blackhole directory does not exist: " + Engine.Server.Config.BlackholeDir);
}
var fileName = DateTime.Now.Ticks + ".torrent";
File.WriteAllBytes(Path.Combine(Engine.Server.Config.BlackholeDir, fileName), downloadBytes);
jsonReply["result"] = "success";
}
catch (Exception ex)
{
logger.Error(ex, "Error downloading to blackhole " + indexerID + " " + path);
jsonReply["result"] = "error";
jsonReply["error"] = ex.Message;
}
return Json(jsonReply);
}
}
}

11
src/Jackett/Controllers/PotatoController.cs
View File

@ -100,15 +100,14 @@ namespace Jackett.Controllers
}
releases = indexer.FilterResults(torznabQuery, releases);
var severUrl = string.Format("{0}://{1}:{2}/", Request.RequestUri.Scheme, Request.RequestUri.Host, Request.RequestUri.Port);
var proxiedReleases = releases.Select(s => Mapper.Map<ReleaseInfo>(s).ConvertToProxyLink(severUrl, indexerID));
var serverUrl = string.Format("{0}://{1}:{2}/", Request.RequestUri.Scheme, Request.RequestUri.Host, Request.RequestUri.Port);
var potatoResponse = new TorrentPotatoResponse();
foreach(var release in proxiedReleases)
foreach(var r in releases)
{
var release = Mapper.Map<ReleaseInfo>(r);
release.Link = release.ConvertToProxyLink(serverUrl, indexerID);
potatoResponse.results.Add(new TorrentPotatoResponseItem()
{
release_name = release.Title + "[" + indexer.DisplayName + "]", // Suffix the indexer so we can see which tracker we are using in CPS as it just says torrentpotato >.>

35
src/Jackett/Controllers/TorznabController.cs
View File

@ -64,21 +64,25 @@ namespace Jackett.Controllers
var releases = await indexer.PerformQuery(torznabQuery);
// Some trackers do not keep their clocks up to date and can be ~20 minutes out!
foreach(var release in releases)
{
if (release.PublishDate > DateTime.Now)
release.PublishDate = DateTime.Now;
}
// Some trackers do not support multiple category filtering so filter the releases that match manually.
var filteredReleases = releases = indexer.FilterResults(torznabQuery, releases);
int? newItemCount = null;
// Cache non query results
if (string.IsNullOrEmpty(torznabQuery.SanitizedSearchTerm))
{
newItemCount = cacheService.CacheRssResults(indexer, releases);
newItemCount = cacheService.GetNewItemCount(indexer, filteredReleases);
cacheService.CacheRssResults(indexer, releases);
}
var releaseCount = releases.Count();
releases = indexer.FilterResults(torznabQuery, releases);
var removedInFilterCount = releaseCount - releases.Count();
if (newItemCount.HasValue)
newItemCount -= removedInFilterCount;
// Log info
var logBuilder = new StringBuilder();
if (newItemCount != null) {
@ -94,20 +98,27 @@ namespace Jackett.Controllers
logger.Info(logBuilder.ToString());
var severUrl = string.Format("{0}://{1}:{2}/", Request.RequestUri.Scheme, Request.RequestUri.Host, Request.RequestUri.Port);
var serverUrl = string.Format("{0}://{1}:{2}/", Request.RequestUri.Scheme, Request.RequestUri.Host, Request.RequestUri.Port);
var resultPage = new ResultPage(new ChannelInfo
{
Title = indexer.DisplayName,
Description = indexer.DisplayDescription,
Link = new Uri(indexer.SiteLink),
ImageUrl = new Uri(severUrl + "logos/" + indexer.ID + ".png"),
ImageUrl = new Uri(serverUrl + "logos/" + indexer.ID + ".png"),
ImageTitle = indexer.DisplayName,
ImageLink = new Uri(indexer.SiteLink),
ImageDescription = indexer.DisplayName
});
resultPage.Releases.AddRange(releases.Select(s=>Mapper.Map<ReleaseInfo>(s).ConvertToProxyLink(severUrl, indexerID)));
var xml = resultPage.ToXml(new Uri(severUrl));
foreach(var result in releases)
{
var clone = Mapper.Map<ReleaseInfo>(result);
clone.Link = clone.ConvertToProxyLink(serverUrl, indexerID);
resultPage.Releases.Add(result);
}
var xml = resultPage.ToXml(new Uri(serverUrl));
// Force the return as XML
return new HttpResponseMessage()
{

1
src/Jackett/Jackett.csproj
View File

@ -172,6 +172,7 @@
</ItemGroup>
<ItemGroup>
<Compile Include="AuthenticationException.cs" />
<Compile Include="Controllers\BlackholeController.cs" />
<Compile Include="Controllers\PotatoController.cs" />
<Compile Include="Controllers\TorznabController.cs" />
<Compile Include="Controllers\DownloadController.cs" />

1
src/Jackett/Models/Config/ServerConfig.cs
View File

@ -20,6 +20,7 @@ namespace Jackett.Models.Config
public string APIKey { get; set; }
public string AdminPassword { get; set; }
public string InstanceId { get; set; }
public string BlackholeDir { get; set; }
public string[] GetListenAddresses(bool? external = null)
{

9
src/Jackett/Models/ReleaseInfo.cs
View File

@ -92,15 +92,14 @@ namespace Jackett.Models
return (long)(kb * 1024f);
}
public ReleaseInfo ConvertToProxyLink(string serverUrl, string indexerId)
public Uri ConvertToProxyLink(string serverUrl, string indexerId, string action = "download")
{
if (Link == null || (Link.IsAbsoluteUri && Link.Scheme == "magnet"))
return this;
return Link;
var originalLink = Link;
var encodedLink = HttpServerUtility.UrlTokenEncode(Encoding.UTF8.GetBytes(originalLink.ToString())) + "/t.torrent";
var proxyLink = string.Format("{0}api/{1}/download/{2}", serverUrl, indexerId, encodedLink);
Link = new Uri(proxyLink);
return this;
var proxyLink = string.Format("{0}api/{1}/{2}/{3}", serverUrl, indexerId, action, encodedLink);
return new Uri(proxyLink);
}
}
}

1
src/Jackett/Models/TrackerCacheResult.cs
View File

@ -11,5 +11,6 @@ namespace Jackett.Models
public DateTime FirstSeen { get; set; }
public string Tracker { get; set; }
public string CategoryDesc { get; set; }
public Uri BlackholeLink { get; set; }
}
}

38
src/Jackett/Services/CacheService.cs
View File

@ -11,21 +11,21 @@ namespace Jackett.Services
{
public interface ICacheService
{
int CacheRssResults(IIndexer indexer, IEnumerable<ReleaseInfo> releases);
void CacheRssResults(IIndexer indexer, IEnumerable<ReleaseInfo> releases);
List<TrackerCacheResult> GetCachedResults(string serverUrl);
int GetNewItemCount(IIndexer indexer, IEnumerable<ReleaseInfo> releases);
}
public class CacheService : ICacheService
{
private readonly List<TrackerCache> cache = new List<TrackerCache>();
private readonly int MAX_RESULTS_PER_TRACKER = 250;
private readonly int MAX_RESULTS_PER_TRACKER = 1000;
private readonly TimeSpan AGE_LIMIT = new TimeSpan(7, 0, 0, 0);
public int CacheRssResults(IIndexer indexer, IEnumerable<ReleaseInfo> releases)
public void CacheRssResults(IIndexer indexer, IEnumerable<ReleaseInfo> releases)
{
lock (cache)
{
int newItemCount = 0;
var trackerCache = cache.Where(c => c.TrackerId == indexer.ID).FirstOrDefault();
if (trackerCache == null)
{
@ -49,7 +49,6 @@ namespace Jackett.Services
existingItem = new CachedResult();
existingItem.Created = DateTime.Now;
trackerCache.Results.Add(existingItem);
newItemCount++;
}
existingItem.Result = release;
@ -60,6 +59,28 @@ namespace Jackett.Services
{
tracker.Results = tracker.Results.OrderByDescending(i => i.Created).Take(MAX_RESULTS_PER_TRACKER).ToList();
}
}
}
public int GetNewItemCount(IIndexer indexer, IEnumerable<ReleaseInfo> releases)
{
lock (cache)
{
int newItemCount = 0;
var trackerCache = cache.Where(c => c.TrackerId == indexer.ID).FirstOrDefault();
if (trackerCache != null)
{
foreach (var release in releases)
{
if (trackerCache.Results.Where(i => i.Result.Guid == release.Guid).Count() == 0)
{
newItemCount++;
}
}
}
else {
newItemCount++;
}
return newItemCount;
}
@ -79,12 +100,15 @@ namespace Jackett.Services
item.FirstSeen = release.Created;
item.Tracker = tracker.TrackerName;
item.Peers = item.Peers - item.Seeders; // Use peers as leechers
item.ConvertToProxyLink(serverUrl, tracker.TrackerId);
item.Link = item.ConvertToProxyLink(serverUrl, tracker.TrackerId);
if(item.Link!=null && item.Link.Scheme != "magnet" && !string.IsNullOrWhiteSpace(Engine.Server.Config.BlackholeDir))
item.BlackholeLink = item.ConvertToProxyLink(serverUrl, tracker.TrackerId, "blackhole");
results.Add(item);
}
}
return results.OrderByDescending(i=>i.PublishDate).ToList();
return results.Take(3000).OrderByDescending(i=>i.PublishDate).ToList();
}
}
}

2
src/Jackett/Services/LogCacheService.cs
View File

@ -30,7 +30,7 @@ namespace Jackett.Services
Message = l.Message,
When = l.TimeStamp
});
logs = logs.Take(100).ToList();
logs = logs.Take(50).ToList();
}
}

6
src/Jackett/Startup.cs
View File

@ -115,6 +115,12 @@ namespace Jackett
defaults: new { controller = "Download", action = "Download" }
);
config.Routes.MapHttpRoute(
name: "blackhole",
routeTemplate: "api/{indexerID}/blackhole/{path}/t.torrent",
defaults: new { controller = "Blackhole", action = "Blackhole" }
);
appBuilder.UseFileServer(new FileServerOptions
{
RequestPath = new PathString(string.Empty),