Add Support for Gazelle based indexers

2017-09-22 22:48:15 -04:00 · 2017-09-22 22:48:15 -04:00 · 1322633d0d
parent 7c3446baab
commit 1322633d0d
7 changed files with 486 additions and 0 deletions
--- a/src/NzbDrone.Core/Indexers/Gazelle/Gazelle.cs
+++ b/src/NzbDrone.Core/Indexers/Gazelle/Gazelle.cs
@ -0,0 +1,81 @@
+using NLog;
+using NzbDrone.Common.Cache;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Configuration;
+using NzbDrone.Core.Parser;
+using NzbDrone.Core.ThingiProvider;
+using System.Collections.Generic;
+
+namespace NzbDrone.Core.Indexers.Gazelle
+{
+    public class Gazelle : HttpIndexerBase<GazelleSettings>
+    {
+        public override string Name => "Gazelle API";
+        public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
+        public override bool SupportsRss => true;
+        public override bool SupportsSearch => true;
+        public override int PageSize => 50;
+
+        private readonly ICached<Dictionary<string, string>> _authCookieCache;
+        private readonly IHttpClient _httpClient;
+        private readonly Logger _logger;
+
+        public Gazelle(IHttpClient httpClient, ICacheManager cacheManager, IIndexerStatusService indexerStatusService,
+            IConfigService configService, IParsingService parsingService, Logger logger)
+            : base(httpClient, indexerStatusService, configService, parsingService, logger)
+        {
+            _httpClient = httpClient;
+            _logger = logger;
+            _authCookieCache = cacheManager.GetCache<Dictionary<string, string>>(GetType(), "authCookies");
+        }
+
+        public override IIndexerRequestGenerator GetRequestGenerator()
+        {
+            return new GazelleRequestGenerator()
+            {
+                Settings = Settings,
+                HttpClient = _httpClient,
+                Logger = _logger,
+                AuthCookieCache = _authCookieCache
+            };
+        }
+
+        public override IParseIndexerResponse GetParser()
+        {
+            return new GazelleParser(Settings);
+        }
+
+        public override IEnumerable<ProviderDefinition> DefaultDefinitions
+        {
+            get
+            {
+                yield return GetDefinition("Apollo.Rip", GetSettings("https://apollo.rip"));
+                yield return GetDefinition("REDacted", GetSettings("https://redacted.ch"));
+                yield return GetDefinition("Not What CD", GetSettings("https://notwhat.cd"));
+
+            }
+        }
+
+        private IndexerDefinition GetDefinition(string name, GazelleSettings settings)
+        {
+            return new IndexerDefinition
+            {
+                EnableRss = false,
+                EnableSearch = false,
+                Name = name,
+                Implementation = GetType().Name,
+                Settings = settings,
+                Protocol = DownloadProtocol.Torrent,
+                SupportsRss = SupportsRss,
+                SupportsSearch = SupportsSearch
+            };
+        }
+
+        private GazelleSettings GetSettings(string url)
+        {
+            var settings = new GazelleSettings { BaseUrl = url };
+
+            return settings;
+        }
+    }
+}
--- a/src/NzbDrone.Core/Indexers/Gazelle/GazelleApi.cs
+++ b/src/NzbDrone.Core/Indexers/Gazelle/GazelleApi.cs
@ -0,0 +1,88 @@
+using System;
+using Newtonsoft.Json;
+using System.Collections.Generic;
+
+namespace NzbDrone.Core.Indexers.Gazelle
+{
+    public class GazelleArtist
+    {
+        public string Name { get; set; }
+        public string Id { get; set; }
+        public string Aliasid { get; set; }
+    }
+
+    public class GazelleTorrent
+    {
+        public int TorrentId { get; set; }
+        public int EditionId { get; set; }
+        public List<GazelleArtist> Artists { get; set; }
+        public bool Remastered { get; set; }
+        public string RemasterYear { get; set; }
+        public string RemasterTitle { get; set; }
+        public string Media { get; set; }
+        public string Encoding { get; set; }
+        public string Format { get; set; }
+        public bool HasLog { get; set; }
+        public int LogScore { get; set; }
+        public bool HasQueue { get; set; }
+        public bool Scene { get; set; }
+        public bool VanityHouse { get; set; }
+        public int FileCount { get; set; }
+        public DateTime Time { get; set; }
+        public string Size { get; set; }
+        public string Snatches { get; set; }
+        public string Seeders { get; set; }
+        public string Leechers { get; set; }
+        public bool IsFreeLeech { get; set; }
+        public bool IsNeutralLeech { get; set; }
+        public bool IsPersonalFreeLeech { get; set; }
+        public bool CanUseToken { get; set; }
+    }
+
+    public class GazelleRelease
+    {
+        public string GroupId { get; set; }
+        public string GroupName { get; set; }
+        public string Artist { get; set; }
+        public string GroupYear { get; set; }
+        public string Cover { get; set; }
+        public List<string> Tags { get; set; }
+        public string ReleaseType { get; set; }
+        public int TotalLeechers { get; set; }
+        public int TotalSeeders { get; set; }
+        public int TotalSnatched { get; set; }
+        public long MaxSize { get; set; }
+        public string GroupTime { get; set; }
+        public List<GazelleTorrent> Torrents { get; set; }
+    }
+
+    public class GazelleResponse
+    {
+        public string Status { get; set; }
+        public GazelleBrowseResponse Response { get; set; }
+    }
+
+    public class GazelleBrowseResponse
+    {
+        public List<GazelleRelease> Results { get; set; }
+        public string CurrentPage { get; set; }
+        public string Pages { get; set; }
+    }
+
+    public class GazelleAuthResponse
+    {
+        public string Status { get; set; }
+        public GazelleIndexResponse Response { get; set; }
+
+    }
+
+    public class GazelleIndexResponse
+    {
+        public string Username { get; set; }
+        public string Id { get; set; }
+        public string Authkey { get; set; }
+        public string Passkey { get; set; }
+
+    }
+
+}
--- a/src/NzbDrone.Core/Indexers/Gazelle/GazelleInfo.cs
+++ b/src/NzbDrone.Core/Indexers/Gazelle/GazelleInfo.cs
@ -0,0 +1,13 @@
+using NzbDrone.Core.Parser.Model;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace NzbDrone.Core.Indexers.Gazelle
+{
+    public class GazelleInfo : TorrentInfo
+    {
+        public bool? Scene { get; set; }
+    }
+}
--- a/src/NzbDrone.Core/Indexers/Gazelle/GazelleParser.cs
+++ b/src/NzbDrone.Core/Indexers/Gazelle/GazelleParser.cs
@ -0,0 +1,112 @@
+using System.Collections.Generic;
+using System.Net;
+using Newtonsoft.Json;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Indexers.Exceptions;
+using NzbDrone.Core.Parser.Model;
+using System.Linq;
+using NzbDrone.Common.Cache;
+using NzbDrone.Common.Extensions;
+
+namespace NzbDrone.Core.Indexers.Gazelle
+{
+    public class GazelleParser : IParseIndexerResponse
+    {
+        private readonly GazelleSettings _settings;
+        public ICached<Dictionary<string, string>> AuthCookieCache { get; set; }
+
+        public GazelleParser(GazelleSettings settings)
+        {
+            _settings = settings;
+        }
+
+        public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
+        {
+            var torrentInfos = new List<ReleaseInfo>();
+
+            if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
+            {
+                // Remove cookie cache
+                AuthCookieCache.Remove(_settings.BaseUrl.Trim().TrimEnd('/'));
+
+                throw new IndexerException(indexerResponse, $"Unexpected response status {indexerResponse.HttpResponse.StatusCode} code from API request");
+            }
+
+            if (!indexerResponse.HttpResponse.Headers.ContentType.Contains(HttpAccept.Json.Value))
+            {
+                // Remove cookie cache
+                AuthCookieCache.Remove(_settings.BaseUrl.Trim().TrimEnd('/'));
+
+                throw new IndexerException(indexerResponse, $"Unexpected response header {indexerResponse.HttpResponse.Headers.ContentType} from API request, expected {HttpAccept.Json.Value}");
+            }
+
+            var jsonResponse = JsonConvert.DeserializeObject<GazelleResponse>(indexerResponse.Content);
+            if (jsonResponse.Status != "success" ||
+                jsonResponse.Status.IsNullOrWhiteSpace() ||
+                jsonResponse.Response == null)
+            {
+                return torrentInfos;
+            }
+
+
+            foreach (var result in jsonResponse.Response.Results)
+            {
+                if (result.Torrents != null)
+                {
+                    foreach (var torrent in result.Torrents)
+                    {
+                        var id = torrent.TorrentId;
+
+                        torrentInfos.Add(new GazelleInfo()
+                        {
+                            Guid = string.Format("Gazelle-{0}", id),
+                            Artist = result.Artist,
+                            // Splice Title from info to avoid calling API again for every torrent.
+                            Title = result.Artist + " - " + result.GroupName + " (" + result.GroupYear +") (" + torrent.Format + " " + torrent.Encoding + ")",
+                            Album = result.GroupName,
+                            Container = torrent.Encoding,
+                            Codec = torrent.Format,
+                            Size = long.Parse(torrent.Size),
+                            DownloadUrl = GetDownloadUrl(id, _settings.AuthKey, _settings.PassKey),
+                            InfoUrl = GetInfoUrl(result.GroupId, id),
+                            Seeders = int.Parse(torrent.Seeders),
+                            Peers = int.Parse(torrent.Leechers) + int.Parse(torrent.Seeders),
+                            PublishDate = torrent.Time.ToUniversalTime(),
+                            Scene = torrent.Scene,
+                        });
+                    }
+                }
+            }
+
+            var torr = torrentInfos;
+            // order by date
+            return
+                torrentInfos
+                    .OrderByDescending(o => o.PublishDate)
+                    .ToArray();
+
+        }
+
+        private string GetDownloadUrl(int torrentId, string authKey, string passKey)
+        {
+            var url = new HttpUri(_settings.BaseUrl)
+                .CombinePath("/torrents.php")
+                .AddQueryParam("action", "download")
+                .AddQueryParam("id", torrentId)
+                .AddQueryParam("authkey", authKey)
+                .AddQueryParam("torrent_pass", passKey);
+
+            return url.FullUri;
+        }
+
+        private string GetInfoUrl(string groupId, int torrentId)
+        {
+            var url = new HttpUri(_settings.BaseUrl)
+                .CombinePath("/torrents.php")
+                .AddQueryParam("id", groupId)
+                .AddQueryParam("torrentid", torrentId);
+
+            return url.FullUri;
+        }
+    }
+}
--- a/src/NzbDrone.Core/Indexers/Gazelle/GazelleRequestGenerator.cs
+++ b/src/NzbDrone.Core/Indexers/Gazelle/GazelleRequestGenerator.cs
@ -0,0 +1,141 @@
+using System;
+using System.Collections.Generic;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.IndexerSearch.Definitions;
+using NzbDrone.Common.Cache;
+using NLog;
+using NzbDrone.Common.Serializer;
+
+namespace NzbDrone.Core.Indexers.Gazelle
+{
+    public class GazelleRequestGenerator : IIndexerRequestGenerator
+    {
+
+        public GazelleSettings Settings { get; set; }
+
+        public ICached<Dictionary<string, string>> AuthCookieCache { get; set; }
+        public IHttpClient HttpClient { get; set; }
+        public Logger Logger { get; set; }
+
+        public virtual IndexerPageableRequestChain GetRecentRequests()
+        {
+            var pageableRequests = new IndexerPageableRequestChain();
+
+            pageableRequests.Add(GetRequest(null));
+
+            return pageableRequests;
+        }
+
+        public IndexerPageableRequestChain GetSearchRequests(AlbumSearchCriteria searchCriteria)
+        {
+            var pageableRequests = new IndexerPageableRequestChain();
+            pageableRequests.Add(GetRequest(string.Format("&artistname={0}&groupname={1}", searchCriteria.Artist.Name, searchCriteria.AlbumTitle)));
+            return pageableRequests;
+        }
+
+        public IndexerPageableRequestChain GetSearchRequests(ArtistSearchCriteria searchCriteria)
+        {
+            var pageableRequests = new IndexerPageableRequestChain();
+            pageableRequests.Add(GetRequest(string.Format("&artistname={0}",searchCriteria.Artist.Name)));
+            return pageableRequests;
+        }
+
+        private IEnumerable<IndexerRequest> GetRequest(string searchParameters)
+        {
+            Authenticate();
+
+            var filter = "";
+            if (searchParameters == null)
+            {
+
+            }
+
+            var request =
+                new IndexerRequest(
+                    $"{Settings.BaseUrl.Trim().TrimEnd('/')}/ajax.php?action=browse&searchstr={searchParameters}{filter}",
+                    HttpAccept.Json);
+
+            var cookies = AuthCookieCache.Find(Settings.BaseUrl.Trim().TrimEnd('/'));
+            foreach (var cookie in cookies)
+            {
+                request.HttpRequest.Cookies[cookie.Key] = cookie.Value;
+            }
+
+            yield return request;
+        }
+
+        private GazelleAuthResponse GetIndex(Dictionary<string,string> cookies)
+        {
+            var indexRequestBuilder = new HttpRequestBuilder($"{Settings.BaseUrl.Trim().TrimEnd('/')}")
+            {
+                LogResponseContent = true
+            };
+
+            indexRequestBuilder.SetCookies(cookies);
+            indexRequestBuilder.Method = HttpMethod.GET;
+            indexRequestBuilder.Resource("ajax.php?action=index");
+
+            var authIndexRequest = indexRequestBuilder
+                .Accept(HttpAccept.Json)
+                .Build();
+
+            var indexResponse = HttpClient.Execute(authIndexRequest);
+
+            var result = Json.Deserialize<GazelleAuthResponse>(indexResponse.Content);
+
+            return result;
+        }
+
+        private void Authenticate()
+        {
+
+            var requestBuilder = new HttpRequestBuilder($"{Settings.BaseUrl.Trim().TrimEnd('/')}")
+            {
+                LogResponseContent = true
+            };
+
+            requestBuilder.Method = HttpMethod.POST;
+            requestBuilder.Resource("login.php");
+            requestBuilder.PostProcess += r => r.RequestTimeout = TimeSpan.FromSeconds(15);
+
+            var authKey = Settings.BaseUrl.Trim().TrimEnd('/');
+            var cookies = AuthCookieCache.Find(authKey);
+
+            if (cookies == null)
+            {
+                AuthCookieCache.Remove(authKey);
+                var authLoginRequest = requestBuilder
+                    .AddFormParameter("username", Settings.Username)
+                    .AddFormParameter("password", Settings.Password)
+                    .AddFormParameter("keeplogged", "1")
+                    .SetHeader("Content-Type", "multipart/form-data")
+                    .Accept(HttpAccept.Json)
+                    .Build();
+
+                var response = HttpClient.Execute(authLoginRequest);
+
+                cookies = response.GetCookies();
+                AuthCookieCache.Set(authKey, cookies, new TimeSpan(7, 0, 0, 0, 0)); // re-auth every 7 days
+                requestBuilder.SetCookies(cookies);
+            }
+            else
+            {
+                requestBuilder.SetCookies(cookies);
+            }
+
+            var index = GetIndex(cookies);
+
+            if (index.Status != "success" || string.IsNullOrWhiteSpace(index.Status))
+            {
+                Logger.Debug("Gazelle authentication failed.");
+                throw new Exception("Failed to authenticate with Gazelle.");
+            }
+
+            Logger.Debug("Gazelle authentication succeeded.");
+
+            Settings.AuthKey = index.Response.Authkey;
+            Settings.PassKey = index.Response.Passkey;
+
+        }        
+    }
+}
--- a/src/NzbDrone.Core/Indexers/Gazelle/GazelleSettings.cs
+++ b/src/NzbDrone.Core/Indexers/Gazelle/GazelleSettings.cs
@ -0,0 +1,45 @@
+using FluentValidation;
+using NzbDrone.Core.Annotations;
+using NzbDrone.Core.ThingiProvider;
+using NzbDrone.Core.Validation;
+using System.Text.RegularExpressions;
+
+namespace NzbDrone.Core.Indexers.Gazelle
+{
+    public class GazelleSettingsValidator : AbstractValidator<GazelleSettings>
+    {
+        public GazelleSettingsValidator()
+        {
+            RuleFor(c => c.BaseUrl).ValidRootUrl();
+            RuleFor(c => c.Username).NotEmpty();
+            RuleFor(c => c.Password).NotEmpty();
+        }
+    }
+
+    public class GazelleSettings : IProviderConfig
+    {
+        private static readonly GazelleSettingsValidator Validator = new GazelleSettingsValidator();
+
+        public GazelleSettings()
+        {
+
+        }
+
+        public string AuthKey;
+        public string PassKey;
+
+        [FieldDefinition(0, Label = "URL", Advanced = true, HelpText = "Do not change this unless you know what you're doing. Since your cookie will be sent to that host.")]
+        public string BaseUrl { get; set; }
+
+        [FieldDefinition(1, Label = "Username", HelpText = "Username")]
+        public string Username { get; set; }
+
+        [FieldDefinition(2, Label = "Password", Type = FieldType.Password, HelpText = "Password")]
+        public string Password { get; set; }
+
+        public NzbDroneValidationResult Validate()
+        {
+            return new NzbDroneValidationResult(Validator.Validate(this));
+        }
+    }
+}
--- a/src/NzbDrone.Core/NzbDrone.Core.csproj
+++ b/src/NzbDrone.Core/NzbDrone.Core.csproj
@ -616,6 +616,12 @@
    <Compile Include="IndexerSearch\CutoffUnmetAlbumSearchCommand.cs" />
    <Compile Include="IndexerSearch\Definitions\AlbumSearchCriteria.cs" />
    <Compile Include="IndexerSearch\Definitions\ArtistSearchCriteria.cs" />
+    <Compile Include="Indexers\Gazelle\Gazelle.cs" />
+    <Compile Include="Indexers\Gazelle\GazelleApi.cs" />
+    <Compile Include="Indexers\Gazelle\GazelleInfo.cs" />
+    <Compile Include="Indexers\Gazelle\GazelleParser.cs" />
+    <Compile Include="Indexers\Gazelle\GazelleRequestGenerator.cs" />
+    <Compile Include="Indexers\Gazelle\GazelleSettings.cs" />
    <Compile Include="Indexers\Waffles\WafflesRssParser.cs" />
    <Compile Include="Indexers\Waffles\Waffles.cs" />
    <Compile Include="Indexers\Waffles\WafflesRequestGenerator.cs" />