diff --git a/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs b/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs index e1cb27775..3e880616e 100644 --- a/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs +++ b/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs @@ -104,9 +104,9 @@ namespace NzbDrone.Common.Test.Http Mocker.SetConstant(Mocker.Resolve()); // Used for manual testing of socks proxies. - //Mocker.GetMock() - // .Setup(v => v.GetProxySettings(It.IsAny())) - // .Returns(new HttpProxySettings(ProxyType.Socks5, "127.0.0.1", 5476, "", false)); + // Mocker.GetMock() + // .Setup(v => v.GetProxySettings(It.IsAny())) + // .Returns(new HttpProxySettings(ProxyType.Socks5, "127.0.0.1", 5476, "", false)); // Roundrobin over the two servers, to reduce the chance of hitting the ratelimiter. _httpBinHost2 = _httpBinHosts[_httpBinRandom++ % _httpBinHosts.Length]; @@ -283,6 +283,38 @@ namespace NzbDrone.Common.Test.Http response.Resource.Headers[header].ToString().Should().Be(value); } + [Test] + public void should_download_file() + { + var file = GetTempFilePath(); + + var url = "https://radarr.video/img/slider/moviedetails.png"; + + Subject.DownloadFile(url, file); + + var fileInfo = new FileInfo(file); + fileInfo.Exists.Should().BeTrue(); + fileInfo.Length.Should().Be(251536); + } + + [Test] + public void should_download_file_with_redirect() + { + var file = GetTempFilePath(); + + var request = new HttpRequestBuilder($"https://{_httpBinHost}/redirect-to") + .AddQueryParam("url", $"https://radarr.video/img/slider/moviedetails.png") + .Build(); + + Subject.DownloadFile(request.Url.FullUri, file); + + ExceptionVerification.ExpectedErrors(0); + + var fileInfo = new FileInfo(file); + fileInfo.Exists.Should().BeTrue(); + fileInfo.Length.Should().Be(251536); + } + [Test] public void should_not_download_file_with_error() { @@ -320,7 +352,7 @@ namespace NzbDrone.Common.Test.Http var oldRequest = new HttpRequest($"https://{_httpBinHost2}/get"); oldRequest.Cookies["my"] = "cookie"; - var oldClient = new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), Mocker.GetMock().Object, Mocker.Resolve()); + var oldClient = new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve()); oldClient.Should().NotBeSameAs(Subject); diff --git a/src/NzbDrone.Common/Http/Dispatchers/IHttpDispatcher.cs b/src/NzbDrone.Common/Http/Dispatchers/IHttpDispatcher.cs index 8e665ceed..b4cae7ff8 100644 --- a/src/NzbDrone.Common/Http/Dispatchers/IHttpDispatcher.cs +++ b/src/NzbDrone.Common/Http/Dispatchers/IHttpDispatcher.cs @@ -5,5 +5,6 @@ namespace NzbDrone.Common.Http.Dispatchers public interface IHttpDispatcher { HttpResponse GetResponse(HttpRequest request, CookieContainer cookies); + void DownloadFile(string url, string fileName); } } diff --git a/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs b/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs index 96762f120..356d684b9 100644 --- a/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs +++ b/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using System.IO; using System.IO.Compression; using System.Net; @@ -57,7 +58,7 @@ namespace NzbDrone.Common.Http.Dispatchers webRequest.Timeout = (int)Math.Ceiling(request.RequestTimeout.TotalMilliseconds); } - AddProxy(webRequest, request); + webRequest.Proxy = GetProxy(request.Url); if (request.Headers != null) { @@ -145,13 +146,54 @@ namespace NzbDrone.Common.Http.Dispatchers return new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), data, httpWebResponse.StatusCode); } - protected virtual void AddProxy(HttpWebRequest webRequest, HttpRequest request) + public void DownloadFile(string url, string fileName) { - var proxySettings = _proxySettingsProvider.GetProxySettings(request); + try + { + var fileInfo = new FileInfo(fileName); + if (fileInfo.Directory != null && !fileInfo.Directory.Exists) + { + fileInfo.Directory.Create(); + } + + _logger.Debug("Downloading [{0}] to [{1}]", url, fileName); + + var stopWatch = Stopwatch.StartNew(); + var uri = new HttpUri(url); + + using (var webClient = new GZipWebClient()) + { + webClient.Headers.Add(HttpRequestHeader.UserAgent, _userAgentBuilder.GetUserAgent()); + webClient.Proxy = GetProxy(uri); + webClient.DownloadFile(uri.FullUri, fileName); + stopWatch.Stop(); + _logger.Debug("Downloading Completed. took {0:0}s", stopWatch.Elapsed.Seconds); + } + } + catch (WebException e) + { + _logger.Warn("Failed to get response from: {0} {1}", url, e.Message); + throw; + } + catch (Exception e) + { + _logger.Warn(e, "Failed to get response from: " + url); + throw; + } + } + + protected virtual IWebProxy GetProxy(HttpUri uri) + { + IWebProxy proxy = null; + + var proxySettings = _proxySettingsProvider.GetProxySettings(uri); + if (proxySettings != null) { - webRequest.Proxy = _createManagedWebProxy.GetWebProxy(proxySettings); + proxy = _createManagedWebProxy.GetWebProxy(proxySettings); } + + return proxy; } protected virtual void AddRequestHeaders(HttpWebRequest webRequest, HttpHeader headers) diff --git a/src/NzbDrone.Common/Http/HttpClient.cs b/src/NzbDrone.Common/Http/HttpClient.cs index cce6d125f..f89307d5e 100644 --- a/src/NzbDrone.Common/Http/HttpClient.cs +++ b/src/NzbDrone.Common/Http/HttpClient.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.IO; using System.Linq; using System.Net; using NLog; @@ -35,19 +34,16 @@ namespace NzbDrone.Common.Http private readonly ICached _cookieContainerCache; private readonly List _requestInterceptors; private readonly IHttpDispatcher _httpDispatcher; - private readonly IUserAgentBuilder _userAgentBuilder; public HttpClient(IEnumerable requestInterceptors, ICacheManager cacheManager, IRateLimitService rateLimitService, IHttpDispatcher httpDispatcher, - IUserAgentBuilder userAgentBuilder, Logger logger) { _requestInterceptors = requestInterceptors.ToList(); _rateLimitService = rateLimitService; _httpDispatcher = httpDispatcher; - _userAgentBuilder = userAgentBuilder; _logger = logger; ServicePointManager.DefaultConnectionLimit = 12; @@ -233,35 +229,7 @@ namespace NzbDrone.Common.Http public void DownloadFile(string url, string fileName) { - try - { - var fileInfo = new FileInfo(fileName); - if (fileInfo.Directory != null && !fileInfo.Directory.Exists) - { - fileInfo.Directory.Create(); - } - - _logger.Debug("Downloading [{0}] to [{1}]", url, fileName); - - var stopWatch = Stopwatch.StartNew(); - using (var webClient = new GZipWebClient()) - { - webClient.Headers.Add(HttpRequestHeader.UserAgent, _userAgentBuilder.GetUserAgent()); - webClient.DownloadFile(url, fileName); - stopWatch.Stop(); - _logger.Debug("Downloading Completed. took {0:0}s", stopWatch.Elapsed.Seconds); - } - } - catch (WebException e) - { - _logger.Warn("Failed to get response from: {0} {1}", url, e.Message); - throw; - } - catch (Exception e) - { - _logger.Warn(e, "Failed to get response from: " + url); - throw; - } + _httpDispatcher.DownloadFile(url, fileName); } public HttpResponse Get(HttpRequest request) diff --git a/src/NzbDrone.Common/Http/Proxy/IHttpProxySettingsProvider.cs b/src/NzbDrone.Common/Http/Proxy/IHttpProxySettingsProvider.cs index bb5909395..ed223fe63 100644 --- a/src/NzbDrone.Common/Http/Proxy/IHttpProxySettingsProvider.cs +++ b/src/NzbDrone.Common/Http/Proxy/IHttpProxySettingsProvider.cs @@ -2,7 +2,7 @@ namespace NzbDrone.Common.Http.Proxy { public interface IHttpProxySettingsProvider { - HttpProxySettings GetProxySettings(HttpRequest request); + HttpProxySettings GetProxySettings(HttpUri uri); HttpProxySettings GetProxySettings(); } } diff --git a/src/NzbDrone.Core.Test/Framework/CoreTest.cs b/src/NzbDrone.Core.Test/Framework/CoreTest.cs index 1036090a4..48dc033d8 100644 --- a/src/NzbDrone.Core.Test/Framework/CoreTest.cs +++ b/src/NzbDrone.Core.Test/Framework/CoreTest.cs @@ -26,7 +26,7 @@ namespace NzbDrone.Core.Test.Framework Mocker.SetConstant(new HttpProxySettingsProvider(Mocker.Resolve())); Mocker.SetConstant(new ManagedWebProxyFactory(Mocker.Resolve())); Mocker.SetConstant(new ManagedHttpDispatcher(Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), TestLogger)); - Mocker.SetConstant(new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), TestLogger)); + Mocker.SetConstant(new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), TestLogger)); Mocker.SetConstant(new RadarrCloudRequestBuilder()); } diff --git a/src/NzbDrone.Core/Http/HttpProxySettingsProvider.cs b/src/NzbDrone.Core/Http/HttpProxySettingsProvider.cs index dbea6dcd0..a56bcf2e6 100644 --- a/src/NzbDrone.Core/Http/HttpProxySettingsProvider.cs +++ b/src/NzbDrone.Core/Http/HttpProxySettingsProvider.cs @@ -15,7 +15,7 @@ namespace NzbDrone.Core.Http _configService = configService; } - public HttpProxySettings GetProxySettings(HttpRequest request) + public HttpProxySettings GetProxySettings(HttpUri uri) { var proxySettings = GetProxySettings(); if (proxySettings == null) @@ -23,7 +23,7 @@ namespace NzbDrone.Core.Http return null; } - if (ShouldProxyBeBypassed(proxySettings, request.Url)) + if (ShouldProxyBeBypassed(proxySettings, uri)) { return null; } @@ -49,7 +49,7 @@ namespace NzbDrone.Core.Http public bool ShouldProxyBeBypassed(HttpProxySettings proxySettings, HttpUri url) { - //We are utilising the WebProxy implementation here to save us having to reimplement it. This way we use Microsofts implementation + //We are utilizing the WebProxy implementation here to save us having to re-implement it. This way we use Microsofts implementation var proxy = new WebProxy(proxySettings.Host + ":" + proxySettings.Port, proxySettings.BypassLocalAddress, proxySettings.BypassListAsArray); return proxy.IsBypassed((Uri)url); diff --git a/src/NzbDrone.Core/Notifications/Trakt/TraktProxy.cs b/src/NzbDrone.Core/Notifications/Trakt/TraktProxy.cs index ed05a36ee..a428335ce 100644 --- a/src/NzbDrone.Core/Notifications/Trakt/TraktProxy.cs +++ b/src/NzbDrone.Core/Notifications/Trakt/TraktProxy.cs @@ -1,6 +1,3 @@ -using System; -using System.Net; -using FluentValidation.Results; using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http;