Fixed: Use Proxy for MediaCovers and Metadata (#4916)

[common]

Co-authored-by: Soroush <soroush@falahati.net>

src/NzbDrone.Common.Test/Http/HttpClientFixture.cs

@@ -104,9 +104,9 @@ public void SetUp()
             Mocker.SetConstant<IHttpDispatcher>(Mocker.Resolve<TDispatcher>());
 
             // Used for manual testing of socks proxies.
-            //Mocker.GetMock<IHttpProxySettingsProvider>()
+            // Mocker.GetMock<IHttpProxySettingsProvider>()
-            //      .Setup(v => v.GetProxySettings(It.IsAny<HttpRequest>()))
+            //       .Setup(v => v.GetProxySettings(It.IsAny<HttpUri>()))
-            //      .Returns(new HttpProxySettings(ProxyType.Socks5, "127.0.0.1", 5476, "", false));
+            //       .Returns(new HttpProxySettings(ProxyType.Socks5, "127.0.0.1", 5476, "", false));
 
             // Roundrobin over the two servers, to reduce the chance of hitting the ratelimiter.
             _httpBinHost2 = _httpBinHosts[_httpBinRandom++ % _httpBinHosts.Length];
@@ -283,6 +283,38 @@ public void should_send_headers(string header, string value)
             response.Resource.Headers[header].ToString().Should().Be(value);
         }
 
+        [Test]
+        public void should_download_file()
+        {
+            var file = GetTempFilePath();
+
+            var url = "https://radarr.video/img/slider/moviedetails.png";
+
+            Subject.DownloadFile(url, file);
+
+            var fileInfo = new FileInfo(file);
+            fileInfo.Exists.Should().BeTrue();
+            fileInfo.Length.Should().Be(251536);
+        }
+
+        [Test]
+        public void should_download_file_with_redirect()
+        {
+            var file = GetTempFilePath();
+
+            var request = new HttpRequestBuilder($"https://{_httpBinHost}/redirect-to")
+                .AddQueryParam("url", $"https://radarr.video/img/slider/moviedetails.png")
+                .Build();
+
+            Subject.DownloadFile(request.Url.FullUri, file);
+
+            ExceptionVerification.ExpectedErrors(0);
+
+            var fileInfo = new FileInfo(file);
+            fileInfo.Exists.Should().BeTrue();
+            fileInfo.Length.Should().Be(251536);
+        }
+
         [Test]
         public void should_not_download_file_with_error()
         {
@@ -320,7 +352,7 @@ public void GivenOldCookie()
             var oldRequest = new HttpRequest($"https://{_httpBinHost2}/get");
             oldRequest.Cookies["my"] = "cookie";
 
-            var oldClient = new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve<ICacheManager>(), Mocker.Resolve<IRateLimitService>(), Mocker.Resolve<IHttpDispatcher>(), Mocker.GetMock<IUserAgentBuilder>().Object, Mocker.Resolve<Logger>());
+            var oldClient = new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve<ICacheManager>(), Mocker.Resolve<IRateLimitService>(), Mocker.Resolve<IHttpDispatcher>(), Mocker.Resolve<Logger>());
 
             oldClient.Should().NotBeSameAs(Subject);
 

src/NzbDrone.Common/Http/Dispatchers/IHttpDispatcher.cs

@@ -5,5 +5,6 @@ namespace NzbDrone.Common.Http.Dispatchers
     public interface IHttpDispatcher
     {
         HttpResponse GetResponse(HttpRequest request, CookieContainer cookies);
+        void DownloadFile(string url, string fileName);
     }
 }

src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Diagnostics;
 using System.IO;
 using System.IO.Compression;
 using System.Net;
@@ -57,7 +58,7 @@ public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies)
                 webRequest.Timeout = (int)Math.Ceiling(request.RequestTimeout.TotalMilliseconds);
             }
 
-            AddProxy(webRequest, request);
+            webRequest.Proxy = GetProxy(request.Url);
 
             if (request.Headers != null)
             {
@@ -145,13 +146,54 @@ public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies)
             return new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), data, httpWebResponse.StatusCode);
         }
 
-        protected virtual void AddProxy(HttpWebRequest webRequest, HttpRequest request)
+        public void DownloadFile(string url, string fileName)
         {
-            var proxySettings = _proxySettingsProvider.GetProxySettings(request);
+            try
+            {
+                var fileInfo = new FileInfo(fileName);
+                if (fileInfo.Directory != null && !fileInfo.Directory.Exists)
+                {
+                    fileInfo.Directory.Create();
+                }
+
+                _logger.Debug("Downloading [{0}] to [{1}]", url, fileName);
+
+                var stopWatch = Stopwatch.StartNew();
+                var uri = new HttpUri(url);
+
+                using (var webClient = new GZipWebClient())
+                {
+                    webClient.Headers.Add(HttpRequestHeader.UserAgent, _userAgentBuilder.GetUserAgent());
+                    webClient.Proxy = GetProxy(uri);
+                    webClient.DownloadFile(uri.FullUri, fileName);
+                    stopWatch.Stop();
+                    _logger.Debug("Downloading Completed. took {0:0}s", stopWatch.Elapsed.Seconds);
+                }
+            }
+            catch (WebException e)
+            {
+                _logger.Warn("Failed to get response from: {0} {1}", url, e.Message);
+                throw;
+            }
+            catch (Exception e)
+            {
+                _logger.Warn(e, "Failed to get response from: " + url);
+                throw;
+            }
+        }
+
+        protected virtual IWebProxy GetProxy(HttpUri uri)
+        {
+            IWebProxy proxy = null;
+
+            var proxySettings = _proxySettingsProvider.GetProxySettings(uri);
+
             if (proxySettings != null)
             {
-                webRequest.Proxy = _createManagedWebProxy.GetWebProxy(proxySettings);
+                proxy = _createManagedWebProxy.GetWebProxy(proxySettings);
             }
+
+            return proxy;
         }
 
         protected virtual void AddRequestHeaders(HttpWebRequest webRequest, HttpHeader headers)

src/NzbDrone.Common/Http/HttpClient.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.IO;
 using System.Linq;
 using System.Net;
 using NLog;
@@ -35,19 +34,16 @@ public class HttpClient : IHttpClient
         private readonly ICached<CookieContainer> _cookieContainerCache;
         private readonly List<IHttpRequestInterceptor> _requestInterceptors;
         private readonly IHttpDispatcher _httpDispatcher;
-        private readonly IUserAgentBuilder _userAgentBuilder;
 
         public HttpClient(IEnumerable<IHttpRequestInterceptor> requestInterceptors,
             ICacheManager cacheManager,
             IRateLimitService rateLimitService,
             IHttpDispatcher httpDispatcher,
-            IUserAgentBuilder userAgentBuilder,
             Logger logger)
         {
             _requestInterceptors = requestInterceptors.ToList();
             _rateLimitService = rateLimitService;
             _httpDispatcher = httpDispatcher;
-            _userAgentBuilder = userAgentBuilder;
             _logger = logger;
 
             ServicePointManager.DefaultConnectionLimit = 12;
@@ -233,35 +229,7 @@ private void HandleResponseCookies(HttpResponse response, CookieContainer cookie
 
         public void DownloadFile(string url, string fileName)
         {
-            try
+            _httpDispatcher.DownloadFile(url, fileName);
-            {
-                var fileInfo = new FileInfo(fileName);
-                if (fileInfo.Directory != null && !fileInfo.Directory.Exists)
-                {
-                    fileInfo.Directory.Create();
-                }
-
-                _logger.Debug("Downloading [{0}] to [{1}]", url, fileName);
-
-                var stopWatch = Stopwatch.StartNew();
-                using (var webClient = new GZipWebClient())
-                {
-                    webClient.Headers.Add(HttpRequestHeader.UserAgent, _userAgentBuilder.GetUserAgent());
-                    webClient.DownloadFile(url, fileName);
-                    stopWatch.Stop();
-                    _logger.Debug("Downloading Completed. took {0:0}s", stopWatch.Elapsed.Seconds);
-                }
-            }
-            catch (WebException e)
-            {
-                _logger.Warn("Failed to get response from: {0} {1}", url, e.Message);
-                throw;
-            }
-            catch (Exception e)
-            {
-                _logger.Warn(e, "Failed to get response from: " + url);
-                throw;
-            }
         }
 
         public HttpResponse Get(HttpRequest request)

src/NzbDrone.Common/Http/Proxy/IHttpProxySettingsProvider.cs

@@ -2,7 +2,7 @@ namespace NzbDrone.Common.Http.Proxy
 {
     public interface IHttpProxySettingsProvider
     {
-        HttpProxySettings GetProxySettings(HttpRequest request);
+        HttpProxySettings GetProxySettings(HttpUri uri);
         HttpProxySettings GetProxySettings();
     }
 }

src/NzbDrone.Core.Test/Framework/CoreTest.cs

@@ -26,7 +26,7 @@ protected void UseRealHttp()
             Mocker.SetConstant<IHttpProxySettingsProvider>(new HttpProxySettingsProvider(Mocker.Resolve<ConfigService>()));
             Mocker.SetConstant<ICreateManagedWebProxy>(new ManagedWebProxyFactory(Mocker.Resolve<CacheManager>()));
             Mocker.SetConstant<IHttpDispatcher>(new ManagedHttpDispatcher(Mocker.Resolve<IHttpProxySettingsProvider>(), Mocker.Resolve<ICreateManagedWebProxy>(), Mocker.Resolve<UserAgentBuilder>(), Mocker.Resolve<IPlatformInfo>(), TestLogger));
-            Mocker.SetConstant<IHttpClient>(new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve<CacheManager>(), Mocker.Resolve<RateLimitService>(), Mocker.Resolve<IHttpDispatcher>(), Mocker.Resolve<UserAgentBuilder>(), TestLogger));
+            Mocker.SetConstant<IHttpClient>(new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve<CacheManager>(), Mocker.Resolve<RateLimitService>(), Mocker.Resolve<IHttpDispatcher>(), TestLogger));
             Mocker.SetConstant<IRadarrCloudRequestBuilder>(new RadarrCloudRequestBuilder());
         }
 

src/NzbDrone.Core/Http/HttpProxySettingsProvider.cs

@@ -15,7 +15,7 @@ public HttpProxySettingsProvider(IConfigService configService)
             _configService = configService;
         }
 
-        public HttpProxySettings GetProxySettings(HttpRequest request)
+        public HttpProxySettings GetProxySettings(HttpUri uri)
         {
             var proxySettings = GetProxySettings();
             if (proxySettings == null)
@@ -23,7 +23,7 @@ public HttpProxySettings GetProxySettings(HttpRequest request)
                 return null;
             }
 
-            if (ShouldProxyBeBypassed(proxySettings, request.Url))
+            if (ShouldProxyBeBypassed(proxySettings, uri))
             {
                 return null;
             }
@@ -49,7 +49,7 @@ public HttpProxySettings GetProxySettings()
 
         public bool ShouldProxyBeBypassed(HttpProxySettings proxySettings, HttpUri url)
         {
-            //We are utilising the WebProxy implementation here to save us having to reimplement it. This way we use Microsofts implementation
+            //We are utilizing the WebProxy implementation here to save us having to re-implement it. This way we use Microsofts implementation
             var proxy = new WebProxy(proxySettings.Host + ":" + proxySettings.Port, proxySettings.BypassLocalAddress, proxySettings.BypassListAsArray);
 
             return proxy.IsBypassed((Uri)url);

src/NzbDrone.Core/Notifications/Trakt/TraktProxy.cs

@@ -1,6 +1,3 @@
-using System;
-using System.Net;
-using FluentValidation.Results;
 using NLog;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;