Fixed: Revised handling of cookies in case of redirects.

src/NzbDrone.Common.Test/Http/HttpClientFixture.cs

@@ -285,19 +285,74 @@ namespace NzbDrone.Common.Test.Http
             response.Resource.Headers.Should().NotContainKey("Cookie");
         }
 
+        [Test]
+        public void should_not_store_request_cookie()
+        {
+            var requestGet = new HttpRequest($"http://{_httpBinHost}/get");
+            requestGet.Cookies.Add("my", "cookie");
+            requestGet.AllowAutoRedirect = false;
+            requestGet.StoreRequestCookie = false;
+            requestGet.StoreResponseCookie = false;
+            var responseGet = Subject.Get<HttpBinResource>(requestGet);
+
+            var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies");
+            requestCookies.AllowAutoRedirect = false;
+            var responseCookies = Subject.Get<HttpCookieResource>(requestCookies);
+
+            responseCookies.Resource.Cookies.Should().BeEmpty();
+
+            ExceptionVerification.IgnoreErrors();
+        }
+
+        [Test]
+        public void should_store_request_cookie()
+        {
+            var requestGet = new HttpRequest($"http://{_httpBinHost}/get");
+            requestGet.Cookies.Add("my", "cookie");
+            requestGet.AllowAutoRedirect = false;
+            requestGet.StoreRequestCookie.Should().BeTrue();
+            requestGet.StoreResponseCookie = false;
+            var responseGet = Subject.Get<HttpBinResource>(requestGet);
+
+            var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies");
+            requestCookies.AllowAutoRedirect = false;
+            var responseCookies = Subject.Get<HttpCookieResource>(requestCookies);
+
+            responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie");
+
+            ExceptionVerification.IgnoreErrors();
+        }
+
+        [Test]
+        public void should_delete_request_cookie()
+        {
+            var requestDelete = new HttpRequest($"http://{_httpBinHost}/cookies/delete?my");
+            requestDelete.Cookies.Add("my", "cookie");
+            requestDelete.AllowAutoRedirect = true;
+            requestDelete.StoreRequestCookie = false;
+            requestDelete.StoreResponseCookie = false;
+
+            // Delete and redirect since that's the only way to check the internal temporary cookie container
+            var responseCookies = Subject.Get<HttpCookieResource>(requestDelete);
+
+            responseCookies.Resource.Cookies.Should().BeEmpty();
+        }
+
         [Test]
         public void should_not_store_response_cookie()
         {
             var requestSet = new HttpRequest($"http://{_httpBinHost}/cookies/set?my=cookie");
             requestSet.AllowAutoRedirect = false;
+            requestSet.StoreRequestCookie = false;
+            requestSet.StoreResponseCookie.Should().BeFalse();
 
             var responseSet = Subject.Get(requestSet);
 
-            var request = new HttpRequest($"http://{_httpBinHost}/get");
+            var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies");
 
-            var response = Subject.Get<HttpBinResource>(request);
+            var responseCookies = Subject.Get<HttpCookieResource>(requestCookies);
 
-            response.Resource.Headers.Should().NotContainKey("Cookie");
+            responseCookies.Resource.Cookies.Should().BeEmpty();
 
             ExceptionVerification.IgnoreErrors();
         }
@@ -307,19 +362,31 @@ namespace NzbDrone.Common.Test.Http
         {
             var requestSet = new HttpRequest($"http://{_httpBinHost}/cookies/set?my=cookie");
             requestSet.AllowAutoRedirect = false;
+            requestSet.StoreRequestCookie = false;
             requestSet.StoreResponseCookie = true;
 
             var responseSet = Subject.Get(requestSet);
 
-            var request = new HttpRequest($"http://{_httpBinHost}/get");
+            var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies");
 
-            var response = Subject.Get<HttpBinResource>(request);
+            var responseCookies = Subject.Get<HttpCookieResource>(requestCookies);
 
-            response.Resource.Headers.Should().ContainKey("Cookie");
+            responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie");
 
-            var cookie = response.Resource.Headers["Cookie"].ToString();
+            ExceptionVerification.IgnoreErrors();
+        }
 
-            cookie.Should().Contain("my=cookie");
+        [Test]
+        public void should_temp_store_response_cookie()
+        {
+            var requestSet = new HttpRequest($"http://{_httpBinHost}/cookies/set?my=cookie");
+            requestSet.AllowAutoRedirect = true;
+            requestSet.StoreRequestCookie = false;
+            requestSet.StoreResponseCookie.Should().BeFalse();
+            var responseSet = Subject.Get<HttpCookieResource>(requestSet);
+
+            // Set and redirect since that's the only way to check the internal temporary cookie container
+            responseSet.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie");
 
             ExceptionVerification.IgnoreErrors();
         }
@@ -328,21 +395,129 @@ namespace NzbDrone.Common.Test.Http
         public void should_overwrite_response_cookie()
         {
             var requestSet = new HttpRequest($"http://{_httpBinHost}/cookies/set?my=cookie");
+            requestSet.Cookies.Add("my", "oldcookie");
             requestSet.AllowAutoRedirect = false;
+            requestSet.StoreRequestCookie = false;
             requestSet.StoreResponseCookie = true;
-            requestSet.Cookies["my"] = "oldcookie";
 
             var responseSet = Subject.Get(requestSet);
 
-            var request = new HttpRequest($"http://{_httpBinHost}/get");
+            var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies");
 
-            var response = Subject.Get<HttpBinResource>(request);
+            var responseCookies = Subject.Get<HttpCookieResource>(requestCookies);
 
-            response.Resource.Headers.Should().ContainKey("Cookie");
+            responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie");
 
-            var cookie = response.Resource.Headers["Cookie"].ToString();
+            ExceptionVerification.IgnoreErrors();
+        }
 
-            cookie.Should().Contain("my=cookie");
+        [Test]
+        public void should_overwrite_temp_response_cookie()
+        {
+            var requestSet = new HttpRequest($"http://{_httpBinHost}/cookies/set?my=cookie");
+            requestSet.Cookies.Add("my", "oldcookie");
+            requestSet.AllowAutoRedirect = true;
+            requestSet.StoreRequestCookie = true;
+            requestSet.StoreResponseCookie = false;
+
+            var responseSet = Subject.Get<HttpCookieResource>(requestSet);
+
+            responseSet.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie");
+
+            var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies");
+
+            var responseCookies = Subject.Get<HttpCookieResource>(requestCookies);
+
+            responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "oldcookie");
+
+            ExceptionVerification.IgnoreErrors();
+        }
+
+        [Test]
+        public void should_not_delete_response_cookie()
+        {
+            var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies");
+            requestCookies.Cookies.Add("my", "cookie");
+            requestCookies.AllowAutoRedirect = false;
+            requestCookies.StoreRequestCookie = true;
+            requestCookies.StoreResponseCookie = false;
+            var responseCookies = Subject.Get<HttpCookieResource>(requestCookies);
+
+            responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie");
+
+            var requestDelete = new HttpRequest($"http://{_httpBinHost}/cookies/delete?my");
+            requestDelete.AllowAutoRedirect = false;
+            requestDelete.StoreRequestCookie = false;
+            requestDelete.StoreResponseCookie = false;
+
+            var responseDelete = Subject.Get(requestDelete);
+
+            requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies");
+            requestCookies.StoreRequestCookie = false;
+            requestCookies.StoreResponseCookie = false;
+
+            responseCookies = Subject.Get<HttpCookieResource>(requestCookies);
+
+            responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie");
+
+            ExceptionVerification.IgnoreErrors();
+        }
+
+        [Test]
+        public void should_delete_response_cookie()
+        {
+            var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies");
+            requestCookies.Cookies.Add("my", "cookie");
+            requestCookies.AllowAutoRedirect = false;
+            requestCookies.StoreRequestCookie = true;
+            requestCookies.StoreResponseCookie = false;
+            var responseCookies = Subject.Get<HttpCookieResource>(requestCookies);
+
+            responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie");
+
+            var requestDelete = new HttpRequest($"http://{_httpBinHost}/cookies/delete?my");
+            requestDelete.AllowAutoRedirect = false;
+            requestDelete.StoreRequestCookie = false;
+            requestDelete.StoreResponseCookie = true;
+
+            var responseDelete = Subject.Get(requestDelete);
+
+            requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies");
+            requestCookies.StoreRequestCookie = false;
+            requestCookies.StoreResponseCookie = false;
+
+            responseCookies = Subject.Get<HttpCookieResource>(requestCookies);
+
+            responseCookies.Resource.Cookies.Should().BeEmpty();
+
+            ExceptionVerification.IgnoreErrors();
+        }
+
+        [Test]
+        public void should_delete_temp_response_cookie()
+        {
+            var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies");
+            requestCookies.Cookies.Add("my", "cookie");
+            requestCookies.AllowAutoRedirect = false;
+            requestCookies.StoreRequestCookie = true;
+            requestCookies.StoreResponseCookie = false;
+            var responseCookies = Subject.Get<HttpCookieResource>(requestCookies);
+
+            responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie");
+
+            var requestDelete = new HttpRequest($"http://{_httpBinHost}/cookies/delete?my");
+            requestDelete.AllowAutoRedirect = true;
+            requestDelete.StoreRequestCookie = false;
+            requestDelete.StoreResponseCookie = false;
+            var responseDelete = Subject.Get<HttpCookieResource>(requestDelete);
+
+            responseDelete.Resource.Cookies.Should().BeEmpty();
+
+            requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies");
+            requestCookies.StoreRequestCookie = false;
+            requestCookies.StoreResponseCookie = false;
+
+            responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie");
 
             ExceptionVerification.IgnoreErrors();
         }
@@ -454,4 +629,9 @@ namespace NzbDrone.Common.Test.Http
         public string Url { get; set; }
         public string Data { get; set; }
     }
+
+    public class HttpCookieResource
+    {
+        public Dictionary<string, string> Cookies { get; set; }
+    }
 }

src/NzbDrone.Common/Http/HttpClient.cs

@@ -52,7 +52,9 @@ namespace NzbDrone.Common.Http
 
         public HttpResponse Execute(HttpRequest request)
         {
-            var response = ExecuteRequest(request);
+            var cookieContainer = InitializeRequestCookies(request);
+
+            var response = ExecuteRequest(request, cookieContainer);
 
             if (request.AllowAutoRedirect && response.HasHttpRedirect)
             {
@@ -71,7 +73,7 @@ namespace NzbDrone.Common.Http
                         throw new WebException($"Too many automatic redirections were attempted for {autoRedirectChain.Join(" -> ")}", WebExceptionStatus.ProtocolError);
                     }
 
-                    response = ExecuteRequest(request);
+                    response = ExecuteRequest(request, cookieContainer);
                 }
                 while (response.HasHttpRedirect);
             }
@@ -98,7 +100,7 @@ namespace NzbDrone.Common.Http
             return response;
         }
 
-        private HttpResponse ExecuteRequest(HttpRequest request)
+        private HttpResponse ExecuteRequest(HttpRequest request, CookieContainer cookieContainer)
         {
             foreach (var interceptor in _requestInterceptors)
             {
@@ -114,11 +116,11 @@ namespace NzbDrone.Common.Http
 
             var stopWatch = Stopwatch.StartNew();
 
-            var cookies = PrepareRequestCookies(request);
+            PrepareRequestCookies(request, cookieContainer);
 
-            var response = _httpDispatcher.GetResponse(request, cookies);
+            var response = _httpDispatcher.GetResponse(request, cookieContainer);
 
-            HandleResponseCookies(request, cookies);
+            HandleResponseCookies(response, cookieContainer);
 
             stopWatch.Stop();
 
@@ -137,49 +139,80 @@ namespace NzbDrone.Common.Http
             return response;
         }
 
-        private CookieContainer PrepareRequestCookies(HttpRequest request)
+        private CookieContainer InitializeRequestCookies(HttpRequest request)
         {
             lock (_cookieContainerCache)
             {
-                var persistentCookieContainer = _cookieContainerCache.Get("container", () => new CookieContainer());
+                var sourceContainer = new CookieContainer();
+
+                var presistentContainer = _cookieContainerCache.Get("container", () => new CookieContainer());
+                var persistentCookies = presistentContainer.GetCookies((Uri)request.Url);
+                sourceContainer.Add(persistentCookies);
 
                 if (request.Cookies.Count != 0)
                 {
                     foreach (var pair in request.Cookies)
                     {
-                        persistentCookieContainer.Add(new Cookie(pair.Key, pair.Value, "/", request.Url.Host)
+                        var cookie = new Cookie(pair.Key, pair.Value, "/")
                         {
                             // Use Now rather than UtcNow to work around Mono cookie expiry bug.
                             // See https://gist.github.com/ta264/7822b1424f72e5b4c961
                             Expires = DateTime.Now.AddHours(1)
-                        });
+                        };
-                    }
-                }
 
-                var requestCookies = persistentCookieContainer.GetCookies((Uri)request.Url);
+                        sourceContainer.Add((Uri)request.Url, cookie);
 
-                var cookieContainer = new CookieContainer();
+                        if (request.StoreRequestCookie)
-
-                cookieContainer.Add(requestCookies);
-
-                return cookieContainer;
-            }
-        }
-
-        private void HandleResponseCookies(HttpRequest request, CookieContainer cookieContainer)
                         {
-            if (!request.StoreResponseCookie)
+                            presistentContainer.Add((Uri)request.Url, cookie);
+                        }
+                    }
+                }
+
+                return sourceContainer;
+            }
+        }
+
+        private void PrepareRequestCookies(HttpRequest request, CookieContainer cookieContainer)
+        {
+            // Don't collect persistnet cookies for intermediate/redirected urls.
+            /*lock (_cookieContainerCache)
+            {
+                var presistentContainer = _cookieContainerCache.Get("container", () => new CookieContainer());
+                var persistentCookies = presistentContainer.GetCookies((Uri)request.Url);
+                var existingCookies = cookieContainer.GetCookies((Uri)request.Url);
+
+                cookieContainer.Add(persistentCookies);
+                cookieContainer.Add(existingCookies);
+            }*/
+        }
+
+        private void HandleResponseCookies(HttpResponse response, CookieContainer cookieContainer)
+        {
+            var cookieHeaders = response.GetCookieHeaders();
+            if (cookieHeaders.Empty())
             {
                 return;
             }
 
+            if (response.Request.StoreResponseCookie)
+            {
                 lock (_cookieContainerCache)
                 {
                     var persistentCookieContainer = _cookieContainerCache.Get("container", () => new CookieContainer());
 
-                var cookies = cookieContainer.GetCookies((Uri)request.Url);
+                    foreach (var cookieHeader in cookieHeaders)
-
+                    {
-                persistentCookieContainer.Add(cookies);
+                        try
+                        {
+                            persistentCookieContainer.SetCookies((Uri)response.Request.Url, cookieHeader);
+                        }
+                        catch (Exception ex)
+                        {
+                            _logger.Debug(ex, "Invalid cookie in {0}", response.Request.Url);
+                        }
+                    }
+                }
             }
         }
 

src/NzbDrone.Common/Http/HttpRequest.cs

@@ -13,8 +13,10 @@ namespace NzbDrone.Common.Http
             Url = new HttpUri(url);
             Headers = new HttpHeader();
             AllowAutoRedirect = true;
+            StoreRequestCookie = true;
             Cookies = new Dictionary<string, string>();
 
+
             if (!RuntimeInfo.IsProduction)
             {
                 AllowAutoRedirect = false;
@@ -37,6 +39,7 @@ namespace NzbDrone.Common.Http
         public bool ConnectionKeepAlive { get; set; }
         public bool LogResponseContent { get; set; }
         public Dictionary<string, string> Cookies { get; private set; }
+        public bool StoreRequestCookie { get; set; }
         public bool StoreResponseCookie { get; set; }
         public TimeSpan RequestTimeout { get; set; }
         public TimeSpan RateLimit { get; set; }

src/NzbDrone.Common/Http/HttpResponse.cs

@@ -55,13 +55,16 @@ namespace NzbDrone.Common.Http
                                        StatusCode == HttpStatusCode.MovedPermanently ||
                                        StatusCode == HttpStatusCode.Found;
 
+        public string[] GetCookieHeaders()
+        {
+            return Headers.GetValues("Set-Cookie") ?? new string[0];
+        }
+
         public Dictionary<string, string> GetCookies()
         {
             var result = new Dictionary<string, string>();
 
-            var setCookieHeaders = Headers.GetValues("Set-Cookie");
+            var setCookieHeaders = GetCookieHeaders();
-            if (setCookieHeaders != null)
-            {
             foreach (var cookie in setCookieHeaders)
             {
                 var match = RegexSetCookie.Match(cookie);
@@ -70,7 +73,6 @@ namespace NzbDrone.Common.Http
                     result[match.Groups[1].Value] = match.Groups[2].Value;
                 }
             }
-            }
 
             return result;
         }