automatically decode CloudFlare protected emails

src/Jackett/Utils/BrowserUtil.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
+using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 
 namespace Jackett.Utils
@@ -21,5 +22,37 @@ namespace Jackett.Utils
                 }
             }
         }
+
+        // This can be used to decode e-mail addresses protected by cloudflare
+        public static string DecodeCloudFlareProtectedEmail(string input)
+        {
+            var key = Convert.ToInt32(input.Substring(0, 2), 16);
+            string result = "";
+            for (var i = 2; i < input.Length - 1; i += 2)
+            {
+                var hexChar = input.Substring(i, 2);
+                var intChar = Convert.ToInt32(hexChar, 16) ^ key;
+                var strChar = Convert.ToChar(intChar);
+                result += strChar;
+            }
+            return result;
+        }
+
+        // decode cloudflare protected emails in a HTML document
+        public static string DecodeCloudFlareProtectedEmailFromHTML(string html)
+        {
+            Regex CFEMailRegex = new Regex("<span class=\"__cf_email__\" data-cfemail=\"(\\w+)\">\\[email&#160;protected\\]<\\/span><script data-cfhash='[\\w]+' type=\"text\\/javascript\">.*?<\\/script>", RegexOptions.Compiled);
+            var CFEMailRegexMatches = CFEMailRegex.Match(html);
+
+            while (CFEMailRegexMatches.Success)
+            {
+                string all = CFEMailRegexMatches.Groups[0].Value;
+                string cfemail = CFEMailRegexMatches.Groups[1].Value;
+                var decoded = DecodeCloudFlareProtectedEmail(cfemail);
+                html = html.Replace(all, decoded);
+                CFEMailRegexMatches = CFEMailRegexMatches.NextMatch();
+            }
+            return html;
+        }
     }
 }

src/Jackett/Utils/Clients/HttpWebClient.cs

@@ -47,7 +47,14 @@ namespace Jackett.Utils.Clients
             logger.Debug(string.Format("WindowsWebClient:GetString(Url:{0})", request.Url));
             var result = await Run(request);
             logger.Debug(string.Format("WindowsWebClient: Returning {0} => {1}", result.Status, (result.Content == null ? "<NULL>" : Encoding.UTF8.GetString(result.Content))));
-            return Mapper.Map<WebClientStringResult>(result);
+            WebClientStringResult stringResult = Mapper.Map<WebClientStringResult>(result);
+            string[] server;
+            if (stringResult.Headers.TryGetValue("server", out server))
+            {
+                if (server[0] == "cloudflare-nginx")
+                    stringResult.Content = BrowserUtil.DecodeCloudFlareProtectedEmailFromHTML(stringResult.Content);
+            }
+            return stringResult;
         }
 
         private async Task<WebClientByteResult> Run(WebRequest webRequest)
@@ -141,6 +148,12 @@ namespace Jackett.Utils.Clients
             var result = new WebClientByteResult();
             result.Content = await response.Content.ReadAsByteArrayAsync();
 
+            foreach (var header in response.Headers)
+            {
+                IEnumerable<string> value = header.Value;
+                result.Headers[header.Key.ToLowerInvariant()] = value.ToArray();
+            }
+
             // some cloudflare clients are using a refresh header
             // Pull it out manually 
             if (response.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
@@ -199,7 +212,6 @@ namespace Jackett.Utils.Clients
                 }
                 result.Cookies = cookieBuilder.ToString().Trim();
             }
-
             ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
             return result;
         }

src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs

@@ -36,8 +36,15 @@ namespace Jackett.Utils.Clients
         {
             logger.Debug(string.Format("UnixLibCurlWebClient:GetString(Url:{0})", request.Url));
             var result = await RunCloudFlare(request);
-            logger.Debug(string.Format("UnixLibCurlWebClient:GetString Returning {0} => {1}", result.Status, (result.Content == null ? "<NULL>" : Encoding.UTF8.GetString(result.Content))));
+            logger.Debug(string.Format("UnixLibCurlWebClient:GetString Returning {0} => {1}", result.Status, (result.Content == null ? "<NULL>" : Encoding.UTF8.GetString(result.Content))));
-            return Mapper.Map<WebClientStringResult>(result);
+            WebClientStringResult stringResult = Mapper.Map<WebClientStringResult>(result);
+            string[] server;
+            if (stringResult.Headers.TryGetValue("server", out server))
+            {
+                if (server[0] == "cloudflare-nginx")
+                    stringResult.Content = BrowserUtil.DecodeCloudFlareProtectedEmailFromHTML(stringResult.Content);
+            }
+            return stringResult;
         }
 
         private string CloudFlareChallengeSolverSolve(string challengePageContent, Uri uri)
@@ -142,7 +149,11 @@ namespace Jackett.Utils.Clients
             {
                 foreach (var header in response.HeaderList)
                 {
-                    switch (header[0].ToLowerInvariant())
+                    var key = header[0].ToLowerInvariant();
+                    
+                    result.Headers[key] = new string[] { header[1] }; // doesn't support multiple identical headers?
+
+                    switch (key)
                     {
                         case "location":
                             result.RedirectingTo = header[1];