2015-08-25 20:30:32 +00:00
|
|
|
|
using AutoMapper;
|
|
|
|
|
using Jackett.Models;
|
2015-09-30 20:21:43 +00:00
|
|
|
|
using Jackett.Services;
|
2015-08-25 20:30:32 +00:00
|
|
|
|
using NLog;
|
|
|
|
|
using System;
|
|
|
|
|
using System.Collections.Generic;
|
|
|
|
|
using System.Linq;
|
|
|
|
|
using System.Net;
|
|
|
|
|
using System.Net.Http;
|
|
|
|
|
using System.Text;
|
|
|
|
|
using System.Threading.Tasks;
|
|
|
|
|
|
|
|
|
|
namespace Jackett.Utils.Clients
|
|
|
|
|
{
|
|
|
|
|
public class HttpWebClient : IWebClient
|
|
|
|
|
{
|
2015-09-30 20:21:43 +00:00
|
|
|
|
Logger logger;
|
|
|
|
|
IConfigurationService configService;
|
2015-08-25 20:30:32 +00:00
|
|
|
|
|
2015-09-30 20:21:43 +00:00
|
|
|
|
public HttpWebClient(Logger l, IConfigurationService c)
|
2015-08-25 20:30:32 +00:00
|
|
|
|
{
|
|
|
|
|
logger = l;
|
2015-09-30 20:21:43 +00:00
|
|
|
|
configService = c;
|
2015-08-25 20:30:32 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
public void Init()
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public async Task<WebClientByteResult> GetBytes(WebRequest request)
|
|
|
|
|
{
|
|
|
|
|
logger.Debug(string.Format("WindowsWebClient:GetBytes(Url:{0})", request.Url));
|
|
|
|
|
var result = await Run(request);
|
|
|
|
|
logger.Debug(string.Format("WindowsWebClient: Returning {0} => {1} bytes", result.Status, (result.Content == null ? "<NULL>" : result.Content.Length.ToString())));
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public async Task<WebClientStringResult> GetString(WebRequest request)
|
|
|
|
|
{
|
|
|
|
|
logger.Debug(string.Format("WindowsWebClient:GetString(Url:{0})", request.Url));
|
|
|
|
|
var result = await Run(request);
|
|
|
|
|
logger.Debug(string.Format("WindowsWebClient: Returning {0} => {1}", result.Status, (result.Content == null ? "<NULL>" : Encoding.UTF8.GetString(result.Content))));
|
|
|
|
|
return Mapper.Map<WebClientStringResult>(result);
|
|
|
|
|
}
|
|
|
|
|
|
2015-09-30 20:21:43 +00:00
|
|
|
|
private async Task<WebClientByteResult> Run(WebRequest webRequest)
|
2015-08-25 20:30:32 +00:00
|
|
|
|
{
|
|
|
|
|
var cookies = new CookieContainer();
|
2015-09-30 20:21:43 +00:00
|
|
|
|
if (!string.IsNullOrEmpty(webRequest.Cookies))
|
2015-08-25 20:30:32 +00:00
|
|
|
|
{
|
2015-09-30 20:21:43 +00:00
|
|
|
|
var uri = new Uri(webRequest.Url);
|
|
|
|
|
foreach (var c in webRequest.Cookies.Split(';'))
|
2015-08-25 20:30:32 +00:00
|
|
|
|
{
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
cookies.SetCookies(uri, c);
|
|
|
|
|
}
|
|
|
|
|
catch (CookieException ex)
|
|
|
|
|
{
|
|
|
|
|
logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var client = new HttpClient(new HttpClientHandler
|
|
|
|
|
{
|
|
|
|
|
CookieContainer = cookies,
|
|
|
|
|
AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
|
|
|
|
|
UseCookies = true,
|
|
|
|
|
});
|
|
|
|
|
|
2015-09-30 20:21:43 +00:00
|
|
|
|
if(webRequest.EmulateBrowser)
|
2015-12-18 17:03:38 +00:00
|
|
|
|
client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
|
2015-09-30 20:21:43 +00:00
|
|
|
|
else
|
2015-12-18 17:03:38 +00:00
|
|
|
|
client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion());
|
2015-08-25 20:30:32 +00:00
|
|
|
|
HttpResponseMessage response = null;
|
2015-09-30 20:21:43 +00:00
|
|
|
|
var request = new HttpRequestMessage();
|
|
|
|
|
request.Headers.ExpectContinue = false;
|
|
|
|
|
request.RequestUri = new Uri(webRequest.Url);
|
2015-08-25 20:30:32 +00:00
|
|
|
|
|
2015-09-30 20:21:43 +00:00
|
|
|
|
if (webRequest.Headers != null)
|
2015-08-25 20:30:32 +00:00
|
|
|
|
{
|
2015-09-30 20:21:43 +00:00
|
|
|
|
foreach (var header in webRequest.Headers)
|
2015-08-25 20:30:32 +00:00
|
|
|
|
{
|
2015-09-30 20:21:43 +00:00
|
|
|
|
if (header.Key != "Content-Type")
|
|
|
|
|
{
|
|
|
|
|
request.Headers.Add(header.Key, header.Value);
|
|
|
|
|
}
|
2015-08-25 20:30:32 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2015-09-30 20:21:43 +00:00
|
|
|
|
if (!string.IsNullOrEmpty(webRequest.RawBody))
|
|
|
|
|
{
|
|
|
|
|
var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast<KeyValuePair<string,string>?>().FirstOrDefault();
|
|
|
|
|
if (type.HasValue)
|
|
|
|
|
{
|
|
|
|
|
var str = new StringContent(webRequest.RawBody);
|
|
|
|
|
str.Headers.Remove("Content-Type");
|
|
|
|
|
str.Headers.Add("Content-Type", type.Value.Value);
|
|
|
|
|
request.Content = str;
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
request.Content = new StringContent(webRequest.RawBody);
|
|
|
|
|
request.Method = HttpMethod.Post;
|
|
|
|
|
}
|
|
|
|
|
else if (webRequest.Type == RequestType.POST)
|
2015-08-25 20:30:32 +00:00
|
|
|
|
{
|
2015-09-30 20:21:43 +00:00
|
|
|
|
request.Content = new FormUrlEncodedContent(webRequest.PostData);
|
|
|
|
|
request.Method = HttpMethod.Post;
|
2015-08-25 20:30:32 +00:00
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
2015-09-30 20:21:43 +00:00
|
|
|
|
request.Method = HttpMethod.Get;
|
2015-08-25 20:30:32 +00:00
|
|
|
|
}
|
|
|
|
|
|
2015-09-30 20:21:43 +00:00
|
|
|
|
response = await client.SendAsync(request);
|
|
|
|
|
|
2015-08-25 20:30:32 +00:00
|
|
|
|
var result = new WebClientByteResult();
|
|
|
|
|
result.Content = await response.Content.ReadAsByteArrayAsync();
|
2015-12-18 17:03:38 +00:00
|
|
|
|
|
|
|
|
|
// some cloudflare clients are using a refresh header
|
|
|
|
|
// Pull it out manually
|
|
|
|
|
if (response.statusCode == System.Net.HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
|
|
|
|
|
{
|
|
|
|
|
var refreshHeaders = response.Headers.GetValues("Refresh");
|
|
|
|
|
var redirval = "";
|
|
|
|
|
var redirtime = 0;
|
|
|
|
|
if (refreshHeaders != null)
|
|
|
|
|
{
|
|
|
|
|
foreach (var value in refreshHeaders)
|
|
|
|
|
{
|
|
|
|
|
var start = value.IndexOf("=");
|
|
|
|
|
var end = value.IndexOf(";");
|
|
|
|
|
var len = value.Length;
|
|
|
|
|
if (start > -1)
|
|
|
|
|
{
|
|
|
|
|
redirval = value.Substring(start + 1);
|
|
|
|
|
result.RedirectingTo = redirval;
|
|
|
|
|
// normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
|
|
|
|
|
// of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
|
|
|
|
|
// it shoudln't include service unavailable..only if we have this redirect header.
|
|
|
|
|
response.StatusCode = System.Net.HttpStatusCode.Redirect;
|
|
|
|
|
redirtime = Int32.Parse(value.Substring(0, end));
|
|
|
|
|
System.Threading.Thread.Sleep(redirtime * 1000);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
2015-08-25 20:30:32 +00:00
|
|
|
|
if (response.Headers.Location != null)
|
|
|
|
|
{
|
|
|
|
|
result.RedirectingTo = response.Headers.Location.ToString();
|
|
|
|
|
}
|
|
|
|
|
result.Status = response.StatusCode;
|
|
|
|
|
|
|
|
|
|
// Compatiblity issue between the cookie format and httpclient
|
|
|
|
|
// Pull it out manually ignoring the expiry date then set it manually
|
|
|
|
|
// http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
|
|
|
|
|
IEnumerable<string> cookieHeaders;
|
2015-09-30 20:21:43 +00:00
|
|
|
|
var responseCookies = new List<Tuple<string, string>>();
|
2015-08-25 20:30:32 +00:00
|
|
|
|
if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
|
|
|
|
|
{
|
2015-09-30 20:21:43 +00:00
|
|
|
|
foreach (var value in cookieHeaders)
|
2015-08-25 20:30:32 +00:00
|
|
|
|
{
|
2015-09-30 20:21:43 +00:00
|
|
|
|
var nameSplit = value.IndexOf('=');
|
|
|
|
|
if (nameSplit > -1)
|
|
|
|
|
{
|
|
|
|
|
responseCookies.Add(new Tuple<string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') + 1)));
|
|
|
|
|
}
|
2015-08-25 20:30:32 +00:00
|
|
|
|
}
|
|
|
|
|
|
2015-09-30 20:21:43 +00:00
|
|
|
|
var cookieBuilder = new StringBuilder();
|
|
|
|
|
foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
|
|
|
|
|
{
|
|
|
|
|
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
|
|
|
|
|
}
|
|
|
|
|
result.Cookies = cookieBuilder.ToString().Trim();
|
2015-08-25 20:30:32 +00:00
|
|
|
|
}
|
|
|
|
|
|
2015-09-30 20:21:43 +00:00
|
|
|
|
ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
|
2015-08-25 20:30:32 +00:00
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|