2015-04-24 02:12:02 +00:00
|
|
|
|
using CurlSharp;
|
|
|
|
|
using System;
|
|
|
|
|
using System.Collections.Concurrent;
|
|
|
|
|
using System.Collections.Generic;
|
|
|
|
|
using System.Linq;
|
|
|
|
|
using System.Net.Http;
|
|
|
|
|
using System.Text;
|
|
|
|
|
using System.Threading.Tasks;
|
|
|
|
|
using System.Net.Http.Headers;
|
2015-07-22 22:00:52 +00:00
|
|
|
|
using Jackett.Utils;
|
|
|
|
|
using System.Net;
|
2015-04-24 02:12:02 +00:00
|
|
|
|
|
|
|
|
|
namespace Jackett
|
|
|
|
|
{
|
2015-04-24 02:21:19 +00:00
|
|
|
|
public static class CurlHelper
|
2015-04-24 02:12:02 +00:00
|
|
|
|
{
|
2015-04-24 02:21:19 +00:00
|
|
|
|
public class CurlRequest
|
2015-04-24 02:12:02 +00:00
|
|
|
|
{
|
|
|
|
|
|
|
|
|
|
public string Url { get; private set; }
|
|
|
|
|
|
|
|
|
|
public string Cookies { get; private set; }
|
|
|
|
|
|
|
|
|
|
public string Referer { get; private set; }
|
|
|
|
|
|
|
|
|
|
public HttpMethod Method { get; private set; }
|
|
|
|
|
|
|
|
|
|
public Dictionary<string, string> PostData { get; set; }
|
|
|
|
|
|
|
|
|
|
public CurlRequest(HttpMethod method, string url, string cookies = null, string referer = null)
|
|
|
|
|
{
|
|
|
|
|
Method = method;
|
|
|
|
|
Url = url;
|
|
|
|
|
Cookies = cookies;
|
|
|
|
|
Referer = referer;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public class CurlResponse
|
|
|
|
|
{
|
|
|
|
|
public Dictionary<string, string> Headers { get; private set; }
|
|
|
|
|
|
|
|
|
|
public List<string[]> HeaderList { get; private set; }
|
|
|
|
|
|
|
|
|
|
public byte[] Content { get; private set; }
|
|
|
|
|
|
2015-07-22 22:00:52 +00:00
|
|
|
|
public HttpStatusCode Status { get; private set;}
|
|
|
|
|
|
2015-04-24 02:12:02 +00:00
|
|
|
|
public Dictionary<string, string> Cookies { get; private set; }
|
|
|
|
|
|
|
|
|
|
public List<string> CookiesFlat { get { return Cookies.Select(c => c.Key + "=" + c.Value).ToList(); } }
|
|
|
|
|
|
|
|
|
|
public string CookieHeader { get { return string.Join("; ", CookiesFlat); } }
|
|
|
|
|
|
2015-07-22 22:00:52 +00:00
|
|
|
|
public CurlResponse(List<string[]> headers, byte[] content, HttpStatusCode s)
|
2015-04-24 02:12:02 +00:00
|
|
|
|
{
|
|
|
|
|
Headers = new Dictionary<string, string>();
|
|
|
|
|
Cookies = new Dictionary<string, string>();
|
|
|
|
|
HeaderList = headers;
|
|
|
|
|
Content = content;
|
2015-07-22 22:00:52 +00:00
|
|
|
|
Status = s;
|
2015-04-24 02:12:02 +00:00
|
|
|
|
foreach (var h in headers)
|
|
|
|
|
{
|
|
|
|
|
Headers[h[0]] = h[1];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2015-04-24 02:21:19 +00:00
|
|
|
|
public void AddCookiesFromHeaderValue(string cookieHeaderValue)
|
|
|
|
|
{
|
|
|
|
|
var rawCookies = cookieHeaderValue.Split(';');
|
|
|
|
|
foreach (var rawCookie in rawCookies)
|
|
|
|
|
{
|
|
|
|
|
var parts = rawCookie.Split(new char[] { '=' }, 2, StringSplitOptions.RemoveEmptyEntries);
|
|
|
|
|
if (parts.Length == 1)
|
|
|
|
|
Cookies[rawCookie.Trim()] = string.Empty;
|
|
|
|
|
else
|
|
|
|
|
Cookies[parts[0].Trim()] = parts[1].Trim();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2015-04-24 02:12:02 +00:00
|
|
|
|
public void AddCookiesFromHeaders(List<string[]> headers)
|
|
|
|
|
{
|
|
|
|
|
foreach (var h in headers)
|
|
|
|
|
{
|
|
|
|
|
if (h[0] == "set-cookie")
|
|
|
|
|
{
|
2015-04-24 02:21:19 +00:00
|
|
|
|
AddCookiesFromHeaderValue(h[1]);
|
2015-04-24 02:12:02 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2015-04-24 02:21:19 +00:00
|
|
|
|
public static async Task<CurlResponse> GetAsync(string url, string cookies = null, string referer = null)
|
2015-04-24 02:12:02 +00:00
|
|
|
|
{
|
|
|
|
|
var curlRequest = new CurlRequest(HttpMethod.Get, url, cookies, referer);
|
2015-04-24 02:21:19 +00:00
|
|
|
|
var result = await PerformCurlAsync(curlRequest);
|
2015-04-24 02:12:02 +00:00
|
|
|
|
var checkedResult = await FollowRedirect(url, result);
|
|
|
|
|
return checkedResult;
|
|
|
|
|
}
|
|
|
|
|
|
2015-04-24 02:21:19 +00:00
|
|
|
|
public static async Task<CurlResponse> PostAsync(string url, Dictionary<string, string> formData, string cookies = null, string referer = null)
|
2015-04-24 02:12:02 +00:00
|
|
|
|
{
|
|
|
|
|
var curlRequest = new CurlRequest(HttpMethod.Post, url, cookies, referer);
|
|
|
|
|
curlRequest.PostData = formData;
|
2015-04-24 02:21:19 +00:00
|
|
|
|
var result = await PerformCurlAsync(curlRequest);
|
2015-04-24 02:12:02 +00:00
|
|
|
|
var checkedResult = await FollowRedirect(url, result);
|
|
|
|
|
return checkedResult;
|
|
|
|
|
}
|
|
|
|
|
|
2015-04-24 02:21:19 +00:00
|
|
|
|
private static async Task<CurlResponse> FollowRedirect(string url, CurlResponse response)
|
2015-04-24 02:12:02 +00:00
|
|
|
|
{
|
|
|
|
|
var uri = new Uri(url);
|
|
|
|
|
string redirect;
|
|
|
|
|
if (response.Headers.TryGetValue("location", out redirect))
|
|
|
|
|
{
|
2015-04-24 02:21:19 +00:00
|
|
|
|
string cookie = response.CookieHeader;
|
|
|
|
|
if (!redirect.StartsWith("http://") && !redirect.StartsWith("https://"))
|
|
|
|
|
{
|
|
|
|
|
if (redirect.StartsWith("/"))
|
|
|
|
|
redirect = string.Format("{0}://{1}{2}", uri.Scheme, uri.Host, redirect);
|
|
|
|
|
else
|
|
|
|
|
redirect = string.Format("{0}://{1}/{2}", uri.Scheme, uri.Host, redirect);
|
|
|
|
|
}
|
|
|
|
|
var newRedirect = await GetAsync(redirect, cookie);
|
|
|
|
|
foreach (var c in response.Cookies)
|
|
|
|
|
newRedirect.Cookies[c.Key] = c.Value;
|
2015-04-24 02:12:02 +00:00
|
|
|
|
newRedirect.AddCookiesFromHeaders(response.HeaderList);
|
|
|
|
|
return newRedirect;
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
return response;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2015-04-24 02:21:19 +00:00
|
|
|
|
public static async Task<CurlResponse> PerformCurlAsync(CurlRequest curlRequest)
|
2015-04-24 02:12:02 +00:00
|
|
|
|
{
|
2015-04-24 02:21:19 +00:00
|
|
|
|
return await Task.Run(() => PerformCurl(curlRequest));
|
2015-04-24 02:12:02 +00:00
|
|
|
|
}
|
|
|
|
|
|
2015-04-24 02:21:19 +00:00
|
|
|
|
public static CurlResponse PerformCurl(CurlRequest curlRequest)
|
2015-04-24 02:12:02 +00:00
|
|
|
|
{
|
2015-04-24 02:21:19 +00:00
|
|
|
|
Curl.GlobalInit(CurlInitFlag.All);
|
|
|
|
|
|
2015-04-24 02:12:02 +00:00
|
|
|
|
var headerBuffers = new List<byte[]>();
|
|
|
|
|
var contentBuffers = new List<byte[]>();
|
2015-04-24 02:21:19 +00:00
|
|
|
|
|
|
|
|
|
using (var easy = new CurlEasy())
|
2015-04-24 02:12:02 +00:00
|
|
|
|
{
|
2015-04-24 02:21:19 +00:00
|
|
|
|
easy.Url = curlRequest.Url;
|
|
|
|
|
easy.BufferSize = 64 * 1024;
|
2015-07-22 22:00:52 +00:00
|
|
|
|
easy.UserAgent = BrowserUtil.ChromeUserAgent;
|
2015-04-24 02:21:19 +00:00
|
|
|
|
easy.WriteFunction = (byte[] buf, int size, int nmemb, object data) =>
|
2015-04-24 02:12:02 +00:00
|
|
|
|
{
|
2015-04-24 02:21:19 +00:00
|
|
|
|
contentBuffers.Add(buf);
|
|
|
|
|
return size * nmemb;
|
|
|
|
|
};
|
|
|
|
|
easy.HeaderFunction = (byte[] buf, int size, int nmemb, object extraData) =>
|
|
|
|
|
{
|
|
|
|
|
headerBuffers.Add(buf);
|
|
|
|
|
return size * nmemb;
|
|
|
|
|
};
|
2015-04-24 02:12:02 +00:00
|
|
|
|
|
2015-04-24 02:21:19 +00:00
|
|
|
|
if (!string.IsNullOrEmpty(curlRequest.Cookies))
|
|
|
|
|
easy.Cookie = curlRequest.Cookies;
|
2015-04-24 02:12:02 +00:00
|
|
|
|
|
2015-04-24 02:21:19 +00:00
|
|
|
|
if (!string.IsNullOrEmpty(curlRequest.Referer))
|
|
|
|
|
easy.Referer = curlRequest.Referer;
|
2015-04-24 02:12:02 +00:00
|
|
|
|
|
2015-04-24 02:21:19 +00:00
|
|
|
|
if (curlRequest.Method == HttpMethod.Post)
|
2015-04-24 02:12:02 +00:00
|
|
|
|
{
|
2015-04-24 02:21:19 +00:00
|
|
|
|
easy.Post = true;
|
|
|
|
|
var postString = new FormUrlEncodedContent(curlRequest.PostData).ReadAsStringAsync().Result;
|
|
|
|
|
easy.PostFields = postString;
|
|
|
|
|
easy.PostFieldSize = Encoding.UTF8.GetByteCount(postString);
|
2015-04-24 02:12:02 +00:00
|
|
|
|
}
|
|
|
|
|
|
2015-04-24 02:21:19 +00:00
|
|
|
|
easy.Perform();
|
2015-04-24 02:12:02 +00:00
|
|
|
|
}
|
2015-07-22 22:00:52 +00:00
|
|
|
|
|
2015-04-24 02:21:19 +00:00
|
|
|
|
var headerBytes = Combine(headerBuffers.ToArray());
|
|
|
|
|
var headerString = Encoding.UTF8.GetString(headerBytes);
|
|
|
|
|
var headerParts = headerString.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
|
|
|
|
|
var headers = new List<string[]>();
|
2015-07-22 22:00:52 +00:00
|
|
|
|
var headerCount = 0;
|
|
|
|
|
HttpStatusCode status = HttpStatusCode.InternalServerError;
|
|
|
|
|
foreach (var headerPart in headerParts)
|
2015-04-24 02:12:02 +00:00
|
|
|
|
{
|
2015-07-22 22:00:52 +00:00
|
|
|
|
if (headerCount == 0)
|
|
|
|
|
{
|
|
|
|
|
var responseCode = int.Parse(headerPart.Split(' ')[1]);
|
|
|
|
|
status = (HttpStatusCode)responseCode;
|
|
|
|
|
}
|
|
|
|
|
else
|
2015-04-24 02:21:19 +00:00
|
|
|
|
{
|
2015-07-22 22:00:52 +00:00
|
|
|
|
var keyVal = headerPart.Split(new char[] { ':' }, 2);
|
|
|
|
|
if (keyVal.Length > 1)
|
|
|
|
|
{
|
|
|
|
|
headers.Add(new[] { keyVal[0].ToLower().Trim(), keyVal[1].Trim() });
|
|
|
|
|
}
|
2015-04-24 02:21:19 +00:00
|
|
|
|
}
|
2015-07-22 22:00:52 +00:00
|
|
|
|
|
|
|
|
|
headerCount++;
|
2015-04-24 02:12:02 +00:00
|
|
|
|
}
|
2015-04-24 02:21:19 +00:00
|
|
|
|
|
|
|
|
|
var contentBytes = Combine(contentBuffers.ToArray());
|
|
|
|
|
|
2015-07-22 22:00:52 +00:00
|
|
|
|
var curlResponse = new CurlResponse(headers, contentBytes, status);
|
2015-04-24 02:21:19 +00:00
|
|
|
|
if (!string.IsNullOrEmpty(curlRequest.Cookies))
|
|
|
|
|
curlResponse.AddCookiesFromHeaderValue(curlRequest.Cookies);
|
|
|
|
|
curlResponse.AddCookiesFromHeaders(headers);
|
|
|
|
|
|
|
|
|
|
return curlResponse;
|
|
|
|
|
|
2015-04-24 02:12:02 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static byte[] Combine(params byte[][] arrays)
|
|
|
|
|
{
|
|
|
|
|
byte[] ret = new byte[arrays.Sum(x => x.Length)];
|
|
|
|
|
int offset = 0;
|
|
|
|
|
foreach (byte[] data in arrays)
|
|
|
|
|
{
|
|
|
|
|
Buffer.BlockCopy(data, 0, ret, offset, data.Length);
|
|
|
|
|
offset += data.Length;
|
|
|
|
|
}
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|