mirror
/
Jackett
mirror of https://github.com/Jackett/Jackett
1
0
Fork 0
Code Issues Releases Wiki Activity

Make lib curl more stable

This commit is contained in:
KZ 2015-07-25 09:34:15 +01:00
parent 2e7fc987ac
commit 65be91c932
4 changed files with 82 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

154
src/Jackett/CurlHelper.cs
View File

@ -12,8 +12,16 @@ using System.Net;
namespace Jackett
{
public static class CurlHelper
public class CurlHelper
{
private static readonly object instance = new object();
static CurlHelper()
{
Engine.Logger.Debug("LibCurl init" + Curl.GlobalInit(CurlInitFlag.All).ToString());
Engine.Logger.Debug("LibCurl version " + Curl.Version);
}
public class CurlRequest
{
@ -39,17 +47,11 @@ namespace Jackett
public class CurlResponse
{
public Dictionary<string, string> Headers { get; private set; }
public List<string[]> HeaderList { get; private set; }
public byte[] Content { get; private set; }
public HttpStatusCode Status { get; private set;}
public Dictionary<string, string> Cookies { get; private set; }
public List<string> CookiesFlat { get { return Cookies.Select(c => c.Key + "=" + c.Value).ToList(); } }
public string CookieHeader { get { return string.Join("; ", CookiesFlat); } }
public CurlResponse(List<string[]> headers, byte[] content, HttpStatusCode s)
@ -94,17 +96,15 @@ namespace Jackett
{
var curlRequest = new CurlRequest(HttpMethod.Get, url, cookies, referer);
var result = await PerformCurlAsync(curlRequest);
var checkedResult = await FollowRedirect(url, result);
return checkedResult;
}
return result;
}
public static async Task<CurlResponse> PostAsync(string url, Dictionary<string, string> formData, string cookies = null, string referer = null)
{
var curlRequest = new CurlRequest(HttpMethod.Post, url, cookies, referer);
curlRequest.PostData = formData;
var result = await PerformCurlAsync(curlRequest);
var checkedResult = await FollowRedirect(url, result);
return checkedResult;
return result;
}
private static async Task<CurlResponse> FollowRedirect(string url, CurlResponse response)
@ -139,78 +139,80 @@ namespace Jackett
public static CurlResponse PerformCurl(CurlRequest curlRequest)
{
Curl.GlobalInit(CurlInitFlag.All);
var headerBuffers = new List<byte[]>();
var contentBuffers = new List<byte[]>();
using (var easy = new CurlEasy())
lock (instance)
{
easy.Url = curlRequest.Url;
easy.BufferSize = 64 * 1024;
easy.UserAgent = BrowserUtil.ChromeUserAgent;
easy.WriteFunction = (byte[] buf, int size, int nmemb, object data) =>
{
contentBuffers.Add(buf);
return size * nmemb;
};
easy.HeaderFunction = (byte[] buf, int size, int nmemb, object extraData) =>
{
headerBuffers.Add(buf);
return size * nmemb;
};
var headerBuffers = new List<byte[]>();
var contentBuffers = new List<byte[]>();
if (!string.IsNullOrEmpty(curlRequest.Cookies))
easy.Cookie = curlRequest.Cookies;
if (!string.IsNullOrEmpty(curlRequest.Referer))
easy.Referer = curlRequest.Referer;
if (curlRequest.Method == HttpMethod.Post)
using (var easy = new CurlEasy())
{
easy.Post = true;
var postString = new FormUrlEncodedContent(curlRequest.PostData).ReadAsStringAsync().Result;
easy.PostFields = postString;
easy.PostFieldSize = Encoding.UTF8.GetByteCount(postString);
}
easy.Perform();
}
var headerBytes = Combine(headerBuffers.ToArray());
var headerString = Encoding.UTF8.GetString(headerBytes);
var headerParts = headerString.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
var headers = new List<string[]>();
var headerCount = 0;
HttpStatusCode status = HttpStatusCode.InternalServerError;
foreach (var headerPart in headerParts)
{
if (headerCount == 0)
{
var responseCode = int.Parse(headerPart.Split(' ')[1]);
status = (HttpStatusCode)responseCode;
}
else
{
var keyVal = headerPart.Split(new char[] { ':' }, 2);
if (keyVal.Length > 1)
easy.Url = curlRequest.Url;
easy.BufferSize = 64 * 1024;
easy.UserAgent = BrowserUtil.ChromeUserAgent;
easy.FollowLocation = false;
easy.ConnectTimeout = 20;
easy.WriteFunction = (byte[] buf, int size, int nmemb, object data) =>
{
headers.Add(new[] { keyVal[0].ToLower().Trim(), keyVal[1].Trim() });
contentBuffers.Add(buf);
return size * nmemb;
};
easy.HeaderFunction = (byte[] buf, int size, int nmemb, object extraData) =>
{
headerBuffers.Add(buf);
return size * nmemb;
};
if (!string.IsNullOrEmpty(curlRequest.Cookies))
easy.Cookie = curlRequest.Cookies;
if (!string.IsNullOrEmpty(curlRequest.Referer))
easy.Referer = curlRequest.Referer;
if (curlRequest.Method == HttpMethod.Post)
{
easy.Post = true;
var postString = new FormUrlEncodedContent(curlRequest.PostData).ReadAsStringAsync().Result;
easy.PostFields = postString;
easy.PostFieldSize = Encoding.UTF8.GetByteCount(postString);
}
easy.Perform();
}
headerCount++;
var headerBytes = Combine(headerBuffers.ToArray());
var headerString = Encoding.UTF8.GetString(headerBytes);
var headerParts = headerString.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
var headers = new List<string[]>();
var headerCount = 0;
HttpStatusCode status = HttpStatusCode.InternalServerError;
foreach (var headerPart in headerParts)
{
if (headerCount == 0)
{
var responseCode = int.Parse(headerPart.Split(' ')[1]);
status = (HttpStatusCode)responseCode;
}
else
{
var keyVal = headerPart.Split(new char[] { ':' }, 2);
if (keyVal.Length > 1)
{
headers.Add(new[] { keyVal[0].ToLower().Trim(), keyVal[1].Trim() });
}
}
headerCount++;
}
var contentBytes = Combine(contentBuffers.ToArray());
var curlResponse = new CurlResponse(headers, contentBytes, status);
if (!string.IsNullOrEmpty(curlRequest.Cookies))
curlResponse.AddCookiesFromHeaderValue(curlRequest.Cookies);
curlResponse.AddCookiesFromHeaders(headers);
return curlResponse;
}
var contentBytes = Combine(contentBuffers.ToArray());
var curlResponse = new CurlResponse(headers, contentBytes, status);
if (!string.IsNullOrEmpty(curlRequest.Cookies))
curlResponse.AddCookiesFromHeaderValue(curlRequest.Cookies);
curlResponse.AddCookiesFromHeaders(headers);
return curlResponse;
}
public static byte[] Combine(params byte[][] arrays)

1
src/Jackett/Indexers/AlphaRatio.cs
View File

@ -105,7 +105,6 @@ namespace Jackett.Indexers
dom["#loginform > table"].Remove();
var errorMessage = dom["#loginform"].Text().Trim().Replace("\n\t", " ");
throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)config);
}
else
{

2
src/Jackett/Indexers/BaseIndexer.cs
View File

@ -24,7 +24,6 @@ namespace Jackett.Indexers
protected Logger logger;
protected IIndexerManagerService indexerService;
// protected IWebClient webClient;
protected static List<CachedResult> cache = new List<CachedResult>();
protected static readonly TimeSpan cacheTime = new TimeSpan(0, 9, 0);
@ -42,7 +41,6 @@ namespace Jackett.Indexers
TorznabCaps = caps;
this.logger = logger;
indexerService = manager;
// webClient = wc;
}
protected void SaveConfig(JToken config)

5
src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs
View File

@ -55,6 +55,7 @@ namespace Jackett.Utils.Clients
}
}
logger.Debug(string.Format("UnixLibCurlWebClient: Returning", result.Status));
return result;
}
@ -63,12 +64,14 @@ namespace Jackett.Utils.Clients
logger.Debug(string.Format("UnixLibCurlWebClient:GetString(Url:{0})", request.Url));
var result = await GetBytes(request);
return new WebClientStringResult()
var sresult = new WebClientStringResult()
{
Content = Encoding.UTF8.GetString(result.Content),
Cookies = result.Cookies,
Status = result.Status
};
return sresult;
}
}
}