using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using AngleSharp.Dom;
using AngleSharp.Html.Dom;
using AngleSharp.Html.Parser;
using Jackett.Common.Helpers;
using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Microsoft.AspNetCore.WebUtilities;
using Newtonsoft.Json.Linq;
using NLog;
using static Jackett.Common.Models.IndexerConfig.ConfigurationData;

namespace Jackett.Common.Indexers
{
    public class CardigannIndexer : BaseWebIndexer
    {
        protected IndexerDefinition Definition;
        protected WebResult landingResult;
        protected IHtmlDocument landingResultDocument;

        protected List<string> DefaultCategories = new List<string>();

        private new ConfigurationData configData
        {
            get => base.configData;
            set => base.configData = value;
        }

        protected readonly string[] OptionalFields = { "imdb", "rageid", "tmdbid", "tvdbid", "poster" };

        private static readonly string[] _SupportedLogicFunctions =
        {
            "and",
            "or",
            "eq",
            "ne"
        };

        private static readonly string[] _LogicFunctionsUsingStringLiterals =
        {
            "eq",
            "ne"
        };

        // Matches a logic function above and 2 or more of (.varname) or .varname or "string literal" in any combination
        private static readonly Regex _LogicFunctionRegex = new Regex(
            @$"\b({string.Join("|", _SupportedLogicFunctions.Select(Regex.Escape))})(?:\s+(\(?\.[^\)\s]+\)?|""[^""]+"")){{2,}}");

        public CardigannIndexer(IIndexerConfigurationService configService, Utils.Clients.WebClient wc, Logger l,
                                IProtectionService ps, ICacheService cs, IndexerDefinition Definition)
            : base(configService: configService,
                   client: wc,
                   logger: l,
                   p: ps,
                   cacheService: cs
                   )
        {
            this.Definition = Definition;
            Id = Definition.Id;

            // Add default data if necessary
            if (Definition.Settings == null)
            {
                Definition.Settings = new List<settingsField>
                {
                    new settingsField { Name = "username", Label = "Username", Type = "text" },
                    new settingsField { Name = "password", Label = "Password", Type = "password" }
                };
            }

            if (Definition.Encoding == null)
                Definition.Encoding = "UTF-8";

            if (Definition.Login != null && Definition.Login.Method == null)
                Definition.Login.Method = "form";

            if (Definition.Search.Paths == null)
            {
                Definition.Search.Paths = new List<searchPathBlock>();
            }

            // convert definitions with a single search Path to a Paths entry
            if (Definition.Search.Path != null)
            {
                Definition.Search.Paths.Add(new searchPathBlock
                {
                    Path = Definition.Search.Path,
                    Inheritinputs = true
                });
            }

            // init missing mandatory attributes
            DisplayName = Definition.Name;
            DisplayDescription = Definition.Description;
            if (Definition.Links.Count > 1)
                AlternativeSiteLinks = Definition.Links.ToArray();
            DefaultSiteLink = Definition.Links[0];
            if (Definition.Legacylinks != null)
                LegacySiteLinks = Definition.Legacylinks.ToArray();
            Encoding = Encoding.GetEncoding(Definition.Encoding);
            if (!DefaultSiteLink.EndsWith("/"))
                DefaultSiteLink += "/";
            Language = Definition.Language;
            Type = Definition.Type;
            TorznabCaps = new TorznabCapabilities();
            TorznabCaps.ParseCardigannSearchModes(Definition.Caps.Modes);

            // init config Data
            configData = new ConfigurationData();
            foreach (var Setting in Definition.Settings)
            {
                ConfigurationItem item;

                var itemName = Setting.Label ?? Setting.Name;

                if (Setting.Type != null)
                {
                    switch (Setting.Type)
                    {
                        case "checkbox":
                            item = new BoolConfigurationItem(itemName) { Value = false };

                            if (Setting.Default != null && Setting.Default == "true")
                            {
                                ((BoolConfigurationItem)item).Value = true;
                            }
                            break;
                        case "password":
                        case "text":
                            item = new StringConfigurationItem(itemName) { Value = Setting.Default };
                            break;
                        case "multi-select":
                            if (Setting.Options == null)
                            {
                                throw new Exception("Options must be given for the 'multi-select' type.");
                            }

                            item = new MultiSelectConfigurationItem(itemName, Setting.Options) { Values = Setting.Defaults };
                            break;
                        case "select":
                            if (Setting.Options == null)
                            {
                                throw new Exception("Options must be given for the 'select' type.");
                            }

                            item = new SingleSelectConfigurationItem(itemName, Setting.Options) { Value = Setting.Default };
                            break;
                        case "info":
                            item = new DisplayInfoConfigurationItem(itemName, Setting.Default);
                            break;
                        default:
                            throw new Exception($"Invalid setting type '{Setting.Type}' specified.");
                    }
                }
                else
                {
                    item = new StringConfigurationItem(itemName) { Value = Setting.Default };
                }

                configData.AddDynamic(Setting.Name, item);
            }

            if (Definition.Caps.Categories != null)
            {
                foreach (var Category in Definition.Caps.Categories)
                {
                    var cat = TorznabCatType.GetCatByName(Category.Value);
                    if (cat == null)
                    {
                        logger.Error(string.Format("CardigannIndexer ({0}): invalid Torznab category for id {1}: {2}", Id, Category.Key, Category.Value));
                        continue;
                    }
                    AddCategoryMapping(Category.Key, cat);
                }
            }

            if (Definition.Caps.Categorymappings != null)
            {
                foreach (var Categorymapping in Definition.Caps.Categorymappings)
                {
                    TorznabCategory TorznabCat = null;

                    if (Categorymapping.cat != null)
                    {
                        TorznabCat = TorznabCatType.GetCatByName(Categorymapping.cat);
                        if (TorznabCat == null)
                        {
                            logger.Error(string.Format("CardigannIndexer ({0}): invalid Torznab category for id {1}: {2}", Id, Categorymapping.id, Categorymapping.cat));
                            continue;
                        }
                    }
                    AddCategoryMapping(Categorymapping.id, TorznabCat, Categorymapping.desc);
                    if (Categorymapping.Default)
                        DefaultCategories.Add(Categorymapping.id);
                }
            }
            LoadValuesFromJson(null);
        }

        public override void LoadValuesFromJson(JToken jsonConfig, bool useProtectionService = false)
        {
            base.LoadValuesFromJson(jsonConfig, useProtectionService);

            // add self signed cert to trusted certs
            if (Definition.Certificates != null)
            {
                foreach (var certificateHash in Definition.Certificates)
                    webclient.AddTrustedCertificate(new Uri(SiteLink).Host, certificateHash);
            }
        }

        protected Dictionary<string, object> GetBaseTemplateVariables()
        {
            var variables = new Dictionary<string, object>
            {
                [".Config.sitelink"] = SiteLink,
                [".True"] = "True",
                [".False"] = null,
                [".Today.Year"] = DateTime.Today.Year.ToString()
            };

            foreach (var setting in Definition.Settings)
            {
                var configurationItem = configData.GetDynamic(setting.Name);
                if (configurationItem == null)
                    continue;

                var variableKey = ".Config." + setting.Name;

                switch (configurationItem)
                {
                    case BoolConfigurationItem boolItem:
                        {
                            variables[variableKey] = variables[boolItem.Value ? ".True" : ".False"];
                            break;
                        }
                    case StringConfigurationItem stringItem:
                        {
                            variables[variableKey] = stringItem.Value;
                            break;
                        }
                    case PasswordConfigurationItem passwordItem:
                        {
                            variables[variableKey] = passwordItem.Value;
                            break;
                        }
                    case SingleSelectConfigurationItem selectItem:
                        {
                            variables[variableKey] = selectItem.Value;
                            break;
                        }
                    case MultiSelectConfigurationItem multiSelectItem:
                        {
                            variables[variableKey] = multiSelectItem.Values;
                            break;
                        }
                    case DisplayImageConfigurationItem displayImageItem:
                        {
                            variables[variableKey] = displayImageItem.Value;
                            break;
                        }
                    case DisplayInfoConfigurationItem displayInfoItem:
                        {
                            variables[variableKey] = displayInfoItem.Value;
                            break;
                        }
                    case HiddenStringConfigurationItem hiddenStringItem:
                        {
                            variables[variableKey] = hiddenStringItem.Value;
                            break;
                        }
                    default:
                        {
                            //TODO Should this throw a NotSupportedException, as it used to?
                            break;
                        }
                }
            }

            return variables;
        }

        // A very bad implementation of the golang template/text templating engine.
        // But it should work for most basic constucts used by Cardigann definitions.
        protected delegate string TemplateTextModifier(string str);
        protected string applyGoTemplateText(string template, Dictionary<string, object> variables = null, TemplateTextModifier modifier = null)
        {
            if (variables == null)
            {
                variables = GetBaseTemplateVariables();
            }

            // handle re_replace expression
            // Example: {{ re_replace .Query.Keywords "[^a-zA-Z0-9]+" "%" }}
            var ReReplaceRegex = new Regex(@"{{\s*re_replace\s+(\..+?)\s+""(.*?)""\s+""(.*?)""\s*}}");
            var ReReplaceRegexMatches = ReReplaceRegex.Match(template);

            while (ReReplaceRegexMatches.Success)
            {
                var all = ReReplaceRegexMatches.Groups[0].Value;
                var variable = ReReplaceRegexMatches.Groups[1].Value;
                var regexp = ReReplaceRegexMatches.Groups[2].Value;
                var newvalue = ReReplaceRegexMatches.Groups[3].Value;

                var ReplaceRegex = new Regex(regexp);
                var input = (string)variables[variable];
                var expanded = ReplaceRegex.Replace(input, newvalue);

                if (modifier != null)
                    expanded = modifier(expanded);

                template = template.Replace(all, expanded);
                ReReplaceRegexMatches = ReReplaceRegexMatches.NextMatch();
            }

            // handle join expression
            // Example: {{ join .Categories "," }}
            var JoinRegex = new Regex(@"{{\s*join\s+(\..+?)\s+""(.*?)""\s*}}");
            var JoinMatches = JoinRegex.Match(template);

            while (JoinMatches.Success)
            {
                var all = JoinMatches.Groups[0].Value;
                var variable = JoinMatches.Groups[1].Value;
                var delimiter = JoinMatches.Groups[2].Value;

                var input = (ICollection<string>)variables[variable];
                var expanded = string.Join(delimiter, input);

                if (modifier != null)
                    expanded = modifier(expanded);

                template = template.Replace(all, expanded);
                JoinMatches = JoinMatches.NextMatch();
            }

            var logicMatch = _LogicFunctionRegex.Match(template);

            while (logicMatch.Success)
            {
                var functionStartIndex = logicMatch.Groups[0].Index;
                var functionLength = logicMatch.Groups[0].Length;
                var functionName = logicMatch.Groups[1].Value;
                // Use Group.Captures to get each matching string in a repeating Match.Group
                // Strip () around variable names here, as they are optional. Use quotes to differentiate variables and literals
                var parameters = logicMatch.Groups[2].Captures.Cast<Capture>().Select(c => c.Value.Trim('(', ')')).ToList();
                var functionResult = "";

                // If the function can't use string literals, fail silently by removing the literals.
                if (!_LogicFunctionsUsingStringLiterals.Contains(functionName))
                    parameters.RemoveAll(param => param.StartsWith("\""));

                switch (functionName)
                {
                    case "and": // returns first null or empty, else last variable
                    case "or": // returns first not null or empty, else last variable
                        var isAnd = functionName == "and";
                        foreach (var parameter in parameters)
                        {
                            functionResult = parameter;
                            // (null as string) == null
                            // (if null or empty) break if and, continue if or
                            // (if neither null nor empty) continue if and, break if or
                            if (string.IsNullOrWhiteSpace(variables[parameter] as string) == isAnd)
                                break;
                        }
                        break;
                    case "eq": // Returns .True if equal
                    case "ne": // Returns .False if equal
                        {
                            var wantEqual = functionName == "eq";
                            // eq/ne take exactly 2 params. Update the length to match
                            // This removes the whitespace between params 2 and 3.
                            // It shouldn't matter because the match starts at a word boundary
                            if (parameters.Count > 2)
                                functionLength = logicMatch.Groups[2].Captures[2].Index - functionStartIndex;

                            // Take first two parameters, convert vars to values and strip quotes on string literals
                            // Counting distinct gives us 1 if equal and 2 if not.
                            var isEqual =
                                parameters.Take(2).Select(param => param.StartsWith("\"") ? param.Trim('"') : variables[param] as string)
                                          .Distinct().Count() == 1;

                            functionResult = isEqual == wantEqual ? ".True" : ".False";
                            break;
                        }
                }

                template = template.Remove(functionStartIndex, functionLength)
                                   .Insert(functionStartIndex, functionResult);
                // Rerunning match instead of using nextMatch allows us to support nested functions
                // like {{if and eq (.Var1) "string1" eq (.Var2) "string2"}}
                // No performance is lost because Match/NextMatch are lazy evaluated and pause execution after first match
                logicMatch = _LogicFunctionRegex.Match(template);
            }

            // handle if ... else ... expression
            var IfElseRegex = new Regex(@"{{\s*if\s*(.+?)\s*}}(.*?){{\s*else\s*}}(.*?){{\s*end\s*}}");
            var IfElseRegexMatches = IfElseRegex.Match(template);

            while (IfElseRegexMatches.Success)
            {
                string conditionResult = null;

                var all = IfElseRegexMatches.Groups[0].Value;
                var condition = IfElseRegexMatches.Groups[1].Value;
                var onTrue = IfElseRegexMatches.Groups[2].Value;
                var onFalse = IfElseRegexMatches.Groups[3].Value;

                if (condition.StartsWith("."))
                {
                    var conditionResultState = false;
                    var value = variables[condition];

                    if (value == null)
                        conditionResultState = false;
                    else if (value is string)
                        conditionResultState = !string.IsNullOrWhiteSpace((string)value);
                    else if (value is ICollection)
                        conditionResultState = ((ICollection)value).Count > 0;
                    else
                        throw new Exception(string.Format("Unexpceted type for variable {0}: {1}", condition, value.GetType()));

                    if (conditionResultState)
                    {
                        conditionResult = onTrue;
                    }
                    else
                    {
                        conditionResult = onFalse;
                    }
                }
                else
                {
                    throw new NotImplementedException("CardigannIndexer: Condition operation '" + condition + "' not implemented");
                }
                template = template.Replace(all, conditionResult);
                IfElseRegexMatches = IfElseRegexMatches.NextMatch();
            }

            // handle range expression
            var RangeRegex = new Regex(@"{{\s*range\s*(.+?)\s*}}(.*?){{\.}}(.*?){{end}}");
            var RangeRegexMatches = RangeRegex.Match(template);

            while (RangeRegexMatches.Success)
            {
                var expanded = string.Empty;

                var all = RangeRegexMatches.Groups[0].Value;
                var variable = RangeRegexMatches.Groups[1].Value;
                var prefix = RangeRegexMatches.Groups[2].Value;
                var postfix = RangeRegexMatches.Groups[3].Value;

                foreach (var value in (ICollection<string>)variables[variable])
                {
                    var newvalue = value;
                    if (modifier != null)
                        newvalue = modifier(newvalue);
                    expanded += prefix + newvalue + postfix;
                }
                template = template.Replace(all, expanded);
                RangeRegexMatches = RangeRegexMatches.NextMatch();
            }

            // handle simple variables
            var VariablesRegEx = new Regex(@"{{\s*(\..+?)\s*}}");
            var VariablesRegExMatches = VariablesRegEx.Match(template);

            while (VariablesRegExMatches.Success)
            {
                var expanded = string.Empty;

                var all = VariablesRegExMatches.Groups[0].Value;
                var variable = VariablesRegExMatches.Groups[1].Value;

                var value = (string)variables[variable];
                if (modifier != null)
                    value = modifier(value);
                template = template.Replace(all, value);
                VariablesRegExMatches = VariablesRegExMatches.NextMatch();
            }

            return template;
        }

        protected bool checkForError(WebResult loginResult, IList<errorBlock> errorBlocks)
        {
            if (loginResult.Status == HttpStatusCode.Unauthorized) // e.g. used by YGGtorrent
                throw new ExceptionWithConfigData("401 Unauthorized, check your credentials", configData);

            if (errorBlocks == null)
                return true; // no error

            var ResultParser = new HtmlParser();
            var ResultDocument = ResultParser.ParseDocument(loginResult.ContentString);
            foreach (var error in errorBlocks)
            {
                var selection = ResultDocument.QuerySelector(error.Selector);
                if (selection != null)
                {
                    var errorMessage = selection.TextContent;
                    if (error.Message != null)
                    {
                        errorMessage = handleSelector(error.Message, ResultDocument.FirstElementChild);
                    }
                    throw new ExceptionWithConfigData(string.Format("Error: {0}", errorMessage.Trim()), configData);
                }
            }
            return true; // no error
        }

        protected async Task<bool> DoLogin()
        {
            var Login = Definition.Login;

            if (Login == null)
                return true;

            if (Login.Method == "post")
            {
                var pairs = new Dictionary<string, string>();
                foreach (var Input in Definition.Login.Inputs)
                {
                    var value = applyGoTemplateText(Input.Value);
                    pairs.Add(Input.Key, value);
                }

                var LoginUrl = resolvePath(Login.Path).ToString();
                configData.CookieHeader.Value = null;
                var loginResult = await RequestLoginAndFollowRedirect(LoginUrl, pairs, null, true, null, SiteLink, true);
                configData.CookieHeader.Value = loginResult.Cookies;

                checkForError(loginResult, Definition.Login.Error);
            }
            else if (Login.Method == "form")
            {
                var LoginUrl = resolvePath(Login.Path).ToString();

                var queryCollection = new NameValueCollection();
                var pairs = new Dictionary<string, string>();

                var FormSelector = Login.Form;
                if (FormSelector == null)
                    FormSelector = "form";

                // landingResultDocument might not be initiated if the login is caused by a relogin during a query
                if (landingResultDocument == null)
                {
                    var ConfigurationResult = await GetConfigurationForSetup(true);
                    if (ConfigurationResult == null) // got captcha
                    {
                        return false;
                    }
                }

                var form = landingResultDocument.QuerySelector(FormSelector);
                if (form == null)
                {
                    throw new ExceptionWithConfigData(string.Format("Login failed: No form found on {0} using form selector {1}", LoginUrl, FormSelector), configData);
                }

                var inputs = form.QuerySelectorAll("input");
                if (inputs == null)
                {
                    throw new ExceptionWithConfigData(string.Format("Login failed: No inputs found on {0} using form selector {1}", LoginUrl, FormSelector), configData);
                }

                var submitUrlstr = form.GetAttribute("action");
                if (Login.Submitpath != null)
                    submitUrlstr = Login.Submitpath;

                foreach (var input in inputs)
                {
                    var name = input.GetAttribute("name");
                    if (name == null)
                        continue;

                    var value = input.GetAttribute("value");
                    if (value == null)
                        value = "";

                    pairs[name] = value;
                }

                foreach (var Input in Definition.Login.Inputs)
                {
                    var value = applyGoTemplateText(Input.Value);
                    var input = Input.Key;
                    if (Login.Selectors)
                    {
                        var inputElement = landingResultDocument.QuerySelector(Input.Key);
                        if (inputElement == null)
                            throw new ExceptionWithConfigData(string.Format("Login failed: No input found using selector {0}", Input.Key), configData);
                        input = inputElement.GetAttribute("name");
                    }
                    pairs[input] = value;
                }

                // selector inputs
                if (Login.Selectorinputs != null)
                {
                    foreach (var Selectorinput in Login.Selectorinputs)
                    {
                        string value = null;
                        try
                        {
                            value = handleSelector(Selectorinput.Value, landingResultDocument.FirstElementChild);
                            pairs[Selectorinput.Key] = value;
                        }
                        catch (Exception ex)
                        {
                            throw new Exception(string.Format("Error while parsing selector input={0}, selector={1}, value={2}: {3}", Selectorinput.Key, Selectorinput.Value.Selector, value, ex.Message));
                        }
                    }
                }

                // getselector inputs
                if (Login.Getselectorinputs != null)
                {
                    foreach (var Selectorinput in Login.Getselectorinputs)
                    {
                        string value = null;
                        try
                        {
                            value = handleSelector(Selectorinput.Value, landingResultDocument.FirstElementChild);
                            queryCollection[Selectorinput.Key] = value;
                        }
                        catch (Exception ex)
                        {
                            throw new Exception(string.Format("Error while parsing get selector input={0}, selector={1}, value={2}: {3}", Selectorinput.Key, Selectorinput.Value.Selector, value, ex.Message));
                        }
                    }
                }
                if (queryCollection.Count > 0)
                    submitUrlstr += "?" + queryCollection.GetQueryString();
                var submitUrl = resolvePath(submitUrlstr, new Uri(LoginUrl));

                // automatically solve simpleCaptchas, if used
                var simpleCaptchaPresent = landingResultDocument.QuerySelector("script[src*=\"simpleCaptcha\"]");
                if (simpleCaptchaPresent != null)
                {
                    var captchaUrl = resolvePath("simpleCaptcha.php?numImages=1");
                    var simpleCaptchaResult = await RequestWithCookiesAsync(captchaUrl.ToString(), referer: LoginUrl);
                    var simpleCaptchaJSON = JObject.Parse(simpleCaptchaResult.ContentString);
                    var captchaSelection = simpleCaptchaJSON["images"][0]["hash"].ToString();
                    pairs["captchaSelection"] = captchaSelection;
                    pairs["submitme"] = "X";
                }

                if (Login.Captcha != null)
                {
                    var Captcha = Login.Captcha;
                    if (Captcha.Type == "image")
                    {
                        var CaptchaText = (StringConfigurationItem)configData.GetDynamic("CaptchaText");
                        if (CaptchaText != null)
                        {
                            var input = Captcha.Input;
                            if (Login.Selectors)
                            {
                                var inputElement = landingResultDocument.QuerySelector(Captcha.Input);
                                if (inputElement == null)
                                    throw new ExceptionWithConfigData(string.Format("Login failed: No captcha input found using {0}", Captcha.Input), configData);
                                input = inputElement.GetAttribute("name");
                            }
                            pairs[input] = CaptchaText.Value;
                        }
                    }
                    if (Captcha.Type == "text")
                    {
                        var CaptchaAnswer = (StringConfigurationItem)configData.GetDynamic("CaptchaAnswer");
                        if (CaptchaAnswer != null)
                        {
                            var input = Captcha.Input;
                            if (Login.Selectors)
                            {
                                var inputElement = landingResultDocument.QuerySelector(Captcha.Input);
                                if (inputElement == null)
                                    throw new ExceptionWithConfigData(string.Format("Login failed: No captcha input found using {0}", Captcha.Input), configData);
                                input = inputElement.GetAttribute("name");
                            }
                            pairs[input] = CaptchaAnswer.Value;
                        }
                    }
                }

                // clear landingResults/Document, otherwise we might use an old version for a new relogin (if GetConfigurationForSetup() wasn't called before)
                landingResult = null;
                landingResultDocument = null;

                WebResult loginResult = null;
                var enctype = form.GetAttribute("enctype");
                if (enctype == "multipart/form-data")
                {
                    var headers = new Dictionary<string, string>();
                    var boundary = "---------------------------" + (DateTime.UtcNow.Subtract(new DateTime(1970, 1, 1))).TotalSeconds.ToString().Replace(".", "");
                    var bodyParts = new List<string>();

                    foreach (var pair in pairs)
                    {
                        var part = "--" + boundary + "\r\n" +
                          "Content-Disposition: form-data; name=\"" + pair.Key + "\"\r\n" +
                          "\r\n" +
                          pair.Value;
                        bodyParts.Add(part);
                    }

                    bodyParts.Add("--" + boundary + "--");

                    headers.Add("Content-Type", "multipart/form-data; boundary=" + boundary);
                    var body = string.Join("\r\n", bodyParts);
                    loginResult = await RequestWithCookiesAsync(
                        submitUrl.ToString(), configData.CookieHeader.Value, RequestType.POST, SiteLink, pairs, headers,
                        body);
                }
                else
                {
                    loginResult = await RequestLoginAndFollowRedirect(submitUrl.ToString(), pairs, configData.CookieHeader.Value, true, null, LoginUrl, true);
                }

                configData.CookieHeader.Value = loginResult.Cookies;

                checkForError(loginResult, Definition.Login.Error);
            }
            else if (Login.Method == "cookie")
            {
                configData.CookieHeader.Value = ((StringConfigurationItem)configData.GetDynamic("cookie")).Value;
            }
            else if (Login.Method == "get")
            {
                var queryCollection = new NameValueCollection();
                foreach (var Input in Definition.Login.Inputs)
                {
                    var value = applyGoTemplateText(Input.Value);
                    queryCollection.Add(Input.Key, value);
                }

                var LoginUrl = resolvePath(Login.Path + "?" + queryCollection.GetQueryString()).ToString();
                configData.CookieHeader.Value = null;
                var loginResult = await RequestWithCookiesAsync(LoginUrl, referer: SiteLink);
                configData.CookieHeader.Value = loginResult.Cookies;

                checkForError(loginResult, Definition.Login.Error);
            }
            else if (Login.Method == "oneurl")
            {
                var OneUrl = applyGoTemplateText(Definition.Login.Inputs["oneurl"]);
                var LoginUrl = resolvePath(Login.Path + OneUrl).ToString();
                configData.CookieHeader.Value = null;
                var loginResult = await RequestWithCookiesAsync(LoginUrl, referer: SiteLink);
                configData.CookieHeader.Value = loginResult.Cookies;

                checkForError(loginResult, Definition.Login.Error);
            }
            else
            {
                throw new NotImplementedException("Login method " + Definition.Login.Method + " not implemented");
            }
            logger.Debug(string.Format("CardigannIndexer ({0}): Cookies after login: {1}", Id, CookieHeader));
            return true;
        }

        protected string getRedirectDomainHint(string requestUrl, string RedirectUrl)
        {
            if (requestUrl.StartsWith(SiteLink) && !RedirectUrl.StartsWith(SiteLink))
            {
                var uri = new Uri(RedirectUrl);
                return uri.Scheme + "://" + uri.Host + "/";
            }
            return null;
        }

        protected string getRedirectDomainHint(WebResult result) => getRedirectDomainHint(result.Request.Url, result.RedirectingTo);

        protected async Task<bool> TestLogin()
        {
            var Login = Definition.Login;

            if (Login == null || Login.Test == null)
                return false;

            // test if login was successful
            var LoginTestUrl = resolvePath(Login.Test.Path).ToString();
            var headers = ParseCustomHeaders(Definition.Search?.Headers, GetBaseTemplateVariables());
            var testResult = await RequestWithCookiesAsync(LoginTestUrl, headers: headers);

            if (testResult.IsRedirect)
            {
                var errormessage = "Login Failed, got redirected.";
                var DomainHint = getRedirectDomainHint(testResult);
                if (DomainHint != null)
                {
                    errormessage += " Try changing the indexer URL to " + DomainHint + ".";
                    if (Definition.Followredirect)
                    {
                        configData.SiteLink.Value = DomainHint;
                        SiteLink = configData.SiteLink.Value;
                        SaveConfig();
                        errormessage += " Updated site link, please try again.";
                    }
                }
                throw new ExceptionWithConfigData(errormessage, configData);
            }

            if (Login.Test.Selector != null)
            {
                var testResultParser = new HtmlParser();
                var testResultDocument = testResultParser.ParseDocument(testResult.ContentString);
                var selection = testResultDocument.QuerySelectorAll(Login.Test.Selector);
                if (selection.Length == 0)
                {
                    throw new ExceptionWithConfigData(string.Format("Login failed: Selector \"{0}\" didn't match", Login.Test.Selector), configData);
                }
            }
            return true;
        }

        protected bool CheckIfLoginIsNeeded(WebResult Result, IHtmlDocument document)
        {
            if (Result.IsRedirect)
            {
                var DomainHint = getRedirectDomainHint(Result);
                if (DomainHint != null)
                {
                    var errormessage = "Got redirected to another domain. Try changing the indexer URL to " + DomainHint + ".";
                    if (Definition.Followredirect)
                    {
                        configData.SiteLink.Value = DomainHint;
                        SiteLink = configData.SiteLink.Value;
                        SaveConfig();
                        errormessage += " Updated site link, please try again.";
                    }
                    throw new ExceptionWithConfigData(errormessage, configData);
                }

                return true;
            }

            if (Definition.Login == null || Definition.Login.Test == null)
                return false;

            if (Definition.Login.Test.Selector != null)
            {
                var selection = document.QuerySelectorAll(Definition.Login.Test.Selector);
                if (selection.Length == 0)
                {
                    return true;
                }
            }
            return false;
        }

        public override async Task<ConfigurationData> GetConfigurationForSetup()
        {
            try
            {
                return await GetConfigurationForSetup(false);
            }
            catch (Exception e)
            {
                logger.Error("Exception in GetConfigurationForSetup (" + Id + "): " + e);
                return configData;
            }
        }

        public async Task<ConfigurationData> GetConfigurationForSetup(bool automaticlogin)
        {
            var Login = Definition.Login;

            if (Login == null || Login.Method != "form")
                return configData;

            var LoginUrl = resolvePath(Login.Path);

            configData.CookieHeader.Value = null;
            if (Login.Cookies != null)
                configData.CookieHeader.Value = string.Join("; ", Login.Cookies);
            landingResult = await RequestWithCookiesAsync(LoginUrl.AbsoluteUri, referer: SiteLink);

            // Some sites have a temporary redirect before the login page, we need to process it.
            if (Definition.Followredirect)
            {
                await FollowIfRedirect(landingResult, LoginUrl.AbsoluteUri, overrideCookies: landingResult.Cookies, accumulateCookies: true);
            }

            var hasCaptcha = false;
            var htmlParser = new HtmlParser();
            landingResultDocument = htmlParser.ParseDocument(landingResult.ContentString);

            if (Login.Captcha != null)
            {
                var Captcha = Login.Captcha;
                if (Captcha.Type == "image")
                {
                    var captchaElement = landingResultDocument.QuerySelector(Captcha.Selector);
                    if (captchaElement != null)
                    {
                        hasCaptcha = true;

                        var CaptchaUrl = resolvePath(captchaElement.GetAttribute("src"), LoginUrl);
                        var captchaImageData = await RequestWithCookiesAsync(
                            CaptchaUrl.ToString(), landingResult.Cookies, referer: LoginUrl.AbsoluteUri);
                        var CaptchaImage = new DisplayImageConfigurationItem("Captcha Image");
                        var CaptchaText = new StringConfigurationItem("Captcha Text");

                        CaptchaImage.Value = captchaImageData.ContentBytes;

                        configData.AddDynamic("CaptchaImage", CaptchaImage);
                        configData.AddDynamic("CaptchaText", CaptchaText);
                    }
                    else
                    {
                        logger.Debug(string.Format("CardigannIndexer ({0}): No captcha image found", Id));
                    }
                }
                else if (Captcha.Type == "text")
                {
                    var captchaElement = landingResultDocument.QuerySelector(Captcha.Selector);
                    if (captchaElement != null)
                    {
                        hasCaptcha = true;

                        var CaptchaChallenge = new DisplayInfoConfigurationItem("Captcha Challenge", captchaElement.TextContent);
                        var CaptchaAnswer = new StringConfigurationItem("Captcha Answer");

                        configData.AddDynamic("CaptchaChallenge", CaptchaChallenge);
                        configData.AddDynamic("CaptchaAnswer", CaptchaAnswer);
                    }
                    else
                    {
                        logger.Debug(string.Format("CardigannIndexer ({0}): No captcha image found", Id));
                    }
                }
                else
                {
                    throw new NotImplementedException(string.Format("Captcha type \"{0}\" is not implemented", Captcha.Type));
                }
            }

            if (hasCaptcha && automaticlogin)
            {
                configData.LastError.Value = "Got captcha during automatic login, please reconfigure manually";
                logger.Error(string.Format("CardigannIndexer ({0}): Found captcha during automatic login, aborting", Id));
                return null;
            }

            return configData;
        }

        public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
        {
            LoadValuesFromJson(configJson);

            await DoLogin();
            await TestLogin();

            IsConfigured = true;
            SaveConfig();
            return IndexerConfigurationStatus.Completed;
        }

        protected string applyFilters(string Data, List<filterBlock> Filters, Dictionary<string, object> variables = null)
        {
            if (Filters == null)
                return Data;

            foreach (var Filter in Filters)
            {
                switch (Filter.Name)
                {
                    case "querystring":
                        var param = (string)Filter.Args;
                        Data = ParseUtil.GetArgumentFromQueryString(Data, param);
                        break;
                    case "timeparse":
                    case "dateparse":
                        var layout = (string)Filter.Args;
                        try
                        {
                            var Date = DateTimeUtil.ParseDateTimeGoLang(Data, layout);
                            Data = Date.ToString(DateTimeUtil.Rfc1123ZPattern);
                        }
                        catch (FormatException ex)
                        {
                            logger.Debug(ex.Message);
                        }
                        break;
                    case "regexp":
                        var pattern = (string)Filter.Args;
                        var Regexp = new Regex(pattern);
                        var Match = Regexp.Match(Data);
                        Data = Match.Groups[1].Value;
                        break;
                    case "re_replace":
                        var regexpreplace_pattern = (string)Filter.Args[0];
                        var regexpreplace_replacement = (string)Filter.Args[1];
                        regexpreplace_replacement = applyGoTemplateText(regexpreplace_replacement, variables);
                        var regexpreplace_regex = new Regex(regexpreplace_pattern);
                        Data = regexpreplace_regex.Replace(Data, regexpreplace_replacement);
                        break;
                    case "split":
                        var sep = (string)Filter.Args[0];
                        var pos = (string)Filter.Args[1];
                        var posInt = int.Parse(pos);
                        var strParts = Data.Split(sep[0]);
                        if (posInt < 0)
                        {
                            posInt += strParts.Length;
                        }
                        Data = strParts[posInt];
                        break;
                    case "replace":
                        var from = (string)Filter.Args[0];
                        var to = (string)Filter.Args[1];
                        to = applyGoTemplateText(to, variables);
                        Data = Data.Replace(from, to);
                        break;
                    case "trim":
                        var cutset = (string)Filter.Args;
                        if (cutset != null)
                            Data = Data.Trim(cutset[0]);
                        else
                            Data = Data.Trim();
                        break;
                    case "prepend":
                        var prependstr = (string)Filter.Args;
                        Data = applyGoTemplateText(prependstr, variables) + Data;
                        break;
                    case "append":
                        var str = (string)Filter.Args;
                        Data += applyGoTemplateText(str, variables);
                        break;
                    case "tolower":
                        Data = Data.ToLower();
                        break;
                    case "toupper":
                        Data = Data.ToUpper();
                        break;
                    case "urldecode":
                        Data = WebUtilityHelpers.UrlDecode(Data, Encoding);
                        break;
                    case "urlencode":
                        Data = WebUtilityHelpers.UrlEncode(Data, Encoding);
                        break;
                    case "timeago":
                    case "reltime":
                        Data = DateTimeUtil.FromTimeAgo(Data).ToString(DateTimeUtil.Rfc1123ZPattern);
                        break;
                    case "fuzzytime":
                        Data = DateTimeUtil.FromUnknown(Data).ToString(DateTimeUtil.Rfc1123ZPattern);
                        break;
                    case "validfilename":
                        Data = StringUtil.MakeValidFileName(Data, '_', false);
                        break;
                    case "diacritics":
                        var diacriticsOp = (string)Filter.Args;
                        if (diacriticsOp == "replace")
                        {
                            // Should replace diacritics charcaters with their base character
                            // It's not perfect, e.g. "ŠĐĆŽ - šđčćž" becomes "SĐCZ-sđccz"
                            var stFormD = Data.Normalize(NormalizationForm.FormD);
                            var len = stFormD.Length;
                            var sb = new StringBuilder();
                            for (var i = 0; i < len; i++)
                            {
                                var uc = System.Globalization.CharUnicodeInfo.GetUnicodeCategory(stFormD[i]);
                                if (uc != System.Globalization.UnicodeCategory.NonSpacingMark)
                                {
                                    sb.Append(stFormD[i]);
                                }
                            }
                            Data = (sb.ToString().Normalize(NormalizationForm.FormC));
                        }
                        else
                            throw new Exception("unsupported diacritics filter argument");
                        break;
                    case "jsonjoinarray":
                        var jsonjoinarrayJSONPath = (string)Filter.Args[0];
                        var jsonjoinarraySeparator = (string)Filter.Args[1];
                        var jsonjoinarrayO = JObject.Parse(Data);
                        var jsonjoinarrayOResult = jsonjoinarrayO.SelectToken(jsonjoinarrayJSONPath);
                        var jsonjoinarrayOResultStrings = jsonjoinarrayOResult.Select(j => j.ToString());
                        Data = string.Join(jsonjoinarraySeparator, jsonjoinarrayOResultStrings);
                        break;
                    case "hexdump":
                        // this is mainly for debugging invisible special char related issues
                        var HexData = string.Join("", Data.Select(c => c + "(" + ((int)c).ToString("X2") + ")"));
                        logger.Debug(string.Format("CardigannIndexer ({0}): strdump: {1}", Id, HexData));
                        break;
                    case "strdump":
                        // for debugging
                        var DebugData = Data.Replace("\r", "\\r").Replace("\n", "\\n").Replace("\xA0", "\\xA0");
                        var strTag = (string)Filter.Args;
                        if (strTag != null)
                            strTag = string.Format("({0}):", strTag);
                        else
                            strTag = ":";
                        logger.Debug(string.Format("CardigannIndexer ({0}): strdump{1} {2}", Id, strTag, DebugData));
                        break;
                    default:
                        break;
                }
            }
            return Data;
        }

        protected IElement QuerySelector(IElement Element, string Selector)
        {
            // AngleSharp doesn't support the :root pseudo selector, so we check for it manually
            if (Selector.StartsWith(":root"))
            {
                Selector = Selector.Substring(5);
                while (Element.ParentElement != null)
                {
                    Element = Element.ParentElement;
                }
            }
            return Element.QuerySelector(Selector);
        }

        protected string handleSelector(selectorBlock Selector, IElement Dom, Dictionary<string, object> variables = null, bool required = true)
        {
            if (Selector.Text != null)
            {
                return applyFilters(applyGoTemplateText(Selector.Text, variables), Selector.Filters, variables);
            }

            var selection = Dom;
            string value = null;

            if (Selector.Selector != null)
            {
                if (Dom.Matches(Selector.Selector))
                    selection = Dom;
                else
                    selection = QuerySelector(Dom, Selector.Selector);
                if (selection == null)
                {
                    if (required)
                        throw new Exception(string.Format("Selector \"{0}\" didn't match {1}", Selector.Selector, Dom.ToHtmlPretty()));
                    return null;
                }
            }

            if (Selector.Remove != null)
            {
                foreach (var i in selection.QuerySelectorAll(Selector.Remove))
                {
                    i.Remove();
                }
            }

            if (Selector.Case != null)
            {
                foreach (var Case in Selector.Case)
                {
                    if (selection.Matches(Case.Key) || QuerySelector(selection, Case.Key) != null)
                    {
                        value = Case.Value;
                        break;
                    }
                }
                if (value == null)
                {
                    if (required)
                        throw new Exception(string.Format("None of the case selectors \"{0}\" matched {1}", string.Join(",", Selector.Case), selection.ToHtmlPretty()));
                    return null;
                }
            }
            else if (Selector.Attribute != null)
            {
                value = selection.GetAttribute(Selector.Attribute);
                if (value == null)
                {
                    if (required)
                        throw new Exception(string.Format("Attribute \"{0}\" is not set for element {1}", Selector.Attribute, selection.ToHtmlPretty()));
                    return null;
                }
            }
            else
            {
                value = selection.TextContent;
            }

            return applyFilters(ParseUtil.NormalizeSpace(value), Selector.Filters, variables);
        }

        protected Uri resolvePath(string path, Uri currentUrl = null) => new Uri(currentUrl ?? new Uri(SiteLink), path);

        protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
        {
            var releases = new List<ReleaseInfo>();

            var Search = Definition.Search;

            // init template context
            var variables = GetBaseTemplateVariables();

            variables[".Query.Type"] = query.QueryType;
            variables[".Query.Q"] = query.SearchTerm;
            variables[".Query.Series"] = null;
            variables[".Query.Ep"] = query.Episode;
            variables[".Query.Season"] = query.Season;
            variables[".Query.Movie"] = null;
            variables[".Query.Year"] = query.Year.ToString();
            variables[".Query.Limit"] = query.Limit.ToString();
            variables[".Query.Offset"] = query.Offset.ToString();
            variables[".Query.Extended"] = query.Extended.ToString();
            variables[".Query.Categories"] = query.Categories;
            variables[".Query.APIKey"] = query.ApiKey;
            variables[".Query.TVDBID"] = query.TvdbID.ToString();
            variables[".Query.TVRageID"] = query.RageID;
            variables[".Query.IMDBID"] = query.ImdbID;
            variables[".Query.IMDBIDShort"] = query.ImdbIDShort;
            variables[".Query.TMDBID"] = query.TmdbID.ToString();
            variables[".Query.TVMazeID"] = null;
            variables[".Query.TraktID"] = null;
            variables[".Query.Album"] = query.Album;
            variables[".Query.Artist"] = query.Artist;
            variables[".Query.Label"] = query.Label;
            variables[".Query.Track"] = query.Track;
            //variables[".Query.Genre"] = query.Genre ?? new List<string>();
            variables[".Query.Episode"] = query.GetEpisodeSearchString();
            variables[".Query.Author"] = query.Author;
            variables[".Query.Title"] = query.Title;

            var mappedCategories = MapTorznabCapsToTrackers(query);
            if (mappedCategories.Count == 0)
            {
                mappedCategories = DefaultCategories;
            }

            variables[".Categories"] = mappedCategories;

            var KeywordTokens = new List<string>();
            var KeywordTokenKeys = new List<string> { "Q", "Series", "Movie", "Year" };
            foreach (var key in KeywordTokenKeys)
            {
                var Value = (string)variables[".Query." + key];
                if (!string.IsNullOrWhiteSpace(Value))
                    KeywordTokens.Add(Value);
            }

            if (!string.IsNullOrWhiteSpace((string)variables[".Query.Episode"]))
                KeywordTokens.Add((string)variables[".Query.Episode"]);
            variables[".Query.Keywords"] = string.Join(" ", KeywordTokens);
            variables[".Keywords"] = applyFilters((string)variables[".Query.Keywords"], Search.Keywordsfilters);

            // TODO: prepare queries first and then send them parallel
            var SearchPaths = Search.Paths;
            foreach (var SearchPath in SearchPaths)
            {
                // skip path if categories don't match
                if (SearchPath.Categories != null && mappedCategories.Count > 0)
                {
                    var invertMatch = (SearchPath.Categories[0] == "!");
                    var hasIntersect = mappedCategories.Intersect(SearchPath.Categories).Any();
                    if (invertMatch)
                        hasIntersect = !hasIntersect;
                    if (!hasIntersect)
                        continue;
                }

                // build search URL
                // HttpUtility.UrlPathEncode seems to only encode spaces, we use UrlEncode and replace + with %20 as a workaround
                var searchUrl = resolvePath(applyGoTemplateText(SearchPath.Path, variables, WebUtility.UrlEncode).Replace("+", "%20")).AbsoluteUri;
                var queryCollection = new List<KeyValuePair<string, string>>();
                var method = RequestType.GET;

                if (string.Equals(SearchPath.Method, "post", StringComparison.OrdinalIgnoreCase))
                {
                    method = RequestType.POST;
                }

                var InputsList = new List<Dictionary<string, string>>();
                if (SearchPath.Inheritinputs)
                    InputsList.Add(Search.Inputs);
                InputsList.Add(SearchPath.Inputs);

                foreach (var Inputs in InputsList)
                {
                    if (Inputs != null)
                    {
                        foreach (var Input in Inputs)
                        {
                            if (Input.Key == "$raw")
                            {
                                var rawStr = applyGoTemplateText(Input.Value, variables, WebUtility.UrlEncode);
                                foreach (var part in rawStr.Split('&'))
                                {
                                    var parts = part.Split(new[] { '=' }, 2);
                                    var key = parts[0];
                                    if (key.Length == 0)
                                        continue;
                                    var value = "";
                                    if (parts.Count() == 2)
                                        value = parts[1];
                                    queryCollection.Add(key, value);
                                }
                            }
                            else
                                queryCollection.Add(Input.Key, applyGoTemplateText(Input.Value, variables));
                        }
                    }
                }

                if (method == RequestType.GET)
                {
                    if (queryCollection.Count > 0)
                        searchUrl += "?" + queryCollection.GetQueryString(Encoding);
                }
                var searchUrlUri = new Uri(searchUrl);

                // send HTTP request
                var headers = ParseCustomHeaders(Search.Headers, variables);
                var response = await RequestWithCookiesAsync(
                    searchUrl, method: method, headers: headers, data: queryCollection);

                if (response.IsRedirect && SearchPath.Followredirect)
                    await FollowIfRedirect(response);

                var results = response.ContentString;


                try
                {
                    var SearchResultParser = new HtmlParser();
                    var SearchResultDocument = SearchResultParser.ParseDocument(results);

                    // check if we need to login again
                    var loginNeeded = CheckIfLoginIsNeeded(response, SearchResultDocument);
                    if (loginNeeded)
                    {
                        logger.Info(string.Format("CardigannIndexer ({0}): Relogin required", Id));
                        var LoginResult = await DoLogin();
                        if (!LoginResult)
                            throw new Exception(string.Format("Relogin failed"));
                        await TestLogin();
                        response = await RequestWithCookiesAsync(searchUrl, method: method, data: queryCollection);
                        if (response.IsRedirect && SearchPath.Followredirect)
                            await FollowIfRedirect(response);

                        results = response.ContentString;
                        SearchResultDocument = SearchResultParser.ParseDocument(results);
                    }

                    checkForError(response, Definition.Search.Error);

                    if (Search.Preprocessingfilters != null)
                    {
                        results = applyFilters(results, Search.Preprocessingfilters, variables);
                        SearchResultDocument = SearchResultParser.ParseDocument(results);
                        logger.Debug(string.Format("CardigannIndexer ({0}): result after preprocessingfilters: {1}", Id, results));
                    }

                    var rowsSelector = applyGoTemplateText(Search.Rows.Selector, variables);
                    var RowsDom = SearchResultDocument.QuerySelectorAll(rowsSelector);
                    var Rows = new List<IElement>();
                    foreach (var RowDom in RowsDom)
                    {
                        Rows.Add(RowDom);
                    }

                    // merge following rows for After selector
                    var After = Definition.Search.Rows.After;
                    if (After > 0)
                    {
                        for (var i = 0; i < Rows.Count; i += 1)
                        {
                            var CurrentRow = Rows[i];
                            for (var j = 0; j < After; j += 1)
                            {
                                var MergeRowIndex = i + j + 1;
                                var MergeRow = Rows[MergeRowIndex];
                                var MergeNodes = new List<INode>();
                                foreach (var node in MergeRow.ChildNodes)
                                {
                                    MergeNodes.Add(node);
                                }
                                CurrentRow.Append(MergeNodes.ToArray());
                            }
                            Rows.RemoveRange(i + 1, After);
                        }
                    }

                    foreach (var Row in Rows)
                    {
                        try
                        {
                            var release = new ReleaseInfo();

                            // Parse fields
                            foreach (var Field in Search.Fields)
                            {
                                var FieldParts = Field.Key.Split('|');
                                var FieldName = FieldParts[0];
                                var FieldModifiers = new List<string>();
                                for (var i = 1; i < FieldParts.Length; i++)
                                    FieldModifiers.Add(FieldParts[i]);

                                string value = null;
                                var variablesKey = ".Result." + FieldName;
                                var isOptional = OptionalFields.Contains(Field.Key) || FieldModifiers.Contains("optional") || Field.Value.Optional;
                                try
                                {
                                    value = handleSelector(Field.Value, Row, variables, !isOptional);
                                    if (isOptional && value == null)
                                    {
                                        variables[variablesKey] = null;
                                        continue;
                                    }

                                    switch (FieldName)
                                    {
                                        case "download":
                                            if (string.IsNullOrEmpty(value))
                                            {
                                                value = null;
                                                release.Link = null;
                                                break;
                                            }
                                            if (value.StartsWith("magnet:"))
                                            {
                                                release.MagnetUri = new Uri(value);
                                                value = release.MagnetUri.ToString();
                                            }
                                            else
                                            {
                                                release.Link = resolvePath(value, searchUrlUri);
                                                value = release.Link.ToString();
                                            }
                                            break;
                                        case "magnet":
                                            var magnetUri = new Uri(value);
                                            release.MagnetUri = magnetUri;
                                            value = magnetUri.ToString();
                                            break;
                                        case "infohash":
                                            release.InfoHash = value;
                                            break;
                                        case "details":
                                            var url = resolvePath(value, searchUrlUri);
                                            release.Details = url;
                                            value = url.ToString();
                                            break;
                                        case "title":
                                            if (FieldModifiers.Contains("append"))
                                                release.Title += value;
                                            else
                                                release.Title = value;
                                            value = release.Title;
                                            break;
                                        case "description":
                                            if (FieldModifiers.Contains("append"))
                                                release.Description += value;
                                            else
                                                release.Description = value;
                                            value = release.Description;
                                            break;
                                        case "category":
                                            var cats = MapTrackerCatToNewznab(value);
                                            if (cats.Any())
                                            {
                                                if (release.Category == null || FieldModifiers.Contains("noappend"))
                                                    release.Category = cats;
                                                else
                                                    release.Category = release.Category.Union(cats).ToList();
                                            }
                                            value = release.Category.ToString();
                                            break;
                                        case "size":
                                            release.Size = ReleaseInfo.GetBytes(value);
                                            value = release.Size.ToString();
                                            break;
                                        case "leechers":
                                            var leechers = ParseUtil.CoerceLong(value);
                                            leechers = leechers < 5000000L ? leechers : 0; // to fix #6558
                                            if (release.Peers == null)
                                                release.Peers = leechers;
                                            else
                                                release.Peers += leechers;
                                            value = leechers.ToString();
                                            break;
                                        case "seeders":
                                            release.Seeders = ParseUtil.CoerceLong(value);
                                            release.Seeders = release.Seeders < 5000000L ? release.Seeders : 0; // to fix #6558
                                            if (release.Peers == null)
                                                release.Peers = release.Seeders;
                                            else
                                                release.Peers += release.Seeders;
                                            value = release.Seeders.ToString();
                                            break;
                                        case "date":
                                            release.PublishDate = DateTimeUtil.FromUnknown(value);
                                            value = release.PublishDate.ToString(DateTimeUtil.Rfc1123ZPattern);
                                            break;
                                        case "files":
                                            release.Files = ParseUtil.CoerceLong(value);
                                            value = release.Files.ToString();
                                            break;
                                        case "grabs":
                                            release.Grabs = ParseUtil.CoerceLong(value);
                                            value = release.Grabs.ToString();
                                            break;
                                        case "downloadvolumefactor":
                                            release.DownloadVolumeFactor = ParseUtil.CoerceDouble(value);
                                            value = release.DownloadVolumeFactor.ToString();
                                            break;
                                        case "uploadvolumefactor":
                                            release.UploadVolumeFactor = ParseUtil.CoerceDouble(value);
                                            value = release.UploadVolumeFactor.ToString();
                                            break;
                                        case "minimumratio":
                                            release.MinimumRatio = ParseUtil.CoerceDouble(value);
                                            value = release.MinimumRatio.ToString();
                                            break;
                                        case "minimumseedtime":
                                            release.MinimumSeedTime = ParseUtil.CoerceLong(value);
                                            value = release.MinimumSeedTime.ToString();
                                            break;
                                        case "imdb":
                                            release.Imdb = ParseUtil.GetLongFromString(value);
                                            value = release.Imdb.ToString();
                                            break;
                                        case "tmdbid":
                                            var TmdbIDRegEx = new Regex(@"(\d+)", RegexOptions.Compiled);
                                            var TmdbIDMatch = TmdbIDRegEx.Match(value);
                                            var TmdbID = TmdbIDMatch.Groups[1].Value;
                                            release.TMDb = ParseUtil.CoerceLong(TmdbID);
                                            value = release.TMDb.ToString();
                                            break;
                                        case "rageid":
                                            var RageIDRegEx = new Regex(@"(\d+)", RegexOptions.Compiled);
                                            var RageIDMatch = RageIDRegEx.Match(value);
                                            var RageID = RageIDMatch.Groups[1].Value;
                                            release.RageID = ParseUtil.CoerceLong(RageID);
                                            value = release.RageID.ToString();
                                            break;
                                        case "tvdbid":
                                            var TVDBIdRegEx = new Regex(@"(\d+)", RegexOptions.Compiled);
                                            var TVDBIdMatch = TVDBIdRegEx.Match(value);
                                            var TVDBId = TVDBIdMatch.Groups[1].Value;
                                            release.TVDBId = ParseUtil.CoerceLong(TVDBId);
                                            value = release.TVDBId.ToString();
                                            break;
                                        case "author":
                                            release.Author = value;
                                            break;
                                        case "booktitle":
                                            release.BookTitle = value;
                                            break;
                                        case "poster":
                                            if (!string.IsNullOrWhiteSpace(value))
                                            {
                                                var poster = resolvePath(value, searchUrlUri);
                                                release.Poster = poster;
                                            }
                                            value = release.Poster.ToString();
                                            break;
                                        default:
                                            break;
                                    }
                                    variables[variablesKey] = value;
                                }
                                catch (Exception ex)
                                {
                                    if (!variables.ContainsKey(variablesKey))
                                        variables[variablesKey] = null;
                                    if (isOptional)
                                    {
                                        variables[variablesKey] = null;
                                        continue;
                                    }
                                    throw new Exception(string.Format("Error while parsing field={0}, selector={1}, value={2}: {3}", Field.Key, Field.Value.Selector, (value == null ? "<null>" : value), ex.Message));
                                }
                            }

                            var Filters = Definition.Search.Rows.Filters;
                            var SkipRelease = false;
                            if (Filters != null)
                            {
                                foreach (var Filter in Filters)
                                {
                                    switch (Filter.Name)
                                    {
                                        case "andmatch":
                                            var CharacterLimit = -1;
                                            if (Filter.Args != null)
                                                CharacterLimit = int.Parse(Filter.Args);

                                            if (query.ImdbID != null && TorznabCaps.MovieSearchImdbAvailable)
                                                break; // skip andmatch filter for imdb searches

                                            if (query.TmdbID != null && TorznabCaps.MovieSearchTmdbAvailable)
                                                break; // skip andmatch filter for tmdb searches

                                            if (query.TvdbID != null && TorznabCaps.TvSearchTvdbAvailable)
                                                break; // skip andmatch filter for tvdb searches

                                            var queryKeywords = variables[".Keywords"] as string;

                                            if (!query.MatchQueryStringAND(release.Title, CharacterLimit, queryKeywords))
                                            {
                                                logger.Debug(string.Format("CardigannIndexer ({0}): skipping {1} (andmatch filter)", Id, release.Title));
                                                SkipRelease = true;
                                            }
                                            break;
                                        case "strdump":
                                            // for debugging
                                            logger.Debug(string.Format("CardigannIndexer ({0}): row strdump: {1}", Id, Row.ToHtmlPretty()));
                                            break;
                                        default:
                                            logger.Error(string.Format("CardigannIndexer ({0}): Unsupported rows filter: {1}", Id, Filter.Name));
                                            break;
                                    }
                                }
                            }

                            if (SkipRelease)
                                continue;

                            // if DateHeaders is set go through the previous rows and look for the header selector
                            var DateHeaders = Definition.Search.Rows.Dateheaders;
                            if (release.PublishDate == DateTime.MinValue && DateHeaders != null)
                            {
                                var PrevRow = Row.PreviousElementSibling;
                                string value = null;
                                if (PrevRow == null) // continue with parent
                                {
                                    var Parent = Row.ParentElement;
                                    if (Parent != null)
                                        PrevRow = Parent.PreviousElementSibling;
                                }
                                while (PrevRow != null)
                                {
                                    var CurRow = PrevRow;
                                    logger.Debug(PrevRow.OuterHtml);
                                    try
                                    {
                                        value = handleSelector(DateHeaders, CurRow);
                                        break;
                                    }
                                    catch (Exception)
                                    {
                                        // do nothing
                                    }
                                    PrevRow = CurRow.PreviousElementSibling;
                                    if (PrevRow == null) // continue with parent
                                    {
                                        var Parent = CurRow.ParentElement;
                                        if (Parent != null)
                                            PrevRow = Parent.PreviousElementSibling;
                                    }
                                }

                                if (value == null && DateHeaders.Optional == false)
                                    throw new Exception(string.Format("No date header row found for {0}", release.ToString()));
                                if (value != null)
                                    release.PublishDate = DateTimeUtil.FromUnknown(value);
                            }

                            releases.Add(release);
                        }
                        catch (Exception ex)
                        {
                            logger.Error(string.Format("CardigannIndexer ({0}): Error while parsing row '{1}':\n\n{2}", Id, Row.ToHtmlPretty(), ex));
                        }
                    }
                }
                catch (Exception ex)
                {
                    OnParseError(results, ex);
                }
            }
            if (query.Limit > 0)
                releases = releases.Take(query.Limit).ToList();
            return releases;
        }

        protected async Task<WebResult> handleRequest(requestBlock request, Dictionary<string, object> variables = null, string referer = null)
        {
            var requestLinkStr = resolvePath(applyGoTemplateText(request.Path, variables)).ToString();
            logger.Debug($"CardigannIndexer ({Id}): handleRequest() requestLinkStr= {requestLinkStr}");

            Dictionary<string, string> pairs = null;
            var queryCollection = new NameValueCollection();

            var method = RequestType.GET;
            if (string.Equals(request.Method, "post", StringComparison.OrdinalIgnoreCase))
            {
                method = RequestType.POST;
                pairs = new Dictionary<string, string>();
            }

            foreach (var Input in request.Inputs)
            {
                var value = applyGoTemplateText(Input.Value, variables);
                if (method == RequestType.GET)
                    queryCollection.Add(Input.Key, value);
                else if (method == RequestType.POST)
                    pairs.Add(Input.Key, value);
            }

            if (queryCollection.Count > 0)
            {
                if (!requestLinkStr.Contains("?"))
                    requestLinkStr += "?" + queryCollection.GetQueryString(Encoding, separator: request.Queryseparator).Substring(1);
                else
                    requestLinkStr += queryCollection.GetQueryString(Encoding, separator: request.Queryseparator);
            }

            var response = await RequestWithCookiesAndRetryAsync(requestLinkStr, null, method, referer, pairs);
            logger.Debug($"CardigannIndexer ({Id}): handleRequest() remote server returned {response.Status.ToString()}" + (response.IsRedirect ? " => " + response.RedirectingTo : ""));
            return response;
        }

        protected IDictionary<string, object> AddTemplateVariablesFromUri(IDictionary<string, object> variables, Uri uri, string prefix = "")
        {
            variables[prefix + ".AbsoluteUri"] = uri.AbsoluteUri;
            variables[prefix + ".AbsolutePath"] = uri.AbsolutePath;
            variables[prefix + ".Scheme"] = uri.Scheme;
            variables[prefix + ".Host"] = uri.Host;
            variables[prefix + ".Port"] = uri.Port.ToString();
            variables[prefix + ".PathAndQuery"] = uri.PathAndQuery;
            variables[prefix + ".Query"] = uri.Query;
            var queryString = QueryHelpers.ParseQuery(uri.Query);
            foreach (var key in queryString.Keys)
            {
                //If we have supplied the same query string multiple time, just take the first.
                variables[prefix + ".Query." + key] = queryString[key].First();
            }
            return variables;
        }

        public override async Task<byte[]> Download(Uri link)
        {
            var method = RequestType.GET;
            if (Definition.Download != null)
            {
                var Download = Definition.Download;
                var variables = GetBaseTemplateVariables();
                AddTemplateVariablesFromUri(variables, link, ".DownloadUri");
                if (Download.Before != null)
                    await handleRequest(Download.Before, variables, link.ToString());

                if (Download.Method == "post")
                    method = RequestType.POST;

                if (Download.Selector != null)
                {
                    var selector = applyGoTemplateText(Download.Selector, variables);
                    var headers = ParseCustomHeaders(Definition.Search?.Headers, variables);
                    var response = await RequestWithCookiesAsync(link.ToString(), headers: headers);
                    if (response.IsRedirect)
                        response = await RequestWithCookiesAsync(response.RedirectingTo, headers: headers);
                    var results = response.ContentString;
                    var searchResultParser = new HtmlParser();
                    var searchResultDocument = searchResultParser.ParseDocument(results);
                    var downloadElement = searchResultDocument.QuerySelector(selector);
                    if (downloadElement != null)
                    {
                        logger.Debug(string.Format("CardigannIndexer ({0}): Download selector {1} matched:{2}", Id, selector, downloadElement.ToHtmlPretty()));
                        var href = "";
                        if (Download.Attribute != null)
                        {
                            href = downloadElement.GetAttribute(Download.Attribute);
                            if (href == null)
                                throw new Exception(string.Format("Attribute \"{0}\" is not set for element {1}", Download.Attribute, downloadElement.ToHtmlPretty()));
                        }
                        else
                        {
                            href = downloadElement.TextContent;
                        }
                        href = applyFilters(href, Download.Filters, variables);
                        link = resolvePath(href, link);
                    }
                    else
                    {
                        logger.Error(string.Format("CardigannIndexer ({0}): Download selector {1} didn't match:\n{2}", Id, Download.Selector, results));
                        throw new Exception(string.Format("Download selector {0} didn't match", Download.Selector));
                    }
                }
            }
            return await base.Download(link, method, link.ToString());
        }

        private Dictionary<string, string> ParseCustomHeaders(Dictionary<string, List<string>> customHeaders,
                                                              Dictionary<string, object> variables)
        {
            if (customHeaders == null)
                return null;

            // FIXME: fix jackett header handling (allow it to specifiy the same header multipe times)
            var headers = new Dictionary<string, string>();
            foreach (var header in customHeaders)
                headers.Add(header.Key, applyGoTemplateText(header.Value[0], variables));

            return headers;
        }
    }
}