From e59a9049363a8b1ce360aa1436112ec756735f7d Mon Sep 17 00:00:00 2001 From: Bogdan Date: Mon, 20 Mar 2023 18:05:25 +0200 Subject: [PATCH] cardigann: add fields default value (#14069) * cardigann: add fields default value * cardigann: re-add inner exception --- azure-pipelines.yml | 2 +- src/Jackett.Common/Definitions/bootytape.yml | 15 +++---- src/Jackett.Common/Definitions/ebookbay.yml | 28 +++++------- src/Jackett.Common/Definitions/joyhd.yml | 7 ++- .../Definitions/satclubbing.yml | 10 ++--- src/Jackett.Common/Definitions/schema.json | 37 ++++++++++++++- .../Indexers/CardigannIndexer.cs | 45 +++++++++++++------ .../Models/IndexerDefinition.cs | 1 + 8 files changed, 94 insertions(+), 51 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3adbf69e5..4ec05c87e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -385,7 +385,7 @@ stages: npm install -g ajv-cli-servarr ajv-formats # set fail as false fail=0 - ajv test -d "src/Jackett.Common/Definitions/*.yml" -s "src/Jackett.Common/Definitions/schema.json" --valid --all-errors -c ajv-formats + ajv test -d "src/Jackett.Common/Definitions/*.yml" -s "src/Jackett.Common/Definitions/schema.json" --valid --all-errors -c ajv-formats --spec=draft2019 if [ "$?" -ne 0 ]; then fail=1 fi diff --git a/src/Jackett.Common/Definitions/bootytape.yml b/src/Jackett.Common/Definitions/bootytape.yml index 97e183c56..cb581e05b 100644 --- a/src/Jackett.Common/Definitions/bootytape.yml +++ b/src/Jackett.Common/Definitions/bootytape.yml @@ -111,21 +111,18 @@ search: optional: true files: text: "{{ if .Result.files_optional }}{{ .Result.files_optional }}{{ else }}1{{ end }}" - size_optional: + size: selector: td:nth-child(3) optional: true - size: - text: "{{ if .Result.size_optional }}{{ .Result.size_optional }}{{ else }}0 B{{ end }}" - seeders_optional: + default: 0 + seeders: selector: a[href$="toseeders=1"] optional: true - seeders: - text: "{{ if .Result.seeders_optional }}{{ .Result.seeders_optional }}{{ else }}0{{ end }}" - leechers_optional: + default: 0 + leechers: selector: a[href$="todlers=1"] optional: true - leechers: - text: "{{ if .Result.leechers_optional }}{{ .Result.leechers_optional }}{{ else }}0{{ end }}" + default: 0 date: text: now downloadvolumefactor: diff --git a/src/Jackett.Common/Definitions/ebookbay.yml b/src/Jackett.Common/Definitions/ebookbay.yml index 3dbdfe954..78652bdfe 100644 --- a/src/Jackett.Common/Definitions/ebookbay.yml +++ b/src/Jackett.Common/Definitions/ebookbay.yml @@ -116,38 +116,34 @@ search: attribute: src date: text: now - size_optional: - optional: true + size: selector: p:contains("File Size") + optional: true + default: 0 filters: - name: regexp args: "File Size: (.+?)s?$" - size: - text: "{{ if .Result.size_optional }}{{ .Result.size_optional }}{{ else }}0 B{{ end }}" - seeders_optional: - optional: true + seeders: selector: p:contains("Seeds") + optional: true + default: 0 filters: - name: regexp args: "Seeds: (\\d+)" - seeders: - text: "{{ if .Result.seeders_optional }}{{ .Result.seeders_optional }}{{ else }}0{{ end }}" - leechers_optional: - optional: true + leechers: selector: p:contains("Peers") + optional: true + default: 0 filters: - name: regexp args: "Peers: (\\d+)" - leechers: - text: "{{ if .Result.leechers_optional }}{{ .Result.leechers_optional }}{{ else }}0{{ end }}" - grabs_optional: - optional: true + grabs: selector: p:contains("Completed Downloads") + optional: true + default: 0 filters: - name: regexp args: "Completed Downloads: (\\d+)" - grabs: - text: "{{ if .Result.grabs_optional }}{{ .Result.grabs_optional }}{{ else }}0{{ end }}" downloadvolumefactor: text: 0 uploadvolumefactor: diff --git a/src/Jackett.Common/Definitions/joyhd.yml b/src/Jackett.Common/Definitions/joyhd.yml index d3508f091..ccf039325 100644 --- a/src/Jackett.Common/Definitions/joyhd.yml +++ b/src/Jackett.Common/Definitions/joyhd.yml @@ -105,12 +105,11 @@ search: args: cat title_default: selector: a[href^="details.php?id="] - title_optional: - optional: true + title: selector: a[title][href^="details.php?id="] attribute: title - title: - text: "{{ if .Result.title_optional }}{{ .Result.title_optional }}{{ else }}{{ .Result.title_default }}{{ end }}" + optional: true + default: "{{ .Result.title_default }}" details: selector: a[href^="details.php?id="] attribute: href diff --git a/src/Jackett.Common/Definitions/satclubbing.yml b/src/Jackett.Common/Definitions/satclubbing.yml index 64d11fa2b..ef40abe9d 100644 --- a/src/Jackett.Common/Definitions/satclubbing.yml +++ b/src/Jackett.Common/Definitions/satclubbing.yml @@ -137,19 +137,17 @@ search: selector: a[href$="filelist=1"] size: selector: td:has(a[href$="filelist=1"]) + td + td - seeders_optional: + seeders: selector: a[href$="toseeders=1"] optional: true - seeders: - text: "{{ if .Result.seeders_optional }}{{ .Result.seeders_optional }}{{ else }}0{{ end }}" - leechers_optional: + default: 0 + leechers: selector: td:has(a[href$="toseeders=1"]) optional: true + default: 0 filters: - name: split args: ["|", 1] - leechers: - text: "{{ if .Result.leechers_optional }}{{ .Result.leechers_optional }}{{ else }}0{{ end }}" downloadvolumefactor: case: img[src$="pic/free_icon.gif"]: 0 diff --git a/src/Jackett.Common/Definitions/schema.json b/src/Jackett.Common/Definitions/schema.json index 255de0502..8b4143589 100644 --- a/src/Jackett.Common/Definitions/schema.json +++ b/src/Jackett.Common/Definitions/schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft/2019-09/schema", "$ref": "#/definitions/SchemaRoot", "definitions": { "SchemaRoot": { @@ -413,6 +413,16 @@ "optional": { "type": "boolean" }, + "default": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + }, "case": { "type": "object", "additionalProperties": false, @@ -441,6 +451,31 @@ } } }, + "dependentRequired": { + "default": [ + "optional" + ] + }, + "allOf": [ + { + "if": { + "properties": { + "default": { + "not": { + "const": null + } + } + } + }, + "then": { + "properties": { + "optional": { + "const": true + } + } + } + } + ], "title": "SelectorBlock" }, "Search": { diff --git a/src/Jackett.Common/Indexers/CardigannIndexer.cs b/src/Jackett.Common/Indexers/CardigannIndexer.cs index e93654b27..948053674 100644 --- a/src/Jackett.Common/Indexers/CardigannIndexer.cs +++ b/src/Jackett.Common/Indexers/CardigannIndexer.cs @@ -1482,6 +1482,7 @@ namespace Jackett.Common.Indexers string value = null; var variablesKey = ".Result." + FieldName; var isOptional = OptionalFields.Contains(Field.Key) || FieldModifiers.Contains("optional") || Field.Value.Optional; + try { var parentObj = mulRow; @@ -1489,27 +1490,34 @@ namespace Jackett.Common.Indexers parentObj = Row.Value(); value = handleJsonSelector(Field.Value, parentObj, variables, !isOptional); + if (isOptional && string.IsNullOrWhiteSpace(value)) { - variables[variablesKey] = null; - continue; + var defaultValue = applyGoTemplateText(Field.Value.Default, variables); + + if (string.IsNullOrWhiteSpace(defaultValue)) + { + variables[variablesKey] = null; + continue; + } + + value = defaultValue; } variables[variablesKey] = ParseFields(value, FieldName, release, FieldModifiers, searchUrlUri); } catch (Exception ex) { - if (!variables.ContainsKey(variablesKey)) + if (!variables.ContainsKey(variablesKey) || isOptional) variables[variablesKey] = null; + if (isOptional) - { - variables[variablesKey] = null; continue; - } + throw new Exception($"Error while parsing field={Field.Key}, selector={Field.Value.Selector}, value={value ?? ""}: {ex.Message}", ex); } - } + var Filters = Definition.Search.Rows.Filters; var SkipRelease = ParseRowFilters(Filters, release, query, variables, Row); @@ -1574,7 +1582,6 @@ namespace Jackett.Common.Indexers var rowsSelector = applyGoTemplateText(Search.Rows.Selector, variables); rowsDom = SearchResultDocument.QuerySelectorAll(rowsSelector); - } var Rows = new List(); @@ -1623,26 +1630,34 @@ namespace Jackett.Common.Indexers string value = null; var variablesKey = ".Result." + FieldName; var isOptional = OptionalFields.Contains(Field.Key) || FieldModifiers.Contains("optional") || Field.Value.Optional; + try { value = handleSelector(Field.Value, Row, variables, !isOptional); + if (isOptional && string.IsNullOrWhiteSpace(value)) { - variables[variablesKey] = null; - continue; + var defaultValue = applyGoTemplateText(Field.Value.Default, variables); + + if (string.IsNullOrWhiteSpace(defaultValue)) + { + variables[variablesKey] = null; + continue; + } + + value = defaultValue; } variables[variablesKey] = ParseFields(value, FieldName, release, FieldModifiers, searchUrlUri); } catch (Exception ex) { - if (!variables.ContainsKey(variablesKey)) + if (!variables.ContainsKey(variablesKey) || isOptional) variables[variablesKey] = null; + if (isOptional) - { - variables[variablesKey] = null; continue; - } + throw new Exception($"Error while parsing field={Field.Key}, selector={Field.Value.Selector}, value={value ?? ""}: {ex.Message}", ex); } } @@ -1707,8 +1722,10 @@ namespace Jackett.Common.Indexers } } } + if (query.Limit > 0) releases = releases.Take(query.Limit).ToList(); + return releases; } diff --git a/src/Jackett.Common/Models/IndexerDefinition.cs b/src/Jackett.Common/Models/IndexerDefinition.cs index b3f39eb78..7704bfbe8 100644 --- a/src/Jackett.Common/Models/IndexerDefinition.cs +++ b/src/Jackett.Common/Models/IndexerDefinition.cs @@ -110,6 +110,7 @@ namespace Jackett.Common.Models { public string Selector { get; set; } public bool Optional { get; set; } = false; + public string Default { get; set; } public string Text { get; set; } public string Attribute { get; set; } public string Remove { get; set; }