cardigann: add fields default value (#14069)

* cardigann: add fields default value

* cardigann: re-add inner exception

azure-pipelines.yml

@@ -385,7 +385,7 @@ stages:
                 npm install -g ajv-cli-servarr ajv-formats
                 # set fail as false
                 fail=0
-                ajv test -d "src/Jackett.Common/Definitions/*.yml" -s "src/Jackett.Common/Definitions/schema.json" --valid --all-errors -c ajv-formats
+                ajv test -d "src/Jackett.Common/Definitions/*.yml" -s "src/Jackett.Common/Definitions/schema.json" --valid --all-errors -c ajv-formats --spec=draft2019
                 if [ "$?" -ne 0 ]; then
                     fail=1
                 fi

src/Jackett.Common/Definitions/bootytape.yml

@@ -111,21 +111,18 @@ search:
       optional: true
     files:
       text: "{{ if .Result.files_optional }}{{ .Result.files_optional }}{{ else }}1{{ end }}"
-    size_optional:
+    size:
       selector: td:nth-child(3)
       optional: true
-    size:
+      default: 0
-      text: "{{ if .Result.size_optional }}{{ .Result.size_optional }}{{ else }}0 B{{ end }}"
+    seeders:
-    seeders_optional:
       selector: a[href$="toseeders=1"]
       optional: true
-    seeders:
+      default: 0
-      text: "{{ if .Result.seeders_optional }}{{ .Result.seeders_optional }}{{ else }}0{{ end }}"
+    leechers:
-    leechers_optional:
       selector: a[href$="todlers=1"]
       optional: true
-    leechers:
+      default: 0
-      text: "{{ if .Result.leechers_optional }}{{ .Result.leechers_optional }}{{ else }}0{{ end }}"
     date:
       text: now
     downloadvolumefactor:

src/Jackett.Common/Definitions/ebookbay.yml

@@ -116,38 +116,34 @@ search:
       attribute: src
     date:
       text: now
-    size_optional:
+    size:
-      optional: true
       selector: p:contains("File Size")
+      optional: true
+      default: 0
       filters:
         - name: regexp
           args: "File Size: (.+?)s?$"
-    size:
+    seeders:
-      text: "{{ if .Result.size_optional }}{{ .Result.size_optional }}{{ else }}0 B{{ end }}"
-    seeders_optional:
-      optional: true
       selector: p:contains("Seeds")
+      optional: true
+      default: 0
       filters:
         - name: regexp
           args: "Seeds: (\\d+)"
-    seeders:
+    leechers:
-      text: "{{ if .Result.seeders_optional }}{{ .Result.seeders_optional }}{{ else }}0{{ end }}"
-    leechers_optional:
-      optional: true
       selector: p:contains("Peers")
+      optional: true
+      default: 0
       filters:
         - name: regexp
           args: "Peers: (\\d+)"
-    leechers:
+    grabs:
-      text: "{{ if .Result.leechers_optional }}{{ .Result.leechers_optional }}{{ else }}0{{ end }}"
-    grabs_optional:
-      optional: true
       selector: p:contains("Completed Downloads")
+      optional: true
+      default: 0
       filters:
         - name: regexp
           args: "Completed Downloads: (\\d+)"
-    grabs:
-      text: "{{ if .Result.grabs_optional }}{{ .Result.grabs_optional }}{{ else }}0{{ end }}"
     downloadvolumefactor:
       text: 0
     uploadvolumefactor:

src/Jackett.Common/Definitions/joyhd.yml

@@ -105,12 +105,11 @@ search:
           args: cat
     title_default:
       selector: a[href^="details.php?id="]
-    title_optional:
+    title:
-      optional: true
       selector: a[title][href^="details.php?id="]
       attribute: title
-    title:
+      optional: true
-      text: "{{ if .Result.title_optional }}{{ .Result.title_optional }}{{ else }}{{ .Result.title_default }}{{ end }}"
+      default: "{{ .Result.title_default }}"
     details:
       selector: a[href^="details.php?id="]
       attribute: href

src/Jackett.Common/Definitions/satclubbing.yml

@@ -137,19 +137,17 @@ search:
       selector: a[href$="filelist=1"]
     size:
       selector: td:has(a[href$="filelist=1"]) + td + td
-    seeders_optional:
+    seeders:
       selector: a[href$="toseeders=1"]
       optional: true
-    seeders:
+      default: 0
-      text: "{{ if .Result.seeders_optional }}{{ .Result.seeders_optional }}{{ else }}0{{ end }}"
+    leechers:
-    leechers_optional:
       selector: td:has(a[href$="toseeders=1"])
       optional: true
+      default: 0
       filters:
         - name: split
           args: ["|", 1]
-    leechers:
-      text: "{{ if .Result.leechers_optional }}{{ .Result.leechers_optional }}{{ else }}0{{ end }}"
     downloadvolumefactor:
       case:
         img[src$="pic/free_icon.gif"]: 0

src/Jackett.Common/Definitions/schema.json

@@ -1,5 +1,5 @@
 {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$schema": "https://json-schema.org/draft/2019-09/schema",
     "$ref": "#/definitions/SchemaRoot",
     "definitions": {
         "SchemaRoot": {
@@ -413,6 +413,16 @@
                 "optional": {
                     "type": "boolean"
                 },
+                "default": {
+                    "oneOf": [
+                        {
+                            "type": "string"
+                        },
+                        {
+                            "type": "number"
+                        }
+                    ]
+                },
                 "case": {
                     "type": "object",
                     "additionalProperties": false,
@@ -441,6 +451,31 @@
                     }
                 }
             },
+            "dependentRequired": {
+                "default": [
+                    "optional"
+                ]
+            },
+            "allOf": [
+                {
+                    "if": {
+                        "properties": {
+                            "default": {
+                                "not": {
+                                    "const": null
+                                }
+                            }
+                        }
+                    },
+                    "then": {
+                        "properties": {
+                            "optional": {
+                                "const": true
+                            }
+                        }
+                    }
+                }
+            ],
             "title": "SelectorBlock"
         },
         "Search": {

src/Jackett.Common/Indexers/CardigannIndexer.cs

@@ -1482,6 +1482,7 @@ namespace Jackett.Common.Indexers
                                 string value = null;
                                 var variablesKey = ".Result." + FieldName;
                                 var isOptional = OptionalFields.Contains(Field.Key) || FieldModifiers.Contains("optional") || Field.Value.Optional;
+
                                 try
                                 {
                                     var parentObj = mulRow;
@@ -1489,27 +1490,34 @@ namespace Jackett.Common.Indexers
                                         parentObj = Row.Value<JObject>();
 
                                     value = handleJsonSelector(Field.Value, parentObj, variables, !isOptional);
+
                                     if (isOptional && string.IsNullOrWhiteSpace(value))
                                     {
-                                        variables[variablesKey] = null;
+                                        var defaultValue = applyGoTemplateText(Field.Value.Default, variables);
-                                        continue;
+
+                                        if (string.IsNullOrWhiteSpace(defaultValue))
+                                        {
+                                            variables[variablesKey] = null;
+                                            continue;
+                                        }
+
+                                        value = defaultValue;
                                     }
 
                                     variables[variablesKey] = ParseFields(value, FieldName, release, FieldModifiers, searchUrlUri);
                                 }
                                 catch (Exception ex)
                                 {
-                                    if (!variables.ContainsKey(variablesKey))
+                                    if (!variables.ContainsKey(variablesKey) || isOptional)
                                         variables[variablesKey] = null;
+
                                     if (isOptional)
-                                    {
-                                        variables[variablesKey] = null;
                                         continue;
-                                    }
+
                                     throw new Exception($"Error while parsing field={Field.Key}, selector={Field.Value.Selector}, value={value ?? "<null>"}: {ex.Message}", ex);
                                 }
-
                             }
+
                             var Filters = Definition.Search.Rows.Filters;
                             var SkipRelease = ParseRowFilters(Filters, release, query, variables, Row);
 
@@ -1574,7 +1582,6 @@ namespace Jackett.Common.Indexers
 
                             var rowsSelector = applyGoTemplateText(Search.Rows.Selector, variables);
                             rowsDom = SearchResultDocument.QuerySelectorAll(rowsSelector);
-
                         }
 
                         var Rows = new List<IElement>();
@@ -1623,26 +1630,34 @@ namespace Jackett.Common.Indexers
                                     string value = null;
                                     var variablesKey = ".Result." + FieldName;
                                     var isOptional = OptionalFields.Contains(Field.Key) || FieldModifiers.Contains("optional") || Field.Value.Optional;
+
                                     try
                                     {
                                         value = handleSelector(Field.Value, Row, variables, !isOptional);
+
                                         if (isOptional && string.IsNullOrWhiteSpace(value))
                                         {
-                                            variables[variablesKey] = null;
+                                            var defaultValue = applyGoTemplateText(Field.Value.Default, variables);
-                                            continue;
+
+                                            if (string.IsNullOrWhiteSpace(defaultValue))
+                                            {
+                                                variables[variablesKey] = null;
+                                                continue;
+                                            }
+
+                                            value = defaultValue;
                                         }
 
                                         variables[variablesKey] = ParseFields(value, FieldName, release, FieldModifiers, searchUrlUri);
                                     }
                                     catch (Exception ex)
                                     {
-                                        if (!variables.ContainsKey(variablesKey))
+                                        if (!variables.ContainsKey(variablesKey) || isOptional)
                                             variables[variablesKey] = null;
+
                                         if (isOptional)
-                                        {
-                                            variables[variablesKey] = null;
                                             continue;
-                                        }
+
                                         throw new Exception($"Error while parsing field={Field.Key}, selector={Field.Value.Selector}, value={value ?? "<null>"}: {ex.Message}", ex);
                                     }
                                 }
@@ -1707,8 +1722,10 @@ namespace Jackett.Common.Indexers
                     }
                 }
             }
+
             if (query.Limit > 0)
                 releases = releases.Take(query.Limit).ToList();
+
             return releases;
         }
 

src/Jackett.Common/Models/IndexerDefinition.cs

@@ -110,6 +110,7 @@ namespace Jackett.Common.Models
     {
         public string Selector { get; set; }
         public bool Optional { get; set; } = false;
+        public string Default { get; set; }
         public string Text { get; set; }
         public string Attribute { get; set; }
         public string Remove { get; set; }