diff --git a/changedetectionio/blueprint/ui/templates/edit.html b/changedetectionio/blueprint/ui/templates/edit.html index b6398638310..0f9c0c6df51 100644 --- a/changedetectionio/blueprint/ui/templates/edit.html +++ b/changedetectionio/blueprint/ui/templates/edit.html @@ -335,6 +335,10 @@

{{ _('Text filtering') }}

{{ render_checkbox_field(form.remove_duplicate_lines) }} {{ _('Remove duplicate lines of text') }} +
+ {{ render_checkbox_field(form.sort_keys_alphabetically) }} + {{ _('Helps reduce changes detected caused by inconsistent JSON formatting') }} +
{{ render_checkbox_field(form.sort_text_alphabetically) }} {{ _('Helps reduce changes detected caused by sites shuffling lines around, combine with') }} {{ _('check unique lines') }} {{ _('below.') }} diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 9ffd111fee6..b9f62d483f8 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -811,6 +811,7 @@ class processor_text_json_diff_form(commonSettingsForm): ignore_status_codes = BooleanField(_l('Ignore status codes (process non-2xx status codes as normal)'), default=False) check_unique_lines = BooleanField(_l('Only trigger when unique lines appear in all history'), default=False) remove_duplicate_lines = BooleanField(_l('Remove duplicate lines of text'), default=False) + sort_keys_alphabetically = BooleanField(_l('Sort JSON keys alphabetically'), default=False) sort_text_alphabetically = BooleanField(_l('Sort text alphabetically'), default=False) strip_ignored_lines = TernaryNoneBooleanField(_l('Strip ignored lines'), default=None) trim_text_whitespace = BooleanField(_l('Trim whitespace before and after text'), default=False) diff --git a/changedetectionio/model/__init__.py b/changedetectionio/model/__init__.py index cfc8aee6495..63f2a0b530e 100644 --- a/changedetectionio/model/__init__.py +++ b/changedetectionio/model/__init__.py @@ -57,6 +57,7 @@ def __init__(self, *arg, **kw): 'price_change_threshold_percent': None, 'proxy': None, # Preferred proxy connection 'remote_server_reply': None, # From 'server' reply header + 'sort_keys_alphabetically': False, 'sort_text_alphabetically': False, 'strip_ignored_lines': None, 'subtractive_selectors': [], diff --git a/changedetectionio/processors/text_json_diff/processor.py b/changedetectionio/processors/text_json_diff/processor.py index 8f999944dbe..b19eb0eacbb 100644 --- a/changedetectionio/processors/text_json_diff/processor.py +++ b/changedetectionio/processors/text_json_diff/processor.py @@ -27,6 +27,15 @@ def _(x): return x JSON_FILTER_PREFIXES = ['json:', 'jq:', 'jqraw:'] + +def sort_json_keys(value): + """Recursively sort JSON object keys while preserving array order.""" + if isinstance(value, dict): + return {key: sort_json_keys(value[key]) for key in sorted(value.keys())} + if isinstance(value, list): + return [sort_json_keys(item) for item in value] + return value + # Assume it's this type if the server says nothing on content-type DEFAULT_WHEN_NO_CONTENT_TYPE_HEADER = 'text/html' @@ -122,6 +131,32 @@ def trim_whitespace(text): # Use generator expression to avoid building intermediate list return '\n'.join(line.strip() for line in text.replace("\n\n", "\n").splitlines()) + @staticmethod + def format_json_output_if_possible(text): + """Pretty-print JSON if the entire text is valid JSON.""" + try: + parsed = json.loads(text) + except Exception: + return text + + try: + return json.dumps(parsed, indent=2, ensure_ascii=False) + except Exception: + return text + + @staticmethod + def sort_json_keys_if_possible(text, indent=2): + """Sort JSON object keys recursively if the entire text is valid JSON.""" + try: + parsed = json.loads(text) + except Exception: + return text + + try: + return json.dumps(sort_json_keys(parsed), indent=indent, ensure_ascii=False) + except Exception: + return text + @staticmethod def remove_duplicate_lines(text): """Remove duplicate lines while preserving order.""" @@ -282,9 +317,12 @@ def preprocess_json(self, raw_content): # Then we re-format it, else it does have filters (later on) which will reformat it anyway content = html_tools.extract_json_as_string(content=raw_content, json_filter="json:$") - # Sort JSON to avoid false alerts from reordering + # Format JSON, optionally sorting keys to avoid false alerts from reordering try: - content = json.dumps(json.loads(content), sort_keys=True, indent=2, ensure_ascii=False) + parsed = json.loads(content) + if self.watch.get('sort_keys_alphabetically'): + parsed = sort_json_keys(parsed) + content = json.dumps(parsed, indent=2, ensure_ascii=False) except Exception: # Might be malformed JSON, continue anyway pass @@ -457,6 +495,12 @@ def run_changedetection(self, watch): stripped_text = html_content # === TEXT TRANSFORMATIONS === + # If JSON/jq filters are used on embedded JSON, reformat output JSON before any text filtering/transformations + if filter_config.has_include_json_filters: + stripped_text = transformer.format_json_output_if_possible(stripped_text) + if watch.get('sort_keys_alphabetically'): + stripped_text = transformer.sort_json_keys_if_possible(stripped_text) + if watch.get('trim_text_whitespace'): stripped_text = transformer.trim_whitespace(stripped_text) diff --git a/changedetectionio/translations/cs/LC_MESSAGES/messages.po b/changedetectionio/translations/cs/LC_MESSAGES/messages.po index 0e5099a0e89..fa98dc207fa 100644 --- a/changedetectionio/translations/cs/LC_MESSAGES/messages.po +++ b/changedetectionio/translations/cs/LC_MESSAGES/messages.po @@ -1404,6 +1404,10 @@ msgid "" "lines against all history for this watch." msgstr "" +#: changedetectionio/blueprint/ui/templates/edit.html +msgid "Helps reduce changes detected caused by inconsistent JSON formatting" +msgstr "" + #: changedetectionio/blueprint/ui/templates/edit.html msgid "Helps reduce changes detected caused by sites shuffling lines around, combine with" msgstr "" diff --git a/changedetectionio/translations/de/LC_MESSAGES/messages.po b/changedetectionio/translations/de/LC_MESSAGES/messages.po index 132639889aa..eec982895c4 100644 --- a/changedetectionio/translations/de/LC_MESSAGES/messages.po +++ b/changedetectionio/translations/de/LC_MESSAGES/messages.po @@ -1436,6 +1436,10 @@ msgstr "" "Gut geeignet für Websites, auf denen nur Inhalte verschoben werden und Sie wissen möchten, wann NEUE Inhalte " "hinzugefügt werden. Vergleicht neue Zeilen mit dem gesamten Verlauf dieser Überwachung." +#: changedetectionio/blueprint/ui/templates/edit.html +msgid "Helps reduce changes detected caused by inconsistent JSON formatting" +msgstr "" + #: changedetectionio/blueprint/ui/templates/edit.html msgid "Helps reduce changes detected caused by sites shuffling lines around, combine with" msgstr "" diff --git a/changedetectionio/translations/en_GB/LC_MESSAGES/messages.po b/changedetectionio/translations/en_GB/LC_MESSAGES/messages.po index e77e4fac531..e4c3e657405 100644 --- a/changedetectionio/translations/en_GB/LC_MESSAGES/messages.po +++ b/changedetectionio/translations/en_GB/LC_MESSAGES/messages.po @@ -1400,6 +1400,10 @@ msgid "" "lines against all history for this watch." msgstr "" +#: changedetectionio/blueprint/ui/templates/edit.html +msgid "Helps reduce changes detected caused by inconsistent JSON formatting" +msgstr "" + #: changedetectionio/blueprint/ui/templates/edit.html msgid "Helps reduce changes detected caused by sites shuffling lines around, combine with" msgstr "" diff --git a/changedetectionio/translations/en_US/LC_MESSAGES/messages.po b/changedetectionio/translations/en_US/LC_MESSAGES/messages.po index 0edefb6c9cc..2bb4fe8aa3c 100644 --- a/changedetectionio/translations/en_US/LC_MESSAGES/messages.po +++ b/changedetectionio/translations/en_US/LC_MESSAGES/messages.po @@ -1400,6 +1400,10 @@ msgid "" "lines against all history for this watch." msgstr "" +#: changedetectionio/blueprint/ui/templates/edit.html +msgid "Helps reduce changes detected caused by inconsistent JSON formatting" +msgstr "" + #: changedetectionio/blueprint/ui/templates/edit.html msgid "Helps reduce changes detected caused by sites shuffling lines around, combine with" msgstr "" diff --git a/changedetectionio/translations/fr/LC_MESSAGES/messages.po b/changedetectionio/translations/fr/LC_MESSAGES/messages.po index 5433b040dbe..32b2fa985cb 100644 --- a/changedetectionio/translations/fr/LC_MESSAGES/messages.po +++ b/changedetectionio/translations/fr/LC_MESSAGES/messages.po @@ -1406,6 +1406,10 @@ msgid "" "lines against all history for this watch." msgstr "" +#: changedetectionio/blueprint/ui/templates/edit.html +msgid "Helps reduce changes detected caused by inconsistent JSON formatting" +msgstr "" + #: changedetectionio/blueprint/ui/templates/edit.html msgid "Helps reduce changes detected caused by sites shuffling lines around, combine with" msgstr "" diff --git a/changedetectionio/translations/it/LC_MESSAGES/messages.po b/changedetectionio/translations/it/LC_MESSAGES/messages.po index 4966096adbf..62ea3fb3053 100644 --- a/changedetectionio/translations/it/LC_MESSAGES/messages.po +++ b/changedetectionio/translations/it/LC_MESSAGES/messages.po @@ -1402,6 +1402,10 @@ msgid "" "lines against all history for this watch." msgstr "" +#: changedetectionio/blueprint/ui/templates/edit.html +msgid "Helps reduce changes detected caused by inconsistent JSON formatting" +msgstr "" + #: changedetectionio/blueprint/ui/templates/edit.html msgid "Helps reduce changes detected caused by sites shuffling lines around, combine with" msgstr "" diff --git a/changedetectionio/translations/ko/LC_MESSAGES/messages.po b/changedetectionio/translations/ko/LC_MESSAGES/messages.po index 72fce1f1ad3..8fadcad2e75 100644 --- a/changedetectionio/translations/ko/LC_MESSAGES/messages.po +++ b/changedetectionio/translations/ko/LC_MESSAGES/messages.po @@ -1400,6 +1400,10 @@ msgid "" "lines against all history for this watch." msgstr "" +#: changedetectionio/blueprint/ui/templates/edit.html +msgid "Helps reduce changes detected caused by inconsistent JSON formatting" +msgstr "" + #: changedetectionio/blueprint/ui/templates/edit.html msgid "Helps reduce changes detected caused by sites shuffling lines around, combine with" msgstr "" diff --git a/changedetectionio/translations/messages.pot b/changedetectionio/translations/messages.pot index 724c5014a5a..498544be32a 100644 --- a/changedetectionio/translations/messages.pot +++ b/changedetectionio/translations/messages.pot @@ -1399,6 +1399,10 @@ msgid "" "lines against all history for this watch." msgstr "" +#: changedetectionio/blueprint/ui/templates/edit.html +msgid "Helps reduce changes detected caused by inconsistent JSON formatting" +msgstr "" + #: changedetectionio/blueprint/ui/templates/edit.html msgid "Helps reduce changes detected caused by sites shuffling lines around, combine with" msgstr "" diff --git a/changedetectionio/translations/zh/LC_MESSAGES/messages.po b/changedetectionio/translations/zh/LC_MESSAGES/messages.po index 9cad9702edd..8283e2e9490 100644 --- a/changedetectionio/translations/zh/LC_MESSAGES/messages.po +++ b/changedetectionio/translations/zh/LC_MESSAGES/messages.po @@ -1400,6 +1400,10 @@ msgid "" "lines against all history for this watch." msgstr "适合仅移动内容的网站，想知道新增内容时使用，会将新行与该监控项的全部历史进行比对。" +#: changedetectionio/blueprint/ui/templates/edit.html +msgid "Helps reduce changes detected caused by inconsistent JSON formatting" +msgstr "" + #: changedetectionio/blueprint/ui/templates/edit.html msgid "Helps reduce changes detected caused by sites shuffling lines around, combine with" msgstr "有助于减少因行顺序变化导致的变更，可结合" diff --git a/changedetectionio/translations/zh_Hant_TW/LC_MESSAGES/messages.po b/changedetectionio/translations/zh_Hant_TW/LC_MESSAGES/messages.po index 7b1a1513931..8016828ad1f 100644 --- a/changedetectionio/translations/zh_Hant_TW/LC_MESSAGES/messages.po +++ b/changedetectionio/translations/zh_Hant_TW/LC_MESSAGES/messages.po @@ -1400,6 +1400,10 @@ msgid "" "lines against all history for this watch." msgstr "適用於內容僅會移動的網站，且您想知道何時新增了「新」內容，此功能會將新行與此監測任務的所有歷史記錄進行比較。" +#: changedetectionio/blueprint/ui/templates/edit.html +msgid "Helps reduce changes detected caused by inconsistent JSON formatting" +msgstr "" + #: changedetectionio/blueprint/ui/templates/edit.html msgid "Helps reduce changes detected caused by sites shuffling lines around, combine with" msgstr "有助於減少因網站重新排列行而檢測到的變更，結合"