Better configuration of browsers

.github/workflows/test-stack-reusable-workflow.yml

@@ -292,8 +292,8 @@ jobs:
 
       - name: Specific tests in built container for Selenium
         run: |
-          docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
-
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest  --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest  --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
 
   # SMTP tests
   smtp-tests:

changedetectionio/__init__.py

@@ -10,7 +10,6 @@
 from loguru import logger
 import getopt
 import logging
-import os
 import platform
 import signal
 import threading

changedetectionio/api/Import.py

@@ -154,11 +154,10 @@ def post(self):
             if extras['processor'] not in available:
                 return f"Invalid processor '{extras['processor']}'. Available processors: {', '.join(available)}", 400
 
-        # Validate fetch_backend if provided
+        # Validate fetch_backend if provided (legacy API compat — still accepted, stored as-is)
         if 'fetch_backend' in extras:
             from changedetectionio.content_fetchers import available_fetchers
             available = [f[0] for f in available_fetchers()]
-            # Also allow 'system' and extra_browser_* patterns
             is_valid = (
                 extras['fetch_backend'] == 'system' or
                 extras['fetch_backend'] in available or
@@ -167,6 +166,14 @@ def post(self):
             if not is_valid:
                 return f"Invalid fetch_backend '{extras['fetch_backend']}'. Available: system, {', '.join(available)}", 400
 
+        # Validate browser_profile if provided
+        if 'browser_profile' in extras:
+            from changedetectionio.model.browser_profile import get_builtin_profiles, RESERVED_MACHINE_NAMES
+            store_profiles = self.datastore.data['settings']['application'].get('browser_profiles', {})
+            known = set(get_builtin_profiles().keys()) | set(store_profiles.keys()) | {'system', None}
+            if extras['browser_profile'] not in known:
+                return f"Invalid browser_profile '{extras['browser_profile']}'. Available: {', '.join(str(k) for k in known)}", 400
+
         # Validate notification_urls if provided
         if 'notification_urls' in extras:
             from wtforms import ValidationError

changedetectionio/api/Tags.py

@@ -85,6 +85,9 @@ def queue_watches_background():
         # Create clean tag dict without Watch-specific fields
         clean_tag = {k: v for k, v in tag.items() if k not in watch_only_fields}
 
+        # fetch_backend is a legacy field superseded by browser_profile — omit from API response
+        clean_tag.pop('fetch_backend', None)
+
         return clean_tag
 
     @auth.check_token

changedetectionio/api/Watch.py

@@ -105,6 +105,9 @@ def get(self, uuid):
         watch['viewed'] = watch_obj.viewed
         watch['link'] = watch_obj.link,
 
+        # fetch_backend is a legacy field superseded by browser_profile — omit from API response
+        watch.pop('fetch_backend', None)
+
         return watch
 
     @auth.check_token

changedetectionio/blueprint/browser_steps/__init__.py

@@ -208,28 +208,23 @@ async def start_browsersteps_session(watch_uuid):
         browsersteps_start_session = {'start_time': time.time()}
 
         # Build proxy dict first — needed by both the CDP path and fetcher-specific launchers
-        proxy_id = datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
+        proxy_url = datastore.get_proxy_url_for_watch(uuid=watch_uuid)
         proxy = None
-        if proxy_id:
-            proxy_url = datastore.proxy_list.get(proxy_id, {}).get('url')
-            if proxy_url:
-                from urllib.parse import urlparse
-                parsed = urlparse(proxy_url)
-                proxy = {'server': proxy_url}
-                if parsed.username:
-                    proxy['username'] = parsed.username
-                if parsed.password:
-                    proxy['password'] = parsed.password
-                logger.debug(f"Browser Steps: UUID {watch_uuid} selected proxy {proxy_url}")
+        if proxy_url:
+            from urllib.parse import urlparse
+            parsed = urlparse(proxy_url)
+            proxy = {'server': proxy_url}
+            if parsed.username:
+                proxy['username'] = parsed.username
+            if parsed.password:
+                proxy['password'] = parsed.password
+            logger.debug(f"Browser Steps: UUID {watch_uuid} selected proxy {proxy_url}")
 
         # Resolve the fetcher class for this watch so we can ask it to launch its own browser
         # if it supports that (e.g. CloakBrowser, which runs locally rather than via CDP)
         watch = datastore.data['watching'][watch_uuid]
         from changedetectionio import content_fetchers
-        fetcher_name = watch.get_fetch_backend or 'system'
-        if fetcher_name == 'system':
-            fetcher_name = datastore.data['settings']['application'].get('fetch_backend', 'html_requests')
-        fetcher_class = getattr(content_fetchers, fetcher_name, None)
+        fetcher_class = content_fetchers.get_fetcher(watch.effective_browser_profile.fetch_backend)
 
         browser = None
         playwright_context = None
@@ -241,7 +236,7 @@ async def start_browsersteps_session(watch_uuid):
             result = await fetcher_class.get_browsersteps_browser(proxy=proxy, keepalive_ms=keepalive_ms)
             if result is not None:
                 browser, playwright_context = result
-                logger.debug(f"Browser Steps: using fetcher-specific browser for '{fetcher_name}'")
+                logger.debug(f"Browser Steps: using fetcher-specific browser for '{fetcher_class.__name__}'")
 
         # Default: connect to the remote Playwright/sockpuppetbrowser via CDP
         if browser is None:

changedetectionio/blueprint/check_proxies/__init__.py

@@ -46,7 +46,8 @@ def long_task(uuid, preferred_proxy):
                                                                  watch_uuid=uuid
                                                                  )
 
-            asyncio.run(update_handler.call_browser(preferred_proxy_id=preferred_proxy))
+            update_handler.preferred_proxy_override = preferred_proxy
+            asyncio.run(update_handler.call_browser())
         # title, size is len contents not len xfer
         except content_fetcher_exceptions.Non200ErrorCodeReceived as e:
             if e.status_code == 404:

changedetectionio/blueprint/imports/importer.py

@@ -175,9 +175,9 @@ def run(self,
                 dynamic_wachet = str(data.get('dynamic wachet', '')).strip().lower()  # Convert bool to str to cover all cases
                 # libreoffice and others can have it as =FALSE() =TRUE(), or bool(true)
                 if 'true' in dynamic_wachet or dynamic_wachet == '1':
-                    extras['fetch_backend'] = 'html_webdriver'
+                    extras['browser_profile'] = 'browser_chromeplaywright'
                 elif 'false' in dynamic_wachet or dynamic_wachet == '0':
-                    extras['fetch_backend'] = 'html_requests'
+                    extras['browser_profile'] = 'direct_http_requests'
 
                 if data.get('xpath'):
                     # @todo split by || ?

changedetectionio/blueprint/settings/__init__.py

@@ -15,6 +15,9 @@
 def construct_blueprint(datastore: ChangeDetectionStore):
     settings_blueprint = Blueprint('settings', __name__, template_folder="templates")
 
+    from changedetectionio.blueprint.settings.browser_profile import construct_blueprint as construct_browser_profile_blueprint
+    settings_blueprint.register_blueprint(construct_browser_profile_blueprint(datastore), url_prefix='/browsers')
+
     @settings_blueprint.route("", methods=['GET', "POST"])
     @login_optionally_required
     def settings_page():

changedetectionio/blueprint/settings/browser_profile/__init__.py

@@ -0,0 +1,200 @@
+import flask_login
+from flask import Blueprint, render_template, request, redirect, url_for, flash
+from flask_babel import gettext
+
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.auth_decorator import login_optionally_required
+
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    settings_browser_profile_blueprint = Blueprint(
+        'settings_browsers',
+        __name__,
+        template_folder="templates"
+    )
+
+    def _render_index(browser_profile_form=None, editing_machine_name=None):
+        from changedetectionio import forms
+        from changedetectionio import content_fetchers as cf
+        from changedetectionio.model.browser_profile import BrowserProfile, RESERVED_MACHINE_NAMES
+
+        # Only browser-capable fetchers are valid profile types
+        fetcher_choices = cf.available_browser_fetchers()
+        if browser_profile_form is None:
+            browser_profile_form = forms.BrowserProfileForm()
+        browser_profile_form.fetch_backend.choices = fetcher_choices
+
+        fetcher_supports_screenshots = {name: True for name, _ in fetcher_choices}
+        fetcher_requires_connection_url = {name: True for name, cls in cf.FETCHERS.items()
+                                           if getattr(cls, 'requires_connection_url', False)}
+
+        # Table shows default built-in profiles first, then user-created profiles
+        store_profiles = datastore.data['settings']['application'].get('browser_profiles', {})
+        user_profiles = dict(cf.DEFAULT_BROWSER_PROFILES)
+        for machine_name, raw in store_profiles.items():
+            try:
+                user_profiles[machine_name] = BrowserProfile(**raw) if isinstance(raw, dict) else raw
+            except Exception:
+                pass
+
+        current_default = datastore.data['settings']['application'].get('browser_profile') or 'direct_http_requests'
+
+        return render_template(
+            "browser_profiles.html",
+            browser_profiles=user_profiles,
+            browser_profile_form=browser_profile_form,
+            reserved_browser_profile_names=RESERVED_MACHINE_NAMES,
+            fetcher_choices=fetcher_choices,
+            fetcher_supports_screenshots=fetcher_supports_screenshots,
+            fetcher_requires_connection_url=fetcher_requires_connection_url,
+            current_default_profile=current_default,
+            editing_machine_name=editing_machine_name,
+        )
+
+    @settings_browser_profile_blueprint.route("", methods=['GET'])
+    @login_optionally_required
+    def index():
+        return _render_index()
+
+    @settings_browser_profile_blueprint.route("/<string:machine_name>/edit", methods=['GET'])
+    @login_optionally_required
+    def edit(machine_name):
+        from changedetectionio import forms
+        from changedetectionio.model.browser_profile import BrowserProfile, RESERVED_MACHINE_NAMES
+
+        if machine_name in RESERVED_MACHINE_NAMES:
+            flash(gettext("Built-in browser profiles cannot be edited."), 'error')
+            return redirect(url_for('settings.settings_browsers.index'))
+
+        store_profiles = datastore.data['settings']['application'].get('browser_profiles', {})
+        raw = store_profiles.get(machine_name)
+        if raw is None:
+            flash(gettext("Browser profile not found."), 'error')
+            return redirect(url_for('settings.settings_browsers.index'))
+
+        profile = BrowserProfile(**raw) if isinstance(raw, dict) else raw
+        form = forms.BrowserProfileForm(data=profile.model_dump())
+        return _render_index(browser_profile_form=form, editing_machine_name=machine_name)
+
+    @settings_browser_profile_blueprint.route("/save", methods=['POST'])
+    @login_optionally_required
+    def save():
+        from changedetectionio import forms
+        from changedetectionio import content_fetchers as cf
+        from changedetectionio.model.browser_profile import BrowserProfile, RESERVED_MACHINE_NAMES
+
+        fetcher_choices = [(name, desc) for name, desc in cf.available_fetchers()]
+        browser_profile_form = forms.BrowserProfileForm(formdata=request.form)
+        browser_profile_form.fetch_backend.choices = fetcher_choices
+
+        if not browser_profile_form.validate():
+            flash(gettext("Browser profile error: {}").format(
+                '; '.join(str(e) for errs in browser_profile_form.errors.values() for e in errs)
+            ), 'error')
+            return redirect(url_for('settings.settings_browsers.index'))
+
+        name = browser_profile_form.name.data.strip()
+        machine_name = BrowserProfile.machine_name_from_str(name)
+
+        if machine_name in RESERVED_MACHINE_NAMES:
+            flash(gettext("Cannot use reserved profile name '{}'. Please choose a different name.").format(name), 'error')
+            return redirect(url_for('settings.settings_browsers.index'))
+
+        original_machine_name = request.form.get('original_machine_name', '').strip()
+        store_profiles = datastore.data['settings']['application'].setdefault('browser_profiles', {})
+
+        if machine_name != original_machine_name and machine_name in store_profiles:
+            flash(gettext("A browser profile named '{}' already exists.").format(name), 'error')
+            return redirect(url_for('settings.settings_browsers.index'))
+
+        profile_data = {
+            'name': name,
+            'fetch_backend': browser_profile_form.fetch_backend.data,
+            'browser_connection_url': browser_profile_form.browser_connection_url.data or None,
+            'viewport_width': browser_profile_form.viewport_width.data or 1280,
+            'viewport_height': browser_profile_form.viewport_height.data or 1000,
+            'block_images': bool(browser_profile_form.block_images.data),
+            'block_fonts': bool(browser_profile_form.block_fonts.data),
+            'ignore_https_errors': bool(browser_profile_form.ignore_https_errors.data),
+            'user_agent': browser_profile_form.user_agent.data or None,
+            'locale': browser_profile_form.locale.data or None,
+            'custom_headers': browser_profile_form.custom_headers.data or '',
+            'is_builtin': False,
+        }
+
+        try:
+            BrowserProfile(**profile_data)
+        except Exception as e:
+            flash(gettext("Browser profile validation error: {}").format(str(e)), 'error')
+            return redirect(url_for('settings.settings_browsers.index'))
+
+        # Handle rename: remove old key, cascade-update watches and tags
+        if original_machine_name and original_machine_name != machine_name and original_machine_name in store_profiles:
+            del store_profiles[original_machine_name]
+            for watch in datastore.data['watching'].values():
+                if watch.get('browser_profile') == original_machine_name:
+                    watch['browser_profile'] = machine_name
+            for tag in datastore.data.get('settings', {}).get('application', {}).get('tags', {}).values():
+                if tag.get('browser_profile') == original_machine_name:
+                    tag['browser_profile'] = machine_name
+
+        store_profiles[machine_name] = profile_data
+        datastore.commit()
+        flash(gettext("Browser profile '{}' saved.").format(name), 'notice')
+        return redirect(url_for('settings.settings_browsers.index'))
+
+    @settings_browser_profile_blueprint.route("/<string:machine_name>/delete", methods=['GET'])
+    @login_optionally_required
+    def delete(machine_name):
+        from changedetectionio.model.browser_profile import RESERVED_MACHINE_NAMES
+
+        if machine_name in RESERVED_MACHINE_NAMES:
+            flash(gettext("Built-in browser profiles cannot be deleted."), 'error')
+            return redirect(url_for('settings.settings_browsers.index'))
+
+        store_profiles = datastore.data['settings']['application'].get('browser_profiles', {})
+        if machine_name not in store_profiles:
+            flash(gettext("Browser profile not found."), 'error')
+            return redirect(url_for('settings.settings_browsers.index'))
+
+        raw = store_profiles[machine_name]
+        profile_name = raw.get('name', machine_name) if isinstance(raw, dict) else machine_name
+
+        for watch in datastore.data['watching'].values():
+            if watch.get('browser_profile') == machine_name:
+                watch['browser_profile'] = None
+
+        for tag in datastore.data.get('settings', {}).get('application', {}).get('tags', {}).values():
+            if tag.get('browser_profile') == machine_name:
+                tag['browser_profile'] = None
+
+        if datastore.data['settings']['application'].get('browser_profile') == machine_name:
+            datastore.data['settings']['application']['browser_profile'] = None
+
+        del store_profiles[machine_name]
+        datastore.commit()
+        flash(gettext("Browser profile '{}' deleted.").format(profile_name), 'notice')
+        return redirect(url_for('settings.settings_browsers.index'))
+
+    @settings_browser_profile_blueprint.route("/set-default", methods=['POST'])
+    @login_optionally_required
+    def set_default():
+        from changedetectionio import content_fetchers as cf
+
+        machine_name = request.form.get('machine_name', '').strip()
+        if not machine_name:
+            flash(gettext("No profile specified."), 'error')
+            return redirect(url_for('settings.settings_browsers.index'))
+
+        from changedetectionio.model.browser_profile import get_profile
+        store_profiles = datastore.data['settings']['application'].get('browser_profiles', {})
+        if get_profile(machine_name, store_profiles) is None:
+            flash(gettext("Unknown browser profile '{}'.").format(machine_name), 'error')
+            return redirect(url_for('settings.settings_browsers.index'))
+
+        datastore.data['settings']['application']['browser_profile'] = machine_name
+        datastore.commit()
+        flash(gettext("Default browser profile set to '{}'.").format(machine_name), 'notice')
+        return redirect(url_for('settings.settings_browsers.index'))
+
+    return settings_browser_profile_blueprint