Mount Experimental Features dialog; add shared-server toggle; clean up flags

CHANGELOG.md

@@ -6,6 +6,13 @@ follow [Semantic Versioning](https://semver.org/).
 
 ## [Unreleased]
 
+### Changed
+
+- Experimental Features dialog is reachable again (Engine → Experimental
+  Features…), now including the shared detection server toggle; dead
+  `AUTOPTZ_INFERENCE_SCHEDULER` references removed; the full flag surface is
+  documented in `docs/flags.md`.
+
 ## [2.2.0-rc9] — 2026-06-30
 
 > Pre-release for testing. Headline: an **AutoPTZ 2.2 reliability overhaul** plus

autoptz/engine/runtime/experimental_flags.py

@@ -139,6 +139,24 @@ class ExperimentalFlag:
         choices=("fastest", "bgra"),
         restart_required=True,
     ),
+    ExperimentalFlag(
+        env_key="AUTOPTZ_MODEL_SERVER",
+        label="Shared detection server (multi-camera)",
+        description=(
+            "Run one shared detection server process that every camera delegates "
+            "to, instead of each camera loading its own model set. Validated as "
+            "the best-scaling mode for many cameras (54 ms end-to-end at 16 "
+            "cameras vs. 1.6 s for the threaded path) and now self-healing: the "
+            "server auto-respawns on crash, fails fast during an outage instead "
+            "of hanging cameras, and cameras fall back to their own local "
+            "detector after repeated failures. Still experimental — off by "
+            "default; validate on your hardware before relying on it."
+        ),
+        default="0",
+        kind="bool",
+        choices=(),
+        restart_required=True,
+    ),
 )
 
 

autoptz/ui/widgets/main_window.py

@@ -40,6 +40,7 @@
 from autoptz.ui.widgets.common import on_theme_changed
 from autoptz.ui.widgets.dialogs import (
     AboutDialog,
+    ExperimentalFeaturesDialog,
     ModelManagerDialog,
     NetworkCameraDialog,
 )
@@ -402,6 +403,17 @@ def _build_menus(self) -> None:
                 ),
             )
         )
+        self._act_experimental = _action(
+            self,
+            "Experimental Features...",
+            self._open_experimental_features,
+            tip=(
+                "Toggle curated experimental engine flags (e.g. the shared "
+                "detection server) and per-camera tracking defaults for new "
+                "cameras. Most changes need a restart to take effect."
+            ),
+        )
+        engine.addAction(self._act_experimental)
         engine.addSeparator()
         self._act_stop_tracking = _action(
             self,
@@ -948,6 +960,9 @@ def _open_model_manager(
         except Exception:  # noqa: BLE001
             log.debug("refresh services after model dialog failed", exc_info=True)
 
+    def _open_experimental_features(self) -> None:
+        ExperimentalFeaturesDialog(self._client, parent=self).exec()
+
     def _maybe_show_model_setup_on_startup(self) -> None:
         from PySide6.QtWidgets import QApplication
 

autoptz/ui/widgets/services_panel.py

@@ -229,6 +229,22 @@ def __init__(self, client: Any, parent: QWidget | None = None) -> None:
         root.addLayout(self._list)
         root.addStretch(1)
 
+        # ── footer (mirrors the Manage Models button pattern) ─────────────────────
+        root.addWidget(hline())
+        experimental_row = QHBoxLayout()
+        experimental_row.setSpacing(6)
+        self._experimental_btn = QPushButton("Experimental...")
+        self._experimental_btn.setToolTip(
+            "Toggle curated experimental engine flags (e.g. the shared detection "
+            "server) and per-camera tracking defaults for new cameras. Most "
+            "changes need a restart to take effect."
+        )
+        self._experimental_btn.clicked.connect(self._open_experimental_features)
+        experimental_row.addWidget(self._experimental_btn)
+        experimental_row.addWidget(HelpBadge(self._experimental_btn.toolTip()))
+        experimental_row.addStretch(1)
+        root.addLayout(experimental_row)
+
         self._restyle()
         on_theme_changed(client, self._restyle)
         _connect(client, "engineStateChanged", self.refresh)
@@ -426,6 +442,11 @@ def _open_model_manager(self) -> None:
         self._refresh_optional_components()
         self.refresh()
 
+    def _open_experimental_features(self) -> None:
+        from autoptz.ui.widgets.dialogs.experimental import ExperimentalFeaturesDialog
+
+        ExperimentalFeaturesDialog(self._client, parent=self).exec()
+
     def _ensure_row(self, key: str) -> None:
         if key in self._rows:
             return

docs/configuration.md

@@ -69,3 +69,6 @@ See [Performance](performance.md) for how these interact with your accelerator.
 | `AUTOPTZ_POSE_MODEL_PATH` | Use this pose ONNX verbatim. |
 | `AUTOPTZ_FORCE_EP` / `AUTOPTZ_PRECISION` / `AUTOPTZ_ORT_INTRA_THREADS` | Hardware prefs (set automatically from config by the supervisor). |
 | `AUTOPTZ_UPDATE_REPO` | Override the GitHub repo the updater checks. |
+
+The full `AUTOPTZ_*` flag surface — including the experimental flags managed by
+Engine → Experimental Features... — is documented in [flags.md](flags.md).

docs/engineering/retired-experiments.md

@@ -205,3 +205,9 @@ not preference.
 ### Reconsideration Criteria
 Only promote after it passes the same release gates as production: 6 and 8 fake
 NDI streams, CPU/RAM stability, clean shutdown, and no app-induced capture drops.
+
+> **Reconsidered 2026-07-03:** the curated Experimental Features dialog is
+> mounted again (Engine → Experimental Features…), by explicit user decision for
+> the 2.2.0 release. The 2026-06-29 rationale above still governs the *shape* of
+> the surface — curated registry entries with honest descriptions and restart
+> semantics, never a generic dumping ground for research switches.

docs/flags.md

@@ -0,0 +1,77 @@
+# AUTOPTZ_* environment flags
+
+Every ``AUTOPTZ_*`` environment variable the app reads, in one place. Verified by
+grep against the source tree — if a flag isn't listed here, it isn't read
+anywhere in `autoptz/`.
+
+Categories:
+
+- **Experimental** — managed by the Engine → Experimental Features... dialog
+  (also reachable from the Services panel). Persisted selections are applied to
+  `os.environ` by the supervisor at the next engine start
+  (`Supervisor._apply_experimental_env`, see
+  `autoptz/engine/runtime/experimental_flags.py` for the full schema). Setting
+  these directly in the shell also works — the dialog only clobbers a key it
+  actually persisted, never an operator-exported var it never touched.
+- **Operations override** — for advanced/offline/CI deployment, not surfaced in
+  any dialog. Safe to set by the operator; the app runs fine without them.
+- **Internal/dev** — set BY the app itself (derived from config, not meant to be
+  hand-set) or intended for CI/local development and debugging only.
+
+## Experimental (Engine → Experimental Features...)
+
+| Name | Meaning | Default |
+| --- | --- | --- |
+| `AUTOPTZ_UNIFIED_POSE` | Use one YOLO11-pose backbone for boxes + keypoints instead of a separate detector and pose pass. | `0` (off) |
+| `AUTOPTZ_ASYNC_APPEARANCE` | Run face recognition and appearance ReID on their own thread, overlapping inference. | `1` (on) |
+| `AUTOPTZ_PTZ_PUMP` | Drive PTZ commands from a dedicated background loop instead of inline on the inference thread. | `0` (off) |
+| `AUTOPTZ_PTZ_SERIAL_AUTOPROBE` | Scan serial ports for a companion VISCA control port when a USB PTZ camera opens. | `1` (on) |
+| `AUTOPTZ_REID_DEVICE` | Force the OSNet appearance (ReID) model onto a specific device (`cpu`/`mps`/`cuda`). | `""` (auto) |
+| `AUTOPTZ_COREML_UNITS` | Target for the CoreML execution provider on Apple/Intel-Mac builds. | `""` (ALL) |
+| `AUTOPTZ_TRUE_LATENCY_LEAD` | Lead the PTZ aim by the measured whole-pipeline dead time instead of just ingest+inference latency. | `0` (off) |
+| `AUTOPTZ_NDI_COLOR_FORMAT` | Color format requested from NDI sources (`fastest`/`bgra`). | `fastest` |
+| `AUTOPTZ_MODEL_SERVER` | Run one shared detection server process that every camera delegates to (best-scaling mode for many cameras; self-healing — auto-respawn, fast-fail, local-detector fallback). Still experimental. | `0` (off) |
+
+The dialog also exposes 4 per-camera `TrackingConfig` defaults (`unified_pose`,
+`use_target_associator`, `stage_spread`, `group_framing`) applied to newly added
+cameras — these are config fields, not env vars, so they aren't in this table.
+
+## Operations override
+
+| Name | Meaning | Default |
+| --- | --- | --- |
+| `AUTOPTZ_MODEL_PATH` | Use this detector ONNX file verbatim; skips download/export. | unset (auto-managed) |
+| `AUTOPTZ_POSE_MODEL_PATH` | Use this pose ONNX file verbatim. | unset (auto-managed) |
+| `AUTOPTZ_MODEL_URL` | Mirror/base URL to fetch a prebuilt detector ONNX from (air-gapped/offline installs); accepts a `{stem}`/`{model}` placeholder. | built-in HuggingFace export URL |
+| `AUTOPTZ_MODEL_URL_<STEM>` | Per-model override of `AUTOPTZ_MODEL_URL` for one specific weight (e.g. `AUTOPTZ_MODEL_URL_YOLO11M`). | unset |
+| `AUTOPTZ_FACE_MODEL` | InsightFace model pack name used for face recognition. | `buffalo_l` |
+| `AUTOPTZ_DB_PATH` | Override the ConfigStore SQLite path. | platform user-data dir |
+| `AUTOPTZ_UPDATE_REPO` | Override the GitHub repo the in-app updater checks for releases. | the AutoPTZ repo |
+| `AUTOPTZ_FORCE_EP` | Force a specific ONNX Runtime execution provider, bypassing auto-detection. | unset (auto) — normally config-driven, see Internal/dev |
+| `AUTOPTZ_PRECISION` | Force inference precision (`auto`/`fp32`/`fp16`/`int8`). | `auto` — normally config-driven, see Internal/dev |
+
+`AUTOPTZ_FORCE_EP` and `AUTOPTZ_PRECISION` are operator-settable overrides, but
+in normal operation the supervisor sets them itself from the hardware/config
+selection at engine start (see Internal/dev) — set them by hand only to
+override that choice for a single run.
+
+## Internal/dev
+
+| Name | Meaning | Default |
+| --- | --- | --- |
+| `AUTOPTZ_ORT_INTRA_THREADS` | ONNX Runtime intra-op thread cap. **Set BY the supervisor** at engine start from the detected core count; hand-set only for benchmarking. | unset (auto) |
+| `AUTOPTZ_CV2_THREADS` | OpenCV thread cap. **Set BY the supervisor** alongside `AUTOPTZ_ORT_INTRA_THREADS` to prevent thread-pool oversubscription. | unset (auto) |
+| `AUTOPTZ_NO_MODEL_EXPORT` | Disable the Ultralytics/Torch ONNX export fallback; used by CI and locked-down installs where `torch` isn't available. | unset (export allowed) |
+| `AUTOPTZ_MS_DIAG` | Verbose diagnostic logging for the model-server process. | unset (off) |
+| `AUTOPTZ_MARK_GT` | Enable AutoPTZ Mark's ground-truth synthetic-camera scoring path. | unset (off) |
+| `AUTOPTZ_MARK_NO_AUTOSTART` | Skip Mark's engine auto-start (dev/test harness convenience). | unset (off) |
+| `AUTOPTZ_START_MARK` | Launch straight into AutoPTZ Mark instead of the normal app on startup. | unset (off) |
+| `AUTOPTZ_SYNTH_DEBUG` | Verbose logging for the synthetic-camera ingest path (bench/Mark tooling). | unset (off) |
+| `AUTOPTZ_SKIP_CAMERA_PREFLIGHT` | Skip the startup camera-availability preflight check. | unset (preflight runs) |
+| `AUTOPTZ_NO_COLOR` / `NO_COLOR` | Disable ANSI color in log output. | unset (color on when a TTY) |
+| `AUTOPTZ_FORCE_COLOR` | Force ANSI color in log output even when not a TTY (e.g. piped CI logs). | unset (off) |
+
+`AUTOPTZ_PROCESS_PER_CAMERA` is **retired and ignored** by the env parser (see
+`docs/engineering/retired-experiments.md`) — the standalone model-per-child
+experiment it gated is superseded by the shared model-server candidate
+(`AUTOPTZ_MODEL_SERVER` above). It is intentionally not listed as a live flag.

tests/test_experimental_dialog.py

@@ -109,6 +109,16 @@ def test_apply_button_click_runs_the_apply_flow(qtapp) -> None:
     dlg.close()
 
 
+def test_dialog_renders_model_server_row(qtapp) -> None:
+    """The registry-driven dialog must render a row for the shared model-server
+    flag once it is registered — no hand-built row, purely from EXPERIMENTAL_FLAGS.
+    """
+    client = _client()
+    dlg, _ = _dialog(client)
+    assert "AUTOPTZ_MODEL_SERVER" in dlg._bool_boxes
+    dlg.close()
+
+
 def test_legacy_apply_still_just_persists(qtapp) -> None:
     """``_apply`` stays a pure persist (no modal) so existing callers/tests hold."""
     client = _client()

tests/test_experimental_flags.py

@@ -44,13 +44,25 @@ def test_expected_flags_inventoried() -> None:
         "AUTOPTZ_NDI_COLOR_FORMAT",
         "AUTOPTZ_PTZ_SERIAL_AUTOPROBE",
         "AUTOPTZ_TRUE_LATENCY_LEAD",
+        "AUTOPTZ_MODEL_SERVER",
     }
 
 
-def test_process_scaling_flags_are_not_in_normal_experimental_ui() -> None:
+def test_process_per_camera_stays_out_of_the_normal_experimental_ui() -> None:
+    # The standalone model-per-child experiment is retired (see
+    # docs/engineering/retired-experiments.md) and is NOT the same thing as the
+    # shared model-server candidate below — it stays out of the registry.
     keys = {f.env_key for f in EXPERIMENTAL_FLAGS}
     assert "AUTOPTZ_PROCESS_PER_CAMERA" not in keys
-    assert "AUTOPTZ_MODEL_SERVER" not in keys
+
+
+def test_model_server_flag_registered() -> None:
+    flag = next(f for f in EXPERIMENTAL_FLAGS if f.env_key == "AUTOPTZ_MODEL_SERVER")
+    assert flag.kind == "bool"
+    assert flag.default == "0"  # off by default — still experimental
+    assert flag.restart_required is True
+    assert flag.label.strip()
+    assert flag.description.strip()
 
 
 def test_true_latency_lead_flag_registered() -> None:

tests/test_supervisor_experimental.py

@@ -6,6 +6,7 @@
 from typing import Any
 
 from autoptz.config.store import ConfigStore
+from autoptz.engine.runtime.flags import env_model_server
 from autoptz.engine.supervisor import Supervisor
 
 
@@ -120,22 +121,37 @@ def test_absent_dict_leaves_unmanaged_env_untouched(tmp_path: Any, monkeypatch:
     # Feature-inactive baseline: with nothing persisted, a directly-set env var
     # (exported by the operator, or by another test) is NOT clobbered.  Only keys
     # the user actually persists in experimental_features are managed.
-    monkeypatch.setenv("AUTOPTZ_MODEL_SERVER", "1")
+    monkeypatch.setenv("AUTOPTZ_PROCESS_PER_CAMERA", "1")
     sup, _store = _sup(tmp_path)  # nothing persisted
     sup._apply_experimental_env()
-    assert os.environ.get("AUTOPTZ_MODEL_SERVER") == "1"
+    assert os.environ.get("AUTOPTZ_PROCESS_PER_CAMERA") == "1"
 
 
-def test_process_scaling_flags_are_not_managed_by_dev_flag_persistence(
+def test_model_server_flag_round_trips_through_dialog_persistence(
     tmp_path: Any, monkeypatch: Any
 ) -> None:
-    # The model-server candidate is env-only. A deliberately exported env var
-    # must not be clobbered by a stale saved dict from an older build.
+    """AUTOPTZ_MODEL_SERVER is now a registered experimental flag: persisting it
+    "on" via the dialog's apply path (``experimental_features`` in ConfigStore)
+    must flow through the supervisor's env-apply at engine start, same as any
+    other registered flag (PR #132 convention).
+    """
+    monkeypatch.delenv("AUTOPTZ_MODEL_SERVER", raising=False)
+    sup, store = _sup(tmp_path)
+    store.set_setting("experimental_features", {"AUTOPTZ_MODEL_SERVER": "1"})
+    sup._apply_experimental_env()
+    assert os.environ.get("AUTOPTZ_MODEL_SERVER") == "1"
+    assert env_model_server() is True
+
+
+def test_model_server_default_value_pops_existing_env(tmp_path: Any, monkeypatch: Any) -> None:
+    # Saving "off" (the registry default) clears a stale env var so the in-code
+    # fallback (off) governs — same contract as every other bool flag.
     monkeypatch.setenv("AUTOPTZ_MODEL_SERVER", "1")
     sup, store = _sup(tmp_path)
     store.set_setting("experimental_features", {"AUTOPTZ_MODEL_SERVER": "0"})
     sup._apply_experimental_env()
-    assert os.environ.get("AUTOPTZ_MODEL_SERVER") == "1"
+    assert "AUTOPTZ_MODEL_SERVER" not in os.environ
+    assert env_model_server() is False
 
 
 def test_tracking_keys_in_dict_are_ignored_for_env(tmp_path: Any, monkeypatch: Any) -> None:

tests/test_ui.py

@@ -1862,6 +1862,58 @@ def test_status_logs_button_controls_logs_dock(self, tmp_path) -> None:
         result = _run_ui_smoke(code, cwd=Path(__file__).resolve().parents[1], env=env, timeout=30)
         assert result.returncode == 0, result.stderr or result.stdout
 
+    def test_engine_menu_opens_experimental_features_dialog(self, tmp_path) -> None:
+        """Engine menu must expose "Experimental Features..." and triggering it
+        must actually open the (registry-driven) ExperimentalFeaturesDialog —
+        the dialog class exists and is exported but was never mounted anywhere.
+        """
+        code = f"""
+import os
+import sys
+from pathlib import Path
+os.environ.setdefault("QT_QPA_PLATFORM", "offscreen")
+from PySide6.QtWidgets import QApplication
+from autoptz.config.store import ConfigStore
+from autoptz.ui.engine_client import EngineClient
+from autoptz.ui.frames import ShmFrameSource
+from autoptz.ui.log_bridge import LogListModel
+from autoptz.ui.widgets import MainWindow
+from autoptz.ui.widgets.dialogs.experimental import ExperimentalFeaturesDialog
+app = QApplication(sys.argv[:1])
+client = EngineClient(store=ConfigStore(db_path=Path({str(tmp_path / "cfg.db")!r}), debounce_s=0))
+win = MainWindow(client, log_model=LogListModel(), frame_source=ShmFrameSource())
+try:
+    act = getattr(win, "_act_experimental", None)
+    assert act is not None, "MainWindow has no _act_experimental action"
+    assert "Experimental Features" in act.text()
+
+    engine_menu = None
+    for menu_action in win.menuBar().actions():
+        if menu_action.text().replace("&", "") == "Engine":
+            engine_menu = menu_action.menu()
+            break
+    assert engine_menu is not None, "no Engine menu found"
+    assert act in engine_menu.actions(), "_act_experimental is not in the Engine menu"
+
+    opened = {{}}
+    orig_exec = ExperimentalFeaturesDialog.exec
+    def fake_exec(self):
+        opened["dialog"] = self
+        return 0
+    ExperimentalFeaturesDialog.exec = fake_exec
+    try:
+        act.trigger()
+    finally:
+        ExperimentalFeaturesDialog.exec = orig_exec
+    assert isinstance(opened.get("dialog"), ExperimentalFeaturesDialog)
+finally:
+    win.close()
+"""
+        env = dict(os.environ)
+        env.setdefault("QT_QPA_PLATFORM", "offscreen")
+        result = _run_ui_smoke(code, cwd=Path(__file__).resolve().parents[1], env=env, timeout=30)
+        assert result.returncode == 0, result.stderr or result.stdout
+
     def test_theme_does_not_strip_mainwindow_desktop_chrome(self, tmp_path) -> None:
         code = f"""
 import os
@@ -2190,7 +2242,12 @@ def test_construction_reads_state_and_apply_persists(self, tmp_path) -> None:
 
 
 class TestNormalMenuSurface:
-    def test_mark_present_but_experimental_features_absent(self, tmp_path) -> None:
+    def test_mark_and_experimental_features_both_present(self, tmp_path) -> None:
+        """Run AutoPTZ Mark and Experimental Features... are both real, reachable
+        menu actions. (Experimental Features was briefly demoted to env-only in
+        rc9's "simplify experimental surfaces" pass — PR-7E brought it back as a
+        proper visible, registry-driven surface; see docs/flags.md.)
+        """
         code = f"""
 import os
 import sys
@@ -2210,7 +2267,7 @@ def test_mark_present_but_experimental_features_absent(self, tmp_path) -> None:
 try:
     texts = [a.text() for a in win.findChildren(QAction)]
     assert any("Run AutoPTZ Mark" in (t or "") for t in texts), texts
-    assert not any("Experimental Features" in (t or "") for t in texts), texts
+    assert any("Experimental Features" in (t or "") for t in texts), texts
 finally:
     win.close()
 """