diff --git a/.github/scripts/windows_test_assignments.json b/.github/scripts/windows_test_assignments.json index ad64ac38f..fd7aa7211 100644 --- a/.github/scripts/windows_test_assignments.json +++ b/.github/scripts/windows_test_assignments.json @@ -81,6 +81,7 @@ "tests/test_engine/test_agent_provider_identity.py", "tests/test_engine/test_agent_transactional_tool_publication.py", "tests/test_engine/test_attachment_aware_routing.py", + "tests/test_engine/test_attachment_replay_ownership.py", "tests/test_engine/test_auxiliary_usage_accounting.py", "tests/test_engine/test_cancelled_turn_segments.py", "tests/test_engine/test_coding_mode.py", @@ -105,6 +106,7 @@ "tests/test_engine/test_resolve_turn_thinking.py", "tests/test_engine/test_route_plan.py", "tests/test_engine/test_router_calibration.py", + "tests/test_engine/test_router_configured_image_policy.py", "tests/test_engine/test_router_decision_event.py", "tests/test_engine/test_routing_policy_parity.py", "tests/test_engine/test_runtime_agent_iteration_timeout.py", @@ -158,6 +160,7 @@ "tests/test_provider/test_context_profile_parity.py", "tests/test_provider/test_error_secret_boundary.py", "tests/test_provider/test_failure_injection.py", + "tests/test_provider/test_image_projection.py", "tests/test_provider/test_list_models_provider_identity.py", "tests/test_provider/test_live_catalog.py", "tests/test_provider/test_preset_registry.py", @@ -857,6 +860,7 @@ "tests/test_session/test_agent_task_storage.py", "tests/test_session/test_artifact_session_lifecycle.py", "tests/test_session/test_assistant_message_identity.py", + "tests/test_session/test_attachment_manifest.py", "tests/test_session/test_branch_media_copy.py", "tests/test_session/test_cascade_pagination.py", "tests/test_session/test_compaction.py", diff --git a/.github/scripts/windows_test_durations.json b/.github/scripts/windows_test_durations.json index 4a40c9f53..b5ab87999 100644 --- a/.github/scripts/windows_test_durations.json +++ b/.github/scripts/windows_test_durations.json @@ -335,6 +335,7 @@ "tests/test_engine/test_artifact_routing_policy.py": 0.01, "tests/test_engine/test_attachment_aware_routing.py": 9.013, "tests/test_engine/test_attachment_messages.py": 1.073, + "tests/test_engine/test_attachment_replay_ownership.py": 0.01, "tests/test_engine/test_auxiliary_usage_accounting.py": 0.028, "tests/test_engine/test_bootstrap_snapshot_invalidation.py": 0.478, "tests/test_engine/test_cache_break_monitor.py": 0.106, @@ -400,6 +401,7 @@ "tests/test_engine/test_router_budget_gate.py": 0.043, "tests/test_engine/test_router_calibration.py": 0.144, "tests/test_engine/test_router_calibration_service.py": 3.359, + "tests/test_engine/test_router_configured_image_policy.py": 0.01, "tests/test_engine/test_router_control.py": 0.062, "tests/test_engine/test_router_decision_event.py": 0.07, "tests/test_engine/test_router_decision_record.py": 0.786, @@ -955,6 +957,7 @@ "tests/test_provider/test_error_secret_boundary.py": 0.194, "tests/test_provider/test_failure_classification_parity.py": 0.353, "tests/test_provider/test_failure_injection.py": 0.066, + "tests/test_provider/test_image_projection.py": 0.01, "tests/test_provider/test_live_catalog.py": 0.089, "tests/test_provider/test_list_models_provider_identity.py": 0.01, "tests/test_provider/test_models_dev_snapshot_costs.py": 0.01, @@ -1205,6 +1208,7 @@ "tests/test_session/test_agent_task_storage.py": 4.576, "tests/test_session/test_artifact_session_lifecycle.py": 0.01, "tests/test_session/test_assistant_message_identity.py": 0.062, + "tests/test_session/test_attachment_manifest.py": 0.01, "tests/test_session/test_branch_media_copy.py": 0.88, "tests/test_session/test_cascade_pagination.py": 0.078, "tests/test_session/test_compaction.py": 0.282, diff --git a/desktop/electron/scripts/test-onboarding-flow.mjs b/desktop/electron/scripts/test-onboarding-flow.mjs index 57f8533cc..1d9ffaa3b 100644 --- a/desktop/electron/scripts/test-onboarding-flow.mjs +++ b/desktop/electron/scripts/test-onboarding-flow.mjs @@ -1135,13 +1135,13 @@ try { assert.equal(credential.routerTiers.c1.model, 'deepseek-v4-pro-0813') assert.equal(credential.routerTiers.c2.model, 'kimi-k2.7-code') assert.equal(credential.routerTiers.c3.model, 'glm-5.2') - assert.equal(credential.routerTiers.c0.supportsImage, false) - assert.equal(credential.routerTiers.c1.supportsImage, false) - assert.equal(credential.routerTiers.c2.supportsImage, false) - assert.equal(credential.routerTiers.c3.supportsImage, false) + assert.equal(Object.hasOwn(credential.routerTiers.c0, 'supportsImage'), false) + assert.equal(Object.hasOwn(credential.routerTiers.c1, 'supportsImage'), false) + assert.equal(Object.hasOwn(credential.routerTiers.c2, 'supportsImage'), false) + assert.equal(Object.hasOwn(credential.routerTiers.c3, 'supportsImage'), false) assert.equal(credential.routerTiers.c3.ensembleEnabled, true) assert.equal(credential.routerTiers.image_model.model, 'kimi-k2.6') - assert.equal(credential.routerTiers.image_model.supportsImage, true) + assert.equal(Object.hasOwn(credential.routerTiers.image_model, 'supportsImage'), false) assert.match(config, /\[squilla_router\]\nenabled = true/) assert.match(config, /\[llm\][\s\S]*?model = "deepseek-v4-pro-0813"/) assert.match(config, /\[squilla_router\.tiers\.c0\]\nprovider = "tokenrhythm"\nmodel = "deepseek-v4-flash-0731"/) @@ -1149,6 +1149,7 @@ try { assert.match(config, /\[squilla_router\.tiers\.c2\]\nprovider = "tokenrhythm"\nmodel = "kimi-k2.7-code"/) assert.match(config, /\[squilla_router\.tiers\.c3\][\s\S]*?model = "glm-5.2"[\s\S]*?ensemble_enabled = true/) assert.doesNotMatch(config, /thinking_level\s*=/) + assert.doesNotMatch(config, /supports_image\s*=/) assert.match(config, /\[llm_ensemble\]\nenabled = false/) assert.equal(successfulProbeServer.requests.length, 1) assert.equal(successfulProbeServer.requests[0].url, '/v1/chat/completions') diff --git a/desktop/electron/scripts/test-router-tier-normalization.mjs b/desktop/electron/scripts/test-router-tier-normalization.mjs index ce1898aaa..28a33a7d8 100644 --- a/desktop/electron/scripts/test-router-tier-normalization.mjs +++ b/desktop/electron/scripts/test-router-tier-normalization.mjs @@ -3,10 +3,10 @@ import { strict as assert } from 'node:assert' import { normalizeRouterTiers } from '../dist/router-tier-normalization.js' const currentFallback = { - c0: { provider: 'tokenrhythm', model: 'deepseek-v4-flash-0731' }, + c0: { provider: 'tokenrhythm', model: 'deepseek-v4-flash-0731', supportsImage: false }, c1: { provider: 'tokenrhythm', model: 'deepseek-v4-pro-0813' }, c2: { provider: 'tokenrhythm', model: 'kimi-k2.7-code' }, - c3: { provider: 'tokenrhythm', model: 'glm-5.2', ensembleEnabled: true }, + c3: { provider: 'tokenrhythm', model: 'glm-5.2', supportsImage: true, ensembleEnabled: true }, } const legacyCredentialTiers = { @@ -22,15 +22,32 @@ assert.deepEqual( Object.fromEntries(Object.entries(legacyCredentialTiers).map(([name, tier]) => [name, tier.model])), ) assert.equal(Object.hasOwn(loaded.c3, 'ensembleEnabled'), false) +assert.equal(Object.hasOwn(loaded.c0, 'supportsImage'), false) +assert.equal(Object.hasOwn(loaded.c3, 'supportsImage'), false) // saveDesktopCredential normalizes the already-loaded same-provider ladder a // second time. The missing legacy opt-in must remain missing on that pass. const resaved = normalizeRouterTiers(loaded, currentFallback) assert.deepEqual(resaved, loaded) assert.equal(Object.hasOwn(resaved.c3, 'ensembleEnabled'), false) +assert.equal(Object.hasOwn(resaved.c0, 'supportsImage'), false) + +for (const key of ['supports_image', 'supportsImage']) { + for (const value of [false, true]) { + const legacyCapability = normalizeRouterTiers( + { ...legacyCredentialTiers, c0: { ...legacyCredentialTiers.c0, [key]: value } }, + currentFallback, + ) + assert.equal(Object.hasOwn(legacyCapability.c0, 'supportsImage'), false) + assert.equal(Object.hasOwn(legacyCapability.c0, 'supports_image'), false) + assert.equal(legacyCapability.c0.model, legacyCredentialTiers.c0.model) + } +} const fresh = normalizeRouterTiers(undefined, currentFallback) assert.equal(fresh.c3.ensembleEnabled, true) +assert.equal(Object.hasOwn(fresh.c0, 'supportsImage'), false) +assert.equal(Object.hasOwn(fresh.c3, 'supportsImage'), false) const explicitSnakeCase = normalizeRouterTiers( { ...legacyCredentialTiers, c3: { ...legacyCredentialTiers.c3, ensemble_enabled: true } }, diff --git a/desktop/electron/src/main.ts b/desktop/electron/src/main.ts index 7825c8b6a..377af8037 100644 --- a/desktop/electron/src/main.ts +++ b/desktop/electron/src/main.ts @@ -1914,18 +1914,18 @@ function minimaxRouterProfile(provider: string): Record { const ROUTER_PROFILES: Record> = { tokenrhythm: { - c0: { provider: 'tokenrhythm', model: 'deepseek-v4-flash-0731', description: 'Fast DeepSeek V4 Flash 0731 route for simple work', supportsImage: false }, - c1: { provider: 'tokenrhythm', model: 'deepseek-v4-pro-0813', description: 'Default DeepSeek V4 Pro 0813 route for normal agent work', supportsImage: false }, - c2: { provider: 'tokenrhythm', model: 'kimi-k2.7-code', description: 'Strong Kimi 2.7 Code route for harder coding and analysis', supportsImage: false }, - c3: { provider: 'tokenrhythm', model: 'glm-5.2', description: 'Highest tier: shared B5 fusion; GLM 5.2 is retained for single-model C3 mode', supportsImage: false, ensembleEnabled: true }, - image_model: { provider: 'tokenrhythm', model: 'kimi-k2.6', description: 'Vision route for image attachments', supportsImage: true, imageOnly: true }, + c0: { provider: 'tokenrhythm', model: 'deepseek-v4-flash-0731', description: 'Fast DeepSeek V4 Flash 0731 route for simple work' }, + c1: { provider: 'tokenrhythm', model: 'deepseek-v4-pro-0813', description: 'Default DeepSeek V4 Pro 0813 route for normal agent work' }, + c2: { provider: 'tokenrhythm', model: 'kimi-k2.7-code', description: 'Strong Kimi 2.7 Code route for harder coding and analysis' }, + c3: { provider: 'tokenrhythm', model: 'glm-5.2', description: 'Highest tier: shared B5 fusion; GLM 5.2 is retained for single-model C3 mode', ensembleEnabled: true }, + image_model: { provider: 'tokenrhythm', model: 'kimi-k2.6', description: 'Vision route for image attachments', imageOnly: true }, }, openrouter: { c0: { provider: 'openrouter', model: 'deepseek/deepseek-v4-flash', description: 'Fast everyday work', thinkingLevel: 'high' }, c1: { provider: 'openrouter', model: 'deepseek/deepseek-v4-pro', description: 'Balanced agent work', thinkingLevel: 'high' }, c2: { provider: 'openrouter', model: 'z-ai/glm-5.2', description: 'Complex reasoning', thinkingLevel: 'high' }, c3: { provider: 'openrouter', model: 'anthropic/claude-opus-4.8', description: 'Highest quality review and planning', thinkingLevel: 'high' }, - image_model: { provider: 'openrouter', model: 'moonshotai/kimi-k2.6', description: 'Vision route for image attachments', supportsImage: true, imageOnly: true, thinkingLevel: 'medium' }, + image_model: { provider: 'openrouter', model: 'moonshotai/kimi-k2.6', description: 'Vision route for image attachments', imageOnly: true, thinkingLevel: 'medium' }, }, openai: { c0: { provider: 'openai', model: 'gpt-5.4-nano', description: 'Fast simple work', thinkingLevel: 'none' }, @@ -1952,10 +1952,10 @@ const ROUTER_PROFILES: Record> = { c3: { provider: 'gemini', model: 'gemini-3.1-pro-preview', description: 'Deep reasoning', thinkingLevel: 'high' }, }, moonshot: { - c0: { provider: 'moonshot', model: 'kimi-k2.6', description: 'Fast multimodal work', supportsImage: true, thinkingLevel: 'low' }, - c1: { provider: 'moonshot', model: 'kimi-k2.6', description: 'Balanced multimodal work', supportsImage: true, thinkingLevel: 'medium' }, - c2: { provider: 'moonshot', model: 'kimi-k2.6', description: 'Complex text and image work', supportsImage: true, thinkingLevel: 'medium' }, - c3: { provider: 'moonshot', model: 'kimi-k2.7-code', description: 'Code-heavy deep reasoning', supportsImage: true, thinkingLevel: 'high' }, + c0: { provider: 'moonshot', model: 'kimi-k2.6', description: 'Fast multimodal work', thinkingLevel: 'low' }, + c1: { provider: 'moonshot', model: 'kimi-k2.6', description: 'Balanced multimodal work', thinkingLevel: 'medium' }, + c2: { provider: 'moonshot', model: 'kimi-k2.6', description: 'Complex text and image work', thinkingLevel: 'medium' }, + c3: { provider: 'moonshot', model: 'kimi-k2.7-code', description: 'Code-heavy deep reasoning', thinkingLevel: 'high' }, }, kimi_coding_openai: textRouterProfile( 'kimi_coding_openai', @@ -2139,7 +2139,6 @@ function routerTierTomlLines(name: string, tier: RouterTier): string[] { `model = ${tomlString(tier.model)}`, ] if (tier.description) lines.push(`description = ${tomlString(tier.description)}`) - if (tier.supportsImage !== undefined) lines.push(`supports_image = ${tier.supportsImage ? 'true' : 'false'}`) if (tier.imageOnly !== undefined) lines.push(`image_only = ${tier.imageOnly ? 'true' : 'false'}`) if (tier.thinkingLevel) lines.push(`thinking_level = ${tomlString(tier.thinkingLevel)}`) if (tier.ensembleEnabled !== undefined) lines.push(`ensemble_enabled = ${tier.ensembleEnabled ? 'true' : 'false'}`) diff --git a/desktop/electron/src/router-tier-normalization.ts b/desktop/electron/src/router-tier-normalization.ts index 0efe80e73..ecca55b88 100644 --- a/desktop/electron/src/router-tier-normalization.ts +++ b/desktop/electron/src/router-tier-normalization.ts @@ -2,6 +2,7 @@ export interface RouterTier { provider: string model: string description?: string + /** Read compatibility only; the gateway resolves model capability. */ supportsImage?: boolean imageOnly?: boolean thinkingLevel?: string @@ -20,7 +21,11 @@ function canonicalTierKey(name: string): string { } function cloneRouterTiers(tiers: Record): Record { - return Object.fromEntries(Object.entries(tiers).map(([name, tier]) => [name, { ...tier }])) + return Object.fromEntries(Object.entries(tiers).map(([name, tier]) => { + const copy = { ...tier } + delete copy.supportsImage + return [name, copy] + })) } function normalizeBooleanSetting(raw: unknown, fallback: boolean): boolean { @@ -58,18 +63,18 @@ export function normalizeRouterTiers( const hasEnsembleEnabled = Object.prototype.hasOwnProperty.call(tier, 'ensembleEnabled') || Object.prototype.hasOwnProperty.call(tier, 'ensemble_enabled') const ensembleEnabled = tier.ensembleEnabled ?? tier.ensemble_enabled - out[name] = { + const normalizedTier: RouterTier = { ...out[name], provider, model, description: String(tier.description || out[name]?.description || ''), - supportsImage: Boolean(tier.supportsImage ?? tier.supports_image ?? out[name]?.supportsImage), imageOnly: Boolean(tier.imageOnly ?? tier.image_only ?? out[name]?.imageOnly), thinkingLevel: String(tier.thinkingLevel ?? tier.thinking_level ?? out[name]?.thinkingLevel ?? ''), ...(hasEnsembleEnabled ? { ensembleEnabled: normalizeBooleanSetting(ensembleEnabled, false) } : {}), } + out[name] = normalizedTier } return out } diff --git a/docs/features/squilla-router.md b/docs/features/squilla-router.md index 5a74f9cf6..8693bec70 100644 --- a/docs/features/squilla-router.md +++ b/docs/features/squilla-router.md @@ -78,10 +78,22 @@ Packaged static B5 lineups use a 120-second total budget per proposer and a 180-second aggregator idle budget. Operator-authored `custom_b5` lineups use 300 and 480 seconds respectively unless explicitly configured otherwise. -C3 fusion itself is excluded from image routing, but the dedicated -`image_model` tier remains eligible and is preferred for image requests. If it -is unavailable, another non-C3 tier with `supports_image = true` may handle the -request. +Image routing considers only the configured C0–C3 single-model deployments; +C3 fusion is excluded. Image input capability is resolved automatically from +the shared provider model catalog, including API-provided deployment metadata +and offline catalog fallback. The web and desktop clients do not expose a +manual image-capability switch. Legacy tier `supports_image` values remain +readable but do not override the deployment's capability, and `image_model` +is retained for compatibility without creating an executable fifth route. + +When capability is unknown, the configured model may receive a native image +request. A recognized image-input rejection can advance through the remaining +configured tiers before falling back to explicit not-analyzed/analysis-failed +text markers. Direct retries only its configured model with markers; Ensemble +uses text markers for all members. These retries stop after visible output or +tool effects. Projection never deletes the canonical attachment, so switching +back to a vision-capable model can recover earlier images after history +compression. Disable routing and use the configured provider/model directly: diff --git a/opensquilla-webui/src/components/setup/SetupModelStrategyPanel.test.ts b/opensquilla-webui/src/components/setup/SetupModelStrategyPanel.test.ts index 557263dd3..88d474eb2 100644 --- a/opensquilla-webui/src/components/setup/SetupModelStrategyPanel.test.ts +++ b/opensquilla-webui/src/components/setup/SetupModelStrategyPanel.test.ts @@ -317,6 +317,9 @@ describe('SetupModelStrategyPanel', () => { expect(el.querySelector('[aria-label="c0 request entry"]')).toBeTruthy() expect(el.querySelector('.setup-tier-table__row.is-head')?.textContent) .toContain('Request entry') + expect(el.querySelector('[aria-label$="supports image"]')).toBeNull() + expect(el.querySelector('[data-testid="router-image-capability-hint"]')?.textContent) + .toContain('detected automatically from provider metadata and request results') app.unmount() }) diff --git a/opensquilla-webui/src/components/setup/SetupModelStrategyPanel.vue b/opensquilla-webui/src/components/setup/SetupModelStrategyPanel.vue index 0bccfb895..6820bfe1f 100644 --- a/opensquilla-webui/src/components/setup/SetupModelStrategyPanel.vue +++ b/opensquilla-webui/src/components/setup/SetupModelStrategyPanel.vue @@ -107,7 +107,7 @@ const emit = defineEmits<{ updateFixedModel: [value: string] updateRouterDefaultTier: [value: string] updateRouterVisualMode: [value: string] - updateTierField: [name: string, key: 'provider' | 'model' | 'thinkingLevel' | 'supportsImage' | 'ensembleEnabled' | 'ensembleSelectionMode', value: string | boolean] + updateTierField: [name: string, key: 'provider' | 'model' | 'thinkingLevel' | 'ensembleEnabled' | 'ensembleSelectionMode', value: string | boolean] updateEnsembleScheme: [value: 'preset' | 'custom'] addEnsembleCandidate: [provider: string, model: string, role: EnsembleCandidateRole] removeEnsembleCandidate: [candidate: EnsembleCandidateView] @@ -825,6 +825,9 @@ function credentialLabel(candidate: EnsembleCandidateView): string { @update-tier-field="(name, key, value) => emit('updateTierField', name, key, value)" @migrate-legacy-ensemble="emit('migrateEnsembleLegacy')" /> +

+ {{ t('setup.router.imageCapabilityAutomatic') }} +

{ app.unmount() }) - it('renders provider, model, thinking, and image controls', async () => { + it('renders routing controls without manual image capability inputs', async () => { const { app, el } = await mountTable() const table = el.querySelector('[role="table"]') @@ -147,7 +147,7 @@ describe('SetupTierTable — editable routing rows', () => { expect(head?.textContent).toContain('Request entry') expect(head?.textContent).toContain('Model') expect(head?.textContent).toContain('Thinking') - expect(head?.textContent).toContain('Image') + expect(head?.textContent).not.toContain('Image') const requestEntry = el.querySelector('[aria-label="c0 request entry"]') expect(requestEntry?.tagName).toBe('SELECT') @@ -157,7 +157,7 @@ describe('SetupTierTable — editable routing rows', () => { expect(model?.value).toBe('deepseek/deepseek-v4-flash') expect(model?.disabled).toBe(false) expect(el.querySelector('select[aria-label="c0 thinking level"]')?.value).toBe('high') - expect(el.querySelector('input[aria-label="c0 supports image"]')).toBeTruthy() + expect(el.querySelector('[aria-label$="supports image"]')).toBeNull() app.unmount() }) @@ -220,7 +220,7 @@ describe('SetupTierTable — editable routing rows', () => { expect(tooltip.textContent).toContain('Current fusion plan is ready') expect(tooltip.textContent) .toContain('Fixed and fallback model: OpenRouter · deepseek/deepseek-v4-pro') - expect(tooltip.textContent).toContain('image requests still use the Image model configuration') + expect(tooltip.textContent).toContain('Router image processing uses only configured C0–C3 models') expect(details.dataset.open).toBe('false') details.parentElement?.dispatchEvent(new MouseEvent('mouseenter')) await nextTick() @@ -703,8 +703,9 @@ describe('SetupTierTable — editable routing rows', () => { app.unmount() }) - it('disables only C3 image input while keeping the dedicated image route editable', async () => { + it.each([false, true])('hides the image row in editable and preset tables (readonly=%s)', async (readonly) => { const { app, el } = await mountTable({ + readonly, rows: [ { ...ROWS[1], @@ -720,20 +721,22 @@ describe('SetupTierTable — editable routing rows', () => { supportsImage: true, }, ], - providerOptions: [{ providerId: 'openai', label: 'OpenAI' }], + providerCredentialStatus: [{ provider: 'openai', available: false }], + providerOptions: [ + { providerId: 'openai', label: 'OpenAI' }, + { providerId: 'openrouter', label: 'OpenRouter' }, + ], }) - const c3Image = el.querySelector('input[aria-label="c3 supports image"]')! - expect(c3Image.checked).toBe(false) - expect(c3Image.disabled).toBe(true) - const imageModel = el.querySelector('input[aria-label="image_model model"]')! - expect(imageModel.value).toBe('vision-model') - expect(imageModel.disabled).toBe(false) - expect(el.querySelector('select[aria-label="image_model thinking level"]')?.disabled).toBe(false) - const imageModelSwitch = el.querySelector('input[aria-label="image_model supports image"]')! - expect(imageModelSwitch.checked).toBe(true) - expect(imageModelSwitch.disabled).toBe(false) - expect(el.textContent).toContain('image requests still use the Image model configuration') + expect(el.querySelector('[aria-label="c3 supports image"]')).toBeNull() + expect(el.querySelectorAll('[role="row"]')).toHaveLength(2) + expect(el.querySelector('[aria-label="image_model model"]')).toBeNull() + expect(el.querySelector('[aria-label="image_model request entry"]')).toBeNull() + expect(el.querySelector('[aria-label="image_model thinking level"]')).toBeNull() + expect(el.querySelector('[aria-label="image_model supports image"]')).toBeNull() + expect(el.querySelector('[aria-label="C3 processing mode or model"]')).toBeTruthy() + expect(el.textContent).not.toContain('vision-model') + expect(el.textContent).not.toContain('Legacy compatibility setting') app.unmount() }) @@ -761,7 +764,7 @@ describe('SetupTierTable — editable routing rows', () => { expect(provider.closest('[role="row"]')?.getAttribute('aria-disabled')).toBeNull() expect(el.querySelector('input[aria-label="c0 model"]')?.disabled).toBe(true) expect(el.querySelector('select[aria-label="c0 thinking level"]')?.disabled).toBe(true) - expect(el.querySelector('input[aria-label="c0 supports image"]')?.disabled).toBe(true) + expect(el.querySelector('[aria-label="c0 supports image"]')).toBeNull() provider.value = 'openrouter' provider.dispatchEvent(new Event('change', { bubbles: true })) expect(onUpdateTierField).toHaveBeenCalledWith('c0', 'provider', 'openrouter') @@ -791,7 +794,7 @@ describe('SetupTierTable — editable routing rows', () => { expect(el.querySelector('select[aria-label="c0 request entry"]')?.disabled).toBe(true) expect(el.querySelector('input[aria-label="c0 model"]')?.disabled).toBe(true) expect(el.querySelector('select[aria-label="c0 thinking level"]')?.disabled).toBe(true) - expect(el.querySelector('input[aria-label="c0 supports image"]')?.disabled).toBe(true) + expect(el.querySelector('[aria-label="c0 supports image"]')).toBeNull() app.unmount() }) @@ -936,7 +939,7 @@ describe('SetupTierTable — readonly preview mode', () => { expect(el.querySelectorAll('select').length).toBe(0) expect(el.querySelector('input[role="combobox"]')).toBeNull() // The image switch stays visible (disabled) so the preview shows state. - expect(el.querySelector('input[aria-label="c0 supports image"]')?.disabled).toBe(true) + expect(el.querySelector('[aria-label="c0 supports image"]')).toBeNull() app.unmount() }) diff --git a/opensquilla-webui/src/components/setup/SetupTierTable.vue b/opensquilla-webui/src/components/setup/SetupTierTable.vue index a0f8aa8db..fb3919ee5 100644 --- a/opensquilla-webui/src/components/setup/SetupTierTable.vue +++ b/opensquilla-webui/src/components/setup/SetupTierTable.vue @@ -10,7 +10,6 @@ // • readonly — preset preview: no editable controls at all. import { computed, nextTick, onBeforeUnmount, ref, watch } from 'vue' import { useI18n } from 'vue-i18n' -import ControlSwitch from '@/components/ControlSwitch.vue' import Icon from '@/components/Icon.vue' import SetupModelCombobox from '@/components/setup/SetupModelCombobox.vue' import type { @@ -24,6 +23,7 @@ import type { DiscoveredModelsByProvider, } from '@/composables/setup/useSetupProviderForm' import { ROUTER_DYNAMIC_SELECTION_MODE } from '@/types/generated/router_tier_contract' +import { IMAGE_TIER } from '@/utils/chat/routerTiers' const { t } = useI18n() @@ -60,7 +60,7 @@ const props = withDefaults(defineProps<{ }) const emit = defineEmits<{ - updateTierField: [name: string, key: 'provider' | 'model' | 'thinkingLevel' | 'supportsImage' | 'ensembleEnabled' | 'ensembleSelectionMode', value: string | boolean] + updateTierField: [name: string, key: 'provider' | 'model' | 'thinkingLevel' | 'ensembleEnabled' | 'ensembleSelectionMode', value: string | boolean] migrateLegacyEnsemble: [] }>() @@ -80,6 +80,7 @@ const ensembleDetailsAnchor = ref(null) const ensembleTooltipUsesViewport = ref(false) const ensembleTooltipPlacement = ref<'top' | 'bottom'>('top') const ensembleTooltipStyle = ref>({}) +const visibleRows = computed(() => props.rows.filter(row => row.name !== IMAGE_TIER)) function catalogFor(row: SetupTierRow): DiscoveredModelCatalog { const provider = row.provider.trim().toLowerCase() @@ -167,22 +168,6 @@ function rowFieldsDisabled(row: SetupTierRow): boolean { return dependentFieldsDisabled(row) } -function providerFieldDisabled(): boolean { - // An invalid/retired saved provider disables its dependent fields, not the - // remediation control itself. C3 fusion only disables C3's own image input; - // the dedicated image route remains an independent editable capability. - return props.disabled -} - -function imageSwitchDisabled(row: SetupTierRow): boolean { - return rowFieldsDisabled(row) || (row.name === 'c3' && tierEnsembleActive(row)) -} - -function displayedImageSupport(row: SetupTierRow): boolean { - if (row.name === 'c3' && tierEnsembleActive(row)) return false - return row.supportsImage -} - function modelChoiceValue(row: SetupTierRow): string { return row.name === 'c3' && tierEnsembleActive(row) ? ENSEMBLE_CHOICE : row.model } @@ -589,7 +574,7 @@ function updateModelChoice(row: SetupTierRow, value: string) { const showProviderColumn = computed(() => { if (props.readonly) return true - if (props.rows.some(row => ( + if (visibleRows.value.some(row => ( !providerManagedByEnsemble(row) && credentialFor(row)?.available === false ))) return true @@ -601,7 +586,7 @@ const showProviderColumn = computed(() => { if (configuredProviders.size !== 1) return true const [onlyProvider] = [...configuredProviders] - return props.rows.some(row => ( + return visibleRows.value.some(row => ( !providerManagedByEnsemble(row) && row.provider.trim().toLowerCase() !== onlyProvider )) @@ -609,9 +594,9 @@ const showProviderColumn = computed(() => { // The combobox dropdown and compact-plan tooltip are absolutely positioned; // the table's rounded-corner overflow clip must open whenever either floats. -const hasCombobox = computed(() => props.rows.some(row => hasLiveCatalog(row))) +const hasCombobox = computed(() => visibleRows.value.some(row => hasLiveCatalog(row))) const allowsFloatingContent = computed(() => ( - hasCombobox.value || props.rows.some(row => compactSharedTierEnsembleActive(row)) + hasCombobox.value || visibleRows.value.some(row => compactSharedTierEnsembleActive(row)) )) @@ -626,15 +611,15 @@ const allowsFloatingContent = computed(() => ( :aria-disabled="disabled ? 'true' : undefined" >
- {{ t('setup.router.colTier') }}{{ t('setup.router.colProvider') }}{{ t('setup.router.colModel') }}{{ t('setup.router.colThinking') }}{{ t('setup.router.colImage') }} + {{ t('setup.router.colTier') }}{{ t('setup.router.colProvider') }}{{ t('setup.router.colModel') }}{{ t('setup.router.colThinking') }}
{{ tierLabel(tier.name) }} ( } .setup-tier-table--without-provider .setup-tier-table__row { - grid-template-columns: 140px minmax(0, 1fr) 120px 60px; + grid-template-columns: 140px minmax(0, 1fr) 120px; } .setup-tier-table__provider-cell { diff --git a/opensquilla-webui/src/composables/chat/useChatFeatureToggles.test.ts b/opensquilla-webui/src/composables/chat/useChatFeatureToggles.test.ts index 694a37e70..76d11c34e 100644 --- a/opensquilla-webui/src/composables/chat/useChatFeatureToggles.test.ts +++ b/opensquilla-webui/src/composables/chat/useChatFeatureToggles.test.ts @@ -26,6 +26,13 @@ const CAPABILITIES_BY_MODE: ModelRoutingCapabilitiesByMode = { }, } +const EFFECTIVE_CAPABILITIES_BY_MODE: ModelRoutingCapabilitiesByMode = { + ...CAPABILITIES_BY_MODE, + ensemble: { + image_input: { admission: 'allowed', reason: 'ensemble_mode_unsupported' }, + }, +} + function deferred() { let resolve!: (value: T | PromiseLike) => void const promise = new Promise((res) => { @@ -340,8 +347,8 @@ describe('useChatFeatureToggles model routing mode', () => { expect(api.modelRoutingMode.value).toBe('off') }) - it('applies canonical image admission and preserves old-Gateway defaults', async () => { - const blocked = createHarness({ + it('allows capability degradation while preserving real Gateway blocks', async () => { + const degraded = createHarness({ configGetResults: [{}], routingGetResults: [{ mode: 'direct', @@ -351,9 +358,23 @@ describe('useChatFeatureToggles model routing mode', () => { }, }], }) - await blocked.api.loadFeatureToggles() - expect(blocked.api.globalImageInputAdmission.value).toBe('blocked') - expect(blocked.api.globalImageInputAdmissionReason.value).toBe('model_vision_unsupported') + await degraded.api.loadFeatureToggles() + expect(degraded.api.globalImageInputAdmission.value).toBe('allowed') + expect(degraded.api.globalImageInputAdmissionReason.value).toBe('model_vision_unsupported') + + const policyBlocked = createHarness({ + configGetResults: [{}], + routingGetResults: [{ + mode: 'direct', + image_input: { + admission: 'blocked', + reason: 'attachment_policy_denied', + }, + }], + }) + await policyBlocked.api.loadFeatureToggles() + expect(policyBlocked.api.globalImageInputAdmission.value).toBe('blocked') + expect(policyBlocked.api.globalImageInputAdmissionReason.value).toBe('attachment_policy_denied') const legacyDirect = createHarness({ configGetResults: [{ llm_ensemble: { enabled: false } }], @@ -367,7 +388,7 @@ describe('useChatFeatureToggles model routing mode', () => { routingGetResults: [{ mode: 'ensemble' }], }) await legacyEnsemble.api.loadFeatureToggles() - expect(legacyEnsemble.api.globalImageInputAdmission.value).toBe('blocked') + expect(legacyEnsemble.api.globalImageInputAdmission.value).toBe('allowed') expect(legacyEnsemble.api.globalImageInputAdmissionReason.value).toBe( 'ensemble_mode_unsupported', ) @@ -417,8 +438,8 @@ describe('useChatFeatureToggles model routing mode', () => { await api.loadFeatureToggles() - expect(api.modelRoutingCapabilitiesByMode.value).toEqual(CAPABILITIES_BY_MODE) - expect(api.globalImageInputAdmission.value).toBe('blocked') + expect(api.modelRoutingCapabilitiesByMode.value).toEqual(EFFECTIVE_CAPABILITIES_BY_MODE) + expect(api.globalImageInputAdmission.value).toBe('allowed') }) it('clears the whole matrix when a later snapshot is missing or partial', async () => { @@ -446,7 +467,7 @@ describe('useChatFeatureToggles model routing mode', () => { }) await api.loadFeatureToggles() - expect(api.modelRoutingCapabilitiesByMode.value).toEqual(CAPABILITIES_BY_MODE) + expect(api.modelRoutingCapabilitiesByMode.value).toEqual(EFFECTIVE_CAPABILITIES_BY_MODE) await api.loadFeatureToggles() expect(api.modelRoutingCapabilitiesByMode.value).toBeNull() @@ -455,6 +476,36 @@ describe('useChatFeatureToggles model routing mode', () => { expect(api.modelRoutingCapabilitiesByMode.value).toBeNull() }) + it('normalizes capability limits in the per-mode matrix while retaining policy blocks', async () => { + const { api } = createHarness({ + configGetResults: [{}], + routingGetResults: [{ + mode: 'direct', + capabilities_by_mode: { + direct: { + image_input: { admission: 'blocked', reason: 'model_vision_unsupported' }, + }, + router: { + image_input: { admission: 'blocked', reason: 'attachment_policy_denied' }, + }, + ensemble: CAPABILITIES_BY_MODE.ensemble, + }, + }], + }) + + await api.loadFeatureToggles() + + expect(api.modelRoutingCapabilitiesByMode.value).toEqual({ + direct: { + image_input: { admission: 'allowed', reason: 'model_vision_unsupported' }, + }, + router: { + image_input: { admission: 'blocked', reason: 'attachment_policy_denied' }, + }, + ensemble: EFFECTIVE_CAPABILITIES_BY_MODE.ensemble, + }) + }) + it('does not let a late routing GET overwrite a newer changed event', async () => { vi.stubGlobal('document', { visibilityState: 'visible', @@ -556,7 +607,7 @@ describe('useChatFeatureToggles model routing mode', () => { expect(api.codingModeEnabled.value).toBe(false) expect(api.modelRoutingMode.value).toBe('squilla_router') - expect(api.modelRoutingCapabilitiesByMode.value).toEqual(CAPABILITIES_BY_MODE) + expect(api.modelRoutingCapabilitiesByMode.value).toEqual(EFFECTIVE_CAPABILITIES_BY_MODE) }) it('clears a prior canonical matrix when the connected Gateway lacks routing RPC', async () => { @@ -577,14 +628,14 @@ describe('useChatFeatureToggles model routing mode', () => { }) await api.loadFeatureToggles() - expect(api.modelRoutingCapabilitiesByMode.value).toEqual(CAPABILITIES_BY_MODE) + expect(api.modelRoutingCapabilitiesByMode.value).toEqual(EFFECTIVE_CAPABILITIES_BY_MODE) supportsRouting = false await api.loadFeatureToggles() expect(api.modelRoutingCapabilitiesByMode.value).toBeNull() expect(api.modelRoutingMode.value).toBe('llm_ensemble') - expect(api.globalImageInputAdmission.value).toBe('blocked') + expect(api.globalImageInputAdmission.value).toBe('allowed') expect(api.globalImageInputAdmissionReason.value).toBe('ensemble_mode_unsupported') }) @@ -607,7 +658,7 @@ describe('useChatFeatureToggles model routing mode', () => { await api.loadFeatureToggles() await api.loadFeatureToggles() - expect(api.modelRoutingCapabilitiesByMode.value).toEqual(CAPABILITIES_BY_MODE) + expect(api.modelRoutingCapabilitiesByMode.value).toEqual(EFFECTIVE_CAPABILITIES_BY_MODE) expect(api.modelRoutingMode.value).toBe('squilla_router') expect(api.globalImageInputAdmission.value).toBe('allowed') }) diff --git a/opensquilla-webui/src/composables/chat/useChatFeatureToggles.ts b/opensquilla-webui/src/composables/chat/useChatFeatureToggles.ts index dde9b6410..089ffeacd 100644 --- a/opensquilla-webui/src/composables/chat/useChatFeatureToggles.ts +++ b/opensquilla-webui/src/composables/chat/useChatFeatureToggles.ts @@ -84,7 +84,7 @@ function parseCapabilitiesByMode(value: unknown): ModelRoutingCapabilitiesByMode ) return null parsed[mode] = { image_input: { - admission, + admission: effectiveImageAdmission(admission, reason), reason, }, } @@ -96,6 +96,24 @@ function isMethodNotFound(error: unknown): boolean { return error instanceof ProviderConfigurationError && error.code === 'unsupported' } +const IMAGE_DEGRADATION_REASONS = new Set([ + 'ensemble_mode_unsupported', + 'model_vision_unsupported', + 'router_image_route_unavailable', +]) + +function effectiveImageAdmission( + admission: ImageInputAdmission, + reason: string, +): ImageInputAdmission { + // Older Gateways reported route/model limitations as a client-side hard + // block. They are now safe degradation signals: the Gateway preserves the + // turn and projects image blocks to truthful markers for text-only routes. + return admission === 'blocked' && IMAGE_DEGRADATION_REASONS.has(reason) + ? 'allowed' + : admission +} + export function useChatFeatureToggles(options: UseChatFeatureTogglesOptions) { const { pushToast } = useToasts() const routerEnabled = ref(false) @@ -146,13 +164,14 @@ export function useChatFeatureToggles(options: UseChatFeatureTogglesOptions) { const admission = snapshot.image_input?.admission if (admission === 'allowed' || admission === 'blocked' || admission === 'unknown') { hasCanonicalImageAdmission = true - globalImageInputAdmission.value = admission - globalImageInputAdmissionReason.value = String( + const reason = String( snapshot.image_input?.reason || 'capability_unknown', ) + globalImageInputAdmission.value = effectiveImageAdmission(admission, reason) + globalImageInputAdmissionReason.value = reason } else if (mode === 'ensemble') { hasCanonicalImageAdmission = false - globalImageInputAdmission.value = 'blocked' + globalImageInputAdmission.value = 'allowed' globalImageInputAdmissionReason.value = 'ensemble_mode_unsupported' } else { hasCanonicalImageAdmission = false @@ -175,7 +194,7 @@ export function useChatFeatureToggles(options: UseChatFeatureTogglesOptions) { llmEnsembleEnabled.value = ensembleEnabled llmEnsembleSelectionMode.value = String(cfg?.llm_ensemble?.selection_mode || '') if (!hasCanonicalImageAdmission) { - globalImageInputAdmission.value = ensembleEnabled ? 'blocked' : 'unknown' + globalImageInputAdmission.value = ensembleEnabled ? 'allowed' : 'unknown' globalImageInputAdmissionReason.value = ensembleEnabled ? 'ensemble_mode_unsupported' : 'capability_unknown' @@ -209,7 +228,6 @@ export function useChatFeatureToggles(options: UseChatFeatureTogglesOptions) { ).trim() tierConfigs[lower] = { model: typeof model === 'string' ? model.trim() : '', - supportsImage: rawTierRecord.supports_image === true || rawTierRecord.supportsImage === true, imageOnly: rawTierRecord.image_only === true || rawTierRecord.imageOnly === true, // New Gateways expose the explicit execution switch. Older PR // snapshots only expose the legacy selection mode, which still diff --git a/opensquilla-webui/src/composables/chat/useChatRenderedMessages.test.ts b/opensquilla-webui/src/composables/chat/useChatRenderedMessages.test.ts index 36cc6c855..fb3d563b9 100644 --- a/opensquilla-webui/src/composables/chat/useChatRenderedMessages.test.ts +++ b/opensquilla-webui/src/composables/chat/useChatRenderedMessages.test.ts @@ -1184,6 +1184,150 @@ describe('useChatRenderedMessages immutable route history', () => { }) }) +describe('useChatRenderedMessages image router candidates', () => { + it('preserves an actually executed legacy winner in restored history', () => { + const api = renderedMessagesFor([{ + role: 'user', + text: 'Describe the shape.', + ts: 1, + attachments: [{ + kind: 'file', + displayId: 'historical-image', + renderKey: 'historical-image', + name: 'shape.png', + mime: 'image/png', + }], + }, { + role: 'assistant', + text: 'A circle.', + ts: 2, + restoredFromHistory: true, + usage: { + routed_tier: 'image_model', + routed_model: 'historical/actual-winner', + routing_source: 'image_route', + router_tier_snapshot: { + version: 1, + request_kind: 'image', + tiers: [{ + tier: 'image_model', + model: 'historical/actual-winner', + execution_kind: 'single_model', + }], + }, + }, + }], undefined, true) + + const strip = api.renderedMessages.value.find(message => message.isRouterStrip) + expect(strip?.gridCells?.map(cell => cell.model)).toEqual(['historical/actual-winner']) + expect(strip?.winnerIdx).toBe(0) + expect(strip?.routerStatic).toBe(true) + }) + + it.each(['live', 'settled', 'restored'])( + 'excludes implicit legacy image candidates from %s image routes', + (stage) => { + for (const snapshotKind of ['none', 'legacy', 'current']) { + const textEntries = [0, 1, 2, 3].map(index => ({ + tier: `c${index}`, + model: `text/configured-${index}`, + execution_kind: 'single_model', + })) + const snapshot = snapshotKind === 'none' ? undefined : { + version: 1, + request_kind: 'image', + tiers: snapshotKind === 'legacy' ? [ + textEntries[1], + { tier: 'image_model', model: 'legacy/unused-vision', execution_kind: 'single_model' }, + ] : textEntries, + } + const route = { + tier: 'c1', + model: 'text/actual-winner', + source: 'image_route', + ...(snapshot ? { router_tier_snapshot: snapshot } : {}), + } + const messages: ChatMessage[] = [{ + role: 'user', + text: 'Describe the attached shapes.', + ts: 1, + turnId: 'turn-image-candidates', + attachments: [{ + kind: 'file', + displayId: 'synthetic-image', + renderKey: 'synthetic-image', + name: 'shapes.png', + mime: 'image/png', + }], + }] + if (stage !== 'restored') { + messages.push({ + role: 'router', + text: '', + ts: 2, + turnId: 'turn-image-candidates', + provenanceKind: 'router_decision', + routerDecision: route, + }) + } + if (stage !== 'live') { + messages.push({ + role: 'assistant', + text: 'The image was not analyzed.', + ts: 3, + turnId: 'turn-image-candidates', + restoredFromHistory: stage === 'restored', + usage: { + routed_tier: route.tier, + routed_model: route.model, + routing_source: route.source, + route_plan: route, + }, + }) + } + const configs: Record = Object.fromEntries( + textEntries.map(entry => [entry.tier, { + model: entry.model, + supportsImage: false, + imageOnly: false, + }]), + ) + configs.image_model = { + model: 'legacy/unused-vision', + supportsImage: true, + imageOnly: true, + } + const before = JSON.stringify(messages) + const api = useChatRenderedMessages({ + messages: ref(messages), + sessionKey: ref('agent:main:webchat:image-candidates'), + routerSlots: ref(Object.keys(configs)), + routerModels: ref({}), + routerTierConfigs: ref(configs), + routerVisualEffectsEnabled: ref(true), + routerVisualMode: ref('real_candidates'), + renderMarkdown: text => text, + stripGeneratedArtifactMarkers: text => text, + stripTimePrefix: text => text, + isSubagentCompletionMessage: () => false, + }) + + const strips = api.renderedMessages.value.filter(message => message.isRouterStrip) + expect(strips).toHaveLength(1) + const strip = strips[0]! + expect(strip.gridCells?.flatMap(cell => cell.tiers).sort()).toEqual( + snapshotKind === 'legacy' ? ['c1'] : ['c0', 'c1', 'c2', 'c3'], + ) + expect(strip.gridCells?.some(cell => cell.model === 'legacy/unused-vision')).toBe(false) + expect(strip.gridCells?.[strip.winnerIdx ?? -1]?.model).toBe('text/actual-winner') + expect(strip.routerStatic).toBe(stage === 'restored') + expect(strip.routerSettled).toBe(stage === 'settled') + expect(JSON.stringify(messages)).toBe(before) + } + }, + ) +}) + describe('useChatRenderedMessages router visual mode', () => { it('keeps real-candidates mode limited to callable router tiers', () => { const api = renderedMessagesForRouterVisualMode('real_candidates') diff --git a/opensquilla-webui/src/composables/chat/useChatRenderedMessages.ts b/opensquilla-webui/src/composables/chat/useChatRenderedMessages.ts index 99de163d2..ff6a95435 100644 --- a/opensquilla-webui/src/composables/chat/useChatRenderedMessages.ts +++ b/opensquilla-webui/src/composables/chat/useChatRenderedMessages.ts @@ -31,6 +31,7 @@ import { toolSecondaryText, } from '@/utils/chat/toolDisplay' import { + IMAGE_TIER, normalizeRouterTextTier, normalizeRouterTier, sortRouterTiers, @@ -703,7 +704,7 @@ export function useChatRenderedMessages(options: UseChatRenderedMessagesOptions) cells.length === 1 && !fixedSessionRoute && !hasRequestSnapshot - && currentRouterCandidatePoolAvailable(requestKind) + && currentRouterCandidatePoolAvailable() ) ) return null return { @@ -1131,11 +1132,14 @@ export function useChatRenderedMessages(options: UseChatRenderedMessagesOptions) ) if (snapshot?.request_kind === requestKind) { return routerCellsFromTierEntries( - snapshot.tiers.map(entry => ({ - tier: entry.tier, - model: entry.model, - executionKind: entry.execution_kind, - })), + snapshot.tiers + // Keep a recorded legacy winner, never an implicit extra candidate. + .filter(entry => entry.tier !== IMAGE_TIER || entry.tier === winnerTier) + .map(entry => ({ + tier: entry.tier, + model: entry.model, + executionKind: entry.execution_kind, + })), winnerTier, winnerModel, true, @@ -1153,8 +1157,9 @@ export function useChatRenderedMessages(options: UseChatRenderedMessagesOptions) executionKind: 'single_model' | 'ensemble' }> = [] for (const tier of sourceTiers) { + if (tier === IMAGE_TIER && tier !== winnerTier) continue const tierConfig = routerTierConfig(tier) - if (tier !== winnerTier && !routerTierMatchesRequestKind(tierConfig, requestKind)) continue + if (tier !== winnerTier && tierConfig.imageOnly) continue const model = tier === winnerTier && winnerModel ? winnerModel : tierConfig.model || options.routerModels.value[tier] || '' @@ -1230,17 +1235,11 @@ export function useChatRenderedMessages(options: UseChatRenderedMessagesOptions) const normalized = normalizeRouterTier(tier) return options.routerTierConfigs.value[normalized] || { model: options.routerModels.value[normalized] || '', - supportsImage: false, imageOnly: false, } } - function routerTierMatchesRequestKind(tierConfig: ChatRouterTierConfig, requestKind: ChatRouterRequestKind): boolean { - if (requestKind === 'image') return tierConfig.supportsImage || tierConfig.imageOnly - return !tierConfig.imageOnly - } - - function currentRouterCandidatePoolAvailable(requestKind: ChatRouterRequestKind): boolean { + function currentRouterCandidatePoolAvailable(): boolean { const configuredTiers = options.routerSlots.value.length ? options.routerSlots.value : Object.keys(options.routerTierConfigs.value) @@ -1248,7 +1247,7 @@ export function useChatRenderedMessages(options: UseChatRenderedMessagesOptions) const tier = normalizeRouterTier(rawTier) const config = routerTierConfig(tier) const model = config.model || options.routerModels.value[tier] || '' - return Boolean(model && routerTierMatchesRequestKind(config, requestKind)) + return Boolean(tier !== IMAGE_TIER && model && !config.imageOnly) }) } diff --git a/opensquilla-webui/src/composables/chat/useChatSend.attachments.test.ts b/opensquilla-webui/src/composables/chat/useChatSend.attachments.test.ts index 622b67aa9..dbe64323b 100644 --- a/opensquilla-webui/src/composables/chat/useChatSend.attachments.test.ts +++ b/opensquilla-webui/src/composables/chat/useChatSend.attachments.test.ts @@ -3866,7 +3866,7 @@ describe('useChatSend attachment payloads', () => { expect(queued.attachments).toEqual([failed]) }) - it('keeps a queued image intact while Ensemble routing cannot send it', async () => { + it('allows a queued Ensemble image to continue through marker degradation', async () => { const image: Attachment = { kind: 'staged', local_id: 32, @@ -3887,9 +3887,11 @@ describe('useChatSend attachment payloads', () => { }) await expect(api.sendQueuedSteer(queued)).resolves.toBe('not_sent') - await expect(api.sendQueuedFollowup(queued)).resolves.toBe('not_sent') + await expect(api.sendQueuedFollowup(queued)).resolves.toBe('accepted') - expect(rpc.call).not.toHaveBeenCalled() + expect(rpc.call).toHaveBeenCalledWith('chat.send', expect.objectContaining({ + attachments: [expect.objectContaining({ mime: 'image/png' })], + })) expect(queued.attachments).toEqual([image]) expect(inputText.value).toBe('unrelated live draft') }) @@ -6706,7 +6708,7 @@ describe('useChatSend attachment payloads', () => { }) }) -describe('useChatSend Ensemble image guard', () => { +describe('useChatSend image admission', () => { function readyAttachment( mime: string, overrides: Partial = {}, @@ -6721,7 +6723,7 @@ describe('useChatSend Ensemble image guard', () => { } } - it('blocks a direct Ensemble image send before any visible or RPC mutation', async () => { + it('allows a direct Ensemble image send for backend marker degradation', async () => { const image = readyAttachment('image/png', { name: 'photo.png' }) const pendingAttachments = ref([image]) const inputText = ref('describe this') @@ -6735,14 +6737,19 @@ describe('useChatSend Ensemble image guard', () => { await api.onSend() - expect(rpc.call).not.toHaveBeenCalled() - expect(prepareAttachmentsForSend).not.toHaveBeenCalled() - expect(options.messages.value).toEqual([]) - expect(inputText.value).toBe('describe this') - expect(pendingAttachments.value).toEqual([image]) + expect(rpc.call).toHaveBeenCalledWith('chat.send', expect.objectContaining({ + attachments: [expect.objectContaining({ mime: 'image/png' })], + })) + expect(prepareAttachmentsForSend).toHaveBeenCalledOnce() + expect(options.messages.value).toContainEqual(expect.objectContaining({ + role: 'user', + text: 'describe this', + })) + expect(inputText.value).toBe('') + expect(pendingAttachments.value).toEqual([]) expect(options.pendingSessionIntent.value).toBeNull() - expect(options.closeSlashMenu).not.toHaveBeenCalled() - expect(stream.startStreaming).not.toHaveBeenCalled() + expect(options.closeSlashMenu).toHaveBeenCalled() + expect(stream.startStreaming).toHaveBeenCalled() }) it('blocks image sends while routing settings are being written', async () => { @@ -6786,7 +6793,7 @@ describe('useChatSend Ensemble image guard', () => { }, ) - it('blocks explicitly unsupported image input before upload or draft mutation', async () => { + it('blocks an explicit image policy rejection before upload or draft mutation', async () => { const image = readyAttachment('image/png', { file_uuid: '' }) const pendingAttachments = ref([image]) const prepareAttachmentsForSend = vi.fn(async () => true) @@ -6822,7 +6829,7 @@ describe('useChatSend Ensemble image guard', () => { })) }) - it('rechecks routing after attachment preparation without consuming the draft', async () => { + it('continues when routing switches to Ensemble during attachment preparation', async () => { const image = readyAttachment('image/gif') const pendingAttachments = ref([image]) const modelRoutingMode = ref<'off' | 'llm_ensemble'>('off') @@ -6839,13 +6846,15 @@ describe('useChatSend Ensemble image guard', () => { await api.onSend() expect(prepareAttachmentsForSend).toHaveBeenCalledOnce() - expect(rpc.call).not.toHaveBeenCalled() - expect(options.messages.value).toEqual([]) - expect(options.inputText.value).toBe('hello') - expect(pendingAttachments.value).toEqual([image]) + expect(rpc.call).toHaveBeenCalledWith('chat.send', expect.objectContaining({ + attachments: [expect.objectContaining({ mime: 'image/gif' })], + })) + expect(options.messages.value).toContainEqual(expect.objectContaining({ role: 'user' })) + expect(options.inputText.value).toBe('') + expect(pendingAttachments.value).toEqual([]) }) - it('blocks a recovered image retry without restoring it after switching to Ensemble', async () => { + it('retries a recovered image after the user switches to Ensemble', async () => { const image = readyAttachment('image/jpg', { name: 'photo.jpg' }) const pendingAttachments = ref([image]) const modelRoutingMode = ref<'off' | 'llm_ensemble'>('off') @@ -6860,12 +6869,12 @@ describe('useChatSend Ensemble image guard', () => { await api.onSend() - expect(rpc.call).toHaveBeenCalledOnce() + expect(rpc.call).toHaveBeenCalledTimes(2) expect(options.inputText.value).toBe('') expect(pendingAttachments.value).toEqual([]) }) - it('preserves an auto-drained queued image after routing switches to Ensemble', async () => { + it('auto-drains a queued image after routing switches to Ensemble', async () => { vi.useFakeTimers() try { const image = readyAttachment('image/png') @@ -6924,10 +6933,12 @@ describe('useChatSend Ensemble image guard', () => { await nextTick() expect(pending.pendingQueue.value).toEqual([]) - expect(rpc.call).not.toHaveBeenCalled() - expect(options.messages.value).toEqual([]) - expect(inputText.value).toBe('queued image') - expect(pendingAttachments.value).toEqual([image]) + expect(rpc.call).toHaveBeenCalledWith('chat.send', expect.objectContaining({ + attachments: [expect.objectContaining({ mime: 'image/png' })], + })) + expect(options.messages.value).toContainEqual(expect.objectContaining({ role: 'user' })) + expect(inputText.value).toBe('') + expect(pendingAttachments.value).toEqual([]) pending.cleanup() } finally { vi.useRealTimers() diff --git a/opensquilla-webui/src/composables/chat/useChatSend.ts b/opensquilla-webui/src/composables/chat/useChatSend.ts index adca87873..4e5b1550f 100644 --- a/opensquilla-webui/src/composables/chat/useChatSend.ts +++ b/opensquilla-webui/src/composables/chat/useChatSend.ts @@ -851,10 +851,6 @@ export function useChatSend(options: UseChatSendOptions) { if (!hasModelInputImageAttachment(attachments)) return false return options.modelRoutingSettingsBusy.value || options.imageInputAdmission?.value === 'blocked' - || ( - options.imageInputAdmission === undefined - && options.modelRoutingMode.value === 'llm_ensemble' - ) } function activeSteerCapability(): ChatSteerCapability | null { @@ -2628,10 +2624,6 @@ export function useChatSend(options: UseChatSendOptions) { } if ( options.imageInputAdmission?.value === 'blocked' - || ( - options.imageInputAdmission === undefined - && options.modelRoutingMode.value === 'llm_ensemble' - ) ) { return preserveRetryState('not_sent') } diff --git a/opensquilla-webui/src/composables/setup/useSetupCatalog.privacy.test.ts b/opensquilla-webui/src/composables/setup/useSetupCatalog.privacy.test.ts index 9c279b22f..1660da1b5 100644 --- a/opensquilla-webui/src/composables/setup/useSetupCatalog.privacy.test.ts +++ b/opensquilla-webui/src/composables/setup/useSetupCatalog.privacy.test.ts @@ -2565,7 +2565,7 @@ describe('useSetupCatalog fresh-install provider semantics', () => { app.unmount() }) - it('invalidates saved C3 readiness when an independent image-model row is edited', async () => { + it('keeps saved C3 readiness when an inactive legacy image row rejects an edit', async () => { mockProviderState( { ...configuredProviderStatus('tokenrhythm'), @@ -2618,7 +2618,10 @@ describe('useSetupCatalog fresh-install provider semantics', () => { expect(api.modelStrategyPanel.value.router.tierEnsembleStatusFresh).toBe(true) api.updateTierField('image_model', 'model', 'vision-model-v2') - expect(api.modelStrategyPanel.value.router.tierEnsembleStatusFresh).toBe(false) + expect(api.modelStrategyPanel.value.router.tierEnsembleStatusFresh).toBe(true) + expect(api.modelStrategyPanel.value.router.tierRows).not.toContainEqual(expect.objectContaining({ + name: 'image_model', + })) expect(api.modelStrategyPanel.value.router.tierEnsembleStatus).toMatchObject({ runtimeStatus: 'blocked', fixedFallbackReady: false, @@ -3302,10 +3305,9 @@ describe('useSetupCatalog configured provider management', () => { expect(api.routerPanel.value.tierRows).toEqual(expect.arrayContaining([ expect.objectContaining({ name: 'c0', provider: 'openrouter', model: 'legacy-model' }), expect.objectContaining({ name: 'c1', provider: 'deepseek', model: 'deepseek-chat' }), - expect.objectContaining({ name: 'image_model', provider: 'deepseek', model: 'deepseek-vision' }), ])) - // The dedicated image route is independent and must never be imported as - // a text proposer when the user converts a legacy dynamic plan. + expect(api.routerPanel.value.tierRows.map(row => row.name)).toEqual(['c0', 'c1']) + // A retained image-model configuration must not become an Ensemble member. expect(api.ensemblePanel.value.tierCandidates).toEqual([ expect.objectContaining({ provider: 'deepseek', model: 'deepseek-chat', source: 'tier' }), ]) diff --git a/opensquilla-webui/src/composables/setup/useSetupCatalog.ts b/opensquilla-webui/src/composables/setup/useSetupCatalog.ts index 0ec47b967..ebf7d47de 100644 --- a/opensquilla-webui/src/composables/setup/useSetupCatalog.ts +++ b/opensquilla-webui/src/composables/setup/useSetupCatalog.ts @@ -1756,7 +1756,6 @@ function routerConflictsWithTarget(value: string): boolean { provider: tier.provider || '', model: tier.model || '', thinkingLevel: tier.thinkingLevel || tier.thinking_level || '', - supportsImage: tier.supportsImage || tier.supports_image || false, ensembleEnabled: typeof tier.ensembleEnabled === 'boolean' ? tier.ensembleEnabled : tier.ensemble_enabled, @@ -3022,7 +3021,7 @@ function setRouterVisualMode(value: string) { function updateTierField( name: string, - key: 'provider' | 'model' | 'thinkingLevel' | 'supportsImage' | 'ensembleEnabled' | 'ensembleSelectionMode', + key: 'provider' | 'model' | 'thinkingLevel' | 'ensembleEnabled' | 'ensembleSelectionMode', value: string | boolean, ) { routerForm.updateTierField(name, key, value) diff --git a/opensquilla-webui/src/composables/setup/useSetupRouterForm.test.ts b/opensquilla-webui/src/composables/setup/useSetupRouterForm.test.ts index add85f16b..585ff9f04 100644 --- a/opensquilla-webui/src/composables/setup/useSetupRouterForm.test.ts +++ b/opensquilla-webui/src/composables/setup/useSetupRouterForm.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from 'vitest' import { computed } from 'vue' -import { routerTierProviderParticipates, useSetupRouterForm } from './useSetupRouterForm' +import { buildRouterPayload, routerTierProviderParticipates, useSetupRouterForm } from './useSetupRouterForm' // openrouter-mix is backend-supported but was unreachable in the WebUI. The // round-trip is subtle: it is the only enabled mode whose tier_profile is null, @@ -19,6 +19,40 @@ function makePanel(form: ReturnType, isOpenrouter: bo } describe('useSetupRouterForm — openrouter-mix round-trip', () => { + it('hides the inactive image row without losing saved config or enabling cross-provider routing', () => { + const form = useSetupRouterForm() + form.initFromConfig({ + enabled: true, + tiers: { + c0: { provider: 'openrouter', model: 'configured/text-model' }, + image_model: { + provider: 'openai', + model: 'saved/vision-model', + thinking_level: 'high', + supports_image: true, + }, + }, + }, {}, 'openrouter', 'custom') + const saved = form.payload() + + form.updateTierField('image_model', 'model', 'replacement/vision-model') + form.updateTierField('image_model', 'provider', 'openrouter') + form.updateTierField('image_model', 'supportsImage', false) + form.updateTierField('image_model', 'thinkingLevel', 'off') + + expect(form.payload()).toEqual(saved) + expect(form.hasMixedTierProviders.value).toBe(false) + expect(form.payload()).not.toHaveProperty('crossProviderTiers') + expect(makePanel(form, true).value.tierRows.map(row => row.name)).toEqual(['c0']) + form.updateTierField('c0', 'model', 'configured/new-text-model') + expect(form.payload()).toMatchObject({ + tiers: { + c0: { model: 'configured/new-text-model' }, + image_model: { model: 'saved/vision-model' }, + }, + }) + }) + it('classifies legacy openrouter mix internally but saves canonical custom mode', () => { const f = useSetupRouterForm() f.initFromConfig({ enabled: true, tier_profile: null }, {}, 'openrouter') @@ -147,12 +181,78 @@ describe('useSetupRouterForm — openrouter-mix round-trip', () => { provider: 'openrouter', model: 'deepseek/deepseek-v4-flash', thinkingLevel: 'high', - supportsImage: false, }, }, }) }) + it('keeps omitted image support unknown through a model-only save', () => { + const f = useSetupRouterForm() + f.initFromConfig({ + enabled: true, + tier_profile: null, + tiers: { + c0: { + provider: 'openrouter', + model: 'operator/custom-model', + }, + }, + }, {}, 'openrouter') + + expect(f.payload()).not.toHaveProperty('tiers.c0.supportsImage') + }) + + it('clears inherited image support when the deployment model changes', () => { + const f = useSetupRouterForm() + f.initFromConfig({ + enabled: true, + tier_profile: 'openai', + }, { + c0: { + provider: 'openai', + model: 'preset-model', + supportsImage: false, + }, + }, 'openai', 'follow_primary') + + f.updateTierField('c0', 'model', 'operator/custom-model') + + expect(f.payload()).not.toHaveProperty('tiers.c0.supportsImage') + }) + + it.each([false, true])('ignores a legacy image declaration of %s when saving', (supportsImage) => { + const f = useSetupRouterForm() + f.initFromConfig({ + enabled: true, + tier_profile: null, + tiers: { + c0: { + provider: 'openrouter', + model: 'operator/text-model', + supports_image: supportsImage, + }, + }, + }, {}, 'openrouter') + + f.updateTierField('c0', 'supportsImage', !supportsImage) + expect(f.payload()).not.toHaveProperty('tiers.c0.supportsImage') + expect(makePanel(f, true).value.tierRows[0]).not.toHaveProperty('supportsImage') + }) + + it('does not serialize retired capability fields supplied by an older caller', () => { + const payload = buildRouterPayload('custom', 'c0', { + c0: { + provider: 'configured-provider', + model: 'configured-model', + thinkingLevel: '', + supportsImage: true, + }, + }) + expect(payload.tiers?.c0).toEqual({ + provider: 'configured-provider', model: 'configured-model', thinkingLevel: '', + }) + }) + it('round-trips a tier-managed ensemble profile from snake case', () => { const f = useSetupRouterForm() f.initFromConfig({ diff --git a/opensquilla-webui/src/composables/setup/useSetupRouterForm.ts b/opensquilla-webui/src/composables/setup/useSetupRouterForm.ts index 3a32544c8..555845412 100644 --- a/opensquilla-webui/src/composables/setup/useSetupRouterForm.ts +++ b/opensquilla-webui/src/composables/setup/useSetupRouterForm.ts @@ -22,7 +22,8 @@ export interface SetupTierValue { provider: string model: string thinkingLevel: string - supportsImage: boolean + /** Accepted for older clients; image capability is resolved by the gateway. */ + supportsImage?: boolean ensembleEnabled?: boolean ensembleSelectionMode?: string } @@ -92,7 +93,6 @@ export function buildRouterPayload( provider: tier.provider, model: tier.model, thinkingLevel: tier.thinkingLevel, - supportsImage: tier.supportsImage, } if (sharedEnsembleTier && typeof tier.ensembleEnabled === 'boolean') { tierPayload.ensembleEnabled = tier.ensembleEnabled @@ -358,6 +358,7 @@ export function useSetupRouterForm() { const tierProviderIds = computed(() => { const ids = new Set() Object.entries(tierValues.value).forEach(([name, tier]) => { + if (name === IMAGE_TIER) return if (!routerTierProviderParticipates(name, tier, routerProviderRoles.value)) return const provider = String(tier.provider || '').trim().toLowerCase() if (provider) ids.add(provider) @@ -447,7 +448,6 @@ export function useSetupRouterForm() { provider: tier.provider || '', model: tier.model || '', thinkingLevel: tier.thinkingLevel || tier.thinking_level || '', - supportsImage: tier.supportsImage || tier.supports_image || false, ensembleEnabled: tierName === 'c3' ? typeof tier.ensembleEnabled === 'boolean' ? tier.ensembleEnabled @@ -465,6 +465,7 @@ export function useSetupRouterForm() { } function updateTierField(name: string, key: keyof SetupTierValue, value: string | boolean) { + if (name === IMAGE_TIER || key === 'supportsImage') return const tier = tierValues.value[name] if (!tier) return if (key === 'ensembleEnabled' && (normalizeRouterTier(name) || name) !== 'c3') return @@ -496,8 +497,10 @@ export function useSetupRouterForm() { } return } - if (key === 'supportsImage') { - tier.supportsImage = Boolean(value) + if (key === 'model') { + const model = String(value) + if (model === tier.model) return + tier.model = model } else if (key === 'ensembleEnabled') { tier.ensembleEnabled = Boolean(value) } else { @@ -522,13 +525,12 @@ export function useSetupRouterForm() { function tierRows(textTiers: readonly string[]): SetupTierRow[] { return Object.entries(tierValues.value) - .filter(([name]) => textTiers.includes(name) || name === IMAGE_TIER) + .filter(([name]) => name !== IMAGE_TIER && textTiers.includes(name)) .map(([name, tier]) => ({ name, provider: tier.provider, model: tier.model, thinkingLevel: tier.thinkingLevel, - supportsImage: tier.supportsImage, ensembleEnabled: tier.ensembleEnabled, ensembleSelectionMode: tier.ensembleSelectionMode, })) diff --git a/opensquilla-webui/src/locales/de.json b/opensquilla-webui/src/locales/de.json index b5420956b..888aee2c7 100644 --- a/opensquilla-webui/src/locales/de.json +++ b/opensquilla-webui/src/locales/de.json @@ -1191,6 +1191,7 @@ "colModel": "Modell", "colThinking": "Denken", "colImage": "Bild", + "imageCapabilityAutomatic": "Die Bildfähigkeit wird automatisch anhand von Anbieter-Metadaten und Anfrageergebnissen erkannt; eine manuelle Einstellung ist nicht erforderlich.", "tierProviderAria": "{tier}-Anfrageeingang", "tierProviderManagedByEnsemble": "Durch Mehrmodell-Fusion bestimmt", "tierProviderManagedByEnsembleAria": "Der {tier}-Anfrageeingang wird durch den Plan der Mehrmodell-Fusion bestimmt", @@ -1221,7 +1222,8 @@ "tierLegacyDynamicEnsembleSummaryGeneric": "Verwendet den zuvor gespeicherten, den Modellstufen folgenden Fusionsplan; bei Fehlern wird das feste und Fallback-Modell verwendet.", "tierLegacyDynamicEnsembleFallbackMissingSummary": "Verwendet den zuvor gespeicherten, den Modellstufen folgenden Fusionsplan; ein nutzbares festes und Fallback-Modell ist noch nicht konfiguriert.", "tierSingleModelAnnouncement": "C3 verwendet ein einzelnes Modell: {model}.", - "tierEnsembleImageRouting": "Die C3-Fusion verarbeitet keine Bilder; Bildanfragen verwenden weiterhin die Bildmodell-Konfiguration.", + "tierEnsembleImageRouting": "Die C3-Fusion erhält Texthinweise für Bilder. Router verwendet zur Bildverarbeitung nur konfigurierte C0–C3-Modelle.", + "tierLegacyImageRouting": "Alte Kompatibilitätseinstellung: Dieses Modell bleibt gespeichert, wird aber nicht für Bildeingaben verwendet. Konfigurieren Sie ein bildfähiges Modell in C0–C3.", "tierThinkingManagedByEnsemble": "Durch Fusionsplan bestimmt", "tierThinkingManagedByEnsembleAria": "Die Denkstufe für {tier} wird durch den Plan der Mehrmodell-Fusion bestimmt", "tierThinkingAria": "{tier}-Denkstufe", diff --git a/opensquilla-webui/src/locales/en.json b/opensquilla-webui/src/locales/en.json index ab32a8607..0514bb174 100644 --- a/opensquilla-webui/src/locales/en.json +++ b/opensquilla-webui/src/locales/en.json @@ -1280,6 +1280,7 @@ "colModel": "Model", "colThinking": "Thinking", "colImage": "Image", + "imageCapabilityAutomatic": "Image capability is detected automatically from provider metadata and request results; no manual setting is needed.", "tierProviderAria": "{tier} request entry", "tierProviderManagedByEnsemble": "Determined by Multi-model fusion", "tierProviderManagedByEnsembleAria": "{tier} request entry is determined by the Multi-model fusion plan", @@ -1310,7 +1311,8 @@ "tierLegacyDynamicEnsembleSummaryGeneric": "Uses the previously saved tier-following fusion plan; if it fails, uses the Fixed and fallback model.", "tierLegacyDynamicEnsembleFallbackMissingSummary": "Uses the previously saved tier-following fusion plan; no usable Fixed and fallback model is configured yet.", "tierSingleModelAnnouncement": "C3 uses one model: {model}.", - "tierEnsembleImageRouting": "C3 fusion does not process images; image requests still use the Image model configuration.", + "tierEnsembleImageRouting": "C3 fusion receives text markers for images. Router image processing uses only configured C0–C3 models.", + "tierLegacyImageRouting": "Legacy compatibility setting: this model is preserved but not used for image input. Configure an image-capable model in C0–C3.", "tierThinkingManagedByEnsemble": "Determined by fusion plan", "tierThinkingManagedByEnsembleAria": "{tier} thinking is determined by the Multi-model fusion plan", "tierThinkingAria": "{tier} thinking level", diff --git a/opensquilla-webui/src/locales/es.json b/opensquilla-webui/src/locales/es.json index c897168a4..2b134cc01 100644 --- a/opensquilla-webui/src/locales/es.json +++ b/opensquilla-webui/src/locales/es.json @@ -1191,6 +1191,7 @@ "colModel": "Modelo", "colThinking": "Razonamiento", "colImage": "Imagen", + "imageCapabilityAutomatic": "La capacidad de imagen se detecta automáticamente a partir de los metadatos del proveedor y los resultados de las solicitudes; no requiere ajustes manuales.", "tierProviderAria": "Entrada de solicitud de {tier}", "tierProviderManagedByEnsemble": "Determinada por la fusión multimodelo", "tierProviderManagedByEnsembleAria": "La entrada de solicitud de {tier} la determina el plan de fusión multimodelo", @@ -1221,7 +1222,8 @@ "tierLegacyDynamicEnsembleSummaryGeneric": "Usa el plan de fusión guardado que sigue los niveles de modelo; si falla, usa el modelo fijo y de respaldo.", "tierLegacyDynamicEnsembleFallbackMissingSummary": "Usa el plan de fusión guardado que sigue los niveles de modelo; todavía no hay un modelo fijo y de respaldo utilizable configurado.", "tierSingleModelAnnouncement": "C3 usa un solo modelo: {model}.", - "tierEnsembleImageRouting": "La fusión C3 no procesa imágenes; las solicitudes de imagen siguen usando la configuración del modelo de imagen.", + "tierEnsembleImageRouting": "La fusión C3 recibe indicaciones de texto para las imágenes. Router solo utiliza los modelos C0–C3 configurados para procesar imágenes.", + "tierLegacyImageRouting": "Configuración antigua de compatibilidad: este modelo se conserva, pero no se utiliza para las imágenes de entrada. Configure un modelo compatible con imágenes en C0–C3.", "tierThinkingManagedByEnsemble": "Determinada por el plan de fusión", "tierThinkingManagedByEnsembleAria": "El razonamiento de {tier} lo determina el plan de fusión multimodelo", "tierThinkingAria": "Nivel de razonamiento de {tier}", diff --git a/opensquilla-webui/src/locales/fr.json b/opensquilla-webui/src/locales/fr.json index 639a7679a..c8d5de62a 100644 --- a/opensquilla-webui/src/locales/fr.json +++ b/opensquilla-webui/src/locales/fr.json @@ -1191,6 +1191,7 @@ "colModel": "Modèle", "colThinking": "Réflexion", "colImage": "Image", + "imageCapabilityAutomatic": "La prise en charge des images est détectée automatiquement à partir des métadonnées du fournisseur et des résultats des requêtes ; aucun réglage manuel n’est nécessaire.", "tierProviderAria": "Entrée de requête {tier}", "tierProviderManagedByEnsemble": "Déterminée par la fusion multimodèle", "tierProviderManagedByEnsembleAria": "L’entrée de requête {tier} est déterminée par le plan de fusion multimodèle", @@ -1221,7 +1222,8 @@ "tierLegacyDynamicEnsembleSummaryGeneric": "Utilise le plan de fusion enregistré qui suit les niveaux de modèle ; en cas d’échec, utilise le modèle fixe et de repli.", "tierLegacyDynamicEnsembleFallbackMissingSummary": "Utilise le plan de fusion enregistré qui suit les niveaux de modèle ; aucun modèle fixe et de repli utilisable n’est encore configuré.", "tierSingleModelAnnouncement": "C3 utilise un seul modèle : {model}.", - "tierEnsembleImageRouting": "La fusion C3 ne traite pas les images ; les requêtes d’image utilisent toujours la configuration du modèle d’image.", + "tierEnsembleImageRouting": "La fusion C3 reçoit des indications textuelles pour les images. Router utilise uniquement les modèles C0–C3 configurés pour traiter les images.", + "tierLegacyImageRouting": "Ancien réglage de compatibilité : ce modèle est conservé mais n’est pas utilisé pour les images en entrée. Configurez un modèle acceptant les images dans C0–C3.", "tierThinkingManagedByEnsemble": "Déterminée par le plan de fusion", "tierThinkingManagedByEnsembleAria": "La réflexion {tier} est déterminée par le plan de fusion multimodèle", "tierThinkingAria": "Niveau de réflexion {tier}", diff --git a/opensquilla-webui/src/locales/ja.json b/opensquilla-webui/src/locales/ja.json index 9853166d0..dfc371518 100644 --- a/opensquilla-webui/src/locales/ja.json +++ b/opensquilla-webui/src/locales/ja.json @@ -1191,6 +1191,7 @@ "colModel": "モデル", "colThinking": "思考", "colImage": "画像", + "imageCapabilityAutomatic": "画像対応はプロバイダーのメタデータとリクエストの結果から自動判定されます。手動設定は不要です。", "tierProviderAria": "{tier} のリクエスト入口", "tierProviderManagedByEnsemble": "マルチモデル融合が決定", "tierProviderManagedByEnsembleAria": "{tier} のリクエスト入口はマルチモデル融合プランによって決まります", @@ -1221,7 +1222,8 @@ "tierLegacyDynamicEnsembleSummaryGeneric": "以前保存したモデル階層追従型の融合プランを使用し、失敗時は「固定・フォールバックモデル」を使用します。", "tierLegacyDynamicEnsembleFallbackMissingSummary": "以前保存したモデル階層追従型の融合プランを使用しますが、利用可能な「固定・フォールバックモデル」がまだ設定されていません。", "tierSingleModelAnnouncement": "C3 は単一モデルを使用します:{model}。", - "tierEnsembleImageRouting": "C3 融合は画像を処理しません。画像リクエストには引き続き画像モデル設定が使用されます。", + "tierEnsembleImageRouting": "C3 融合は画像の代わりにテキストの注記を受け取ります。Router の画像処理には、設定済みの C0–C3 モデルのみを使用します。", + "tierLegacyImageRouting": "旧バージョンとの互換設定:このモデルは保持されますが、画像入力には使用されません。C0–C3 に画像対応モデルを設定してください。", "tierThinkingManagedByEnsemble": "融合プランが決定", "tierThinkingManagedByEnsembleAria": "{tier} の思考レベルはマルチモデル融合プランによって決まります", "tierThinkingAria": "{tier} の思考レベル", diff --git a/opensquilla-webui/src/locales/zh-Hans.json b/opensquilla-webui/src/locales/zh-Hans.json index d3f9f9698..fca0f9da5 100644 --- a/opensquilla-webui/src/locales/zh-Hans.json +++ b/opensquilla-webui/src/locales/zh-Hans.json @@ -1279,6 +1279,7 @@ "colModel": "模型", "colThinking": "思考", "colImage": "图像", + "imageCapabilityAutomatic": "图片能力根据服务商元数据和实际请求结果自动判断,无需手动设置。", "tierProviderAria": "{tier} 请求入口", "tierProviderManagedByEnsemble": "由多模型融合决定", "tierProviderManagedByEnsembleAria": "{tier} 请求入口由多模型融合方案决定", @@ -1309,7 +1310,8 @@ "tierLegacyDynamicEnsembleSummaryGeneric": "使用之前保存的“跟随模型分层”融合方案;失败时使用“固定与回退模型”。", "tierLegacyDynamicEnsembleFallbackMissingSummary": "使用之前保存的“跟随模型分层”融合方案;尚未配置可用的“固定与回退模型”。", "tierSingleModelAnnouncement": "C3 使用单模型:{model}。", - "tierEnsembleImageRouting": "C3 融合不处理图像;图像请求仍按图像模型配置处理。", + "tierEnsembleImageRouting": "C3 融合接收图片的文字标记。Router 仅使用已配置的 C0–C3 模型处理图片。", + "tierLegacyImageRouting": "旧版兼容配置:保留此模型,但不用于图片输入。请在 C0–C3 中配置支持图片的模型。", "tierThinkingManagedByEnsemble": "由融合方案决定", "tierThinkingManagedByEnsembleAria": "{tier} 的思考级别由多模型融合方案决定", "tierThinkingAria": "{tier} 思考级别", diff --git a/opensquilla-webui/src/styles/settings-forms.css b/opensquilla-webui/src/styles/settings-forms.css index d6000c98b..16321eab6 100644 --- a/opensquilla-webui/src/styles/settings-forms.css +++ b/opensquilla-webui/src/styles/settings-forms.css @@ -66,7 +66,7 @@ border-bottom: 1px solid var(--hairline); display: grid; gap: var(--sp-2); - grid-template-columns: 140px 1fr 1fr 120px 60px; + grid-template-columns: 140px 1fr 1fr 120px; padding: var(--sp-2) var(--sp-1); } @@ -266,8 +266,8 @@ /* Responsive */ @media (max-width: 760px) { - /* Don't clip the tier table: shrinking the fixed columns cut MODEL / THINKING / - IMAGE off the right edge with no way to reach them. Keep desktop column sizes + /* Don't clip the tier table: shrinking the fixed columns cut MODEL / THINKING + off the right edge with no way to reach them. Keep desktop column sizes and let the table scroll horizontally instead. */ .setup-tier-table { overflow-x: auto; diff --git a/opensquilla-webui/src/types/chat.ts b/opensquilla-webui/src/types/chat.ts index 39c0cb037..b96b24e43 100644 --- a/opensquilla-webui/src/types/chat.ts +++ b/opensquilla-webui/src/types/chat.ts @@ -243,7 +243,8 @@ export interface ChatRouterCell { export interface ChatRouterTierConfig { model: string - supportsImage: boolean + /** Accepted from legacy snapshots, not used to determine capability. */ + supportsImage?: boolean imageOnly: boolean ensembleEnabled?: boolean } diff --git a/opensquilla-webui/src/utils/chat/routerShapeCache.test.ts b/opensquilla-webui/src/utils/chat/routerShapeCache.test.ts index 2778b2262..c27afd55c 100644 --- a/opensquilla-webui/src/utils/chat/routerShapeCache.test.ts +++ b/opensquilla-webui/src/utils/chat/routerShapeCache.test.ts @@ -7,9 +7,9 @@ function shape(overrides: Partial = {}): RouterShape { slots: ['light', 'standard', 'heavy'], models: { light: 'a/x', standard: 'b/y', heavy: 'c/z' }, configs: { - light: { model: 'a/x', supportsImage: false, imageOnly: false, ensembleEnabled: false }, - standard: { model: 'b/y', supportsImage: true, imageOnly: false, ensembleEnabled: false }, - heavy: { model: 'c/z', supportsImage: false, imageOnly: false, ensembleEnabled: true }, + light: { model: 'a/x', imageOnly: false, ensembleEnabled: false }, + standard: { model: 'b/y', imageOnly: false, ensembleEnabled: false }, + heavy: { model: 'c/z', imageOnly: false, ensembleEnabled: true }, }, ...overrides, } @@ -68,7 +68,15 @@ describe('routerShapeCache — forward compatibility + normalization', () => { models: { standard: 'b/y' }, configs: { standard: { /* no model */ supportsImage: true } }, })) - expect(decoded?.configs.standard).toEqual({ model: '', supportsImage: true, imageOnly: false }) + expect(decoded?.configs.standard).toEqual({ model: '', imageOnly: false }) + }) + + it('drops legacy image declarations while preserving cached route identity', () => { + const s = shape() + s.configs.standard!.supportsImage = true + const legacyCache = JSON.stringify({ v: 1, ...s }) + expect(decodeRouterShape(legacyCache)).toEqual(shape()) + expect(JSON.parse(encodeRouterShape(s)).configs.standard).not.toHaveProperty('supportsImage') }) it('preserves the tier-scoped ensemble flag', () => { diff --git a/opensquilla-webui/src/utils/chat/routerShapeCache.ts b/opensquilla-webui/src/utils/chat/routerShapeCache.ts index e8c75a3ca..9294172bc 100644 --- a/opensquilla-webui/src/utils/chat/routerShapeCache.ts +++ b/opensquilla-webui/src/utils/chat/routerShapeCache.ts @@ -21,7 +21,7 @@ export function encodeRouterShape(shape: RouterShape): string { enabled: shape.enabled === true, slots: shape.slots, models: shape.models, - configs: shape.configs, + configs: asTierConfigRecord(shape.configs), }) } @@ -80,7 +80,6 @@ function asTierConfigRecord(value: unknown): Record dict[str, dict[str, Any]]: raise RuntimeError("TokenRhythm preset has no image_model tier") if image_tier.get("model") != ATTACHMENT_CAPACITY_MODEL: raise RuntimeError("TokenRhythm image_model does not match the verified live fixture") - unsafe_fallback_slots = [ + missing_slots = [ slot for slot in TEXT_PROFILE_SLOTS if not isinstance(tiers.get(slot), dict) - or tiers[slot].get("supports_image") is not False + or not str(tiers[slot].get("model") or "").strip() ] - if unsafe_fallback_slots: - raise RuntimeError( - "TokenRhythm attachment gate requires every text fallback to be explicitly " - "non-vision before any live request" - ) + if missing_slots: + raise RuntimeError("TokenRhythm attachment gate requires configured c0-c3 models") + tiers["c2"] = {**image_tier, "image_only": False} + del tiers["image_model"] + for tier in tiers.values(): + tier.pop("supports_image", None) return tiers @@ -647,6 +654,8 @@ def _attachment_capacity_fixture() -> dict[str, Any]: _inline_image("history-3a.png", payloads[2]), _inline_image("history-3b.png", payloads[3]), ] + images[2]["attachment_id"] = "att_capacity_history_3a" + images[3]["attachment_id"] = "att_capacity_history_3b" turns = [ { "user": _inline_history_envelope("Historical image turn one.", [images[0]]), @@ -691,6 +700,7 @@ def _attachment_capacity_fixture() -> dict[str, Any]: "current_attachment": _inline_image("current.png", current_payload), "excluded_base64": [images[0]["data"], images[1]["data"]], "retained_base64": [images[2]["data"], images[3]["data"]], + "retained_attachment_ids": [images[2]["attachment_id"], images[3]["attachment_id"]], "metrics": { "history_turn_count": len(turns), "history_image_count": len(images), @@ -1696,9 +1706,11 @@ def _run_tokenrhythm_attachment_capacity_in_temp( { "sessionKey": session_key, "message": ( - "请简短描述当前图片,不要调用工具,最后单独输出 " + "Briefly compare the current image with historical attachments " + + ", ".join(fixture["retained_attachment_ids"]) + + ". Do not call tools. End with " + marker - + "。" + + "." ), "attachments": [ { diff --git a/src/opensquilla/engine/agent.py b/src/opensquilla/engine/agent.py index ca116a01e..0b205837f 100644 --- a/src/opensquilla/engine/agent.py +++ b/src/opensquilla/engine/agent.py @@ -18,7 +18,7 @@ import stat import time import uuid -from collections.abc import AsyncIterator, Callable, Mapping +from collections.abc import AsyncIterator, Callable, Mapping, Sequence from contextvars import ContextVar from dataclasses import dataclass, field, replace from datetime import datetime @@ -201,12 +201,19 @@ ) from opensquilla.provider.correlation_context import bind_provider_request_correlation from opensquilla.provider.failures import ProviderFailureKind, classify_provider_error +from opensquilla.provider.image_projection import ( + ImageMarkerState, + ImageProjectionMode, + assert_text_only_messages, + classify_image_failure, + project_messages, +) +from opensquilla.provider.image_projection import ( + count_image_blocks as count_projected_image_blocks, +) from opensquilla.provider.model_identity import is_deepseek_v4_model_id from opensquilla.provider.protocol import ( - IMAGE_INPUT_UNSUPPORTED_CODE, - IMAGE_INPUT_UNSUPPORTED_MESSAGE, count_provider_image_blocks, - image_input_admission_error, project_provider_final_request, project_provider_message_count, provider_metadata, @@ -2741,31 +2748,19 @@ def _strip_historical_image_blocks( from replaying stale image input to a text-only route. """ if preserve_images: + # Keep the historical object graph intact for a vision-capable route. + # The outbound projection below still deep-copies it before a physical + # provider call, so callers cannot mutate the canonical transcript. return messages - sanitized: list[Message] = [] - for msg in messages: - content = msg.content - if not isinstance(content, list): - sanitized.append(msg) - continue - - kept: list[Any] = [] - omitted: list[str] = [] - for block in content: - if isinstance(block, ContentBlockImage): - media_type = block.media_type or "image" - omitted.append(f"[historical image omitted: {media_type}]") - continue - kept.append(block) - - if not omitted: - sanitized.append(msg) - continue - - kept.extend(ContentBlockText(text=marker) for marker in omitted) - sanitized.append(Message(role=msg.role, content=kept)) - return sanitized + # Historical images are not silently deleted. Project them into a + # truthful marker, recursively (including images nested in tool results), + # while keeping the original transcript available for a later vision turn. + return project_messages( + messages, + mode=ImageProjectionMode.MARKER, + marker_state=ImageMarkerState.NOT_REREAD, + ).messages def _trusted_meta_replay_seed_outputs( @@ -3029,6 +3024,7 @@ def __init__( self._state: AgentState = AgentState.IDLE self._history: list[Message] = [] + self._request_image_context: list[Message] = [] self._context: ContextAssembly | None = None # Typed dependency surface. Either constructor injection or legacy # attribute assignment from the runtime is accepted; both reach the same @@ -3635,11 +3631,12 @@ def _history_messages_for_compaction_admission( preserve_tool_call_reasoning=thinking_enabled, preserve_reasoning_content=preserve_reasoning_content, ) + declared_vision_support = str( + getattr(self.config, "model_vision_support", "unknown") or "unknown" + ).strip().lower() preserve_historical_images = bool( self.config.preserve_historical_images - and getattr(effective_capabilities, "supports_vision", False) - if effective_capabilities is not None - else False + and declared_vision_support != "unsupported" ) history = _strip_historical_image_blocks( history, @@ -3679,6 +3676,7 @@ def _assemble_compaction_consumer_request( turn_messages.append(skills_message) request_context_insert_index = len(turn_messages) runtime_context_insert_index = len(turn_messages) + turn_messages.extend(self._request_image_context) if attachment_messages: turn_messages.extend(attachment_messages) elif active_user_message: @@ -4227,6 +4225,142 @@ def _write_context_stage( **payload, ) + @staticmethod + def _image_attachment_ids_from_metadata(metadata: Mapping[str, Any]) -> tuple[str, ...]: + """Read optional attachment IDs without making them part of ChatConfig. + + Attachment IDs are session/runtime metadata, not provider wire data. A + few callers already use slightly different spellings, so accept the + known aliases while keeping the value bounded and deterministic. + """ + + values: list[str] = [] + seen: set[str] = set() + for key in ( + "image_attachment_ids", + "image_intent_attachment_ids", + "attachment_ids", + ): + raw = metadata.get(key) + if isinstance(raw, str): + raw_values: Sequence[Any] = (raw,) + elif isinstance(raw, Sequence) and not isinstance(raw, (bytes, bytearray)): + raw_values = raw + else: + continue + for value in raw_values: + if not isinstance(value, str) or not value.strip(): + continue + normalized = value.strip()[:164] + if normalized in seen: + continue + seen.add(normalized) + values.append(normalized) + return tuple(values) + + def _active_model_vision_support_for_call(self, config: Any) -> str: + """Resolve tri-state evidence for the exact physical selector leg.""" + + support: Any = getattr(config, "model_vision_support", "unknown") + resolver = getattr(self.provider, "active_model_vision_support", None) + if callable(resolver): + try: + support = resolver(config) + except Exception: # noqa: BLE001 - optional selector refinement + support = getattr(config, "model_vision_support", "unknown") + normalized = str(support or "unknown").strip().lower() + return ( + normalized + if normalized in {"supported", "unsupported", "unknown"} + else "unknown" + ) + + def _project_image_input_for_provider( + self, + messages: list[Message], + *, + chat_config: ChatConfig | None = None, + force_marker: bool = False, + marker_state: ImageMarkerState | str = ImageMarkerState.NOT_ANALYZED, + stage: str = "primary", + reason_override: str | None = None, + ) -> tuple[list[Message], Any]: + """Build one physical request view without mutating canonical messages. + + ``Agent`` owns the logical transcript while providers consume a + request-local view. Explicitly unsupported deployments and the + Ensemble contract receive markers; unknown deployments remain native so + the exact configured provider gets one capability probe. A subsequent + precise image rejection can call this helper again with ``force_marker`` + to retry the same configured model. + """ + + config = chat_config or self.config + support = self._active_model_vision_support_for_call(config) + try: + identity = provider_metadata(self.provider) + except Exception: # noqa: BLE001 - metadata is advisory at this boundary + identity = None + provider_name = str( + getattr(identity, "provider_name", "") + or getattr(self.provider, "provider_name", "") + or "" + ).strip().casefold() + provider_kind = str( + getattr(identity, "provider_kind", "") + or getattr(self.provider, "provider_kind", "") + or "" + ).strip().casefold() + is_ensemble = provider_name == "ensemble" or provider_kind == "ensemble" + mode = ( + ImageProjectionMode.MARKER + if force_marker or is_ensemble or support == "unsupported" + else ImageProjectionMode.NATIVE + ) + result = project_messages( + messages, + mode=mode, + marker_state=marker_state, + attachment_ids=self._image_attachment_ids_from_metadata( + self.config.metadata + ), + ) + if result.input_image_count or force_marker or is_ensemble: + reason = ( + "ensemble_text_only" + if is_ensemble + else reason_override + or str( + self.config.metadata.get("image_input_forced_rejection_reason") + or ( + "model_vision_unsupported" + if support == "unsupported" + else "image_capability_probe_failed" + ) + ) + ) + self.config.metadata["image_input_mode"] = mode.value + self.config.metadata["image_input_reason"] = reason + self.config.metadata["image_input_count"] = result.input_image_count + self.config.metadata["image_input_output_count"] = result.output_image_count + self.config.metadata["image_input_marker_count"] = result.marker_count + self.config.metadata["image_input_stage"] = stage + self._write_turn_call_log( + "image_input_projection", + action=("project" if result.marker_count else "preserve"), + mode=mode.value, + reason=reason, + stage=stage, + image_count=result.input_image_count, + marker_count=result.marker_count, + ) + if mode is ImageProjectionMode.MARKER: + # Keep this assertion close to the physical boundary. It catches a + # future nested content shape that the pure projector forgot while + # guaranteeing text-only providers never receive an image block. + assert_text_only_messages(result.messages) + return result.messages, result + def _switch_to_invalid_response_fallback( self, reason: str, @@ -4965,12 +5099,9 @@ def _record_projection_signal_hint_event( @staticmethod def _count_image_blocks(messages: list[Message]) -> int: - count = 0 - for message in messages: - if not isinstance(message.content, list): - continue - count += sum(1 for block in message.content if isinstance(block, ContentBlockImage)) - return count + # Use the same recursive accounting as request projection so images + # nested in tool-result content cannot bypass a text-only boundary. + return count_projected_image_blocks(messages) def _dedup_repeated_tool_results_for_provider( self, @@ -6530,10 +6661,21 @@ def refresh_system_prompt(self, new_prompt: str) -> None: def clear_history(self) -> None: self._history = [] + self._request_image_context = [] def set_history(self, messages: list[Message]) -> None: self._history = list(messages) + def set_request_image_context(self, messages: list[Message]) -> None: + """Bind recovered attachments to the next request's protected input. + + These messages are selected from the canonical transcript by the + runner. They share current-upload budgeting and projection, rather + than competing with ordinary history for the recent-turn window. + """ + + self._request_image_context = [message.model_copy(deep=True) for message in messages] + def history_snapshot(self) -> list[Message]: """Return a detached history list for read-only session forks.""" @@ -6683,12 +6825,13 @@ async def run_turn( with bind_usage_accounting_scope(scope): async for event in self._turn_generator( message, - extra_messages, + [*self._request_image_context, *(extra_messages or [])] or None, semantic_message, pending_input_provider=pending_input_provider, ): yield event finally: + self._request_image_context = [] # A staged candidate is never an implicit commit. If the turn is # cancelled, times out, or exits without document_finish, reject # the draft before releasing the rest of the turn authorities. @@ -6869,37 +7012,51 @@ async def _turn_generator( _ = terminates # always terminates today; reserved for future return - current_turn_image_count = count_provider_image_blocks(extra_messages or []) + current_turn_image_count = count_projected_image_blocks(extra_messages or []) forced_image_rejection = str( self.config.metadata.get("image_input_forced_rejection_reason") or "" ).strip() - image_admission_error = image_input_admission_error( - extra_messages or [], - vision_support=( - "unsupported" - if forced_image_rejection - else self.config.model_vision_support - ), - ) - if forced_image_rejection or image_admission_error is not None: - image_input_reason = forced_image_rejection or "model_vision_unsupported" - self.config.metadata["image_input_mode"] = "rejected" + # Unsupported capability is a request-shaping decision, not a terminal + # turn error. Keep the image in the canonical turn and project a + # marker into the physical request below. Unknown deployments remain + # native so the configured provider can be probed once. + try: + provider_identity = provider_metadata(self.provider) + except Exception: # noqa: BLE001 - provider identity is advisory here + provider_identity = None + provider_is_ensemble = ( + str(getattr(provider_identity, "provider_name", "") or "") + .strip() + .casefold() + == "ensemble" + or str(getattr(provider_identity, "provider_kind", "") or "") + .strip() + .casefold() + == "ensemble" + ) + image_projection_forced = bool( + forced_image_rejection + or self.config.model_vision_support == "unsupported" + or provider_is_ensemble + ) + image_projection_marker_state: ImageMarkerState = ImageMarkerState.NOT_ANALYZED + if image_projection_forced: + image_input_reason = ( + "ensemble_text_only" + if provider_is_ensemble and not forced_image_rejection + else forced_image_rejection or "model_vision_unsupported" + ) + self.config.metadata["image_input_mode"] = ImageProjectionMode.MARKER.value self.config.metadata["image_input_reason"] = image_input_reason self.config.metadata["image_input_count"] = current_turn_image_count self.config.metadata["image_input_stage"] = "primary" self._write_turn_call_log( "image_input_preflight", - action="reject", + action="project", reason=image_input_reason, model=self.config.model_id or "", image_count=current_turn_image_count, ) - yield self._transition(AgentState.ERROR) - yield ErrorEvent( - message=IMAGE_INPUT_UNSUPPORTED_MESSAGE, - code=IMAGE_INPUT_UNSUPPORTED_CODE, - ) - return # Use the system prompt from config (wired by gateway via identity.prompt) if self._context is None: @@ -6958,11 +7115,29 @@ async def _turn_generator( preserve_tool_call_reasoning=thinking_enabled, preserve_reasoning_content=preserve_reasoning_content, ) + # Preserve the sanitized-but-still-image-bearing history separately + # from the physical text-model view. The marker projection below is + # request-local; it must not become the Agent's canonical in-memory + # history and make a later vision-capable turn unable to recover the + # original attachment. + canonical_sanitized_history = list(sanitized_history) + declared_vision_support = str( + self.config.model_vision_support or "unknown" + ).strip().lower() preserve_historical_images = bool( self.config.preserve_historical_images - and getattr(self.config.model_capabilities, "supports_vision", False) - if self.config.model_capabilities is not None - else False + and declared_vision_support != "unsupported" + and ( + declared_vision_support in {"supported", "unknown"} + or ( + self.config.model_capabilities is not None + and getattr( + self.config.model_capabilities, + "supports_vision", + False, + ) + ) + ) ) sanitized_history = _strip_historical_image_blocks( sanitized_history, @@ -6981,6 +7156,13 @@ async def _turn_generator( ) history = limit_turns(sanitized_history, self.config.max_history_turns) history = repair_tool_pairing(history) + initial_provider_history = tuple(history) + canonical_history = repair_tool_pairing( + limit_turns( + canonical_sanitized_history, + self.config.max_history_turns, + ) + ) self._write_context_stage( "session:limited", history, @@ -7146,6 +7328,12 @@ def _accumulate_turn_cost( # boundary. The usage call index supplies the durable half of this # proof; this flag supplies the live turn half. turn_irreversible_effect_started = False + # Image capability recovery has a stricter whole-turn boundary than + # ordinary provider retry accounting: once any model output becomes + # visible or any tool starts executing, replaying the image request + # (or replacing it with a marker request) could duplicate observable + # work. Keep this latch across provider iterations and attempts. + turn_image_retry_barrier_crossed = False # A durable inline candidate is installed only after the rebuilt # request crosses the provider adapter's final admission boundary. self._pending_durable_compaction_event = None @@ -7858,14 +8046,11 @@ def _continuation_request_fits( execution_leg=leg_kind, ) return False - if self._count_image_blocks(turn_messages) > 0 and not supports_vision: - self._write_turn_call_log( - "same_turn_steer_admission", - action="defer_to_follow_up", - reason="vision_unsupported", - execution_leg=leg_kind, - ) - return False + # Image capability never blocks a same-turn continuation. The + # next physical request is independently projected: supported + # deployments receive the canonical image, unknown deployments + # receive one probe, and text-only deployments receive a marker. + del supports_vision if message_count_request_view is not None: base_messages = message_count_request_view.materialize(turn_messages) @@ -8456,6 +8641,11 @@ def _finish_goal_terminal_without_provider(*, reason: str, code: str) -> None: _attempt_retries_used = _retry_policy.used_attempts() _invalid_response_fallback_done = False _message_limit_recovery_done = False + # A precise provider image-capability rejection gets one + # request-local marker retry on the same configured model. It + # must not consume the generic retry budget or select an + # unconfigured model. + _image_marker_retry_done = False provider_activity_id = uuid.uuid4().hex next_provider_activity_reason: _ProviderActivityReason = "initial" while _retry_attempt <= _fallback.max_retries: @@ -8717,12 +8907,110 @@ def _finish_reasoning_block( self._provider_call_tool_result_retrieval_available = ( previous_call_retrieval ) + # Project only this physical request. ``turn_messages`` and + # ``request_turn_messages`` remain image-bearing canonical + # views so a later vision-capable turn can recover the + # original attachment. The projection runs after all + # provider-view sanitizers because tool-result adapters may + # introduce nested image blocks of their own. + active_vision_support = ( + self._active_model_vision_support_for_call(chat_cfg) + ) + barrier_requires_image_marker = bool( + turn_image_retry_barrier_crossed + and active_vision_support == "unknown" + and count_projected_image_blocks(request_messages) > 0 + ) + if barrier_requires_image_marker: + # Once a tool has executed (or output has escaped), an + # unknown-capability native probe cannot be retried + # safely. Shape this request as text up front so the + # same configured model can still finish the turn. + self.config.metadata["image_input_mode"] = ( + ImageProjectionMode.MARKER.value + ) + self.config.metadata["image_input_reason"] = ( + "image_probe_unsafe_after_irreversible_effect" + ) + self.config.metadata["image_input_stage"] = "primary" + request_messages, image_projection_result = ( + self._project_image_input_for_provider( + request_messages, + chat_config=chat_cfg, + force_marker=( + image_projection_forced + or barrier_requires_image_marker + ), + marker_state=image_projection_marker_state, + stage="primary", + reason_override=( + "image_probe_unsafe_after_irreversible_effect" + if barrier_requires_image_marker + else None + ), + ) + ) validation_error = validate_provider_chat_admission( self.provider, request_messages, chat_cfg, ) if validation_error is not None: + validation_image_failure = classify_image_failure( + validation_error, + provider_name=getattr( + self.provider, + "provider_name", + "", + ), + ) + if ( + validation_image_failure.is_unsupported + and not _image_marker_retry_done + and not turn_image_retry_barrier_crossed + ): + image_fallback = getattr( + self.provider, + "fallback_after_image_rejection", + None, + ) + if callable(image_fallback) and image_fallback( + "provider preflight rejected image input" + ): + # Router owns an explicit c0-c3 probe chain. + # Keep the canonical request image-bearing for + # the next configured leg; the wrapper rebinds + # capability and request budgets per call. + image_projection_forced = False + image_projection_marker_state = ( + ImageMarkerState.NOT_ANALYZED + ) + _call_attempt += 1 + continue + # A provider-side preflight can know more than the + # catalog (for example an unlisted deployment). + # Retry once with a truthful marker before any + # visible output or side effect is emitted. + _image_marker_retry_done = True + image_projection_forced = True + self.config.metadata["image_input_mode"] = ( + ImageProjectionMode.MARKER.value + ) + self.config.metadata["image_input_reason"] = ( + "image_capability_probe_failed" + ) + self.config.metadata["image_input_stage"] = "preflight" + self._write_turn_call_log( + "image_input_projection", + action="retry_marker", + reason="provider_preflight_rejected_image", + stage="preflight", + image_count=count_projected_image_blocks( + request_messages + ), + ) + _call_attempt += 1 + continue terminal_error = ErrorEvent( message=validation_error.message, code=validation_error.code, @@ -8739,7 +9027,7 @@ def _finish_reasoning_block( terminal_error = None yield TextDeltaEvent(text=response_text) else: - if terminal_error.code == IMAGE_INPUT_UNSUPPORTED_CODE: + if validation_image_failure.is_unsupported: exact_image_count = count_provider_image_blocks( request_messages ) @@ -9595,6 +9883,7 @@ def _active_stream_deadline() -> float | None: if raw_ev.text and not buffer_document_finalizer: attempt_user_visible_emitted = True attempt_irreversible_output_emitted = True + turn_image_retry_barrier_crossed = True if buffer_document_finalizer: # A mutation finalizer is an untrusted presentation # call. Hold its complete response behind the runtime @@ -9660,6 +9949,7 @@ def _active_stream_deadline() -> float | None: # boundary immediately and cannot later be # discarded in favour of another attempt. attempt_irreversible_output_emitted = True + turn_image_retry_barrier_crossed = True now_monotonic = time.monotonic() first_reasoning_activity = reasoning_activity_started_at_ms == 0 if first_reasoning_activity: @@ -10507,11 +10797,22 @@ def _active_stream_deadline() -> float | None: turn_has_error_usage_receipt = True # One-shot thinking/reasoning fallback _err_lower = raw_ev.message.lower() + _stream_image_failure = classify_image_failure( + raw_ev, + provider_name=getattr( + self.provider, + "provider_name", + "", + ), + ) if ( thinking_enabled and not _thinking_fallback_done and self.config.provider_error_thinking_fallback and not goal_terminal_final_response_pending + and not _stream_image_failure.is_unsupported + and not attempt_irreversible_output_emitted + and not turn_image_retry_barrier_crossed and ("thinking" in _err_lower or "reasoning" in _err_lower) ): _thinking_fallback_done = True @@ -11178,6 +11479,7 @@ def _active_stream_deadline() -> float | None: assistant_text_parts.append(response_text) attempt_user_visible_emitted = True attempt_irreversible_output_emitted = True + turn_image_retry_barrier_crossed = True yield TextDeltaEvent( text=response_text, generation_epoch=generation_epoch, @@ -12212,6 +12514,25 @@ def _active_stream_deadline() -> float | None: ) if not _got_error: + if ( + _got_done_event + and image_projection_result.output_image_count > 0 + ): + # A completed native image request is exact runtime + # evidence for this deployment. Preserve it across + # later tool iterations so the no-retry barrier + # does not unnecessarily downgrade a proven leg. + self.config.model_vision_support = "supported" + chat_cfg = chat_cfg.model_copy( + update={"model_vision_support": "supported"} + ) + mark_vision_supported = getattr( + self.provider, + "mark_active_model_vision_supported", + None, + ) + if callable(mark_vision_supported): + mark_vision_supported() break # stream OK, exit retry loop if provider_error is None: @@ -12228,6 +12549,10 @@ def _active_stream_deadline() -> float | None: raw_code=provider_error.code, message=provider_error.message, ) + image_failure = classify_image_failure( + provider_error, + provider_name=getattr(self.provider, "provider_name", ""), + ) safe_provider_error_code = safe_provider_failure_code( provider_error.code, failure_kind.value, @@ -12238,6 +12563,102 @@ def _active_stream_deadline() -> float | None: status_code=provider_error_status_code, raw_code=provider_error.code, ) + if ( + image_failure.is_unsupported + and not _image_marker_retry_done + and not attempt_irreversible_output_emitted + and not turn_image_retry_barrier_crossed + ): + image_fallback = getattr( + self.provider, + "fallback_after_image_rejection", + None, + ) + if callable(image_fallback) and image_fallback( + "provider rejected image input" + ): + image_projection_forced = False + image_projection_marker_state = ( + ImageMarkerState.NOT_ANALYZED + ) + self.config.metadata["image_input_mode"] = ( + ImageProjectionMode.NATIVE.value + ) + self.config.metadata["image_input_reason"] = ( + "router_next_configured_image_probe" + ) + self.config.metadata["image_input_stage"] = "fallback" + _got_error = False + provider_error = None + _call_attempt += 1 + continue + # The model was not known to be text-only until the + # physical request proved it. Keep the configured + # deployment, preserve the canonical image, and + # retry once with an analysis-failed marker. This + # branch intentionally precedes generic fallback so + # no unconfigured model is introduced. + _image_marker_retry_done = True + image_projection_forced = True + image_projection_marker_state = ( + ImageMarkerState.ANALYSIS_FAILED + ) + self.config.metadata["image_input_mode"] = ( + ImageProjectionMode.MARKER.value + ) + self.config.metadata["image_input_reason"] = ( + "provider_image_capability_rejection" + ) + self.config.metadata["image_input_stage"] = "provider" + self.config.metadata["image_input_failure_code"] = str( + provider_error.code or "" + )[:128] + self._write_turn_call_log( + "image_input_projection", + action="retry_marker", + reason="provider_image_capability_rejection", + stage="provider", + image_count=count_projected_image_blocks( + request_messages + ), + provider_error_code=safe_provider_error_code, + ) + _got_error = False + provider_error = None + _call_attempt += 1 + continue + if image_failure.is_unsupported: + # A precise image rejection is owned exclusively + # by the image policy. Once its safe same-turn + # recovery is unavailable, never let a coincident + # generic classification (for example + # ``empty_response``) replay the request after a + # visible output or tool side effect. + _log.warning( + "provider.image_retry_suppressed", + reason=( + "image_retry_barrier_crossed" + if turn_image_retry_barrier_crossed + or attempt_irreversible_output_emitted + else "image_marker_retry_exhausted" + ), + provider=getattr( + self.provider, + "provider_name", + "", + ), + ) + yield self._transition(AgentState.ERROR) + terminal_error = ErrorEvent( + message=_safe_provider_terminal_message( + failure_kind, + provider_error.code, + ), + code=safe_provider_error_code, + failure_kind=failure_kind.value, + ) + yield terminal_error + break if attempt_irreversible_output_emitted: # Text, reasoning, and tool lifecycle frames are # streamed to the client immediately and cannot be @@ -13744,6 +14165,7 @@ def _active_stream_deadline() -> float | None: provider_done_for_log, ) for pending_tool_event in pending_tool_events: + turn_image_retry_barrier_crossed = True yield pending_tool_event pending_tool_events.clear() @@ -14938,6 +15360,7 @@ def _cap_timeout_by_deadlines(timeout: float) -> float: async def _run_one(tc: ToolCall) -> ToolResult: nonlocal turn_irreversible_effect_started + nonlocal turn_image_retry_barrier_crossed nonlocal workspace_edit_gate_details nonlocal workspace_edit_gate_recovery_read_paths nonlocal workspace_edit_gate_recovery_reads_remaining @@ -15048,6 +15471,7 @@ async def _run_one(tc: ToolCall) -> ToolResult: cancellation_started = False try: turn_irreversible_effect_started = True + turn_image_retry_barrier_crossed = True execution_task = asyncio.create_task( self._execute_tool(execution_tc) ) @@ -15977,18 +16401,13 @@ async def _run_after_key_lock() -> ToolResult: if isinstance(media_by_call, dict) else [] ) - vision_capabilities = getattr(self.config, "model_capabilities", None) - vision_enabled = ( - self.config.model_vision_support == "supported" - or getattr(vision_capabilities, "supports_vision", False) is True - ) - provider_name = str( - getattr(self.provider, "provider_name", "") or "" - ).casefold() - if provider_name == "ensemble": - vision_enabled = False image_blocks: list[ContentBlockImage] = [] - if vision_enabled and isinstance(raw_media, list): + # Keep authenticated tool media in the logical turn for a + # possible later vision-capable request. The shared + # physical-call projection, not tool execution, decides + # whether this exact deployment receives image bytes or a + # truthful marker. + if isinstance(raw_media, list): for item in raw_media[:1]: if not isinstance(item, dict): continue @@ -17022,6 +17441,15 @@ async def _run_after_key_lock() -> ToolResult: # Persist successful turns into in-memory history. Error turns are # persisted by TurnRunner as system errors, while their usage still # flows through the final DoneEvent below when provider usage exists. + # Restore only the unchanged historical prefix from its canonical, + # sanitized image-bearing view. This is deliberately conservative: + # compaction or recovery may replace a prefix, in which case its + # authoritative rebuilt form wins instead of being overwritten. + if len(turn_messages) >= len(initial_provider_history) and all( + turn_messages[index] is projected_message + for index, projected_message in enumerate(initial_provider_history) + ): + turn_messages[: len(history)] = canonical_history self._history = list(turn_messages) self._write_context_stage("session:after", self._history) diff --git a/src/opensquilla/engine/route_plan.py b/src/opensquilla/engine/route_plan.py index b824a877c..521463d07 100644 --- a/src/opensquilla/engine/route_plan.py +++ b/src/opensquilla/engine/route_plan.py @@ -13,7 +13,6 @@ from opensquilla.provider.types import ModelCapabilities, ProviderRequestCorrelation from opensquilla.router_tiers import ( - IMAGE_TIER, TEXT_TIERS, TierConfig, effective_ensemble_selection_mode, @@ -23,8 +22,6 @@ tier_ensemble_execution, ) -_ROUTER_SNAPSHOT_TIER_ORDER = (*TEXT_TIERS, IMAGE_TIER) - @dataclass(frozen=True, slots=True) class RouteFallback: @@ -257,7 +254,12 @@ def _router_tier_snapshot( router = getattr(config, "squilla_router", None) tiers = normalize_tier_mapping(getattr(router, "tiers", None)) normalized_winner = normalize_tier_id(winner_tier) - if not tiers or normalized_winner is None or not winner_model: + if ( + not tiers + or normalized_winner is None + or normalized_winner not in TEXT_TIERS + or not winner_model + ): return None request_kind: Literal["text", "image"] = ( @@ -265,7 +267,6 @@ def _router_tier_snapshot( if _text(metadata.get("routing_source")) == "image_route" or bool(metadata.get("image_route_reason")) or bool(metadata.get("router_vision_followup_needs_image")) - or normalized_winner == IMAGE_TIER else "text" ) shared_selection_mode = effective_ensemble_selection_mode(config) @@ -276,17 +277,14 @@ def _router_tier_snapshot( ) entries: list[RouterTierSnapshotEntry] = [] - for tier in _ROUTER_SNAPSHOT_TIER_ORDER: + # Text-only tiers remain eligible for the image-not-analyzed marker path. + for tier in TEXT_TIERS: tier_config = TierConfig.from_value(tiers.get(tier)) - if not tier_config.model: + if not tier_config.model or tier_config.image_only: continue if request_kind == "image": - if not tier_config.supports_image: - continue if tier == TEXT_TIERS[-1] and c3_fusion_active: continue - elif tier_config.image_only or tier == IMAGE_TIER: - continue selection_mode, _binding = tier_ensemble_execution( tiers, @@ -318,7 +316,7 @@ def _router_tier_snapshot( ) if winner_index is None: entries.append(winner_entry) - entries.sort(key=lambda item: _ROUTER_SNAPSHOT_TIER_ORDER.index(item.tier)) + entries.sort(key=lambda item: TEXT_TIERS.index(item.tier)) else: entries[winner_index] = winner_entry diff --git a/src/opensquilla/engine/runtime.py b/src/opensquilla/engine/runtime.py index fbd967827..a9a42c0ab 100644 --- a/src/opensquilla/engine/runtime.py +++ b/src/opensquilla/engine/runtime.py @@ -233,6 +233,7 @@ ErrorEvent as ProviderErrorEvent, ) from opensquilla.provider import ( + ImageMarkerState, ModelCapabilities, ProviderActivityEvent, ProviderFailureKind, @@ -240,6 +241,7 @@ ProviderRecoveryAction, classify_provider_error, decide_recovery_action, + image_marker, ) from opensquilla.provider import ( ReasoningDeltaEvent as ProviderReasoningDeltaEvent, @@ -253,13 +255,19 @@ from opensquilla.provider import ( ToolUseStartEvent as ProviderToolUseStartEvent, ) +from opensquilla.provider.image_projection import ( + ImageProjectionMode, + assert_text_only_messages, + bind_image_attachment_ids, + classify_image_failure, + project_messages, +) from opensquilla.provider.model_catalog import ( resolve_effective_context_window, shared_catalog, ) from opensquilla.provider.protocol import ( count_provider_image_blocks, - image_input_admission_error, project_provider_final_request, project_provider_message_count, provider_metadata, @@ -2454,6 +2462,147 @@ def _fallback_candidate_accepts_tools(self, deployment: Any) -> bool: or getattr(capabilities, "supports_tools", None) is not False ) + @staticmethod + def _image_attachment_ids_from_metadata(config: Any) -> tuple[str, ...]: + """Read stable attachment ids without making a provider call. + + The selector wrapper is deliberately a transport boundary and must + not inspect or mutate the canonical transcript. It only needs the + optional ids already stamped on the per-turn config so a marker can + point back to the preserved attachment. + """ + + metadata = config if isinstance(config, Mapping) else getattr(config, "metadata", None) + if not isinstance(metadata, Mapping): + return () + result: list[str] = [] + seen: set[str] = set() + for key in ( + "image_attachment_ids", + "image_intent_attachment_ids", + "attachment_ids", + ): + raw_ids = metadata.get(key) + if isinstance(raw_ids, str): + values: Sequence[Any] = (raw_ids,) + elif isinstance(raw_ids, Sequence) and not isinstance( + raw_ids, + (bytes, bytearray), + ): + values = raw_ids + else: + continue + for value in values: + if not isinstance(value, str) or not value.strip(): + continue + normalized = value.strip()[:164] + if normalized in seen: + continue + seen.add(normalized) + result.append(normalized) + return tuple(result) + + def _project_image_messages_for_active_leg( + self, + messages: Sequence[Any], + config: Any, + *, + stage: str, + ) -> list[Any]: + """Build a fresh request view for the currently selected deployment. + + ``Agent`` performs the same projection at its retry boundary. The + selector wrapper repeats it here because a fallback is a new physical + request with a different capability fact. In particular, a known + text-only fallback receives a truthful marker instead of a terminal + admission error; an unknown deployment remains native and can be + probed once by the provider. + """ + + active_config = self._config_for_active_leg(config) + support = str( + getattr(active_config, "model_vision_support", "unknown") or "unknown" + ).strip().lower() + if support not in {"supported", "unsupported", "unknown"}: + support = "unknown" + + provider_kind = "" + provider_name = "" + try: + identity = provider_metadata(self._provider) + provider_kind = str( + getattr(identity, "provider_kind", "") or "" + ).strip().lower() + provider_name = str( + getattr(identity, "provider_name", "") or "" + ).strip().lower() + except Exception: # noqa: BLE001 - metadata is optional at this boundary + provider_kind = str( + getattr(self._provider, "provider_kind", "") or "" + ).strip().lower() + provider_name = str( + getattr(self._provider, "provider_name", "") or "" + ).strip().lower() + + turn_metadata = self._turn_metadata + force_marker = bool( + isinstance(turn_metadata, Mapping) + and ( + turn_metadata.get("image_input_projection_required") is True + or str(turn_metadata.get("image_input_mode") or "") + in {"marker", "text_only"} + ) + ) + ensemble_text_only = provider_kind == "ensemble" or provider_name == "ensemble" + should_marker = force_marker or support == "unsupported" or ensemble_text_only + + # A native projection is still a deep copy. This prevents a provider + # from mutating the list it received and contaminating the next + # selector leg. + if not should_marker: + return project_messages( + messages, + mode=ImageProjectionMode.NATIVE, + ).messages + + if ensemble_text_only: + reason = "ensemble_text_only" + elif support == "unsupported": + reason = "model_vision_unsupported" + else: + reason = str( + turn_metadata.get("image_input_reason") + if isinstance(turn_metadata, Mapping) + else "" + ).strip() or "configured_marker_fallback" + marker_state = ImageMarkerState.NOT_ANALYZED + if isinstance(turn_metadata, Mapping): + existing_state = str( + turn_metadata.get("image_input_marker_state") or "" + ).strip().lower() + if existing_state in {state.value for state in ImageMarkerState}: + marker_state = ImageMarkerState(existing_state) + + projection = project_messages( + messages, + mode=ImageProjectionMode.MARKER, + marker_state=marker_state, + attachment_ids=self._image_attachment_ids_from_metadata( + self._turn_metadata or active_config + ), + ) + if projection.marker_count and isinstance(turn_metadata, dict): + # These fields describe the physical leg that is about to run. + # A previous native probe must not leave stale metadata after a + # configured text-only fallback receives marker projection. + turn_metadata["image_input_mode"] = ImageProjectionMode.MARKER.value + turn_metadata["image_input_reason"] = reason + turn_metadata["image_input_count"] = projection.input_image_count + turn_metadata["image_input_stage"] = stage + turn_metadata["image_input_marker_state"] = marker_state.value + assert_text_only_messages(projection.messages) + return projection.messages + def _advance_past_explicit_tool_denials( self, *, @@ -2525,6 +2674,30 @@ def active_deployment_config(self) -> Any | None: return getattr(self._selector, "current_config", None) + def active_model_vision_support(self, config: Any) -> VisionSupport: + """Return exact tri-state evidence for the current physical leg.""" + + active_config = self._config_for_active_leg(config) + raw_support: Any = getattr( + active_config, + "model_vision_support", + "unknown", + ) + return ( + cast(VisionSupport, raw_support) + if raw_support in {"supported", "unsupported", "unknown"} + else "unknown" + ) + + def mark_active_model_vision_supported(self) -> None: + """Remember a successful native image request for this exact leg.""" + + current_config = getattr(self._selector, "current_config", None) + if current_config is not None: + self._fallback_deployment_vision_support[ + _fallback_deployment_identity(current_config) + ] = "supported" + def configure_fallback_deployment_limits( self, limits: Sequence[tuple[Any, int, int] | tuple[Any, int, int, Any]], @@ -2973,6 +3146,66 @@ def fallback_after_invalid_response(self, reason: str) -> bool: requires_tools=self._last_request_had_tools, ) + def fallback_after_image_rejection(self, reason: str) -> bool: + """Advance to the next configured Router image probe, if any. + + Router image routes install a strict c0-c3-only selector chain. This + method deliberately uses that static chain instead of plugin failover, + records the exact rejected deployment as text-only for the rest of the + turn, and accepts unknown candidates for one native probe. Direct and + Ensemble requests return ``False`` so their same-model marker policy + remains intact. + """ + + metadata = self._turn_metadata + if not isinstance(metadata, dict) or not ( + metadata.get("router_fallback_strict") is True + and metadata.get("routing_source") == "image_route" + and metadata.get("image_input_mode") != "marker" + ): + return False + + current_config = getattr(self._selector, "current_config", None) + if current_config is not None: + self._fallback_deployment_vision_support[ + _fallback_deployment_identity(current_config) + ] = "unsupported" + + next_matching = getattr(self._selector, "next_fallback_matching", None) + if not callable(next_matching): + return False + + def _probeable(candidate: Any) -> bool: + return ( + self._fallback_deployment_vision_support.get( + _fallback_deployment_identity(candidate), + "unknown", + ) + != "unsupported" + ) + + try: + self._provider = next_matching(predicate=_probeable) + except Exception: # noqa: BLE001 - exhaustion selects marker fallback + return False + self._note_fallback_hop() + metadata["router_image_probe_failure_count"] = ( + int(metadata.get("router_image_probe_failure_count") or 0) + 1 + ) + metadata["router_fallback_reason"] = "image_capability_rejection" + metadata["image_input_reason"] = "router_next_configured_image_probe" + metadata["image_input_stage"] = "fallback" + log.info( + "selector.image_probe_fallback", + reason=reason, + provider=self.active_provider_id, + model=str( + getattr(getattr(self._selector, "current_config", None), "model", "") + or "" + ), + ) + return True + def fallback_after_invalid_response_with_capabilities( self, reason: str, @@ -3052,7 +3285,14 @@ def _reject_unsupported_image_input( *, reject_unknown_capability: bool, ) -> ProviderErrorEvent | None: - """Return and record an image admission error for the active physical leg.""" + """Keep legacy admission API while making image handling non-terminal. + + Images are projected by :meth:`_project_image_messages_for_active_leg` + immediately before this check. Unknown capability is intentionally + probed once, and an explicit text-only fact is represented by a marker + rather than an ``ErrorEvent``. The keyword is retained for callers + and third-party subclasses compiled against the old seam. + """ raw_vision_support = getattr(config, "model_vision_support", "unknown") vision_support: VisionSupport = ( @@ -3060,26 +3300,28 @@ def _reject_unsupported_image_input( if raw_vision_support in {"supported", "unsupported", "unknown"} else "unknown" ) - error = image_input_admission_error( - messages, - vision_support=vision_support, - reject_unknown=reject_unknown_capability, - ) - if error is None: - return None image_count = _count_image_blocks(messages) - if self._turn_metadata is not None: - self._turn_metadata["image_input_mode"] = "rejected" - self._turn_metadata["image_input_reason"] = ( - "capability_unknown" - if vision_support == "unknown" - else "model_vision_unsupported" + if image_count and self._turn_metadata is not None: + # A residual image here indicates a caller bypassed the projection + # helper. Do not turn that programming seam into a user-visible + # terminal error; retain bounded diagnostics and let the provider + # (or Agent's precise image-failure retry) remain authoritative. + self._turn_metadata.setdefault( + "image_input_mode", + "native" if vision_support != "unsupported" else "marker", + ) + self._turn_metadata.setdefault( + "image_input_reason", + "capability_probe" if vision_support == "unknown" else "model_vision_unsupported", ) self._turn_metadata["image_input_count"] = image_count self._turn_metadata["image_input_stage"] = ( "fallback" if self._used_fallback else "primary" ) - return error + # ``reject_unknown_capability`` is deliberately ignored. Unknown is + # not evidence of unsupported capability and must not strand a turn. + del reject_unknown_capability + return None def validate_chat_admission( self, @@ -3169,6 +3411,11 @@ async def _chat( active_provider = self._provider active_provider_id, active_model = self._active_deployment() active_config = self._config_for_active_leg(config) + physical_messages = self._project_image_messages_for_active_leg( + messages, + active_config, + stage="fallback" if self._used_fallback else "primary", + ) if ( tools and getattr( @@ -3183,7 +3430,7 @@ async def _chat( code="model_tools_unsupported", ) return - validation_error = self.validate_chat_admission(messages, config) + validation_error = self.validate_chat_admission(physical_messages, config) if validation_error is not None: yield validation_error return @@ -3212,7 +3459,7 @@ async def _chat( def primary_stream_factory() -> AsyncGenerator[Any, None]: return _selector_safe_stream( - lambda: active_provider.chat(messages, **primary_chat_kwargs), + lambda: active_provider.chat(physical_messages, **primary_chat_kwargs), content_started=lambda: emitted_user_visible_content, ) @@ -3348,10 +3595,21 @@ def primary_stream_factory() -> AsyncGenerator[Any, None]: else 0 ) local_admission_escalation = local_admission_fallback_index > 0 - if isinstance(event, ProviderErrorEvent) and ( - _should_use_selector_fallback(self.provider_name, event) - or event.code == "invalid_stream_order" - or local_admission_escalation + precise_image_capability_rejection = bool( + isinstance(event, ProviderErrorEvent) + and classify_image_failure( + event, + provider_name=active_provider_id or self.provider_name, + ).is_unsupported + ) + if ( + isinstance(event, ProviderErrorEvent) + and not precise_image_capability_rejection + and ( + _should_use_selector_fallback(self.provider_name, event) + or event.code == "invalid_stream_order" + or local_admission_escalation + ) ): if not local_admission_escalation: self._record_health_failure(event) @@ -3461,6 +3719,11 @@ def _local_candidate_predicate(candidate: Any) -> bool: fallback_provider = self._provider fallback_provider_id, fallback_model = self._active_deployment() fallback_config = self._config_for_active_leg(config) + fallback_messages = self._project_image_messages_for_active_leg( + messages, + fallback_config, + stage="fallback", + ) if ( tools and getattr( @@ -3478,7 +3741,7 @@ def _local_candidate_predicate(candidate: Any) -> bool: ) return fallback_admission_error = self._reject_unsupported_image_input( - messages, + fallback_messages, fallback_config, reject_unknown_capability=True, ) @@ -3487,7 +3750,7 @@ def _local_candidate_predicate(candidate: Any) -> bool: return fallback_validation_error = validate_provider_chat_admission( fallback_provider, - messages, + fallback_messages, fallback_config, ) if fallback_validation_error is not None: @@ -3577,7 +3840,7 @@ def _local_candidate_predicate(candidate: Any) -> bool: def fallback_stream_factory() -> AsyncGenerator[Any, None]: return _selector_safe_stream( lambda: fallback_provider.chat( - messages, + fallback_messages, tools=tools, config=fallback_config, **( @@ -5805,6 +6068,20 @@ async def _load_turn_transcript() -> Sequence[Any]: att_out.extra_messages, effective_runtime_message, ) + current_attachment_ids = turn.metadata.get("image_attachment_ids") + if ( + extra_msgs + and isinstance(current_attachment_ids, Sequence) + and not isinstance(current_attachment_ids, (str, bytes, bytearray)) + ): + extra_msgs = bind_image_attachment_ids( + extra_msgs, + [ + value + for value in current_attachment_ids + if isinstance(value, str) and value.strip() + ], + ) attachment_turn_input = ( effective_runtime_message if extra_msgs is None else "" ) @@ -5999,20 +6276,26 @@ async def _load_turn_transcript() -> Sequence[Any]: forced_image_rejection_reason = str( turn.metadata.get("image_input_forced_rejection_reason", "") or "" ).strip() - image_input_preflight_blocked = bool( + image_input_projection_required = bool( forced_image_rejection_reason or ( current_turn_image_count > 0 and agent_config.model_vision_support == "unsupported" ) + or turn.metadata.get("image_input_projection_required") is True ) - if image_input_preflight_blocked: - turn.metadata["image_input_mode"] = "rejected" + if image_input_projection_required: + turn.metadata["image_input_mode"] = "marker" turn.metadata["image_input_reason"] = ( forced_image_rejection_reason or "model_vision_unsupported" ) turn.metadata["image_input_count"] = current_turn_image_count turn.metadata["image_input_stage"] = "primary" + # Kept as a named local for frame-walking compatibility tests. A + # missing image capability is no longer an admission block and + # must never suppress compaction; the physical request is shaped + # to text later at the shared provider boundary. + image_input_preflight_blocked = False # 6. Compaction (t3 + preflight) + history load + request-context # prepend. CompactionAndHistoryStage owns the four-call sequence # (t3_upgrade → preflight → load_history → prepend_request_context_prompt). @@ -9299,6 +9582,10 @@ async def _run_pipeline( usage_execution_context: UsageExecutionContext | None = None, provider_request_correlation: ProviderRequestCorrelation | None = None, router_history_replay_request: RouterHistoryReplayRequest | None = None, + bound_user_message_id: str | None = None, + transcript_snapshot: TurnTranscriptSnapshot[Any] | None = None, + expected_session_id: str | None = None, + expected_session_epoch: int | None = None, ) -> tuple[Any, Any]: """Run the pre-turn pipeline and re-resolve provider if model changed. @@ -9547,6 +9834,96 @@ def _run_router_step_sync() -> TurnContext: initial_metadata["attachment_image_count"] = int( attachment_materialization.image_count ) + if not restricted_tool_boundary: + attachment_reference_text = "\n".join( + value + for value in (semantic_message, message) + if isinstance(value, str) and value.strip() + ) + candidate_attachment_ids = tuple( + dict.fromkeys( + ( + *self._attachment_ids_from_text(attachment_reference_text), + *self._attachment_ids_from_resource_refs(attachments), + ) + ) + ) + explicit_attachment_ids = await self._validated_image_attachment_ids( + session_key, + candidate_attachment_ids, + transcript_snapshot=transcript_snapshot, + expected_session_id=expected_session_id, + expected_session_epoch=expected_session_epoch, + ) + if explicit_attachment_ids: + # A canonical occurrence ID is deterministic image intent. It + # must survive even when the source row is outside the normal + # history lookback, and it must reach Router before the archive + # is rehydrated later in ``_load_history``. + initial_metadata["image_intent_attachment_ids"] = list( + explicit_attachment_ids + ) + initial_metadata["router_vision_followup_needs_image"] = True + initial_metadata["router_vision_followup_gate_source"] = ( + "explicit_attachment_id" + ) + if bound_user_message_id: + try: + bound_entries: Sequence[Any] + if transcript_snapshot is not None: + bound_entries = await transcript_snapshot.get_entries() + elif self._session_manager is not None: + owner_kwargs: dict[str, Any] = {} + if _require_optional_exact_session_owner( + expected_session_id, expected_session_epoch + ): + getter = self._session_manager.get_transcript + if all( + _accepts_explicit_keyword_arg(getter, name) + for name in ("expected_session_id", "expected_session_epoch") + ): + owner_kwargs = { + "expected_session_id": expected_session_id, + "expected_session_epoch": expected_session_epoch, + } + elif _has_session_storage(self._session_manager): + raise RuntimeError( + "Bound image replay requires exact session ownership" + ) + bound_entries = await self._session_manager.get_transcript( + session_key, **owner_kwargs + ) + else: + bound_entries = [] + bound_entry = next( + ( + entry + for entry in bound_entries + if str(getattr(entry, "message_id", "") or "") + == bound_user_message_id + ), + None, + ) + if bound_entry is not None: + bound_attachment_ids = self._attachment_ids_from_envelope( + str(getattr(bound_entry, "content", "") or ""), + image_only=True, + ) + if bound_attachment_ids: + initial_metadata["image_attachment_ids"] = list( + bound_attachment_ids + ) + except Exception as exc: # noqa: BLE001 - marker IDs are additive + if ( + (expected_session_id is not None or expected_session_epoch is not None) + and _has_session_storage(self._session_manager) + ): + raise + log.debug( + "turn_runner.bound_attachment_ids_unavailable", + message_id=bound_user_message_id, + error=type(exc).__name__, + ) if input_provenance: if isinstance(input_provenance, dict): normalized_provenance = dict(input_provenance) @@ -9734,24 +10111,35 @@ def _run_router_step_sync() -> TurnContext: from opensquilla.engine.selector_override import ( apply_model_override, cross_provider_tier_config, + resolve_strict_router_fallback_chain, ) + turn_config = self._turn_config() + active_provider_id = getattr( + cloned_selector, + "active_provider_id", + "", + ) provider = apply_model_override( cloned_selector, turn.model, turn_metadata=turn.metadata, realign_routed_model=False, tier_provider_config=cross_provider_tier_config( - self._turn_config(), + turn_config, turn.metadata, turn.model, - active_provider_id=getattr( - cloned_selector, - "active_provider_id", - "", - ), + active_provider_id=active_provider_id, session_key=turn.session_key, ), + strict_router_fallback_chain=( + resolve_strict_router_fallback_chain( + turn_config, + turn.metadata, + active_provider_id=active_provider_id, + session_key=turn.session_key, + ) + ), ) return turn, provider @@ -9961,18 +10349,31 @@ def _record_fixed_ensemble_execution(reason: str) -> None: from opensquilla.engine.selector_override import ( apply_model_override, cross_provider_tier_config, + resolve_strict_router_fallback_chain, ) + turn_config = self._turn_config() + active_provider_id = getattr( + cloned_selector, + "active_provider_id", + "", + ) provider = apply_model_override( cloned_selector, turn.model, turn_metadata=turn.metadata, realign_routed_model=False, tier_provider_config=cross_provider_tier_config( - self._turn_config(), + turn_config, turn.metadata, turn.model, - active_provider_id=getattr(cloned_selector, "active_provider_id", ""), + active_provider_id=active_provider_id, + session_key=turn.session_key, + ), + strict_router_fallback_chain=resolve_strict_router_fallback_chain( + turn_config, + turn.metadata, + active_provider_id=active_provider_id, session_key=turn.session_key, ), ) @@ -10257,13 +10658,18 @@ def _add(value: Any) -> None: "gate_history", } if isinstance(tiers, Mapping): - for raw_tier in tiers.values(): + for tier_name, raw_tier in tiers.items(): if not isinstance(raw_tier, Mapping): continue - if requires_image and not bool(raw_tier.get("supports_image", False)): + if requires_image and normalize_text_tier(tier_name) is None: continue - if not requires_image and bool(raw_tier.get("image_only", False)): + if bool(raw_tier.get("image_only", False)): continue + if not str(raw_tier.get("model") or "").strip(): + continue + # Every configured c-tier may be reached by a native probe + # or the final marker fallback. Legacy image switches do + # not establish the physical deployment's capabilities. _add(raw_tier.get("provider") or active_provider) # Unknown/legacy selector shapes retain the previous conservative @@ -10436,6 +10842,7 @@ def _project_history_replay( trim_last_user: bool, bound_slice_applied: bool, image_replay_entry_indexes: Collection[int] = (), + allowed_image_attachment_ids: frozenset[str] | None = None, media_root: Path | None = None, session_id: str | None = None, materialize_historical_attachments: bool = False, @@ -10520,6 +10927,8 @@ def _entry_projector(entry: Any, entry_index: int) -> HistoryReplayEntryProjecti session_id=session_id, workspace_dir=workspace_dir, historical_materializer=historical_materializer, + allowed_image_attachment_ids=allowed_image_attachment_ids, + source_message_id=getattr(entry, "message_id", None), ) if require_capacity_proof and recognized and valid: persisted_token_count = ( @@ -13665,6 +14074,233 @@ async def _rollback_cancelled_prompt( ) return False + async def _canonical_transcript_for_attachment_replay( + self, + session_key: str, + active_entries: Sequence[Any], + *, + expected_session_id: str | None = None, + expected_session_epoch: int | None = None, + ) -> list[Any]: + """Read the raw archive only when image replay needs it. + + Ordinary provider history intentionally remains the compacted active + tail plus durable summaries. A vision follow-up, however, may need an + image row that compaction moved to ``compacted_transcript_entries``. + This helper keeps that recovery read-only and falls back to the active + snapshot for older/fake session managers that do not expose the + canonical API. + """ + + exact_owner = _require_optional_exact_session_owner( + expected_session_id, expected_session_epoch + ) + manager = self._session_manager + getter = getattr(manager, "get_canonical_transcript", None) + if not callable(getter): + if exact_owner and _has_session_storage(manager): + raise RuntimeError("canonical history reader does not support exact ownership") + return list(active_entries) + getter_kwargs: dict[str, Any] = {} + if exact_owner: + if all( + _accepts_explicit_keyword_arg(getter, name) + for name in ("expected_session_id", "expected_session_epoch") + ): + getter_kwargs["expected_session_id"] = expected_session_id + getter_kwargs["expected_session_epoch"] = expected_session_epoch + elif _has_session_storage(manager): + raise RuntimeError("canonical history reader does not support exact ownership") + try: + canonical = getter(session_key, **getter_kwargs) + if inspect.isawaitable(canonical): + canonical = await canonical + if canonical: + return list(canonical) + except Exception as exc: # noqa: BLE001 - replay must not block a turn + if exact_owner and _has_session_storage(manager): + raise + log.warning( + "turn_runner.canonical_attachment_replay_failed", + session_key=session_key, + error=type(exc).__name__, + ) + return list(active_entries) + + async def _validated_image_attachment_ids( + self, + session_key: str, + candidate_ids: Sequence[str], + *, + transcript_snapshot: TurnTranscriptSnapshot[Any] | None = None, + expected_session_id: str | None = None, + expected_session_epoch: int | None = None, + ) -> tuple[str, ...]: + """Resolve current-input references to image occurrences in this session.""" + + exact_owner = _require_optional_exact_session_owner( + expected_session_id, expected_session_epoch + ) + if not candidate_ids or self._session_manager is None: + return () + try: + if transcript_snapshot is not None: + active_entries = list(await transcript_snapshot.get_entries()) + else: + get_transcript = self._session_manager.get_transcript + getter_kwargs: dict[str, Any] = {} + if exact_owner: + if all( + _accepts_explicit_keyword_arg(get_transcript, name) + for name in ("expected_session_id", "expected_session_epoch") + ): + getter_kwargs["expected_session_id"] = expected_session_id + getter_kwargs["expected_session_epoch"] = expected_session_epoch + elif _has_session_storage(self._session_manager): + raise RuntimeError( + "session history reader does not support exact ownership" + ) + active_entries = list(await get_transcript(session_key, **getter_kwargs)) + canonical_entries = await self._canonical_transcript_for_attachment_replay( + session_key, + active_entries, + expected_session_id=expected_session_id, + expected_session_epoch=expected_session_epoch, + ) + session_id = ( + expected_session_id + if exact_owner + else await self._resolve_session_id_for_log(session_key) + ) + if not session_id: + return () + from opensquilla.session.attachment_manifest import build_attachment_manifest + + manifest = build_attachment_manifest( + canonical_entries, + session_id=session_id, + session_key=session_key, + ) + by_id = { + occurrence.attachment_id: occurrence + for occurrence in manifest.occurrences + } + return tuple( + attachment_id + for attachment_id in candidate_ids + if attachment_id in by_id + and str(by_id[attachment_id].mime).lower().startswith("image/") + ) + except Exception as exc: # noqa: BLE001 - an unverified ID stays text-only + if exact_owner and _has_session_storage(self._session_manager): + raise + log.debug( + "turn_runner.image_attachment_reference_unverified", + session_key=session_key, + error=type(exc).__name__, + ) + return () + + async def _persist_attachment_manifest_best_effort( + self, + session_key: str, + entries: Sequence[Any], + *, + expected_session_id: str | None = None, + expected_session_epoch: int | None = None, + ) -> None: + """Lazily backfill the portable attachment index for this session. + + Older sessions have no manifest row. Building it on the first replay + read keeps migration online and lets later compaction commits merge the + same metadata atomically. Equality is checked against the newest + valid row so ordinary turns do not append unbounded duplicate states. + """ + + exact_owner = _require_optional_exact_session_owner( + expected_session_id, expected_session_epoch + ) + manager = self._session_manager + saver = getattr(manager, "save_context_state", None) + getter = getattr(manager, "get_context_states", None) + if not callable(saver) or not callable(getter) or not entries: + return + owner_kwargs: dict[str, Any] = {} + if exact_owner: + if all( + _accepts_explicit_keyword_arg(method, name) + for method in (getter, saver) + for name in ("expected_session_id", "expected_session_epoch") + ): + owner_kwargs["expected_session_id"] = expected_session_id + owner_kwargs["expected_session_epoch"] = expected_session_epoch + elif _has_session_storage(manager): + raise RuntimeError("attachment manifest storage does not support exact ownership") + try: + from opensquilla.session.attachment_manifest import ( + ATTACHMENT_MANIFEST_PROVIDER, + ATTACHMENT_MANIFEST_STATE_KIND, + AttachmentManifestError, + attachment_manifest_from_context_state, + build_attachment_manifest, + manifest_context_state, + ) + + session_id = ( + expected_session_id + if exact_owner + else await self._resolve_session_id_for_log(session_key) + ) + if not session_id: + return + manifest = build_attachment_manifest( + entries, + session_id=session_id, + session_key=session_key, + ) + if not manifest.occurrences: + return + states = await getter( + session_key, + provider=ATTACHMENT_MANIFEST_PROVIDER, + state_kind=ATTACHMENT_MANIFEST_STATE_KIND, + **owner_kwargs, + ) + latest = None + for state in sorted( + states or [], + key=lambda item: ( + int(getattr(item, "created_at", 0) or 0), + int(getattr(item, "id", 0) or 0), + ), + reverse=True, + ): + try: + latest = attachment_manifest_from_context_state(state) + break + except (AttachmentManifestError, TypeError, ValueError): + continue + if latest is not None: + manifest = latest.merge( + manifest.occurrences, + covered_through_id=manifest.covered_through_id, + ) + if ( + latest is not None + and latest.occurrences == manifest.occurrences + and latest.covered_through_id == manifest.covered_through_id + ): + return + await saver(manifest_context_state(manifest), **owner_kwargs) + except Exception as exc: # noqa: BLE001 - manifest is additive + if exact_owner and _has_session_storage(manager): + raise + log.debug( + "turn_runner.attachment_manifest_persist_skipped", + session_key=session_key, + error=type(exc).__name__, + ) + async def _load_history( self, agent: Agent, @@ -13690,6 +14326,7 @@ async def _load_history( assistant replies. When the id is absent or not found, fall back to the positional trim. """ + agent.set_request_image_context([]) if self._session_manager is None: return None @@ -13718,7 +14355,9 @@ async def _load_history( transcript_kwargs["expected_session_epoch"] = expected_session_epoch transcript = await get_transcript(session_key, **transcript_kwargs) + from opensquilla.engine.history import reconstruct_messages_from_entry from opensquilla.provider import Message + from opensquilla.provider.types import ContentBlockImage, ContentBlockText history: list[Message] = [] summary_markers: list[str] = [] @@ -13771,11 +14410,147 @@ async def _load_history( transcript_len=len(transcript), ) bound_slice_applied = bool(bound_skip_indexes) - model_caps = getattr(getattr(agent, "config", None), "model_capabilities", None) + agent_config = getattr(agent, "config", None) + model_caps = getattr(agent_config, "model_capabilities", None) + declared_vision_support = str( + getattr(agent_config, "model_vision_support", "unknown") or "unknown" + ).strip().lower() + # ``supports_vision`` is the legacy boolean catalog field. The + # tri-state deployment fact is authoritative when present: an unknown + # deployment may still be probed once, while an explicit unsupported + # deployment must receive markers. preserve_image_history = bool( - getattr(getattr(agent, "config", None), "preserve_historical_images", False) - and getattr(model_caps, "supports_vision", False) + getattr(agent_config, "preserve_historical_images", False) + and declared_vision_support != "unsupported" + and ( + declared_vision_support in {"supported", "unknown"} + or getattr(model_caps, "supports_vision", False) + ) + ) + metadata = getattr(agent_config, "metadata", {}) + if not isinstance(metadata, Mapping): + metadata = {} + + requested_attachment_id_list: list[str] = [] + seen_requested_attachment_ids: set[str] = set() + for key in ( + "image_attachment_ids", + "image_intent_attachment_ids", + "attachment_ids", + ): + raw_ids = metadata.get(key) + if isinstance(raw_ids, str): + values: Sequence[Any] = (raw_ids,) + elif isinstance(raw_ids, Sequence) and not isinstance( + raw_ids, (bytes, bytearray) + ): + values = raw_ids + else: + continue + for value in values: + if not isinstance(value, str) or not value.strip(): + continue + normalized_id = value.strip()[:164] + if normalized_id in seen_requested_attachment_ids: + continue + seen_requested_attachment_ids.add(normalized_id) + requested_attachment_id_list.append(normalized_id) + requested_attachment_ids = tuple(requested_attachment_id_list) + requested_image_id_filter = ( + frozenset(requested_attachment_ids) + if requested_attachment_ids + else None + ) + + # A queued/retried task may keep the original persisted user-message + # id while the client sends no attachment bytes on the second + # execution (for example after changing to a vision model). Treat it + # as an attachment replay only when that exact persisted row is an + # image envelope; every ordinary bound text turn also has zero current + # attachments and must not pull unrelated archived images into scope. + current_attachment_count = _non_negative_int( + metadata.get("attachment_count") + ) + bound_attachment_replay_candidate = bool( + bound_user_message_id and current_attachment_count == 0 + ) + bound_row_has_image: bool | None = None + if bound_attachment_replay_candidate: + for entry in transcript: + if ( + getattr(entry, "role", None) == "user" + and str(getattr(entry, "message_id", "") or "") + == bound_user_message_id + ): + bound_row_has_image = self._attachment_envelope_has_image( + str(getattr(entry, "content", "") or "") + ) + break + + # A compacted image is outside the active transcript. Read the + # canonical archive for an explicit ID, a routed image follow-up, or a + # vision-capable model whose route requested historical replay. + independent_replay_signal = bool( + requested_attachment_ids + or metadata.get("image_route_reason") in {"current_turn", "gate_history"} + or metadata.get("router_vision_followup_needs_image") is True + or preserve_image_history + ) + canonical_lookup_required = bool( + independent_replay_signal + or ( + bound_attachment_replay_candidate + and bound_row_has_image is None + ) + ) + canonical_transcript = ( + await self._canonical_transcript_for_attachment_replay( + session_key, + transcript, + expected_session_id=expected_session_id, + expected_session_epoch=expected_session_epoch, + ) + if canonical_lookup_required and not restricted_turn + else list(transcript) + ) + if bound_attachment_replay_candidate and bound_row_has_image is None: + bound_row_has_image = any( + getattr(entry, "role", None) == "user" + and str(getattr(entry, "message_id", "") or "") + == bound_user_message_id + and self._attachment_envelope_has_image( + str(getattr(entry, "content", "") or "") + ) + for entry in canonical_transcript + ) + bound_attachment_replay_requested = bool( + bound_attachment_replay_candidate and bound_row_has_image is True + ) + replay_signal = bool( + independent_replay_signal or bound_attachment_replay_requested ) + replay_images_natively = bool( + declared_vision_support != "unsupported" + and ( + preserve_image_history + or requested_attachment_ids + or bound_attachment_replay_requested + or metadata.get("router_vision_followup_needs_image") is True + ) + ) + if replay_images_natively and agent_config is not None: + # Agent performs a final history sanitation pass immediately + # before provider projection. Carry the resolved image intent to + # that pass so an explicitly rehydrated canonical image is not + # downgraded a second time. + agent_config.preserve_historical_images = True + if not restricted_turn: + await self._persist_attachment_manifest_best_effort( + session_key, + canonical_transcript, + expected_session_id=expected_session_id, + expected_session_epoch=expected_session_epoch, + ) workspace_dir = getattr(getattr(agent, "config", None), "workspace_dir", None) materialize_historical_attachments = bool( getattr( @@ -13794,8 +14569,15 @@ async def _load_history( or 0 ) image_replay_entry_indexes: set[int] = set() + request_image_replay_entries: list[Any] = [] + bound_image_replay_entries: list[Any] = [] + requested_source_message_ids: set[str] = set() image_replay_session_id: str | None = None - if preserve_image_history and lookback > 0: + if replay_signal and ( + lookback > 0 + or bound_attachment_replay_requested + or requested_attachment_ids + ): current_user_entry_index = bound_index if current_user_entry_index is None: current_user_entry_index = ( @@ -13814,19 +14596,154 @@ async def _load_history( and isinstance(getattr(entry, "content", None), str) and bool(str(getattr(entry, "content", "")).strip()) ] - image_replay_entry_indexes = set(user_entry_indexes[-lookback:]) + if preserve_image_history and lookback > 0: + image_replay_entry_indexes = set(user_entry_indexes[-lookback:]) image_replay_session_id = expected_session_id if image_replay_session_id is None: image_replay_session_id = await self._resolve_session_id_for_log(session_key) if image_replay_session_id is None: image_replay_session_id = session_key + + if bound_attachment_replay_requested: + bound_image_replay_entries = [ + entry + for entry in canonical_transcript + if getattr(entry, "role", None) == "user" + and str(getattr(entry, "message_id", "") or "") + == bound_user_message_id + and self._attachment_envelope_has_image( + str(getattr(entry, "content", "") or "") + ) + ][:1] + # The id-bound prompt and later queued user prompts are never + # historical replay candidates. On the simple path exclude the + # final active user row, matching the normal trim behavior. + excluded_canonical_message_ids: set[str] = set() + if bound_user_message_id: + bound_found = False + for entry in canonical_transcript: + message_id = str(getattr(entry, "message_id", "") or "") + if getattr(entry, "role", None) != "user": + continue + if message_id == bound_user_message_id: + bound_found = True + if bound_found and message_id: + excluded_canonical_message_ids.add(message_id) + elif trim_last_user: + for entry in reversed(canonical_transcript): + if getattr(entry, "role", None) == "user": + message_id = str(getattr(entry, "message_id", "") or "") + if message_id: + excluded_canonical_message_ids.add(message_id) + break + + candidate_entries = ( + [ + entry + for entry in canonical_transcript + if getattr(entry, "role", None) == "user" + and str(getattr(entry, "message_id", "") or "") + not in excluded_canonical_message_ids + and self._attachment_envelope_has_image( + str(getattr(entry, "content", "") or "") + ) + ] + if lookback > 0 + else [] + ) + if requested_attachment_ids: + try: + from opensquilla.session.attachment_manifest import ( + build_attachment_manifest, + ) + + requested_manifest = build_attachment_manifest( + canonical_transcript, + session_id=image_replay_session_id or session_key, + session_key=session_key, + ) + requested_source_message_ids = { + occurrence.source_message_id + for occurrence in requested_manifest.by_ids( + requested_attachment_ids + ) + } + except Exception: # noqa: BLE001 - raw envelope IDs still work + requested_source_message_ids = set() + explicit_matches: list[Any] = [] + for entry in canonical_transcript: + if getattr(entry, "role", None) != "user": + continue + if ( + str(getattr(entry, "message_id", "") or "") + in excluded_canonical_message_ids + ): + continue + entry_message_id = str( + getattr(entry, "message_id", "") or "" + ) + if ( + entry_message_id in requested_source_message_ids + or self._attachment_envelope_contains_ids( + str(getattr(entry, "content", "") or ""), + requested_attachment_ids, + ) + ): + explicit_matches.append(entry) + if entry_message_id: + requested_source_message_ids.add(entry_message_id) + candidate_entries = explicit_matches + if replay_images_natively and requested_source_message_ids: + image_replay_entry_indexes.update( + index + for index, entry in enumerate(transcript) + if str(getattr(entry, "message_id", "") or "") + in requested_source_message_ids + ) + else: + candidate_entries = candidate_entries[-lookback:] + active_message_ids = { + str(getattr(entry, "message_id", "") or "") + for entry in transcript + if getattr(entry, "message_id", None) + } + replayed_active_message_ids = { + str(getattr(transcript[index], "message_id", "") or "") + for index in image_replay_entry_indexes + } + request_image_replay_entries = [ + entry + for entry in candidate_entries + if requested_attachment_ids + or str(getattr(entry, "message_id", "") or "") not in active_message_ids + or str(getattr(entry, "message_id", "") or "") in replayed_active_message_ids + ] + # Requested attachments are injected after ordinary history is + # limited. Do not also send their bytes from an active raw row. + request_image_message_ids = { + str(getattr(entry, "message_id", "") or "") + for entry in request_image_replay_entries + } + image_replay_entry_indexes.difference_update( + index + for index, entry in enumerate(transcript) + if str(getattr(entry, "message_id", "") or "") + in request_image_message_ids + ) attachment_replay_session_id = image_replay_session_id - if attachment_replay_session_id is None and materialize_historical_attachments: + history_has_image_envelope = any( + getattr(entry, "role", None) == "user" + and self._attachment_envelope_has_image( + str(getattr(entry, "content", "") or "") + ) + for entry in transcript + ) + if attachment_replay_session_id is None and ( + materialize_historical_attachments or history_has_image_envelope + ): attachment_replay_session_id = expected_session_id if attachment_replay_session_id is None: - attachment_replay_session_id = await self._resolve_session_id_for_log( - session_key - ) + attachment_replay_session_id = await self._resolve_session_id_for_log(session_key) if attachment_replay_session_id is None: attachment_replay_session_id = session_key history_materializer: AttachmentWorkspaceMaterializer | None = None @@ -13860,6 +14777,7 @@ async def _load_history( image_replay_entry_indexes=image_replay_entry_indexes, media_root=self._attachment_media_root(), session_id=attachment_replay_session_id, + allowed_image_attachment_ids=requested_image_id_filter, materialize_historical_attachments=materialize_historical_attachments, workspace_dir=workspace_dir, historical_materializer=history_materializer, @@ -13867,6 +14785,89 @@ async def _load_history( ) history = list(replay.messages) summary_markers.extend(replay.legacy_summary_markers) + + # Image selection belongs to this request, even when its source row + # lives in the archive or outside the ordinary history window. Bind + # only attachment content as protected input; do not replay old user + # instructions as new instructions alongside it. + request_image_context: list[Message] = [] + if request_image_replay_entries: + for entry in request_image_replay_entries: + raw_content = str(getattr(entry, "content", "") or "") + if not raw_content: + continue + replay_content = self._maybe_unpack_attachments( + raw_content, + preserve_image_attachments=replay_images_natively, + allowed_image_attachment_ids=requested_image_id_filter, + materialize_historical_attachments=materialize_historical_attachments, + media_root=self._attachment_media_root(), + session_id=attachment_replay_session_id, + workspace_dir=workspace_dir, + historical_materializer=history_materializer, + source_message_id=getattr(entry, "message_id", None), + include_envelope_text=False, + ) + request_image_context.extend( + reconstruct_messages_from_entry( + "user", + replay_content, + None, + None, + ) + ) + if bound_image_replay_entries: + # The caller re-appends the bound prompt text, so inject only its + # attachment projection here. This works for both active and + # compacted rows and avoids duplicating the user instruction. + bound_attachment_history: list[Message] = [] + for entry in bound_image_replay_entries: + replay_content = self._maybe_unpack_attachments( + str(getattr(entry, "content", "") or ""), + preserve_image_attachments=( + declared_vision_support != "unsupported" + ), + materialize_historical_attachments=( + materialize_historical_attachments + ), + media_root=self._attachment_media_root(), + session_id=attachment_replay_session_id, + workspace_dir=workspace_dir, + historical_materializer=history_materializer, + source_message_id=getattr(entry, "message_id", None), + include_envelope_text=False, + ) + if replay_content: + bound_attachment_history.extend( + reconstruct_messages_from_entry( + "user", + replay_content, + None, + None, + ) + ) + if bound_attachment_history: + request_image_context.extend(bound_attachment_history) + for replay_message in request_image_context: + if isinstance(replay_message.content, list) and any( + isinstance(block, ContentBlockImage) for block in replay_message.content + ): + # This note belongs only to the request-local projection. A + # later text fallback may replace the blocks with markers. + replay_message.content = [ + ContentBlockText( + text=( + "Image replay context for this request: native image blocks below " + "are preserved originals reattached from earlier conversation turns; " + "no new upload is required. Determine current image availability " + "from these blocks or their fallback markers, not prior assistant " + "claims that an image was not analyzed." + ) + ), + *replay_message.content, + ] + break + agent.set_request_image_context(request_image_context) if restricted_turn: # Context states, durable summaries, and legacy summary markers # were produced before this turn's restricted provider projection. @@ -14056,17 +15057,141 @@ def _attachment_envelope_has_image(content: str) -> bool: return True return False + @staticmethod + def _attachment_envelope_contains_ids( + content: str, + attachment_ids: Sequence[str], + ) -> bool: + """Return whether an envelope names one of the requested occurrences.""" + + if not content or not content.lstrip().startswith("{"): + return False + try: + parsed = json.loads(content) + except (json.JSONDecodeError, ValueError): + return False + if not isinstance(parsed, dict): + return False + wanted = {value.strip() for value in attachment_ids if value.strip()} + if not wanted: + return False + atts = parsed.get("attachments") or [] + if not isinstance(atts, list): + return False + return any( + isinstance(att, dict) + and isinstance(att.get("attachment_id"), str) + and att["attachment_id"].strip() in wanted + for att in atts + ) + + @staticmethod + def _attachment_ids_from_text(content: str) -> tuple[str, ...]: + """Extract bounded canonical attachment references from current input.""" + + if not isinstance(content, str) or "att_" not in content: + return () + from opensquilla.session.attachment_manifest import valid_attachment_id + + result: list[str] = [] + seen: set[str] = set() + for match in re.finditer( + r"(? tuple[str, ...]: + """Read canonical attachment IDs from structured resource references.""" + + from opensquilla.session.attachment_manifest import valid_attachment_id + + result: list[str] = [] + seen: set[str] = set() + for attachment in attachments: + if not isinstance(attachment, Mapping): + continue + for key in ("resourceRef", "resource_ref", "resource"): + raw_ref = attachment.get(key) + if not isinstance(raw_ref, Mapping): + continue + resource_type = str( + raw_ref.get("type") or raw_ref.get("resource_type") or "" + ).strip().lower() + if resource_type != "attachment": + continue + attachment_id = valid_attachment_id( + raw_ref.get("id") or raw_ref.get("resource_id") + ) + if attachment_id is None or attachment_id in seen: + continue + seen.add(attachment_id) + result.append(attachment_id) + return tuple(result) + + @staticmethod + def _attachment_ids_from_envelope( + content: str, + *, + image_only: bool = False, + ) -> tuple[str, ...]: + """Read bounded logical IDs from a persisted attachment envelope.""" + + if not content or not content.lstrip().startswith("{"): + return () + try: + parsed = json.loads(content) + except (json.JSONDecodeError, ValueError): + return () + if not isinstance(parsed, dict): + return () + attachments = parsed.get("attachments") + if not isinstance(attachments, list): + return () + from opensquilla.session.attachment_manifest import valid_attachment_id + + result: list[str] = [] + for attachment in attachments: + if not isinstance(attachment, dict): + continue + if image_only: + media_type = ( + attachment.get("type") + or attachment.get("mime") + or attachment.get("media_type") + ) + if not ( + isinstance(media_type, str) + and media_type.startswith("image/") + ): + continue + attachment_id = valid_attachment_id(attachment.get("attachment_id")) + if attachment_id is not None: + result.append(attachment_id) + return tuple(result) + @staticmethod def _maybe_unpack_attachments( content: str, *, preserve_image_attachments: bool = False, + allowed_image_attachment_ids: frozenset[str] | None = None, materialize_historical_attachments: bool = False, media_root: Path | None = None, session_id: str | None = None, workspace_dir: str | Path | None = None, workspace_attachment_budget_bytes: int | None = None, historical_materializer: AttachmentWorkspaceMaterializer | None = None, + source_message_id: str | None = None, + include_envelope_text: bool = True, ) -> Any: """Reduce persisted attachment envelopes to text-only history. @@ -14114,6 +15239,23 @@ def _maybe_unpack_attachments( omitted: list[str] = [] replay_blocks: list[Any] = [] preserved_image = False + occurrence_ids: dict[int, str] = {} + attachment_identity_session_id = session_id or "history" + try: + from opensquilla.session.attachment_manifest import ( + extract_attachment_occurrences_from_envelope, + ) + + occurrence_ids = { + occurrence.ordinal: occurrence.attachment_id + for occurrence in extract_attachment_occurrences_from_envelope( + content, + session_id=attachment_identity_session_id, + source_message_id=source_message_id or "unknown", + ) + } + except (TypeError, ValueError): + occurrence_ids = {} if not materialize_historical_attachments: historical_materializer = None elif historical_materializer is None and session_id and workspace_dir: @@ -14126,11 +15268,11 @@ def _maybe_unpack_attachments( materializable_mimes=None, disk_budget_bytes=workspace_attachment_budget_bytes, ) - if preserve_image_attachments and text: + if preserve_image_attachments and include_envelope_text and text: from opensquilla.provider.types import ContentBlockText replay_blocks.append(ContentBlockText(text=text)) - for att in atts: + for ordinal, att in enumerate(atts): if not isinstance(att, dict): continue media_type = att.get("type") or att.get("mime") or att.get("media_type") @@ -14151,17 +15293,64 @@ def _maybe_unpack_attachments( name = att.get("name") fallback = "image" if media_type.startswith("image/") else "attachment" label = name if isinstance(name, str) and name.strip() else fallback - if preserve_image_attachments and media_type in _IMAGE_ATTACHMENT_MIMES: + attachment_id = occurrence_ids.get(ordinal) + if attachment_id is None: + # Keep legacy IDs deterministic when an old envelope omitted + # them. This is metadata-only; no bytes or paths enter the + # marker or persisted state. + try: + from opensquilla.session.attachment_manifest import legacy_attachment_id + + attachment_id = legacy_attachment_id( + session_id=attachment_identity_session_id, + message_id=source_message_id or "unknown", + index=ordinal, + sha256=(sha_ref if isinstance(sha_ref, str) else None), + ) + except Exception: # noqa: BLE001 - marker identity is advisory + attachment_id = None + image_replay_allowed = ( + allowed_image_attachment_ids is None + or attachment_id in allowed_image_attachment_ids + ) + if ( + preserve_image_attachments + and image_replay_allowed + and media_type in _IMAGE_ATTACHMENT_MIMES + ): from opensquilla.provider.types import ContentBlockImage if isinstance(data, str) and data: try: base64.b64decode(data, validate=True) except (binascii.Error, ValueError): - omitted.append(f"[attachment unavailable: {label} ({media_type})]") + omitted.append( + image_marker( + ImageMarkerState.UNAVAILABLE, + attachment_id=attachment_id, + ) + ) else: + if ( + allowed_image_attachment_ids is not None + and attachment_id is not None + ): + from opensquilla.provider.types import ContentBlockText + + replay_blocks.append( + ContentBlockText( + text=( + "[historical image " + f"attachment_id={attachment_id}]" + ) + ) + ) replay_blocks.append( - ContentBlockImage(media_type=media_type, data=data) + ContentBlockImage( + media_type=media_type, + data=data, + attachment_id=attachment_id, + ) ) preserved_image = True continue @@ -14178,13 +15367,33 @@ def _maybe_unpack_attachments( ) try: raw_bytes = read_attachment_ref_bytes(ref, media_root=media_root) - except (FileNotFoundError, ValueError) as exc: - omitted.append(f"[attachment unavailable: {label}: {exc}]") + except (FileNotFoundError, ValueError): + omitted.append( + image_marker( + ImageMarkerState.UNAVAILABLE, + attachment_id=attachment_id, + ) + ) else: + if ( + allowed_image_attachment_ids is not None + and attachment_id is not None + ): + from opensquilla.provider.types import ContentBlockText + + replay_blocks.append( + ContentBlockText( + text=( + "[historical image " + f"attachment_id={attachment_id}]" + ) + ) + ) replay_blocks.append( ContentBlockImage( media_type=media_type, data=base64.b64encode(raw_bytes).decode("ascii"), + attachment_id=attachment_id, ) ) preserved_image = True @@ -14231,7 +15440,24 @@ def _maybe_unpack_attachments( ) omitted.append(render_attachment_material_marker(result, prefix=prefix)) continue - omitted.append(f"[historical attachment omitted: {label} ({media_type})]") + if media_type in _IMAGE_ATTACHMENT_MIMES: + marker = image_marker( + ( + ImageMarkerState.UNAVAILABLE + if missing_reason and not data and not sha_ref + else ImageMarkerState.NOT_REREAD + ), + attachment_id=attachment_id, + ) + # Retain the legacy phrase for clients/tests that recognize + # it, while adding the explicit state and stable ID required + # for a model switch after compaction. + omitted.append( + f"[historical attachment omitted: {label} ({media_type}); " + f"{marker[1:-1]}]" + ) + else: + omitted.append(f"[historical attachment omitted: {label} ({media_type})]") if preserved_image: if omitted: from opensquilla.provider.types import ContentBlockText @@ -14239,8 +15465,10 @@ def _maybe_unpack_attachments( replay_blocks.extend(ContentBlockText(text=marker) for marker in omitted) return replay_blocks if not omitted: - return text - return "\n".join([text, *omitted]).strip() + return text if include_envelope_text else "" + return "\n".join( + [*((text,) if include_envelope_text and text else ()), *omitted] + ).strip() @staticmethod def _maybe_unpack_assistant_artifacts(content: str) -> str: @@ -14407,7 +15635,19 @@ def _build_attachment_messages( continue if media_type in _IMAGE_ATTACHMENT_MIMES: - attachment_blocks.append(ContentBlockImage(media_type=media_type, data=data)) + raw_attachment_id = att.get("attachment_id") + attachment_blocks.append( + ContentBlockImage( + media_type=media_type, + data=data, + attachment_id=( + raw_attachment_id.strip()[:164] + if isinstance(raw_attachment_id, str) + and raw_attachment_id.strip() + else None + ), + ) + ) if material_marker: attachment_blocks.append(ContentBlockText(text=material_marker)) elif media_type == "application/pdf": diff --git a/src/opensquilla/engine/selector_override.py b/src/opensquilla/engine/selector_override.py index a1d5dcca0..886267034 100644 --- a/src/opensquilla/engine/selector_override.py +++ b/src/opensquilla/engine/selector_override.py @@ -532,6 +532,74 @@ def _disable_selector_provider_state_replay( disable() +def resolve_strict_router_fallback_chain( + config: Any, + turn_metadata: dict[str, Any], + *, + active_provider_id: str, + session_key: str = "", +) -> list[object] | None: + """Resolve the private executable form of a strict Router fallback chain. + + Turn metadata retains only provider/model identifiers. Cross-provider + entries need their own configured credentials before the selector can + install them; same-provider entries remain compact mappings and reuse the + selector's active credentials. Unresolvable cross-provider entries are + omitted instead of borrowing another provider's authority. + """ + + if turn_metadata.get("router_fallback_strict") is not True: + return None + raw_chain = turn_metadata.get("router_fallback_chain") + if not isinstance(raw_chain, list): + return None + + active_provider = str(active_provider_id or "").strip().lower() + router_cfg = getattr(config, "squilla_router", None) + cross_provider_enabled = bool( + getattr(router_cfg, "cross_provider_tiers", False) + ) + mismatch_policy = str( + getattr(router_cfg, "tier_provider_mismatch", "route") or "route" + ).strip().lower() + resolved_chain: list[object] = [] + for raw_entry in raw_chain: + if not isinstance(raw_entry, dict): + continue + model = str(raw_entry.get("model") or "").strip() + if not model: + continue + provider = str(raw_entry.get("provider") or active_provider).strip().lower() + if not provider: + continue + if provider == active_provider: + entry = dict(raw_entry) + entry["provider"] = active_provider + resolved_chain.append(entry) + continue + if not cross_provider_enabled: + if mismatch_policy != "veto": + # Preserve the existing route-mode contract: foreign model ids + # execute through the configured active aggregator/provider. + entry = dict(raw_entry) + entry["provider"] = active_provider + resolved_chain.append(entry) + continue + + # Keep fallback credential resolution private. In particular, do not + # overwrite the selected head's routed_provider_resolution telemetry. + resolved = resolve_tier_provider_config( + config, + provider, + model, + session_key=session_key, + turn_metadata=None, + ) + if resolved is not None: + resolved_chain.append(resolved) + return resolved_chain + + def apply_model_override( selector: Any, model: str, @@ -539,6 +607,7 @@ def apply_model_override( turn_metadata: dict[str, Any], realign_routed_model: bool, tier_provider_config: Any | None = None, + strict_router_fallback_chain: Sequence[object] | None = None, ) -> Any: """Apply ``model`` to the cloned selector and resolve the provider. @@ -550,8 +619,9 @@ def apply_model_override( ``routed_model`` intentionally records the would-be routed choice. ``tier_provider_config`` switches the turn to a cross-provider tier's - full ProviderConfig; the router fallback chain is skipped in that case - (its entries are same-provider models of the provider being left). + full ProviderConfig. A strict Router call may also supply an independently + resolved fallback chain whose cross-provider entries retain their own + configured credentials. """ if turn_metadata.get("large_context_capacity_blocked") is True: from opensquilla.engine.capacity_admission import LargeContextCapacityError @@ -562,6 +632,48 @@ def apply_model_override( ) raise LargeContextCapacityError(reason) + router_fallback_chain = ( + list(strict_router_fallback_chain) + if strict_router_fallback_chain is not None + else ( + turn_metadata.get("router_fallback_chain") + if turn_metadata.get("routing_applied") is True + else None + ) + ) + + def install_strict_model_chain(strict_model: str) -> None: + """Install a verifiably isolated Router chain or fail closed. + + The optional third-party selector seam predates + ``preserve_existing_tail``. Calling its legacy two-argument hook would + leave an opaque configured/plugin tail executable, which violates the + image route's authorization boundary. A selector that cannot prove it + cleared that tail must not execute the turn. + """ + + if not isinstance(router_fallback_chain, list): + raise RuntimeError("strict router fallback chain is unavailable") + override_with_chain = getattr( + selector, + "override_model_with_fallback_chain", + None, + ) + if not callable(override_with_chain): + raise RuntimeError( + "selector does not support strict router fallback isolation" + ) + try: + override_with_chain( + strict_model, + router_fallback_chain, + preserve_existing_tail=False, + ) + except TypeError as exc: + raise RuntimeError( + "selector does not support strict router fallback isolation" + ) from exc + if tier_provider_config is not None and hasattr(selector, "override_provider_config"): _require_provider_config_capacity( tier_provider_config, @@ -572,15 +684,39 @@ def apply_model_override( ), ) if turn_metadata.get("router_fallback_strict") is True: - try: - selector.override_provider_config( - tier_provider_config, - preserve_existing_tail=False, - ) - except TypeError as exc: - raise RuntimeError( - "selector does not support strict artifact fallback isolation" - ) from exc + override_provider_with_chain = getattr( + selector, + "override_provider_config_with_fallback_chain", + None, + ) + if callable(override_provider_with_chain) and isinstance( + router_fallback_chain, + list, + ): + try: + override_provider_with_chain( + tier_provider_config, + router_fallback_chain, + preserve_existing_tail=False, + ) + except TypeError as exc: + raise RuntimeError( + "selector does not support strict router fallback isolation" + ) from exc + else: + if router_fallback_chain: + raise RuntimeError( + "selector does not support strict router fallback isolation" + ) + try: + selector.override_provider_config( + tier_provider_config, + preserve_existing_tail=False, + ) + except TypeError as exc: + raise RuntimeError( + "selector does not support strict artifact fallback isolation" + ) from exc else: bounded_provider_override = getattr( selector, @@ -662,13 +798,13 @@ def apply_model_override( turn_metadata["routed_provider_fallback_model"] = str( getattr(current_config, "model", "") or "" ) + if turn_metadata.get("router_fallback_strict") is True: + current_model = str(getattr(current_config, "model", "") or "").strip() + if not current_model: + raise RuntimeError("strict router fallback head is unavailable") + install_strict_model_chain(current_model) return _resolve_and_record_execution(selector, turn_metadata) - router_fallback_chain = ( - turn_metadata.get("router_fallback_chain") - if turn_metadata.get("routing_applied") is True - else None - ) override_with_fallback_chain = getattr( selector, "override_model_with_fallback_chain", @@ -680,22 +816,8 @@ def apply_model_override( None, ) bounded_fallbacks_required = _bounded_fallback_chain_required(turn_metadata) - if ( - turn_metadata.get("router_fallback_strict") is True - and callable(override_with_fallback_chain) - and isinstance(router_fallback_chain, list) - ): - try: - override_with_fallback_chain( - model, - router_fallback_chain, - preserve_existing_tail=False, - ) - except TypeError: - # Compatibility for third-party selector shims that implement - # the older two-argument hook. A strict Artifact turn must not - # silently retain their unknown fallback tail. - selector.override_model(model) + if turn_metadata.get("router_fallback_strict") is True: + install_strict_model_chain(model) elif bounded_fallbacks_required and callable(override_with_bounded_fallback_chain): approved_router_fallbacks = _capacity_approved_fallback_entries( selector, diff --git a/src/opensquilla/engine/steps/squilla_router.py b/src/opensquilla/engine/steps/squilla_router.py index 96dfeae20..1f24c9428 100644 --- a/src/opensquilla/engine/steps/squilla_router.py +++ b/src/opensquilla/engine/steps/squilla_router.py @@ -12,6 +12,7 @@ import os import threading import time +from collections.abc import Mapping from inspect import Parameter, signature from pathlib import Path from typing import Any, Protocol, cast @@ -48,7 +49,6 @@ from opensquilla.engine.steps.router_decision_record import stage_router_decision from opensquilla.provider.context_capabilities import provider_state_continuity_diagnostic from opensquilla.provider.model_catalog import shared_catalog -from opensquilla.provider.types import ModelCapabilities from opensquilla.router_control import RouterControlHoldStore from opensquilla.router_runtime_diagnostics import ( classify_router_runtime_error, @@ -57,7 +57,6 @@ from opensquilla.router_tiers import ( DEFAULT_TEXT_TIER, HIGHEST_TEXT_TIER, - IMAGE_TIER, TEXT_TIERS, TierConfig, effective_ensemble_selection_mode, @@ -163,6 +162,126 @@ def _router_text_fallback_chain( return chain +def _configured_text_tiers(tiers: Mapping[str, Any] | object) -> list[str]: + """Return executable, user-configured ``c0``-``c3`` tier ids. + + ``image_model`` is intentionally not part of this list. It is retained in + the configuration contract for backwards compatibility, but Router image + execution must never manufacture a fifth deployment behind the user's + four-tier ladder. A tier with a blank model is likewise not executable. + """ + + if not isinstance(tiers, Mapping): + return [] + configured: list[str] = [] + for tier_name in TEXT_TIERS: + raw = tiers.get(tier_name) + if not isinstance(raw, Mapping): + continue + if bool(raw.get("image_only", False)): + continue + if str(raw.get("model") or "").strip(): + configured.append(tier_name) + return configured + + +def _tier_deployment_vision_support(ctx: TurnContext, raw: Mapping[str, Any]) -> str: + """Resolve one configured c-tier's tri-state vision evidence. + + The shared deployment catalog owns capability evidence. Legacy tier + switches remain readable but do not override it. Resolver failures remain + ``unknown`` so the provider can be probed exactly once by the execution + layer. This helper deliberately does not infer support from a model name. + """ + + model = str(raw.get("model") or "").strip() + if not model: + return "unsupported" + llm = getattr(getattr(ctx, "config", None), "llm", None) + active_provider = str(getattr(llm, "provider", "") or "").strip() + router = getattr(getattr(ctx, "config", None), "squilla_router", None) + tier_provider = str(raw.get("provider") or "").strip() + provider = ( + tier_provider + if bool(getattr(router, "cross_provider_tiers", False)) and tier_provider + else active_provider or tier_provider + ) + if not provider: + return "unknown" + resolver = getattr(shared_catalog(), "resolve_deployment_vision_support", None) + if not callable(resolver): + return "unknown" + # Credentials are only authoritative for the active provider. Cross- + # provider tiers still get a catalog/snapshot answer, but must not inherit + # another deployment's secrets. + same_authority = provider.lower() == active_provider.lower() + try: + resolved = resolver( + model, + provider=provider, + api_key=str(getattr(llm, "api_key", "") or "") if same_authority else "", + base_url=str(getattr(llm, "base_url", "") or "") if same_authority else "", + proxy=str(getattr(llm, "proxy", "") or "") if same_authority else "", + ) + except Exception: # noqa: BLE001 - capability discovery is best effort + return "unknown" + return resolved if resolved in {"supported", "unsupported"} else "unknown" + + +def _router_image_fallback_chain( + selected_tier: object, + tiers: Mapping[str, Any] | object, + *, + tier_support: Mapping[str, str] | None = None, + c3_fusion_active: bool = False, + minimum_tier: object | None = None, +) -> list[dict[str, str]]: + """Build the remaining authorized native-image probe chain. + + Image requests may start on a tier with known support or unknown + capability. Catalog-proven text-only tiers have already supplied a + definitive answer and therefore need no physical image request. Supported and + unknown configured tiers remain probeable; after the last rejection the + Agent applies its Direct-style marker projection. The chain is canonical + and deterministic, independent of TOML declaration order, and never + includes ``image_model``. + """ + + if not isinstance(tiers, Mapping): + return [] + selected = normalize_text_tier(selected_tier) + minimum_index = ( + tier_index(normalize_text_tier(minimum_tier)) + if minimum_tier is not None + else -1 + ) + result: list[dict[str, str]] = [] + for tier_name in _configured_text_tiers(tiers): + if tier_name == selected: + continue + if minimum_index >= 0 and tier_index(tier_name) < minimum_index: + continue + if c3_fusion_active and tier_name == HIGHEST_TEXT_TIER: + continue + if tier_support is not None and tier_support.get(tier_name) == "unsupported": + continue + raw = tiers.get(tier_name) + if not isinstance(raw, Mapping): + continue + model = str(raw.get("model") or "").strip() + if not model: + continue + entry = {"tier": tier_name, "model": model} + support = tier_support.get(tier_name) if tier_support is not None else None + if support in {"supported", "unsupported", "unknown"}: + entry["vision_support"] = support + provider = str(raw.get("provider") or "").strip() + if provider: + entry["provider"] = provider + result.append(entry) + return result + + class RoutingHistoryStore: """Per-session routing history with bounded size and eviction. @@ -1188,7 +1307,9 @@ def _capacity_safe_tier( if not isinstance(raw, dict): continue tier = TierConfig.from_value(raw) - if not tier.model or (requires_image and not tier.supports_image): + if not tier.model or ( + requires_image and _tier_deployment_vision_support(ctx, raw) == "unsupported" + ): continue declared_provider = (tier.provider or active_provider).strip().lower() if active_provider_only and declared_provider != active_provider: @@ -1260,29 +1381,68 @@ async def finalize_squilla_router_capacity(ctx: TurnContext) -> TurnContext: request_input_tokens = _complete_request_estimated_tokens(ctx, semantic_message) ctx.metadata["large_context_capacity_required"] = True - minimum_tier = normalize_text_tier( + minimum_context_tier = normalize_text_tier( ctx.metadata.get("large_context_floor_min_tier") ) + configured_text_tiers = _configured_text_tiers(tiers) + artifact_facts = _artifact_routing_facts_for_turn(ctx) + artifact_floor = effective_artifact_floor(artifact_facts, configured_text_tiers) + if artifact_facts is not None and artifact_floor is None: + raise ArtifactRoutingUnavailableError(artifact_facts, configured_text_tiers) + execution_floor_candidates = [ + tier + for tier in (artifact_floor, minimum_context_tier) + if tier is not None + ] + minimum_tier = ( + max(execution_floor_candidates, key=tier_index) + if execution_floor_candidates + else None + ) selected_raw = str(ctx.metadata.get("routed_tier") or "").strip() selected_tier = selected_raw if selected_raw in tiers else None requires_image = _attachments_include_image(ctx.attachments) or ( ctx.metadata.get("router_vision_followup_needs_image") is True ) - valid_tiers = [ - name - for name, raw in tiers.items() - if isinstance(raw, dict) - and not raw.get("image_only", False) - and (not requires_image or raw.get("supports_image", False)) - ] - if requires_image: - valid_tiers.extend( + # A marker downgrade is a text request from the capacity stage's point of + # view. Keep compaction/capacity admission active and consider every + # configured c-tier, rather than filtering the ladder down to image-only + # declarations (which would incorrectly fail when all four are text-only). + projection_required = ctx.metadata.get("image_input_projection_required") is True + if requires_image and projection_required and minimum_tier is None: + # Without an Artifact or large-context floor, a marker-only request + # needs no special attachment-capacity revalidation; the ordinary + # provider admission path still checks its exact text payload. + return ctx + if requires_image and not projection_required: + support_facts = ctx.metadata.get("router_image_tier_support") + c3_fusion_active = bool( + getattr(getattr(ctx.config, "llm_ensemble", None), "enabled", False) + ) or tier_ensemble_active(tiers, HIGHEST_TEXT_TIER) + valid_tiers = [ + name + for name in _configured_text_tiers(tiers) + if ( + ( + not isinstance(support_facts, Mapping) + or support_facts.get(name) in {"supported", "unknown"} + ) + and not (c3_fusion_active and name == HIGHEST_TEXT_TIER) + ) + ] + requires_image_for_capacity = False + elif requires_image and projection_required: + valid_tiers = _configured_text_tiers(tiers) + requires_image_for_capacity = False + else: + valid_tiers = [ name for name, raw in tiers.items() if isinstance(raw, dict) - and raw.get("image_only", False) - and raw.get("supports_image", False) - ) + and not raw.get("image_only", False) + and str(raw.get("model") or "").strip() + ] + requires_image_for_capacity = False valid_tiers = sorted( dict.fromkeys(valid_tiers), key=lambda name: (0, tier_index(name)) if tier_index(name) >= 0 else (1, 0), @@ -1304,7 +1464,7 @@ async def finalize_squilla_router_capacity(ctx: TurnContext) -> TurnContext: minimum_tier=admission_minimum_tier, material_tokens=material_tokens, request_input_tokens=request_input_tokens, - requires_image=requires_image, + requires_image=requires_image_for_capacity, active_provider_only=_capacity_active_provider_only(router_cfg), thinking_mode=(thinking_mode if isinstance(thinking_mode, str) else None), rollout_phase=str(ctx.metadata.get("rollout_phase") or "full"), @@ -1329,10 +1489,44 @@ async def finalize_squilla_router_capacity(ctx: TurnContext) -> TurnContext: ctx.metadata["routed_model"] = ctx.model ctx.metadata["routing_applied"] = True ctx.metadata["applied_model"] = ctx.model - ctx.metadata["router_fallback_chain"] = _router_text_fallback_chain( - capacity_tier, - tiers, - minimum_tier, + if requires_image: + raw_tier_support = ctx.metadata.get("router_image_tier_support") + selected_vision_support = ( + raw_tier_support.get(capacity_tier, "unknown") + if isinstance(raw_tier_support, Mapping) + else "unknown" + ) + ctx.metadata["routed_model_vision_support"] = ( + selected_vision_support + if selected_vision_support + in {"supported", "unsupported", "unknown"} + and ctx.metadata.get("image_input_mode") != "marker" + else "unsupported" + ) + ctx.metadata["router_fallback_chain"] = ( + _router_image_fallback_chain( + capacity_tier, + tiers, + tier_support=( + ctx.metadata.get("router_image_tier_support") + if isinstance( + ctx.metadata.get("router_image_tier_support"), + Mapping, + ) + else None + ), + c3_fusion_active=bool( + getattr(getattr(ctx.config, "llm_ensemble", None), "enabled", False) + ) + or tier_ensemble_active(tiers, HIGHEST_TEXT_TIER), + minimum_tier=minimum_tier, + ) + if requires_image + else _router_text_fallback_chain( + capacity_tier, + tiers, + minimum_tier, + ) ) ctx.metadata["large_context_thinking_budget_tokens"] = ( _route_thinking_budget_tokens( @@ -1348,6 +1542,7 @@ async def finalize_squilla_router_capacity(ctx: TurnContext) -> TurnContext: tiers, capacity_tier, routing_applied=True, + force_direct=requires_image and not projection_required, ) stage_router_decision( ctx, @@ -1374,6 +1569,7 @@ def _context_window_tokens(ctx: TurnContext, router_cfg: object) -> int: def _tier_capability_facts( + ctx: TurnContext, tiers: dict, valid_tiers: list[str], active_provider: str, @@ -1384,11 +1580,8 @@ def _tier_capability_facts( data only). Every field is emitted as ``None`` unless the shared model catalog gives a definite signal, so the gate never acts on ignorance: - - ``supports_vision`` is known only when the resolved entry is NOT - synthesized and ``get_capabilities`` returned something other than - the empty :class:`ModelCapabilities` — an empty result covers both - "no layer knew any capability" and the anthropic/ollama flag-gated - early return, none of which is a definite non-vision signal. + - ``supports_vision`` uses the same deployment-scoped tri-state evidence + as image routing. Missing metadata is not a definite non-vision signal. - ``context_window`` is known only when ``resolve_context_window_with_source`` attributes the value to the catalog (live/snapshot/corrections) or to a per-model ``[models.*]`` @@ -1396,7 +1589,6 @@ def _tier_capability_facts( estimates, not knowledge. """ catalog = shared_catalog() - empty_capabilities = ModelCapabilities() facts: dict[str, TierCapability] = {} for name in valid_tiers: tier = TierConfig.from_value(tiers.get(name)) @@ -1404,12 +1596,8 @@ def _tier_capability_facts( facts[name] = TierCapability() continue provider = (tier.provider or active_provider or "").strip().lower() - supports_vision: bool | None = None - entry = catalog.resolve_entry(tier.model, provider=provider) - if entry.source != "synthesized": - capabilities = catalog.get_capabilities(tier.model, provider_name=provider) - if capabilities != empty_capabilities: - supports_vision = capabilities.supports_vision + support = _tier_deployment_vision_support(ctx, tiers[name]) + supports_vision = None if support == "unknown" else support == "supported" window, window_source = catalog.resolve_context_window_with_source(tier.model, provider) # Operator-declared [models.*] windows count as definite facts for the # capability gate, same as catalog knowledge; only engine defaults @@ -1581,6 +1769,7 @@ def _flag_tier_provider_mismatch( tier_name: str, *, routing_applied: bool, + force_direct: bool = False, ) -> None: """Record the routed tier's provider; warn on unexecutable mismatches. @@ -1597,16 +1786,20 @@ def _flag_tier_provider_mismatch( getattr(getattr(ctx.config, "llm_ensemble", None), "enabled", False) ) shared_selection_mode = effective_ensemble_selection_mode(ctx.config) - provider_role = tier_provider_role( - tier_name, - tiers.get(tier_name), - shared_selection_mode=shared_selection_mode, - router_dynamic_members_active=router_dynamic_tier_members_active( - tiers, + provider_role = ( + "direct" + if force_direct + else tier_provider_role( + tier_name, + tiers.get(tier_name), shared_selection_mode=shared_selection_mode, + router_dynamic_members_active=router_dynamic_tier_members_active( + tiers, + shared_selection_mode=shared_selection_mode, + ensemble_globally_enabled=ensemble_globally_enabled, + ), ensemble_globally_enabled=ensemble_globally_enabled, - ), - ensemble_globally_enabled=ensemble_globally_enabled, + ) ) ctx.metadata["router_tier_provider_role"] = provider_role if provider_role in {"dormant_draft", "blocked"}: @@ -1657,6 +1850,7 @@ def _apply_provider_mismatch_veto( prompt_policy: str | None, *, routing_applied: bool, + force_direct: bool = False, ) -> tuple[RoutingDecision, str | None, str | None]: """Rebind a mismatched classify-path decision when veto mode is on. @@ -1682,12 +1876,16 @@ def _apply_provider_mismatch_veto( shared_selection_mode=shared_selection_mode, ensemble_globally_enabled=ensemble_globally_enabled, ) - selected_role = tier_provider_role( - decision.tier, - tiers.get(decision.tier), - shared_selection_mode=shared_selection_mode, - router_dynamic_members_active=dynamic_members_active, - ensemble_globally_enabled=ensemble_globally_enabled, + selected_role = ( + "direct" + if force_direct + else tier_provider_role( + decision.tier, + tiers.get(decision.tier), + shared_selection_mode=shared_selection_mode, + router_dynamic_members_active=dynamic_members_active, + ensemble_globally_enabled=ensemble_globally_enabled, + ) ) if selected_role in {"dormant_draft", "blocked"}: return decision, thinking_mode, prompt_policy @@ -1698,12 +1896,16 @@ def _apply_provider_mismatch_veto( policy_tiers = {name: dict(value) for name, value in tiers.items()} policy_valid_tiers: list[str] = [] for tier_name in valid_tiers: - role = tier_provider_role( - tier_name, - policy_tiers.get(tier_name), - shared_selection_mode=shared_selection_mode, - router_dynamic_members_active=dynamic_members_active, - ensemble_globally_enabled=ensemble_globally_enabled, + role = ( + "direct" + if force_direct + else tier_provider_role( + tier_name, + policy_tiers.get(tier_name), + shared_selection_mode=shared_selection_mode, + router_dynamic_members_active=dynamic_members_active, + ensemble_globally_enabled=ensemble_globally_enabled, + ) ) if role == "blocked": continue @@ -1874,10 +2076,12 @@ async def apply_squilla_router(ctx: TurnContext) -> TurnContext: proof_max_chars ) - # Image-aware routing: skip ML and pick directly from supports_image tiers - # for current uploads. Historical images require the upstream semantic - # follow-up gate; recent-image/sticky metadata alone is observability and - # replay context, not enough to force vision. + # Image-aware routing: skip ML and pick directly from the user's + # configured c0-c3 deployments for current uploads. ``image_model`` is a + # legacy presentation/configuration field, not an executable fifth leg. + # Historical images require the upstream semantic follow-up gate; + # recent-image/sticky metadata alone is observability and replay context, + # not enough to force vision. # # This runs BEFORE the empty-text guard below: the image route is # deterministic and never consumes the message text, so an image turn with @@ -1887,88 +2091,223 @@ async def apply_squilla_router(ctx: TurnContext) -> TurnContext: history_gate_needs_image = ( ctx.metadata.get("router_vision_followup_needs_image") is True ) - # Computed once and reused below by both the bypass and the policy - # engine's capability gate (which must not recompute the signal). On the - # classify path this is always False today — the bypass routes or raises - # for every image turn — which is exactly the gate's no-op default. + # Computed once and reused below by both the bypass and the policy engine's + # capability gate (which must not recompute the signal). turn_needs_image = current_turn_has_image or history_gate_needs_image if turn_needs_image: c3_fusion_active = bool( getattr(getattr(ctx.config, "llm_ensemble", None), "enabled", False) ) or tier_ensemble_active(tiers, HIGHEST_TEXT_TIER) - image_capable_tiers = { - name: tier - for name, tier in tiers.items() - if tier.get("supports_image", False) + configured_tiers = _configured_text_tiers(tiers) + minimum_execution_index = tier_index(minimum_execution_tier) + execution_eligible_tiers = [ + name + for name in configured_tiers + if minimum_execution_index < 0 + or tier_index(name) >= minimum_execution_index + ] + tier_support: dict[str, str] = {} + for name in configured_tiers: + raw = tiers.get(name) + if isinstance(raw, Mapping): + tier_support[name] = _tier_deployment_vision_support(ctx, raw) + + # A fusion C3 deployment is an Ensemble text leg and therefore cannot + # consume image blocks, even when its draft row says supports_image. + executable_image_tiers = [ + name + for name in execution_eligible_tiers + if tier_support.get(name) in {"supported", "unknown"} and not (c3_fusion_active and name == HIGHEST_TEXT_TIER) - } - image_tiers = { - name: tier - for name, tier in image_capable_tiers.items() - if str(tier.get("model") or "").strip() - } - if not image_tiers: - log.warning( - "squilla_router.no_image_tier", - note="image detected but no executable supports_image tier", - c3_fusion_active=c3_fusion_active, - empty_model_tiers=sorted(image_capable_tiers), - ) - ctx.metadata["image_input_forced_rejection_reason"] = ( - "router_image_route_unavailable" + ] + explicitly_unsupported = [ + name + for name in execution_eligible_tiers + if tier_support.get(name) == "unsupported" + ] + + image_route_reason = "current_turn" if current_turn_has_image else "gate_history" + history_turns = 1 + if image_route_reason == "gate_history": + history_turns = max( + 1, + int(getattr(router_cfg, "vision_history_lookback_turns", 8) or 1), ) - ctx.metadata["image_input_mode"] = "rejected" - ctx.metadata["image_input_reason"] = "router_image_route_unavailable" - return ctx - ordered_image_tiers = sorted( - image_tiers, - key=lambda name: ( - tier_index(name) < 0, - tier_index(name), - ), - ) - if IMAGE_TIER in image_tiers: - ordered_image_tiers = [ - IMAGE_TIER, - *(name for name in ordered_image_tiers if name != IMAGE_TIER), - ] - # The dedicated image tier owns image requests regardless of TOML - # declaration order. Other image-capable tiers remain deterministic - # fallbacks, while an active C3 fusion tier is never one of them. - tier_name = ordered_image_tiers[0] - if minimum_context_tier is not None: - safe_image_tier = _capacity_safe_tier( - ctx, - router_cfg, - tiers, - ordered_image_tiers, - minimum_tier=minimum_context_tier, - material_tokens=material_estimated_tokens, - request_input_tokens=material_estimated_tokens, - requires_image=True, - active_provider_only=_capacity_active_provider_only(router_cfg), - rollout_phase=rollout_phase, + ctx.metadata["image_route_reason"] = image_route_reason + ctx.metadata["route_max_history_turns"] = history_turns + ctx.metadata["router_image_tier_support"] = dict(tier_support) + ctx.metadata["router_image_configured_tiers"] = list(configured_tiers) + + if executable_image_tiers: + # Prefer catalog-proven support over an unknown probe, then + # use canonical c0 TurnContext: ctx.metadata["applied_model"] = ctx.model ctx.metadata["routing_confidence"] = decision.confidence ctx.metadata["routing_source"] = decision.source - if attachment_capacity_required or minimum_context_tier is not None: - ctx.metadata["router_fallback_chain"] = [ - entry - for entry in _router_text_fallback_chain( - decision.tier, - tiers, - minimum_context_tier, - allow_stronger_fallbacks=artifact_facts is not None, - ) - if bool(tiers.get(entry["tier"], {}).get("supports_image", False)) - ] - image_route_reason = "current_turn" if current_turn_has_image else "gate_history" - ctx.metadata["image_route_reason"] = image_route_reason - history_turns = 1 - if image_route_reason == "gate_history": - history_turns = max( - 1, - int(getattr(router_cfg, "vision_history_lookback_turns", 8) or 1), - ) - ctx.metadata["route_max_history_turns"] = history_turns + ctx.metadata["image_input_mode"] = image_input_mode + ctx.metadata["image_input_reason"] = image_input_reason + selected_vision_support = tier_support.get(decision.tier, "unknown") + ctx.metadata["routed_model_vision_support"] = ( + selected_vision_support + if image_input_mode == "native" + and selected_vision_support in {"supported", "unsupported", "unknown"} + else "unsupported" + ) + # Image routes are strict: selector configuration must not append an + # unrelated default tail. Only supported/unknown c-tier deployments + # remain physical probes; explicit denials already count toward the + # four-tier exhaustion decision. + ctx.metadata["router_fallback_strict"] = True + ctx.metadata["router_fallback_chain"] = _router_image_fallback_chain( + decision.tier, + tiers, + tier_support=tier_support, + c3_fusion_active=c3_fusion_active, + minimum_tier=minimum_execution_tier, + ) ctx.metadata.update(_compute_savings(decision.model, tiers)) - # Record the image tier's provider (and assess cross-provider/mismatch) - # like the hold and classify paths — without this, a vision tier that - # declares provider=X never executes the cross-provider switch and no - # mismatch telemetry is emitted. - _flag_tier_provider_mismatch(ctx, tiers, decision.tier, routing_applied=True) - _record_thinking_metadata(ctx, router_cfg, image_tiers[tier_name]) + _flag_tier_provider_mismatch( + ctx, + tiers, + decision.tier, + routing_applied=True, + force_direct=True, + ) + _record_thinking_metadata(ctx, router_cfg, tier_cfg) if attachment_capacity_required or minimum_context_tier is not None: ctx.metadata["large_context_thinking_budget_tokens"] = ( _route_thinking_budget_tokens( ctx, router_cfg, - image_tiers[tier_name], + tier_cfg, rollout_phase=rollout_phase, ) ) stage_router_decision(ctx, decision=decision) - log.debug("squilla_router.image_routed", tier=decision.tier, model=decision.model) return ctx # Empty routing text cannot be classified, but a validated Artifact mutation @@ -2283,9 +2624,10 @@ async def apply_squilla_router(ctx: TurnContext) -> TurnContext: default = normalize_text_tier(getattr(router_cfg, "default_tier", DEFAULT_TEXT_TIER)) if default is None: default = DEFAULT_TEXT_TIER - tier_name = default if default in tiers else next(iter(tiers), None) - if tier_name is None: + fallback_tier = default if default in tiers else next(iter(tiers), "") + if not fallback_tier: return ctx + tier_name = fallback_tier confidence = 0.0 source = "default" probs = synthetic_one_hot(tier_name) @@ -2351,6 +2693,7 @@ async def apply_squilla_router(ctx: TurnContext) -> TurnContext: context_window_tokens=_context_window_tokens(ctx, router_cfg), turn_has_image=turn_needs_image, tier_capabilities=_tier_capability_facts( + ctx, tiers, valid_tiers, str(getattr(getattr(ctx.config, "llm", None), "provider", "") or ""), diff --git a/src/opensquilla/engine/steps/vision_followup_gate.py b/src/opensquilla/engine/steps/vision_followup_gate.py index a66a0d903..c2385b131 100644 --- a/src/opensquilla/engine/steps/vision_followup_gate.py +++ b/src/opensquilla/engine/steps/vision_followup_gate.py @@ -360,6 +360,18 @@ async def apply_vision_followup_gate(ctx: TurnContext) -> TurnContext: if _attachments_include_image(ctx.attachments): ctx.metadata["router_vision_followup_gate_decision"] = "current_image" return ctx + if ctx.metadata.get("image_intent_attachment_ids"): + if _current_text_explicitly_opts_out_image(ctx): + _apply_explicit_opt_out(ctx) + else: + _apply_explicit_previous_image_request(ctx) + ctx.metadata["router_vision_followup_gate_source"] = ( + "explicit_attachment_id" + ) + ctx.metadata["router_vision_followup_gate_reason"] = ( + "current turn references a canonical attachment ID" + ) + return ctx if ctx.metadata.get("router_history_has_recent_image") is not True: ctx.metadata["router_vision_followup_gate_decision"] = "not_applicable" return ctx diff --git a/src/opensquilla/engine/turn_runner/agent_bootstrap_stage.py b/src/opensquilla/engine/turn_runner/agent_bootstrap_stage.py index b0171d6aa..5e9ef6ad5 100644 --- a/src/opensquilla/engine/turn_runner/agent_bootstrap_stage.py +++ b/src/opensquilla/engine/turn_runner/agent_bootstrap_stage.py @@ -562,7 +562,11 @@ def _route_max_history_turns(metadata: dict[str, Any]) -> int: def _preserve_historical_images(metadata: dict[str, Any]) -> bool: image_route_reason = metadata.get("image_route_reason") - return image_route_reason in {"current_turn", "gate_history"} + return bool( + image_route_reason == "gate_history" + or metadata.get("router_vision_followup_needs_image") is True + or metadata.get("image_intent_attachment_ids") + ) @runtime_checkable @@ -858,6 +862,68 @@ async def run( turn=inp.turn, ) agent_metadata = inp.turn.metadata + routed_model_vision_support = agent_metadata.get( + "routed_model_vision_support" + ) + effective_model_vision_support: Literal[ + "supported", "unsupported", "unknown" + ] = ( + routed_model_vision_support + if routed_model_vision_support + in {"supported", "unsupported", "unknown"} + else catalog.vision_support + ) + route_provider = str( + agent_metadata.get("routed_provider") + or inp.active_provider_id + or "" + ) + fallback_sources = ( + agent_metadata.get("router_fallback_chain"), + agent_metadata.get("selector_execution_chain"), + ) + fallback_vision_support: dict[ + tuple[str, str], + Literal["supported", "unsupported", "unknown"], + ] = {} + fallback_vision_support_by_model: dict[ + str, + Literal["supported", "unsupported", "unknown"] | None, + ] = {} + for raw_fallbacks in fallback_sources: + if not isinstance(raw_fallbacks, list): + continue + for raw_fallback in raw_fallbacks: + if not isinstance(raw_fallback, dict): + continue + fallback_model = str(raw_fallback.get("model") or "").strip() + fallback_provider = str( + raw_fallback.get("provider") or route_provider + ).strip() + support = raw_fallback.get("vision_support") + if ( + fallback_model + and fallback_provider + and support in {"supported", "unsupported", "unknown"} + ): + fallback_vision_support.setdefault( + (fallback_provider.lower(), fallback_model), + support, + ) + prior_support = fallback_vision_support_by_model.get( + fallback_model + ) + if ( + fallback_model not in fallback_vision_support_by_model + or prior_support == support + ): + fallback_vision_support_by_model[fallback_model] = support + else: + # Provider-mismatch route mode may execute a configured + # foreign model id through the active aggregator. Use + # model-only evidence only when every declaration for + # that id agrees. + fallback_vision_support_by_model[fallback_model] = None fallback_capabilities: dict[ tuple[str, str], tuple[int, int, ModelCapabilities | None], @@ -913,7 +979,17 @@ async def run( ) ) private_fallback_vision_support.append( - (deployment, fallback_catalog.vision_support) + ( + deployment, + fallback_vision_support.get( + (fallback_provider.lower(), fallback_model), + fallback_vision_support_by_model.get( + fallback_model, + fallback_catalog.vision_support, + ) + or fallback_catalog.vision_support, + ), + ) ) fallback_capabilities.setdefault( (fallback_provider, fallback_model), @@ -923,15 +999,6 @@ async def run( fallback_catalog.capabilities, ), ) - route_provider = str( - agent_metadata.get("routed_provider") - or inp.active_provider_id - or "" - ) - fallback_sources = ( - agent_metadata.get("router_fallback_chain"), - agent_metadata.get("selector_execution_chain"), - ) for raw_fallbacks in fallback_sources: if not isinstance(raw_fallbacks, list): continue @@ -1078,7 +1145,7 @@ async def run( flush_workspace_dir=aux.flush_workspace_dir, model_capabilities=catalog.capabilities, model_tools_capability_verified=active_artifact_tools_verified, - model_vision_support=catalog.vision_support, + model_vision_support=effective_model_vision_support, thinking=aux.thinking, tool_result_projection_max_inline_chars=(aux.tool_result_projection_max_inline_chars), tool_result_fresh_diagnostic_policy_enabled=( diff --git a/src/opensquilla/engine/turn_runner/harness.py b/src/opensquilla/engine/turn_runner/harness.py index 9a6845251..78fc80204 100644 --- a/src/opensquilla/engine/turn_runner/harness.py +++ b/src/opensquilla/engine/turn_runner/harness.py @@ -311,6 +311,10 @@ async def run_pipeline( "usage_execution_context": request.usage_execution_context, "provider_request_correlation": request.provider_request_correlation, "router_history_replay_request": request.router_history_replay_request, + "bound_user_message_id": request.bound_user_message_id, + "transcript_snapshot": request.transcript_snapshot, + "expected_session_id": request.expected_session_id, + "expected_session_epoch": request.expected_session_epoch, } accepted_kwargs = { name: value diff --git a/src/opensquilla/engine/turn_runner/prompt_assembler_stage.py b/src/opensquilla/engine/turn_runner/prompt_assembler_stage.py index 01ccea391..6ad2668ac 100644 --- a/src/opensquilla/engine/turn_runner/prompt_assembler_stage.py +++ b/src/opensquilla/engine/turn_runner/prompt_assembler_stage.py @@ -121,6 +121,13 @@ class RunPipelineRequest: default=None, repr=False, ) + bound_user_message_id: str | None = field(default=None, repr=False) + expected_session_id: str | None = field(default=None, repr=False) + expected_session_epoch: int | None = field(default=None, repr=False) + transcript_snapshot: TurnTranscriptSnapshot[Any] | None = field( + default=None, + repr=False, + ) # --------------------------------------------------------------------------- # Ports — narrow Protocols so the stage is unit-testable without the full @@ -577,6 +584,10 @@ async def run( if inp.attachments else None ), + bound_user_message_id=inp.bound_user_message_id, + transcript_snapshot=inp.transcript_snapshot, + expected_session_id=inp.expected_session_id, + expected_session_epoch=inp.expected_session_epoch, ) turn, provider = await self._pipeline_executor.run_pipeline(request) diff --git a/src/opensquilla/gateway/model_routing.py b/src/opensquilla/gateway/model_routing.py index 621e31069..fe2133213 100644 --- a/src/opensquilla/gateway/model_routing.py +++ b/src/opensquilla/gateway/model_routing.py @@ -15,44 +15,64 @@ from opensquilla.router_tiers import ( CUSTOM_B5_SELECTION_MODE, HIGHEST_TEXT_TIER, - IMAGE_TIER, INDEPENDENT_ENSEMBLE_SELECTION_MODES, STATIC_B5_PROFILES, + TEXT_TIERS, effective_ensemble_selection_mode, ensemble_selection_configured, normalize_tier_mapping, static_b5_profile, tier_ensemble_active, - tier_index, ) ModelRoutingMode = Literal["direct", "router", "ensemble"] -def _router_image_route(config: Any) -> tuple[str, dict[str, Any]] | None: +def _router_image_route(config: Any) -> tuple[str, dict[str, Any], str] | None: + """Return the first executable configured c-tier for an image turn. + + This is a public admission snapshot of the runtime Router policy, not a + separate routing implementation: only user-configured c0-c3 deployments + participate, deployment capability denials are skipped, and proven support + is preferred over an unknown deployment that the execution layer may probe. + The legacy ``image_model`` row is intentionally non-executable. + """ + router = getattr(config, "squilla_router", None) tiers = normalize_tier_mapping(getattr(router, "tiers", {}) or {}) c3_fusion_active = bool( getattr(getattr(config, "llm_ensemble", None), "enabled", False) ) or tier_ensemble_active(tiers, HIGHEST_TEXT_TIER) - image_tiers = { - name: tier - for name, tier in tiers.items() - if isinstance(tier, dict) - and bool(tier.get("supports_image", False)) - and bool(str(tier.get("model") or "").strip()) - and not (c3_fusion_active and name == HIGHEST_TEXT_TIER) - } - if not image_tiers: + llm = getattr(config, "llm", None) + active_provider = _clean(getattr(llm, "provider", "")) + cross_provider = bool(getattr(router, "cross_provider_tiers", False)) + candidates: list[tuple[str, dict[str, Any], str]] = [] + for name in TEXT_TIERS: + tier = tiers.get(name) + if not isinstance(tier, dict) or bool(tier.get("image_only", False)): + continue + model = str(tier.get("model") or "").strip() + if not model or (c3_fusion_active and name == HIGHEST_TEXT_TIER): + continue + + tier_provider = _clean(tier.get("provider")) + provider = tier_provider if cross_provider and tier_provider else active_provider + provider = provider or tier_provider + use_active_authority = not provider or provider == active_provider + support = _deployment_vision_support( + model=model, + provider=provider, + api_key=(str(getattr(llm, "api_key", "") or "") if use_active_authority else ""), + base_url=(str(getattr(llm, "base_url", "") or "") if use_active_authority else ""), + proxy=(str(getattr(llm, "proxy", "") or "") if use_active_authority else ""), + ) + if support in {"supported", "unknown"}: + candidates.append((name, tier, support)) + + if not candidates: return None - ordered = sorted( - image_tiers, - key=lambda name: (tier_index(name) < 0, tier_index(name)), - ) - if IMAGE_TIER in image_tiers: - ordered = [IMAGE_TIER, *(name for name in ordered if name != IMAGE_TIER)] - selected = ordered[0] - return selected, image_tiers[selected] + candidates.sort(key=lambda item: item[2] != "supported") + return candidates[0] def _deployment_vision_support( @@ -93,40 +113,20 @@ def _image_input_routing_snapshot( *, router_enabled: bool, ensemble_enabled: bool, - selection_mode: str, ) -> dict[str, str]: - independent_ensemble = bool( - ensemble_enabled and selection_mode in INDEPENDENT_ENSEMBLE_SELECTION_MODES - ) - if independent_ensemble or (ensemble_enabled and not router_enabled): + if ensemble_enabled: return { - "admission": "blocked", + "admission": "allowed", "reason": "ensemble_mode_unsupported", } if router_enabled: image_route = _router_image_route(config) if image_route is None: return { - "admission": "blocked", + "admission": "allowed", "reason": "router_image_route_unavailable", } - _, image_tier = image_route - llm = getattr(config, "llm", None) - active_provider = _clean(getattr(llm, "provider", "")) - tier_provider = _clean(image_tier.get("provider")) - cross_provider = bool( - getattr(getattr(config, "squilla_router", None), "cross_provider_tiers", False) - ) - provider = tier_provider if cross_provider and tier_provider else active_provider - provider = provider or tier_provider - use_active_authority = not provider or provider == active_provider - vision_support = _deployment_vision_support( - model=_clean(image_tier.get("model")), - provider=provider, - api_key=str(getattr(llm, "api_key", "") or "") if use_active_authority else "", - base_url=str(getattr(llm, "base_url", "") or "") if use_active_authority else "", - proxy=str(getattr(llm, "proxy", "") or "") if use_active_authority else "", - ) + _, _, vision_support = image_route if vision_support == "supported": return { "admission": "allowed", @@ -134,7 +134,7 @@ def _image_input_routing_snapshot( } if vision_support == "unsupported": return { - "admission": "blocked", + "admission": "allowed", "reason": "model_vision_unsupported", } return {"admission": "unknown", "reason": "capability_unknown"} @@ -153,7 +153,7 @@ def _image_input_routing_snapshot( return {"admission": "allowed", "reason": "model_vision_supported"} if vision_support == "unsupported": return { - "admission": "blocked", + "admission": "allowed", "reason": "model_vision_unsupported", } return {"admission": "unknown", "reason": "capability_unknown"} @@ -442,7 +442,6 @@ def model_routing_snapshot(config: Any) -> dict[str, Any]: config, router_enabled=router_enabled, ensemble_enabled=ensemble_enabled, - selection_mode=selection_mode, ), "applies_to": "next_accepted_turn", } diff --git a/src/opensquilla/gateway/workbench_resource_runtime.py b/src/opensquilla/gateway/workbench_resource_runtime.py index 0d8177c26..ae7e2ba1c 100644 --- a/src/opensquilla/gateway/workbench_resource_runtime.py +++ b/src/opensquilla/gateway/workbench_resource_runtime.py @@ -72,6 +72,7 @@ ) from opensquilla.gateway.websocket import get_registry from opensquilla.paths import media_root_from_config, native_io_path +from opensquilla.session.attachment_manifest import legacy_attachment_id from opensquilla.session.keys import canonicalize_session_key from opensquilla.tools.builtin.document_format_adapters import ( DocumentAdapterError, @@ -316,18 +317,6 @@ def _safe_mime(value: object) -> str: return normalized[:120] -def _legacy_attachment_id( - *, - session_id: str, - message_id: str, - index: int, - sha256: str, -) -> str: - digest = hashlib.sha256(f"{session_id}\0{message_id}\0{index}\0{sha256}".encode()).digest()[:18] - token = base64.urlsafe_b64encode(digest).decode("ascii").rstrip("=") - return f"att_legacy_{token}" - - def _attachment_download_url( *, session_key: str, @@ -407,7 +396,7 @@ async def _attachment_occurrences( continue attachment_id = str(item.get("attachment_id") or "") if not _ATTACHMENT_ID_RE.fullmatch(attachment_id): - attachment_id = _legacy_attachment_id( + attachment_id = legacy_attachment_id( session_id=session_id, message_id=message_id, index=index, diff --git a/src/opensquilla/onboarding/mutations.py b/src/opensquilla/onboarding/mutations.py index 07d1705bd..459f65e8a 100644 --- a/src/opensquilla/onboarding/mutations.py +++ b/src/opensquilla/onboarding/mutations.py @@ -312,6 +312,19 @@ def _merge_router_tiers( tier_name = normalize_text_tier(name) or str(name) override = _normalize_tier_payload(tier_name, raw_override) current = dict(merged.get(tier_name, {})) + deployment_changed = any( + field_name in override + and str(override.get(field_name) or "").strip() + != str(current.get(field_name) or "").strip() + for field_name in ("provider", "model") + ) + if deployment_changed and "supports_image" not in override: + # Capability evidence belongs to a deployment identity. A model- + # only/provider-only override must not inherit the managed preset's + # legacy declaration for a different deployment. These fields + # remain readable for older clients, not runtime capability facts. + current.pop("supports_image", None) + current.pop("supportsImage", None) # A pre-``ensemble_enabled`` client can still submit an explicit # per-tier selection mode. That legacy field is an ownership # boundary: do not let a managed preset's new shared-plan flag turn @@ -340,12 +353,13 @@ def _canonical_tier_value(tier: Mapping[str, Any]) -> dict[str, Any]: legacy_selection_mode = str( tier.get("ensemble_selection_mode", tier.get("ensembleSelectionMode", "")) or "" ).strip() + # Retired image switches do not change the semantic routing preset. + # Keep old values in the stored mapping for client compatibility only. return { "provider": str(tier.get("provider") or "").strip().lower(), "model": str(tier.get("model") or "").strip(), "description": str(tier.get("description") or "").strip(), "thinking_level": (str(thinking or "").strip() or None), - "supports_image": bool(tier.get("supports_image", tier.get("supportsImage", False))), "image_only": bool(tier.get("image_only", tier.get("imageOnly", False))), "ensemble_enabled": ensemble_enabled, # Once the new tri-state field exists it owns execution. Retained @@ -507,6 +521,8 @@ def _cross_provider_tier_warnings( ensemble_globally_enabled=ensemble_globally_enabled, ) for tier_name in sorted(tiers): + if tier_name == "image_model": + continue tier = tiers.get(tier_name) if not isinstance(tier, dict): continue @@ -1258,6 +1274,14 @@ def upsert_router( ), llm_profiles=getattr(config, "llm_profiles", None), ) + legacy_image_tier = (router_payload.get("tiers") or {}).get("image_model") + if isinstance(legacy_image_tier, dict) and legacy_image_tier.get("model"): + warnings.append( + "The legacy image_model setting is preserved for compatibility " + "but is not used for image input. Configure an image-capable " + "model in c0-c3; if none can process images, the turn continues " + "with an image-not-analyzed marker and keeps the original image." + ) new_cfg = _clone(config) new_cfg.squilla_router = SquillaRouterConfig(**router_payload) diff --git a/src/opensquilla/onboarding/router_specs.py b/src/opensquilla/onboarding/router_specs.py index b7f08b9c7..f01208996 100644 --- a/src/opensquilla/onboarding/router_specs.py +++ b/src/opensquilla/onboarding/router_specs.py @@ -65,8 +65,11 @@ def _tier_payload(tier: dict[str, Any]) -> dict[str, Any]: "model": tier.get("model", ""), "description": tier.get("description", ""), "thinkingLevel": tier.get("thinking_level", ""), - "supportsImage": bool(tier.get("supports_image", False)), } + if "supports_image" in tier: + payload["supportsImage"] = bool(tier.get("supports_image")) + elif "supportsImage" in tier: + payload["supportsImage"] = bool(tier.get("supportsImage")) ensemble_enabled = tier.get("ensemble_enabled", tier.get("ensembleEnabled")) if isinstance(ensemble_enabled, bool): payload["ensembleEnabled"] = ensemble_enabled diff --git a/src/opensquilla/provider/__init__.py b/src/opensquilla/provider/__init__.py index b38f9f74c..e1ab9b096 100644 --- a/src/opensquilla/provider/__init__.py +++ b/src/opensquilla/provider/__init__.py @@ -13,6 +13,39 @@ classify_provider_error, decide_recovery_action, ) +from .image_projection import ( + ImageFailureClassification, + ImageFailureKind, + ImageInputFailureKind, + ImageIntent, + ImageIntentKind, + ImageMarkerState, + ImageProjectionDecision, + ImageProjectionMode, + ImageProjectionPolicy, + ImageProjectionResult, + MediaProjectionResult, + VisionSupportEvidence, + VisionSupportSource, + VisionSupportStatus, + assert_text_only_messages, + bind_image_attachment_ids, + build_image_marker, + classify_image_failure, + classify_image_input_error, + classify_provider_image_error, + count_image_blocks, + has_image_blocks, + image_marker, + marker_for_image, + normalize_marker_state, + normalize_vision_support, + project_image_messages, + project_messages, + project_messages_for_model, + projection_mode_for_support, + resolve_vision_support, +) from .ollama import OllamaProvider from .openai import OpenAIProvider from .openai_responses import OpenAIResponsesProvider @@ -49,6 +82,7 @@ ChatConfig, ContentBlockCompaction, ContentBlockDocument, + ContentBlockImage, ContentBlockText, ContentBlockThinking, ContentBlockToolResult, @@ -76,6 +110,7 @@ ToolUseDeltaEvent, ToolUseEndEvent, ToolUseStartEvent, + VisionSupport, derive_provider_request_correlation, synthetic_failure_event, ) @@ -147,6 +182,7 @@ "derive_provider_request_correlation", "ModelCapabilities", "ModelInfo", + "VisionSupport", "ChatConfig", "Message", "QuotaStatus", @@ -158,6 +194,39 @@ "ContentBlockToolResult", "ContentBlockCompaction", "ContentBlockDocument", + "ContentBlockImage", + # Request-local image projection + "VisionSupportEvidence", + "VisionSupportSource", + "VisionSupportStatus", + "normalize_vision_support", + "resolve_vision_support", + "ImageProjectionMode", + "ImageProjectionPolicy", + "ImageProjectionDecision", + "MediaProjectionResult", + "ImageProjectionResult", + "ImageIntentKind", + "ImageIntent", + "ImageMarkerState", + "normalize_marker_state", + "image_marker", + "build_image_marker", + "marker_for_image", + "bind_image_attachment_ids", + "projection_mode_for_support", + "project_messages", + "project_messages_for_model", + "project_image_messages", + "count_image_blocks", + "has_image_blocks", + "assert_text_only_messages", + "ImageFailureKind", + "ImageInputFailureKind", + "ImageFailureClassification", + "classify_image_input_error", + "classify_provider_image_error", + "classify_image_failure", # Test-only failure injection seam "FailureInjector", "synthetic_failure_event", diff --git a/src/opensquilla/provider/ensemble.py b/src/opensquilla/provider/ensemble.py index f5d4ea241..d37f207ac 100644 --- a/src/opensquilla/provider/ensemble.py +++ b/src/opensquilla/provider/ensemble.py @@ -58,6 +58,11 @@ ) from .error_redaction import redact_upstream_error_code, redact_upstream_error_text from .failures import ProviderFailureKind, classify_provider_error +from .image_projection import ( + ImageMarkerState, + assert_text_only_messages, + project_messages, +) from .model_catalog import resolve_effective_context_window, shared_catalog from .protocol import ( LLMProvider, @@ -120,6 +125,24 @@ "model also failed. Check the fixed provider, model, and credentials, " "then try again." ) + + +def _ensemble_request_messages(messages: list[Message]) -> list[Message]: + """Return a fresh text-only object graph for one physical member call. + + Ensemble is a text-only virtual model. Re-project at every provider + boundary instead of sharing the coordinator's list across concurrent + proposers or retries: a provider adapter that mutates its input must not + contaminate a sibling leg, and no nested image may reach a member. + """ + + projection = project_messages( + messages, + mode="marker", + marker_state=ImageMarkerState.NOT_ANALYZED, + ) + assert_text_only_messages(projection.messages) + return projection.messages log = structlog.get_logger(__name__) @@ -2225,14 +2248,19 @@ def provider_metadata(self) -> ProviderMetadata: ) def validate_chat_request(self, messages: list[Message]) -> ErrorEvent | None: - """Reject typed image input before any ensemble leg can start.""" + """Validate the already-projected outer Ensemble request. + + Ensemble is intentionally a text-only virtual model. Its public + ``chat`` boundary projects image blocks to truthful markers before any + member is called, so a residual image here indicates a programming + error rather than a user-facing capability failure. + """ if count_provider_image_blocks(messages) <= 0: return None - return ErrorEvent( - message=ENSEMBLE_MULTIMODAL_UNSUPPORTED_MESSAGE, - code=ENSEMBLE_MULTIMODAL_UNSUPPORTED_CODE, - ) + # Keep this method side-effect free for callers that use it as an + # admission probe; ``_chat_unbounded`` performs the actual projection. + return None async def list_models(self) -> list[ModelInfo]: models: list[ModelInfo] = [] @@ -2398,6 +2426,22 @@ async def _chat_unbounded( *, execution_context: TurnExecutionContext | None = None, ) -> AsyncGenerator[StreamEvent, None]: + projection = project_messages( + messages, + mode="marker", + marker_state=ImageMarkerState.NOT_ANALYZED, + ) + if projection.marker_count: + if config is not None and isinstance(getattr(config, "__dict__", None), dict): + metadata = getattr(config, "metadata", None) + if isinstance(metadata, dict): + metadata["image_input_mode"] = "marker" + metadata["image_input_reason"] = "ensemble_text_only" + metadata["image_input_count"] = projection.input_image_count + metadata["image_input_marker_count"] = projection.marker_count + messages = projection.messages + assert_text_only_messages(messages) + validation_error = self.validate_chat_request(messages) if validation_error is not None: yield validation_error @@ -3074,7 +3118,11 @@ async def mark_request_started() -> None: provider_stream = _provider_stream_with_lifecycle( lambda: self._account_physical_stream( - lambda: provider.chat(messages, tools=tools, config=chat_cfg), + lambda: provider.chat( + _ensemble_request_messages(messages), + tools=tools, + config=chat_cfg, + ), provider=member.provider_config.provider, model=member.provider_config.model, ), @@ -3512,7 +3560,11 @@ async def mark_aggregator_request_started() -> None: ) heartbeat_stream = _provider_stream_with_lifecycle( lambda: self._account_physical_stream( - lambda: provider.chat(messages, tools=tools, config=config), + lambda: provider.chat( + _ensemble_request_messages(messages), + tools=tools, + config=config, + ), provider=self.aggregator.provider_config.provider, model=self.aggregator.provider_config.model, ), @@ -4118,7 +4170,7 @@ async def mark_fixed_request_started() -> None: async for event in _provider_stream_with_lifecycle( lambda: self._account_physical_stream( lambda: provider.chat( - fixed_messages, + _ensemble_request_messages(fixed_messages), tools=tools, config=config, ), diff --git a/src/opensquilla/provider/image_projection.py b/src/opensquilla/provider/image_projection.py new file mode 100644 index 000000000..14be91086 --- /dev/null +++ b/src/opensquilla/provider/image_projection.py @@ -0,0 +1,1036 @@ +"""Pure, request-local projection of image-bearing provider messages. + +The transcript is the source of truth for user input and attachments. This +module deliberately operates on a deep copy of that transcript and produces +the view for one *physical* provider call. In particular, replacing an image +with a text marker here never changes the persisted message, which lets a +later turn (or a different configured model) recover the original image. + +Only the projection boundary belongs here. Session storage, model selection, +and retry orchestration can consume the value objects below without making the +projection module depend on the runtime. +""" + +from __future__ import annotations + +import copy +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from enum import StrEnum +from typing import Any, cast + +from .types import ( + ContentBlockImage, + ContentBlockText, + ContentBlockToolResult, + Message, + VisionSupport, +) + +# --------------------------------------------------------------------------- +# Capability and projection value objects +# --------------------------------------------------------------------------- + + +class VisionSupportSource(StrEnum): + """Where a model's tri-state vision fact came from. + + The source is diagnostic only. It intentionally does not imply that a + model may be selected; authorization remains owned by the caller. + """ + + ENSEMBLE_CONTRACT = "ensemble_contract" + USER_CONFIG = "user_config" + RUNTIME_OBSERVATION = "runtime_observation" + CATALOG = "catalog" + NONE = "none" + UNKNOWN = "unknown" + + +class VisionSupportStatus(StrEnum): + """Enum spelling of :data:`VisionSupport` for runtime consumers.""" + + SUPPORTED = "supported" + UNSUPPORTED = "unsupported" + UNKNOWN = "unknown" + + +type VisionSupportValue = VisionSupport | bool | None + + +@dataclass(frozen=True, slots=True) +class VisionSupportEvidence: + """Tri-state capability evidence for one exact model deployment. + + ``unsupported`` is authoritative only when ``source`` carries an + explicit user/configuration fact or a precise runtime observation. A + missing catalog row should be represented as ``unknown`` rather than as a + false value. + """ + + status: VisionSupport = "unknown" + source: VisionSupportSource | str = VisionSupportSource.UNKNOWN + deployment: str = "" + reason: str = "" + + def __post_init__(self) -> None: + status = normalize_vision_support(self.status) + source = str(self.source or VisionSupportSource.UNKNOWN).strip().lower() + if not source: + source = VisionSupportSource.UNKNOWN + object.__setattr__(self, "status", status) + object.__setattr__(self, "source", source) + object.__setattr__(self, "deployment", str(self.deployment or "").strip()) + object.__setattr__(self, "reason", str(self.reason or "").strip()) + + @property + def supports_images(self) -> bool: + """Whether this evidence authorizes sending an image natively.""" + + return self.status == "supported" + + @property + def rejects_images(self) -> bool: + return self.status == "unsupported" + + @property + def is_unknown(self) -> bool: + return self.status == "unknown" + + @classmethod + def from_value( + cls, + value: VisionSupportValue | VisionSupportEvidence, + *, + source: VisionSupportSource | str = VisionSupportSource.UNKNOWN, + deployment: str = "", + reason: str = "", + ) -> VisionSupportEvidence: + if isinstance(value, cls): + return value + return cls( + status=normalize_vision_support(value), + source=source, + deployment=deployment, + reason=reason, + ) + + +def normalize_vision_support(value: object, *, field_present: bool | None = None) -> VisionSupport: + """Normalize legacy booleans and absent fields without collapsing unknown. + + ``field_present=False`` is useful when reading old Router/tier mappings: + an omitted ``supports_image`` key means *unknown*, while an explicit + ``False`` means *unsupported*. + """ + + if field_present is False: + return "unknown" + if isinstance(value, VisionSupportEvidence): + return value.status + if value is True: + return "supported" + if value is False: + return "unsupported" + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in {"supported", "support", "true", "yes", "vision", "native"}: + return "supported" + if normalized in { + "unsupported", + "unsupported_feature", + "false", + "no", + "text_only", + "text-only", + }: + return "unsupported" + return "unknown" + + +def resolve_vision_support( + value: VisionSupportValue | VisionSupportEvidence, + *, + source: VisionSupportSource | str = VisionSupportSource.UNKNOWN, + deployment: str = "", + reason: str = "", + field_present: bool | None = None, +) -> VisionSupportEvidence: + """Return normalized evidence while retaining source and deployment facts.""" + + if isinstance(value, VisionSupportEvidence): + return value + return VisionSupportEvidence( + status=normalize_vision_support(value, field_present=field_present), + source=source, + deployment=deployment, + reason=reason, + ) + + +class ImageProjectionMode(StrEnum): + """How an image is represented in one outbound request.""" + + NATIVE = "native" + MARKER = "marker" + TEXT_ONLY = "marker" + SURROGATE = "surrogate" + + +class ImageIntentKind(StrEnum): + """The caller's reason for including historical/current image context.""" + + NONE = "none" + CURRENT_UPLOAD = "current_upload" + EXPLICIT_HISTORY = "explicit_history" + IMPLICIT_RECENT_FOLLOWUP = "implicit_recent_followup" + EXPLICITLY_IGNORED = "explicitly_ignored" + + +@dataclass(frozen=True, slots=True) +class ImageIntent: + """Request-local image intent, independent of model capability.""" + + kind: ImageIntentKind | str = ImageIntentKind.NONE + attachment_ids: tuple[str, ...] = () + reason: str = "" + + def __post_init__(self) -> None: + raw_kind = str(self.kind or ImageIntentKind.NONE).strip().lower() + aliases = { + "current": ImageIntentKind.CURRENT_UPLOAD, + "upload": ImageIntentKind.CURRENT_UPLOAD, + "history": ImageIntentKind.EXPLICIT_HISTORY, + "explicit": ImageIntentKind.EXPLICIT_HISTORY, + "recent": ImageIntentKind.IMPLICIT_RECENT_FOLLOWUP, + "implicit": ImageIntentKind.IMPLICIT_RECENT_FOLLOWUP, + "ignored": ImageIntentKind.EXPLICITLY_IGNORED, + } + normalized = aliases.get(raw_kind, raw_kind) + try: + normalized_kind: ImageIntentKind | str = ImageIntentKind(normalized) + except ValueError: + # Preserve forward-compatible values for callers that add a new + # intent kind before this package is upgraded. + normalized_kind = raw_kind or ImageIntentKind.NONE + raw_ids = self.attachment_ids or () + ids = tuple( + item.strip() + for item in raw_ids + if isinstance(item, str) and item.strip() + ) + object.__setattr__(self, "kind", normalized_kind) + object.__setattr__(self, "attachment_ids", ids) + object.__setattr__(self, "reason", str(self.reason or "").strip()) + + +class ImageMarkerState(StrEnum): + """Truthful explanation used when an image is absent from a text request.""" + + NOT_ANALYZED = "not_analyzed" + ANALYSIS_FAILED = "analysis_failed" + NOT_REREAD = "not_reread" + UNAVAILABLE = "unavailable" + NOT_SENT = "not_sent" + + +_MARKER_STATE_ALIASES: dict[str, ImageMarkerState] = { + "failed": ImageMarkerState.ANALYSIS_FAILED, + "analysis_error": ImageMarkerState.ANALYSIS_FAILED, + "historical_not_read": ImageMarkerState.NOT_REREAD, + "not_read": ImageMarkerState.NOT_REREAD, + "unread": ImageMarkerState.NOT_REREAD, + "missing": ImageMarkerState.UNAVAILABLE, + "invalid": ImageMarkerState.UNAVAILABLE, + "omitted": ImageMarkerState.NOT_SENT, +} + + +def normalize_marker_state(value: ImageMarkerState | str | None) -> ImageMarkerState: + if isinstance(value, ImageMarkerState): + return value + raw = str(value or ImageMarkerState.NOT_ANALYZED).strip().lower() + try: + return ImageMarkerState(raw) + except ValueError: + return _MARKER_STATE_ALIASES.get(raw, ImageMarkerState.NOT_ANALYZED) + + +def _safe_attachment_id(value: object) -> str: + """Keep IDs in a marker bounded and free of control/injection characters.""" + + text = str(value or "").strip() + if not text: + return "" + sanitized = re.sub(r"[^A-Za-z0-9_.:-]", "_", text) + # Manifest IDs allow the ``att_`` prefix plus up to 160 payload + # characters. Preserve a valid maximum-length ID exactly so a marker can + # be used for a later explicit archive lookup. + return sanitized[:164] + + +def image_marker( + state: ImageMarkerState | str = ImageMarkerState.NOT_ANALYZED, + *, + attachment_id: str | None = None, +) -> str: + """Build the stable, model-visible marker for one omitted image.""" + + normalized = normalize_marker_state(state) + safe_id = _safe_attachment_id(attachment_id) + suffix = f"原图已保留:{safe_id}" if safe_id else "原图已保留" + if normalized is ImageMarkerState.ANALYSIS_FAILED: + return f"[图片分析失败:本回合无法读取原图;{suffix}]" + if normalized is ImageMarkerState.NOT_REREAD: + return f"[历史图片本回合未重新读取;可参考先前回合文字分析;{suffix}]" + if normalized is ImageMarkerState.UNAVAILABLE: + if safe_id: + return f"[历史图片不可用:{safe_id};如需重新分析请重新上传]" + return "[历史图片不可用;如需重新分析请重新上传]" + if normalized is ImageMarkerState.NOT_SENT: + return f"[图片本回合未发送;{suffix}]" + return f"[图片未分析:当前模型不支持图片输入;{suffix}]" + + +# Explicit aliases make call sites read naturally and keep the helper easy to +# discover without committing callers to one spelling. +build_image_marker = image_marker +marker_for_image = image_marker + + +@dataclass(frozen=True, slots=True) +class ImageProjectionPolicy: + """Immutable policy used to project one exact physical request.""" + + mode: ImageProjectionMode | str = ImageProjectionMode.NATIVE + marker_state: ImageMarkerState | str = ImageMarkerState.NOT_ANALYZED + attachment_ids: tuple[str, ...] = () + marker_states: Mapping[str, ImageMarkerState | str] = field(default_factory=dict) + surrogate_by_attachment_id: Mapping[str, str] = field(default_factory=dict) + + def __post_init__(self) -> None: + try: + mode = ImageProjectionMode(str(self.mode).strip().lower()) + except ValueError: + mode = ImageProjectionMode.NATIVE + raw_ids = self.attachment_ids or () + ids = tuple( + item.strip() + for item in raw_ids + if isinstance(item, str) and item.strip() + ) + states = { + str(key): normalize_marker_state(value) + for key, value in dict(self.marker_states or {}).items() + } + surrogates = { + str(key): str(value) + for key, value in dict(self.surrogate_by_attachment_id or {}).items() + if str(value) + } + object.__setattr__(self, "mode", mode) + object.__setattr__(self, "marker_state", normalize_marker_state(self.marker_state)) + object.__setattr__(self, "attachment_ids", ids) + object.__setattr__(self, "marker_states", states) + object.__setattr__(self, "surrogate_by_attachment_id", surrogates) + + @classmethod + def for_support( + cls, + support: VisionSupportValue | VisionSupportEvidence, + *, + marker_state: ImageMarkerState | str = ImageMarkerState.NOT_ANALYZED, + **kwargs: Any, + ) -> ImageProjectionPolicy: + evidence = resolve_vision_support(support) + mode = ( + ImageProjectionMode.MARKER + if evidence.status == "unsupported" + else ImageProjectionMode.NATIVE + ) + return cls(mode=mode, marker_state=marker_state, **kwargs) + + +def projection_mode_for_support( + support: VisionSupportValue | VisionSupportEvidence, + *, + force_text_only: bool = False, +) -> ImageProjectionMode: + """Resolve the request projection without selecting another model. + + Unknown capability deliberately stays native: the provider gets one + chance to prove whether this exact deployment accepts the image. The + caller may pass ``force_text_only=True`` for Ensemble's outer contract. + """ + + if force_text_only: + return ImageProjectionMode.MARKER + evidence = resolve_vision_support(support) + return ( + ImageProjectionMode.MARKER + if evidence.status == "unsupported" + else ImageProjectionMode.NATIVE + ) + + +@dataclass(frozen=True, slots=True) +class ImageProjectionDecision: + """Per-image accounting emitted by :func:`project_messages`.""" + + ordinal: int + attachment_id: str | None + mode: ImageProjectionMode + marker_state: ImageMarkerState | None = None + marker: str | None = None + + +@dataclass(frozen=True, slots=True) +class MediaProjectionResult: + """Projected messages and sanitized image accounting.""" + + messages: list[Message] + mode: ImageProjectionMode + input_image_count: int + output_image_count: int + marker_count: int + decisions: tuple[ImageProjectionDecision, ...] = () + + @property + def projected_messages(self) -> list[Message]: + """Compatibility/readability alias for callers using that spelling.""" + + return self.messages + + @property + def changed(self) -> bool: + return self.mode is not ImageProjectionMode.NATIVE and self.marker_count > 0 + + @property + def text_only(self) -> bool: + return self.output_image_count == 0 + + +ImageProjectionResult = MediaProjectionResult + + +# --------------------------------------------------------------------------- +# Recursive image accounting and transformation +# --------------------------------------------------------------------------- + + +def _is_image_mapping(value: Mapping[str, Any]) -> bool: + raw_type = value.get("type") + return isinstance(raw_type, str) and raw_type.strip().lower() == "image" + + +def _count_nested_images(value: object, *, include_mapping_blocks: bool = True) -> int: + if isinstance(value, ContentBlockImage): + return 1 + if isinstance(value, ContentBlockToolResult): + return _count_nested_images(value.content, include_mapping_blocks=True) + if isinstance(value, Message): + return _count_nested_images(value.content, include_mapping_blocks=True) + if include_mapping_blocks and isinstance(value, Mapping): + if _is_image_mapping(value): + return 1 + if str(value.get("type", "")).strip().lower() == "tool_result": + return _count_nested_images(value.get("content"), include_mapping_blocks=True) + return 0 + if isinstance(value, (list, tuple)): + return sum(_count_nested_images(item, include_mapping_blocks=True) for item in value) + return 0 + + +def count_image_blocks(messages: Sequence[object]) -> int: + """Count image blocks at any depth in message/tool-result content. + + Tool-use arguments are intentionally not traversed. An application JSON + object with ``{"type": "image"}`` is not a provider content block unless + it is inside a message or tool-result content list. + """ + + return sum(_count_nested_images(message) for message in messages) + + +def has_image_blocks(messages: Sequence[object]) -> bool: + return count_image_blocks(messages) > 0 + + +def _bound_image_attachment_ids(value: object) -> set[str]: + if isinstance(value, ContentBlockImage): + return {value.attachment_id} if value.attachment_id else set() + if isinstance(value, ContentBlockToolResult): + return _bound_image_attachment_ids(value.content) + if isinstance(value, Message): + return _bound_image_attachment_ids(value.content) + if isinstance(value, (list, tuple)): + result: set[str] = set() + for item in value: + result.update(_bound_image_attachment_ids(item)) + return result + if isinstance(value, Mapping): + if _is_image_mapping(value): + attachment_id = value.get("attachment_id") + return ( + {attachment_id.strip()[:164]} + if isinstance(attachment_id, str) and attachment_id.strip() + else set() + ) + if str(value.get("type", "")).strip().lower() == "tool_result": + return _bound_image_attachment_ids(value.get("content")) + return set() + + +def bind_image_attachment_ids( + messages: Sequence[Message], + attachment_ids: Sequence[str], +) -> list[Message]: + """Return a deep copy with IDs bound to otherwise-unbound typed images. + + This is used for the current upload envelope after persistence assigned its + canonical occurrence IDs. The field is internal/excluded from provider + serialization; it exists only to keep marker decisions correct when a + request also contains historical or nested tool-result images. + """ + + ids = [value.strip()[:164] for value in attachment_ids if value.strip()] + next_id = 0 + + def visit(value: object) -> object: + nonlocal next_id + if isinstance(value, ContentBlockImage): + attachment_id = value.attachment_id + if not attachment_id and next_id < len(ids): + attachment_id = ids[next_id] + next_id += 1 + return value.model_copy( + deep=True, + update={"attachment_id": attachment_id}, + ) + if isinstance(value, ContentBlockToolResult): + return value.model_copy(deep=True, update={"content": visit(value.content)}) + if isinstance(value, Message): + return value.model_copy(deep=True, update={"content": visit(value.content)}) + if isinstance(value, list): + return [visit(item) for item in value] + if isinstance(value, tuple): + return tuple(visit(item) for item in value) + return copy.deepcopy(value) + + return [cast(Message, visit(message)) for message in messages] + + +@dataclass +class _ProjectionContext: + policy: ImageProjectionPolicy + next_ordinal: int = 0 + next_fallback_id: int = 0 + reserved_attachment_ids: set[str] = field(default_factory=set) + decisions: list[ImageProjectionDecision] = field(default_factory=list) + + def occurrence( + self, + explicit_attachment_id: object = None, + ) -> tuple[int, str | None, ImageMarkerState]: + ordinal = self.next_ordinal + self.next_ordinal += 1 + attachment_id = ( + str(explicit_attachment_id).strip()[:164] + if isinstance(explicit_attachment_id, str) + and explicit_attachment_id.strip() + else None + ) + if attachment_id is None: + while self.next_fallback_id < len(self.policy.attachment_ids): + candidate = self.policy.attachment_ids[self.next_fallback_id] + self.next_fallback_id += 1 + if candidate not in self.reserved_attachment_ids: + attachment_id = candidate + break + state = self.policy.marker_state + if attachment_id is not None: + state = self.policy.marker_states.get(attachment_id, state) + return ordinal, attachment_id, normalize_marker_state(state) + + +def _project_content_value(value: object, context: _ProjectionContext) -> tuple[object, bool]: + """Project a content value while preserving non-content tool arguments.""" + + if isinstance(value, ContentBlockImage): + ordinal, attachment_id, state = context.occurrence(value.attachment_id) + mode = cast(ImageProjectionMode, context.policy.mode) + if mode is ImageProjectionMode.NATIVE: + cloned = value.model_copy(deep=True) + context.decisions.append(ImageProjectionDecision(ordinal, attachment_id, mode)) + return cloned, False + + if mode is ImageProjectionMode.SURROGATE and attachment_id: + surrogate = context.policy.surrogate_by_attachment_id.get(attachment_id) + if surrogate: + marker_text = ( + f"[图片派生描述({_safe_attachment_id(attachment_id)}):{surrogate}]" + ) + else: + marker_text = image_marker(state, attachment_id=attachment_id) + else: + marker_text = image_marker(state, attachment_id=attachment_id) + context.decisions.append( + ImageProjectionDecision( + ordinal, + attachment_id, + mode, + marker_state=state, + marker=marker_text, + ) + ) + return ContentBlockText(text=marker_text), True + + if isinstance(value, ContentBlockToolResult): + projected_content, changed = _project_content_value(value.content, context) + # Always deep-copy a tool result, even when it contains no image, so + # callers can safely mutate the returned request without touching the + # canonical object graph. + if changed: + projected = value.model_copy(deep=True, update={"content": projected_content}) + else: + projected = value.model_copy(deep=True) + return projected, changed + + if isinstance(value, Message): + if isinstance(value.content, (list, tuple)): + projected_content, changed = _project_content_value(value.content, context) + if changed: + return value.model_copy(deep=True, update={"content": projected_content}), True + return value.model_copy(deep=True), False + + if isinstance(value, list): + projected_items: list[object] = [] + changed = False + for item in value: + projected_item, item_changed = _project_content_value(item, context) + projected_items.append(projected_item) + changed = changed or item_changed + return projected_items, changed + + if isinstance(value, tuple): + projected_tuple_items: list[object] = [] + changed = False + for item in value: + projected_item, item_changed = _project_content_value(item, context) + projected_tuple_items.append(projected_item) + changed = changed or item_changed + return tuple(projected_tuple_items), changed + + if isinstance(value, Mapping): + if _is_image_mapping(value): + ordinal, attachment_id, state = context.occurrence( + value.get("attachment_id") + ) + mode = cast(ImageProjectionMode, context.policy.mode) + surrogate = ( + context.policy.surrogate_by_attachment_id.get(attachment_id or "") + if mode is ImageProjectionMode.SURROGATE + else None + ) + marker_text = ( + f"[图片派生描述({_safe_attachment_id(attachment_id)}):{surrogate}]" + if surrogate and attachment_id + else image_marker(state, attachment_id=attachment_id) + ) + context.decisions.append( + ImageProjectionDecision( + ordinal, + attachment_id, + mode, + marker_state=state, + marker=marker_text, + ) + ) + if mode is ImageProjectionMode.NATIVE: + cloned_mapping = copy.deepcopy(dict(value)) + # Mapping-shaped compatibility blocks cannot express a + # Pydantic excluded field, so remove request-local provenance + # explicitly before the native provider boundary. + cloned_mapping.pop("attachment_id", None) + return cloned_mapping, False + # Keep dictionary-shaped content dictionary-shaped. Adapters that + # accept untyped tool-result blocks can serialize this naturally. + return {"type": "text", "text": marker_text}, True + if str(value.get("type", "")).strip().lower() == "tool_result": + projected_content, changed = _project_content_value(value.get("content"), context) + projected_mapping = copy.deepcopy(dict(value)) + if changed: + projected_mapping["content"] = projected_content + return projected_mapping, changed + # Do not inspect arbitrary dictionaries (especially tool-use input). + return copy.deepcopy(value), False + + # A content list can contain provider-compatible custom block objects. A + # deep copy keeps their identity/value intact without making assumptions. + return copy.deepcopy(value), False + + +def project_messages( + messages: Sequence[Message], + mode: ImageProjectionMode | str | None = None, + *, + policy: ImageProjectionPolicy | None = None, + vision_support: VisionSupportValue | VisionSupportEvidence | None = None, + marker_state: ImageMarkerState | str = ImageMarkerState.NOT_ANALYZED, + attachment_ids: Sequence[str] = (), + marker_states: Mapping[str, ImageMarkerState | str] | None = None, + surrogate_by_attachment_id: Mapping[str, str] | None = None, + force_text_only: bool = False, +) -> MediaProjectionResult: + """Deep-copy ``messages`` and project image blocks for one provider call. + + ``mode`` takes precedence over ``vision_support``. When neither is + supplied, native projection is used. ``unknown`` capability also uses a + native request so the provider can be probed once; callers can retry with + ``mode="marker"`` after a precise unsupported-image error. + """ + + if policy is None: + if mode is None: + selected_mode = projection_mode_for_support( + "unknown" if vision_support is None else vision_support, + force_text_only=force_text_only, + ) + else: + try: + selected_mode = ImageProjectionMode(str(mode).strip().lower()) + except ValueError: + selected_mode = ImageProjectionMode.NATIVE + policy = ImageProjectionPolicy( + mode=selected_mode, + marker_state=marker_state, + attachment_ids=tuple(attachment_ids), + marker_states=marker_states or {}, + surrogate_by_attachment_id=surrogate_by_attachment_id or {}, + ) + elif force_text_only and policy.mode is not ImageProjectionMode.MARKER: + policy = ImageProjectionPolicy( + mode=ImageProjectionMode.MARKER, + marker_state=policy.marker_state, + attachment_ids=policy.attachment_ids, + marker_states=policy.marker_states, + surrogate_by_attachment_id=policy.surrogate_by_attachment_id, + ) + + input_count = count_image_blocks(messages) + reserved_attachment_ids: set[str] = set() + for message in messages: + reserved_attachment_ids.update(_bound_image_attachment_ids(message)) + context = _ProjectionContext( + policy=policy, + reserved_attachment_ids=reserved_attachment_ids, + ) + projected_messages: list[Message] = [] + for message in messages: + projected, _ = _project_content_value(message, context) + # The public type is Message, but preserving a malformed custom value + # is safer than silently dropping it. Normal callers always pass + # Message instances. + projected_messages.append(cast(Message, projected)) + + marker_count = sum( + 1 + for decision in context.decisions + if decision.mode is not ImageProjectionMode.NATIVE + and decision.marker is not None + ) + output_count = count_image_blocks(projected_messages) + return MediaProjectionResult( + messages=projected_messages, + mode=cast(ImageProjectionMode, policy.mode), + input_image_count=input_count, + output_image_count=output_count, + marker_count=marker_count, + decisions=tuple(context.decisions), + ) + + +def project_messages_for_model( + messages: Sequence[Message], + *, + vision_support: VisionSupportValue | VisionSupportEvidence = "unknown", + **kwargs: Any, +) -> MediaProjectionResult: + """Named façade for the common exact-deployment projection call.""" + + return project_messages(messages, vision_support=vision_support, **kwargs) + + +def project_image_messages( + messages: Sequence[Message], + mode: ImageProjectionMode | str | None = None, + **kwargs: Any, +) -> list[Message]: + """Compatibility façade returning only the projected message list.""" + + return project_messages(messages, mode, **kwargs).messages + + +def assert_text_only_messages(messages: Sequence[object]) -> None: + """Raise if a supposedly text-only physical request still has an image.""" + + count = count_image_blocks(messages) + if count: + raise ValueError(f"text-only provider request contains {count} image block(s)") + + +# --------------------------------------------------------------------------- +# Image-specific provider failure classification +# --------------------------------------------------------------------------- + + +class ImageFailureKind(StrEnum): + """Failure classes relevant to image projection/retry decisions.""" + + UNSUPPORTED_INPUT = "unsupported_input" + # Readable aliases for callers that use provider terminology. + IMAGE_UNSUPPORTED = "unsupported_input" + INVALID_MEDIA = "invalid_media" + CONTEXT_OVERFLOW = "context_overflow" + AUTHENTICATION = "authentication" + INSUFFICIENT_CREDITS = "insufficient_credits" + RATE_LIMITED = "rate_limited" + TRANSIENT = "transient" + MODEL_NOT_FOUND = "model_not_found" + POLICY_REFUSAL = "policy_refusal" + BAD_REQUEST = "bad_request" + UNKNOWN = "unknown" + + +@dataclass(frozen=True, slots=True) +class ImageFailureClassification: + """Image error class plus the safe retry/cache decision.""" + + kind: ImageFailureKind + caches_unsupported: bool = False + retry_without_image: bool = False + reason: str = "" + + @property + def is_unsupported(self) -> bool: + return self.kind is ImageFailureKind.UNSUPPORTED_INPUT + + +def _error_fields(error: object) -> tuple[int | None, str, str]: + if isinstance(error, Mapping): + status = error.get("status_code", error.get("status")) + code = error.get("code", error.get("error_code", "")) + message = error.get("message", error.get("error", "")) + else: + status = getattr(error, "status_code", None) + code = getattr(error, "code", getattr(error, "error_code", "")) + message = getattr(error, "message", "") + if not message: + message = str(error or "") + raw_code = str(code or "").strip().lower() + try: + status_code = int(status) if status is not None and str(status).strip() else None + except (TypeError, ValueError): + status_code = None + # Provider adapters commonly normalize HTTP failures into ``ErrorEvent`` + # and keep the status only in its string ``code`` field. Preserve that + # stronger signal so an incidental "image unsupported" phrase in a + # 401/429/5xx body cannot poison the exact deployment's vision cache. + if status_code is None and raw_code.isascii() and raw_code.isdigit(): + candidate = int(raw_code) + if 100 <= candidate <= 599: + status_code = candidate + return status_code, raw_code, str(message or "").strip().lower() + + +_IMAGE_UNSUPPORTED_CODES = frozenset( + { + "image_input_unsupported", + "image_not_supported", + "images_not_supported", + "vision_not_supported", + "multimodal_not_supported", + "unsupported_image", + "unsupported_images", + "ensemble_multimodal_unsupported", + } +) +_IMAGE_UNSUPPORTED_RE = re.compile( + r"(?:image|images|vision|multimodal|picture|图片|图像|视觉).{0,80}" + r"(?:not supported|unsupported|does not support|cannot process|" + r"unable to (?:process|handle|accept|analy[sz]e|read)|不支持|无法处理|不能处理)" + r"|(?:does not support|unsupported|not supported|不支持|无法处理).{0,80}" + r"(?:image|images|vision|multimodal|picture|图片|图像|视觉)", +) +_IMAGE_ENDPOINT_UNAVAILABLE_RE = re.compile( + r"\bno endpoints found that support image inputs?\b", +) +_INVALID_MEDIA_RE = re.compile( + r"(?:image|images|picture|图片|图像).{0,80}" + r"(?:invalid|corrupt|malformed|decode|format|mime|media type|too large|" + r"download|fetch|load|inaccessible|尺寸|大小|损坏|格式)", +) + +_INVALID_MEDIA_CODES = frozenset( + { + "invalid_image", + "invalid_media", + "image_too_large", + "unsupported_media_type", + } +) + +_TRANSIENT_STATUS_CODES = frozenset( + {408, 409, 425, 499, 500, 502, 503, 504, 520, 521, 522, 523, 524, 529} +) + + +def classify_image_input_error( + error: object, + *, + provider_name: str = "", +) -> ImageFailureKind: + """Classify only precise image-related evidence. + + Generic ``unsupported`` text is not enough: the message must identify an + image/vision input. This prevents authentication, rate-limit, transport, + and ordinary bad-request failures from poisoning a deployment's capability + cache. + """ + + status_code, raw_code, message = _error_fields(error) + joined = f"{raw_code} {message}".strip() + + # HTTP admission/transport status is stronger evidence than incidental + # image wording in a gateway message. In particular, a 401/429/503 must + # never poison the exact deployment's vision-capability cache, even if the + # body repeats an upstream "image unsupported" sentence. + if status_code in {401, 403}: + return ImageFailureKind.AUTHENTICATION + if status_code == 402: + return ImageFailureKind.INSUFFICIENT_CREDITS + if status_code == 429: + return ImageFailureKind.RATE_LIMITED + if status_code in _TRANSIENT_STATUS_CODES: + return ImageFailureKind.TRANSIENT + + # Keep this import lazy: failures.py imports the provider registry, while + # this low-level module is also imported by provider package initialisation. + try: + from .failures import ProviderFailureKind, classify_provider_error + + provider_kind = classify_provider_error(provider_name, status_code, raw_code, message) + except Exception: # noqa: BLE001 - classification must never break recovery + provider_kind = None + + if provider_kind is not None: + # These provider-wide failures outrank all message-level image text. + # BAD_REQUEST is intentionally handled later: real vision capability + # rejections are commonly delivered as HTTP 400. + if provider_kind is ProviderFailureKind.CONTEXT_OVERFLOW: + return ImageFailureKind.CONTEXT_OVERFLOW + if provider_kind is ProviderFailureKind.AUTH_INVALID: + return ImageFailureKind.AUTHENTICATION + if provider_kind is ProviderFailureKind.INSUFFICIENT_CREDITS: + return ImageFailureKind.INSUFFICIENT_CREDITS + if provider_kind is ProviderFailureKind.RATE_LIMITED: + return ImageFailureKind.RATE_LIMITED + if provider_kind is ProviderFailureKind.MODEL_NOT_FOUND: + # An aggregator can use 404 for a valid model whose endpoints + # cannot accept the requested modality. Only this precise image + # admission response overrides the ordinary missing-model path. + if status_code == 404 and _IMAGE_ENDPOINT_UNAVAILABLE_RE.search(message): + return ImageFailureKind.UNSUPPORTED_INPUT + return ImageFailureKind.MODEL_NOT_FOUND + if provider_kind is ProviderFailureKind.POLICY_REFUSAL: + return ImageFailureKind.POLICY_REFUSAL + if provider_kind in { + ProviderFailureKind.PROVIDER_OVERLOADED, + ProviderFailureKind.TRANSPORT_TRANSIENT, + }: + return ImageFailureKind.TRANSIENT + + # Invalid/corrupt input is not evidence that the configured model lacks + # multimodal capability. Exact media codes therefore precede the + # unsupported-input matcher. + if raw_code in _INVALID_MEDIA_CODES: + return ImageFailureKind.INVALID_MEDIA + if raw_code in _IMAGE_UNSUPPORTED_CODES: + return ImageFailureKind.UNSUPPORTED_INPUT + if _INVALID_MEDIA_RE.search(joined): + return ImageFailureKind.INVALID_MEDIA + if _IMAGE_UNSUPPORTED_RE.search(joined): + return ImageFailureKind.UNSUPPORTED_INPUT + + if provider_kind is not None: + if provider_kind is ProviderFailureKind.BAD_REQUEST: + return ImageFailureKind.BAD_REQUEST + return ImageFailureKind.UNKNOWN + + +def classify_image_failure( + error: object, + *, + provider_name: str = "", +) -> ImageFailureClassification: + """Return image classification and the corresponding safe action.""" + + kind = classify_image_input_error(error, provider_name=provider_name) + if kind is ImageFailureKind.UNSUPPORTED_INPUT: + return ImageFailureClassification( + kind=kind, + caches_unsupported=True, + retry_without_image=True, + reason="precise image-input capability rejection", + ) + if kind is ImageFailureKind.INVALID_MEDIA: + return ImageFailureClassification( + kind=kind, + reason="the image material is invalid or exceeds media limits", + ) + if kind is ImageFailureKind.CONTEXT_OVERFLOW: + return ImageFailureClassification(kind=kind, reason="rebuild after context compaction") + return ImageFailureClassification(kind=kind) + + +# Alternate names used by orchestration code and tests. +classify_provider_image_error = classify_image_input_error +ImageInputFailureKind = ImageFailureKind + + +__all__ = [ + "VisionSupport", + "VisionSupportValue", + "VisionSupportSource", + "VisionSupportStatus", + "VisionSupportEvidence", + "normalize_vision_support", + "resolve_vision_support", + "ImageProjectionMode", + "ImageIntentKind", + "ImageIntent", + "ImageMarkerState", + "normalize_marker_state", + "image_marker", + "build_image_marker", + "marker_for_image", + "ImageProjectionPolicy", + "projection_mode_for_support", + "ImageProjectionDecision", + "MediaProjectionResult", + "ImageProjectionResult", + "count_image_blocks", + "has_image_blocks", + "bind_image_attachment_ids", + "project_messages", + "project_messages_for_model", + "project_image_messages", + "assert_text_only_messages", + "ImageFailureKind", + "ImageInputFailureKind", + "ImageFailureClassification", + "classify_image_input_error", + "classify_provider_image_error", + "classify_image_failure", +] diff --git a/src/opensquilla/provider/model_catalog.py b/src/opensquilla/provider/model_catalog.py index c64927b43..9ecc53ada 100644 --- a/src/opensquilla/provider/model_catalog.py +++ b/src/opensquilla/provider/model_catalog.py @@ -281,9 +281,9 @@ def _corrections_budget_fallback(model_id: str) -> tuple[int, int] | None: def _live_layer_fields(info: ModelInfo | None) -> dict[str, Any]: """Fields the live provider catalog knows, adapted per-1k → per-Mtok. - Capability booleans are computed deterministically from the provider - response at populate time, so they are emitted as known whenever the - model is in the cache. A 0.0 per-1k price is the live cache's "free or + Vision is known only when the provider actually supplied input modalities; + the compatibility boolean default is not evidence of a text-only model. + A 0.0 per-1k price is the live cache's "free or unknown" sentinel, so costs are emitted only when positive — this layer never claims a known $0 price. """ @@ -292,8 +292,9 @@ def _live_layer_fields(info: ModelInfo | None) -> dict[str, Any]: fields: dict[str, Any] = { "supports_reasoning": info.supports_reasoning, "supports_tools": info.supports_tools, - "supports_vision": info.supports_vision, } + if "supports_vision" in info.model_fields_set: + fields["supports_vision"] = info.supports_vision if info.display_name: fields["display_name"] = info.display_name if info.context_window > 0: @@ -474,9 +475,16 @@ def _populate_from_data(self, models: list[dict]) -> None: max_completion = top_provider.get("max_completion_tokens") or 0 supported = set(m.get("supported_parameters", [])) architecture = m.get("architecture") or {} - input_modalities = { - str(item).lower() for item in architecture.get("input_modalities", []) - } + modalities = architecture.get("input_modalities") + vision_fields: dict[str, Any] = {} + if ( + isinstance(modalities, list) + and modalities + and all(isinstance(item, str) and item.strip() for item in modalities) + ): + vision_fields["supports_vision"] = "image" in { + item.strip().lower() for item in modalities + } pricing = m.get("pricing") or {} self._models[model_id] = ModelInfo( provider="openrouter", @@ -486,7 +494,7 @@ def _populate_from_data(self, models: list[dict]) -> None: max_output_tokens=max_completion, supports_reasoning="reasoning" in supported or "reasoning_effort" in supported, supports_tools="tools" in supported or "tool_choice" in supported, - supports_vision="image" in input_modalities, + **vision_fields, input_cost_per_1k=_price_per_1k(pricing.get("prompt")), output_cost_per_1k=_price_per_1k(pricing.get("completion")), ) diff --git a/src/opensquilla/provider/preset_registry.py b/src/opensquilla/provider/preset_registry.py index 11e755b31..62ad90a43 100644 --- a/src/opensquilla/provider/preset_registry.py +++ b/src/opensquilla/provider/preset_registry.py @@ -105,15 +105,16 @@ def _tier( description: str, *, thinking_level: str = "", - supports_image: bool = False, + supports_image: bool | None = None, image_only: bool = False, ) -> dict: entry: dict[str, object] = { "provider": provider_id, "model": model, "description": description, - "supports_image": supports_image, } + if supports_image is not None: + entry["supports_image"] = supports_image if thinking_level: entry["thinking_level"] = thinking_level if image_only: @@ -303,7 +304,6 @@ def _synthesized_tiers(provider_id: str, default_model: str) -> dict[str, dict]: f"{provider_id} {role} route (synthesized default; no curated " f"per-tier model ladder)." ), - "supports_image": False, } return tiers diff --git a/src/opensquilla/provider/selector.py b/src/opensquilla/provider/selector.py index 31d0ee850..05ce953e7 100644 --- a/src/opensquilla/provider/selector.py +++ b/src/opensquilla/provider/selector.py @@ -388,6 +388,10 @@ def __init__( # a new failover chain later cannot re-enable provider-private replay. self._provider_state_replay_disabled = False self._capacity_bounded_fallbacks: frozenset[_CapacityConfigIdentity] | None = None + # A strict per-turn Router chain is an authorization boundary. Once + # installed, failure-specific plugin hooks must not replace it with a + # deployment that was not present in the routed c-tier ladder. + self._static_fallback_chain_only = False def _apply_capacity_fallback_bound(self) -> None: allowed = self._capacity_bounded_fallbacks @@ -512,7 +516,11 @@ def _fallback_chain_after_failure( """Resolve and constrain a failure-specific chain without mutating state.""" current = self._chain[self._index] - if self._plugin is not None and hasattr(self._plugin, "failover_hook"): + if ( + not self._static_fallback_chain_only + and self._plugin is not None + and hasattr(self._plugin, "failover_hook") + ): chain = resolve_failover_chain(primary_failure, self._config, self._plugin) else: chain = list(self._chain[self._index + 1 :]) @@ -593,6 +601,90 @@ def override_provider_config( deduped_fallbacks.append(candidate) self._chain = [cfg, *deduped_fallbacks] self._index = 0 + self._static_fallback_chain_only = not preserve_existing_tail + self._apply_capacity_fallback_bound() + + def override_provider_config_with_fallback_chain( + self, + cfg: ProviderConfig, + fallback_chain: list[object], + *, + preserve_existing_tail: bool = True, + ) -> None: + """Install a full cross-provider head plus an authorized Router tail. + + Resolved ``ProviderConfig`` entries retain their own credentials. + Compact metadata entries can reuse credentials only from the selected + provider or from the selector's original configured provider. Other + cross-provider entries are ignored unless the caller resolved them + first, so this boundary never guesses credentials. + """ + + if self._provider_state_replay_disabled: + cfg = _without_provider_state_replay(cfg) + original_primary = self._chain[0] + existing_candidates = [ + original_primary, + *self._chain[1:], + self._config.primary, + *self._config.fallbacks, + ] + existing_by_provider_model = { + (candidate.provider, candidate.model): candidate + for candidate in existing_candidates + } + + router_fallbacks: list[ProviderConfig] = [] + for entry in fallback_chain: + candidate: ProviderConfig | None = None + if isinstance(entry, ProviderConfig): + candidate = entry + elif isinstance(entry, Mapping): + candidate_model = str(entry.get("model") or "").strip() + if not candidate_model: + continue + candidate_provider = ( + str(entry.get("provider") or original_primary.provider).strip() + or original_primary.provider + ) + candidate = existing_by_provider_model.get( + (candidate_provider, candidate_model) + ) + if candidate is None: + credential_source: ProviderConfig | None = None + if candidate_provider == cfg.provider: + credential_source = cfg + elif candidate_provider == original_primary.provider: + credential_source = original_primary + if credential_source is not None: + candidate = replace( + credential_source, + model=candidate_model, + provider_routing=dict(credential_source.provider_routing), + ) + if candidate is None: + continue + if self._provider_state_replay_disabled: + candidate = _without_provider_state_replay(candidate) + router_fallbacks.append(candidate) + + existing_tail = list(self._chain) + deduped_tail: list[ProviderConfig] = [] + seen: set[_ProviderConfigIdentity] = {_provider_config_identity(cfg)} + candidates = ( + [*router_fallbacks, *existing_tail] + if preserve_existing_tail + else router_fallbacks + ) + for candidate in candidates: + identity = _provider_config_identity(candidate) + if identity in seen: + continue + seen.add(identity) + deduped_tail.append(candidate) + self._chain = [cfg, *deduped_tail] + self._index = 0 + self._static_fallback_chain_only = not preserve_existing_tail self._apply_capacity_fallback_bound() def override_provider_config_with_bounded_fallbacks( @@ -701,6 +793,17 @@ def override_model_with_fallback_chain( router_fallbacks: list[ProviderConfig] = [] for entry in fallback_chain: + if isinstance(entry, ProviderConfig): + candidate = replace( + entry, + provider_routing=dict(entry.provider_routing), + ) + if not candidate.provider.strip() or not candidate.model.strip(): + continue + if candidate.provider.lower() != current.provider.lower(): + candidate = _without_provider_state_replay(candidate) + router_fallbacks.append(candidate) + continue if not isinstance(entry, Mapping): continue candidate_model = str(entry.get("model") or "").strip() @@ -745,6 +848,7 @@ def override_model_with_fallback_chain( deduped_tail.append(cfg) self._chain = [current, *deduped_tail] self._index = 0 + self._static_fallback_chain_only = not preserve_existing_tail def override_model_with_bounded_fallback_chain( self, @@ -783,6 +887,7 @@ def sync_primary(self, cfg: ProviderConfig) -> None: cfg = _without_provider_state_replay(cfg) self._config.primary = cfg self._chain[0] = cfg + self._static_fallback_chain_only = False self.reset() def reset(self) -> None: @@ -811,6 +916,7 @@ def clone(self) -> ModelSelector: cloned = ModelSelector(config_copy, plugin=self._plugin) cloned._provider_state_replay_disabled = self._provider_state_replay_disabled cloned._capacity_bounded_fallbacks = self._capacity_bounded_fallbacks + cloned._static_fallback_chain_only = self._static_fallback_chain_only cloned._apply_capacity_fallback_bound() return cloned diff --git a/src/opensquilla/provider/types.py b/src/opensquilla/provider/types.py index 124467abd..c2f122242 100644 --- a/src/opensquilla/provider/types.py +++ b/src/opensquilla/provider/types.py @@ -677,6 +677,9 @@ class ContentBlockImage(BaseModel): source_type: Literal["base64", "url"] = "base64" media_type: str # "image/png", "image/jpeg", etc. data: str # base64 data or URL + # Request-local provenance only. It binds marker/retry decisions to the + # canonical occurrence without ever entering a provider wire payload. + attachment_id: str | None = Field(default=None, exclude=True, repr=False) class ContentBlockDocument(BaseModel): diff --git a/src/opensquilla/session/__init__.py b/src/opensquilla/session/__init__.py index 5de4f72cd..0e7127e8d 100644 --- a/src/opensquilla/session/__init__.py +++ b/src/opensquilla/session/__init__.py @@ -1,5 +1,26 @@ """opensquilla.session — Session management: lifecycle, storage, key construction, compaction.""" +from opensquilla.session.attachment_manifest import ( + ATTACHMENT_MANIFEST_PROVIDER, + ATTACHMENT_MANIFEST_SCHEMA_VERSION, + ATTACHMENT_MANIFEST_STATE_KIND, + MATERIAL_AVAILABLE, + MATERIAL_INVALID, + MATERIAL_MISSING, + AttachmentManifest, + AttachmentManifestError, + AttachmentManifestStore, + AttachmentOccurrence, + attachment_manifest_from_context_state, + build_attachment_manifest, + deterministic_attachment_id, + extract_attachment_occurrences, + extract_attachment_occurrences_from_envelope, + legacy_attachment_id, + lookup_attachment_occurrence, + manifest_context_state, + merge_attachment_occurrences, +) from opensquilla.session.compaction import ( CompactionConfig, CompactionRequest, @@ -83,4 +104,24 @@ "call_compact_with_optional_config", "compact_accepts_config", "compact_context", + # Attachment occurrence manifest + "ATTACHMENT_MANIFEST_PROVIDER", + "ATTACHMENT_MANIFEST_SCHEMA_VERSION", + "ATTACHMENT_MANIFEST_STATE_KIND", + "MATERIAL_AVAILABLE", + "MATERIAL_INVALID", + "MATERIAL_MISSING", + "AttachmentManifest", + "AttachmentManifestError", + "AttachmentManifestStore", + "AttachmentOccurrence", + "attachment_manifest_from_context_state", + "build_attachment_manifest", + "deterministic_attachment_id", + "extract_attachment_occurrences", + "extract_attachment_occurrences_from_envelope", + "legacy_attachment_id", + "lookup_attachment_occurrence", + "manifest_context_state", + "merge_attachment_occurrences", ] diff --git a/src/opensquilla/session/attachment_manifest.py b/src/opensquilla/session/attachment_manifest.py new file mode 100644 index 000000000..377cde1dd --- /dev/null +++ b/src/opensquilla/session/attachment_manifest.py @@ -0,0 +1,959 @@ +"""Durable attachment occurrence indexing for session history. + +The transcript is the authority for user-visible content. This module keeps +the smaller, machine-readable index needed to find an attachment after a +compaction moved its source row out of the active transcript. The index is +stored as a ``SessionContextState`` snapshot rather than a second SQL schema; +that makes it safe to deploy alongside older databases and lets the existing +session fork/reset code carry the state forward. + +No image bytes are stored in the manifest. Inline legacy envelopes are only +decoded long enough to derive a content hash and a deterministic occurrence +identifier. Request-time image projection belongs to the engine layer and +must never mutate this index. +""" + +from __future__ import annotations + +import base64 +import binascii +import hashlib +import json +import re +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, replace +from datetime import UTC, datetime +from typing import Any, Protocol, cast + +from opensquilla.session.keys import canonicalize_session_key +from opensquilla.session.models import SessionContextState + +ATTACHMENT_MANIFEST_STATE_KIND = "attachment_manifest_v1" +ATTACHMENT_MANIFEST_PROVIDER = "portable" +ATTACHMENT_MANIFEST_SCHEMA_VERSION = 1 + +MATERIAL_AVAILABLE = "available" +MATERIAL_MISSING = "missing" +MATERIAL_INVALID = "invalid" +MATERIAL_STATES = frozenset( + {MATERIAL_AVAILABLE, MATERIAL_MISSING, MATERIAL_INVALID} +) + +_ATTACHMENT_ID_RE = re.compile(r"^att_[A-Za-z0-9_-]{8,160}$") +_SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$") +_MAX_NAME_BYTES = 160 +_MAX_MIME_BYTES = 120 +_MAX_REASON_BYTES = 256 +_MAX_MESSAGE_ID_BYTES = 512 +_MAX_MANIFEST_OCCURRENCES = 100_000 +_MAX_INLINE_BYTES = 64 * 1024 * 1024 + + +class AttachmentManifestError(ValueError): + """Raised when an attachment occurrence cannot be indexed safely.""" + + +class AttachmentManifestStorage(Protocol): + """Small storage surface used by :class:`AttachmentManifestStore`. + + ``SessionStorage`` and ``SessionManager`` both expose these methods. A + protocol keeps this module independent of either high-level owner and + makes the persistence adapter straightforward to exercise with a fake. + """ + + async def save_context_state( + self, state: SessionContextState + ) -> SessionContextState: ... + + async def get_context_states( + self, + session_key: str, + *, + provider: str | None = None, + state_kind: str | None = None, + valid_only: bool = True, + ) -> list[SessionContextState]: ... + + async def get_canonical_transcript( + self, + session_id: str, + limit: int | None = None, + offset: int = 0, + ) -> list[object]: ... + + +def _now_ms() -> int: + return int(datetime.now(UTC).timestamp() * 1000) + + +def _field(value: object, name: str, default: object = None) -> object: + """Read a field from either a mapping or a SQLModel/dataclass row.""" + + if isinstance(value, Mapping): + return value.get(name, default) + return getattr(value, name, default) + + +def _bounded_text(value: object, *, fallback: str, max_bytes: int) -> str: + if not isinstance(value, str): + return fallback + normalized = " ".join(value.strip().split()) + if not normalized: + return fallback + # Slice by encoded bytes, not code points, so the serialized state stays + # bounded for non-ASCII filenames and MIME-like values. + encoded = normalized.encode("utf-8") + if len(encoded) <= max_bytes: + return normalized + return encoded[:max_bytes].decode("utf-8", errors="ignore") or fallback + + +def normalize_attachment_name(value: object, *, fallback: str = "attachment") -> str: + """Return a bounded display name safe for a model-visible descriptor.""" + + if isinstance(value, str): + # Persisted attachment names are display metadata, never storage + # locations. Treat both separators as path separators so manifests + # created on one platform cannot expose a path when read on another. + value = value.strip().replace("\\", "/").rsplit("/", 1)[-1] + return _bounded_text(value, fallback=fallback, max_bytes=_MAX_NAME_BYTES) + + +def normalize_attachment_mime(value: object) -> str: + """Normalize a MIME value without retaining parameters or control chars.""" + + if not isinstance(value, str): + return "application/octet-stream" + normalized = value.split(";", 1)[0].strip().lower() + if "/" not in normalized or any(char in normalized for char in "\r\n"): + return "application/octet-stream" + return _bounded_text( + normalized, + fallback="application/octet-stream", + max_bytes=_MAX_MIME_BYTES, + ) + + +def valid_sha256(value: object) -> str | None: + """Return a canonical lower-case SHA-256 digest, or ``None``.""" + + if not isinstance(value, str) or _SHA256_RE.fullmatch(value) is None: + return None + return value.lower() + + +def valid_attachment_id(value: object) -> str | None: + """Return a valid persisted occurrence ID, or ``None`` for legacy data.""" + + if not isinstance(value, str) or _ATTACHMENT_ID_RE.fullmatch(value) is None: + return None + return value + + +def legacy_attachment_id( + *, + session_id: str, + message_id: str, + index: int, + sha256: str | None = None, +) -> str: + """Derive the stable ID used when an old envelope has no occurrence ID. + + The input uses the source message, ordinal and content hash. Attachment + lookup remains session-scoped, while omitting the session identity keeps + the logical occurrence stable when a transcript is fully forked. The + ``session_id`` argument is retained for call-site compatibility. + """ + + if isinstance(index, bool) or not isinstance(index, int) or index < 0: + raise AttachmentManifestError("attachment index must be a non-negative integer") + digest = hashlib.sha256( + f"{message_id}\0{index}\0{sha256 or ''}".encode() + ).digest()[:18] + token = base64.urlsafe_b64encode(digest).decode("ascii").rstrip("=") + return f"att_legacy_{token}" + + +# Name used by a few call sites that describe this as a deterministic rather +# than legacy identity. Keep both names public so migration code need not +# duplicate the algorithm. +deterministic_attachment_id = legacy_attachment_id + + +def _decode_inline_data(value: object) -> bytes | None: + if isinstance(value, bytes): + if len(value) > _MAX_INLINE_BYTES: + return None + return value + if not isinstance(value, str): + return None + try: + decoded = base64.b64decode(value, validate=True) + except (binascii.Error, ValueError, TypeError): + return None + if len(decoded) > _MAX_INLINE_BYTES: + return None + return decoded + + +def _declared_size(value: object) -> int | None: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + return None + return value + + +def _entry_id(value: object) -> int | None: + raw = value + if isinstance(raw, bool): + return None + if isinstance(raw, int): + return raw if raw >= 0 else None + if isinstance(raw, str) and raw.strip().isdigit(): + try: + parsed = int(raw.strip()) + except ValueError: + return None + return parsed if parsed >= 0 else None + return None + + +def _message_id(value: object, *, source_entry_id: int | None, fallback_index: int) -> str: + raw = value if isinstance(value, str) else "" + normalized = raw.strip() + if not normalized: + normalized = ( + f"entry-{source_entry_id}" + if source_entry_id is not None + else f"legacy-entry-{fallback_index}" + ) + encoded = normalized.encode("utf-8") + if len(encoded) > _MAX_MESSAGE_ID_BYTES: + normalized = encoded[:_MAX_MESSAGE_ID_BYTES].decode("utf-8", errors="ignore") + return normalized + + +def _envelope_from_content(content: object) -> Mapping[str, Any] | None: + if isinstance(content, Mapping): + return cast(Mapping[str, Any], content) + if not isinstance(content, str): + return None + try: + parsed = json.loads(content) + except (TypeError, ValueError, json.JSONDecodeError): + return None + if not isinstance(parsed, Mapping): + return None + return cast(Mapping[str, Any], parsed) + + +def _occurrence_from_item( + item: Mapping[str, Any], + *, + session_id: str, + source_message_id: str, + source_entry_id: int | None, + ordinal: int, + created_at: int, +) -> AttachmentOccurrence: + explicit_sha = item.get("sha256_ref") + if explicit_sha is None: + # A few pre-envelope writers used ``sha256``/``material_id``. Read + # those aliases but never emit them in the new manifest payload. + explicit_sha = item.get("sha256") or item.get("material_id") + sha256_ref = valid_sha256(explicit_sha) + raw_data = item.get("data") + decoded = _decode_inline_data(raw_data) + has_data_field = "data" in item + + material_state = MATERIAL_MISSING + missing_reason: str | None = None + if isinstance(item.get("missing_reason"), str) and item["missing_reason"].strip(): + material_state = MATERIAL_MISSING + missing_reason = _bounded_text( + item["missing_reason"], + fallback="attachment unavailable", + max_bytes=_MAX_REASON_BYTES, + ) + elif explicit_sha is not None and sha256_ref is None: + material_state = MATERIAL_INVALID + missing_reason = "invalid sha256 reference" + elif has_data_field and decoded is None: + material_state = MATERIAL_INVALID + missing_reason = "invalid inline attachment data" + elif sha256_ref is not None or decoded is not None: + material_state = MATERIAL_AVAILABLE + + computed_sha = hashlib.sha256(decoded).hexdigest() if decoded is not None else None + if sha256_ref is None and computed_sha is not None: + sha256_ref = computed_sha + elif sha256_ref is not None and computed_sha is not None and sha256_ref != computed_sha: + material_state = MATERIAL_INVALID + missing_reason = "attachment hash mismatch" + + declared = _declared_size(item.get("size")) + actual_size = len(decoded) if decoded is not None else None + size = declared if declared is not None else actual_size + if declared is not None and actual_size is not None and declared != actual_size: + material_state = MATERIAL_INVALID + missing_reason = "attachment size mismatch" + + # A missing/invalid item still needs a stable identity. Empty SHA is + # intentional: source message + ordinal distinguish occurrences, while + # malformed bytes never become part of an identifier. + attachment_id = valid_attachment_id(item.get("attachment_id")) + if attachment_id is None: + attachment_id = legacy_attachment_id( + session_id=session_id, + message_id=source_message_id, + index=ordinal, + sha256=sha256_ref, + ) + + return AttachmentOccurrence( + attachment_id=attachment_id, + source_entry_id=source_entry_id, + source_message_id=source_message_id, + ordinal=ordinal, + sha256_ref=sha256_ref, + name=normalize_attachment_name(item.get("name")), + mime=normalize_attachment_mime( + item.get("mime") or item.get("type") or item.get("media_type") + ), + size=size, + material_state=material_state, + created_at=created_at, + missing_reason=missing_reason, + ) + + +def extract_attachment_occurrences_from_envelope( + envelope: object, + *, + session_id: str, + source_message_id: str, + source_entry_id: int | None = None, + created_at: int = 0, +) -> tuple[AttachmentOccurrence, ...]: + """Extract attachment occurrences from one canonical envelope. + + The function accepts either the persisted JSON string or an already parsed + mapping, which is useful during legacy backfill and in storage tests. + Invalid/non-object attachment list members are ignored because they do not + identify a material occurrence; malformed material inside an object is + retained as ``material_state='invalid'`` for deterministic degradation. + """ + + parsed = _envelope_from_content(envelope) + if parsed is None: + return () + raw_attachments = parsed.get("attachments") + if not isinstance(raw_attachments, (list, tuple)): + return () + result: list[AttachmentOccurrence] = [] + for ordinal, raw_item in enumerate(raw_attachments): + if not isinstance(raw_item, Mapping): + continue + result.append( + _occurrence_from_item( + cast(Mapping[str, Any], raw_item), + session_id=session_id, + source_message_id=source_message_id, + source_entry_id=source_entry_id, + ordinal=ordinal, + created_at=created_at, + ) + ) + return tuple(result) + + +def extract_attachment_occurrences( + entries: Iterable[object], + *, + session_id: str | None = None, + include_roles: Iterable[str] = ("user",), +) -> tuple[AttachmentOccurrence, ...]: + """Extract occurrences from active and/or compacted transcript rows. + + ``SessionStorage.get_canonical_transcript`` already merges the active and + archived tables. Passing that result here therefore gives one lookup + path for both states. By default only user rows are considered, matching + the canonical attachment envelope contract and avoiding accidental index + entries from arbitrary assistant/tool metadata. + """ + + allowed_roles = {str(role).strip().lower() for role in include_roles} + result: list[AttachmentOccurrence] = [] + for fallback_index, entry in enumerate(entries): + role = _field(entry, "role", "user") + if isinstance(role, str) and role.strip().lower() not in allowed_roles: + continue + entry_session_id = _field(entry, "session_id", "") + resolved_session_id = session_id or ( + entry_session_id if isinstance(entry_session_id, str) else "" + ) + physical_id = _entry_id(_field(entry, "id")) + message_id = _message_id( + _field(entry, "message_id"), + source_entry_id=physical_id, + fallback_index=fallback_index, + ) + created_at_raw = _field(entry, "created_at", 0) + created_at = _entry_id(created_at_raw) or 0 + content = _field(entry, "content") + result.extend( + extract_attachment_occurrences_from_envelope( + content, + session_id=resolved_session_id, + source_message_id=message_id, + source_entry_id=physical_id, + created_at=created_at, + ) + ) + return tuple(result) + + +def preserve_attachment_occurrence_ids( + content: str | None, + *, + session_id: str, + source_message_id: str, +) -> str | None: + """Bind legacy occurrence IDs before copying an envelope to a new message. + + Prefix forks intentionally allocate new message identities. Persist the + source occurrence identity in the copied envelope so existing attachment + references remain valid independently of the copied message's new ID. + Only missing or invalid attachment IDs change; material and user text are + retained, and an already bound envelope keeps its original serialization. + """ + + parsed = _envelope_from_content(content) + if parsed is None: + return content + attachments = parsed.get("attachments") + if not isinstance(attachments, list) or not any( + isinstance(item, Mapping) and valid_attachment_id(item.get("attachment_id")) is None + for item in attachments + ): + return content + occurrences = extract_attachment_occurrences_from_envelope( + parsed, + session_id=session_id, + source_message_id=source_message_id, + ) + copied_attachments = list(attachments) + for occurrence in occurrences: + item = attachments[occurrence.ordinal] + copied_attachments[occurrence.ordinal] = { + **item, + "attachment_id": occurrence.attachment_id, + } + return json.dumps( + {**parsed, "attachments": copied_attachments}, + ensure_ascii=False, + ) + + +@dataclass(frozen=True, slots=True) +class AttachmentOccurrence: + """One logical attachment occurrence in canonical session history.""" + + attachment_id: str + source_message_id: str + ordinal: int + sha256_ref: str | None = None + name: str = "attachment" + mime: str = "application/octet-stream" + size: int | None = None + material_state: str = MATERIAL_MISSING + source_entry_id: int | None = None + created_at: int = 0 + missing_reason: str | None = None + + @property + def message_id(self) -> str: + """Compatibility alias used by existing attachment resource code.""" + + return self.source_message_id + + @property + def index(self) -> int: + """Compatibility alias for the envelope ordinal.""" + + return self.ordinal + + @property + def sha256(self) -> str | None: + """Compatibility alias for the canonical content reference.""" + + return self.sha256_ref + + def to_payload(self) -> dict[str, Any]: + """Serialize metadata only; never include inline bytes or paths.""" + + payload: dict[str, Any] = { + "attachment_id": self.attachment_id, + "source_entry_id": self.source_entry_id, + "source_message_id": self.source_message_id, + "ordinal": self.ordinal, + "sha256_ref": self.sha256_ref, + "name": self.name, + "mime": self.mime, + "size": self.size, + "material_state": self.material_state, + "created_at": self.created_at, + } + if self.missing_reason: + payload["missing_reason"] = self.missing_reason + return payload + + @classmethod + def from_payload(cls, raw: object) -> AttachmentOccurrence: + if not isinstance(raw, Mapping): + raise AttachmentManifestError("attachment occurrence payload must be an object") + attachment_id = valid_attachment_id(raw.get("attachment_id")) + source_message_id = raw.get("source_message_id") + ordinal = raw.get("ordinal") + if attachment_id is None: + raise AttachmentManifestError("attachment occurrence ID is invalid") + if not isinstance(source_message_id, str) or not source_message_id: + raise AttachmentManifestError("attachment source message ID is missing") + if isinstance(ordinal, bool) or not isinstance(ordinal, int) or ordinal < 0: + raise AttachmentManifestError("attachment ordinal is invalid") + state = raw.get("material_state", MATERIAL_MISSING) + if state not in MATERIAL_STATES: + raise AttachmentManifestError("attachment material state is invalid") + source_entry_id = _entry_id(raw.get("source_entry_id")) + size = _declared_size(raw.get("size")) + created_at = _entry_id(raw.get("created_at")) or 0 + missing_reason_raw = raw.get("missing_reason") + missing_reason = ( + _bounded_text( + missing_reason_raw, + fallback="attachment unavailable", + max_bytes=_MAX_REASON_BYTES, + ) + if isinstance(missing_reason_raw, str) and missing_reason_raw.strip() + else None + ) + return cls( + attachment_id=attachment_id, + source_entry_id=source_entry_id, + source_message_id=source_message_id, + ordinal=ordinal, + sha256_ref=valid_sha256(raw.get("sha256_ref")), + name=normalize_attachment_name(raw.get("name")), + mime=normalize_attachment_mime(raw.get("mime")), + size=size, + material_state=str(state), + created_at=created_at, + missing_reason=missing_reason, + ) + + +def _logical_identity(occurrence: AttachmentOccurrence) -> tuple[object, ...]: + """Identity independent of a physical transcript row ID. + + Forked sessions preserve message IDs and attachment IDs but allocate new + database row IDs. Consequently ``source_entry_id`` is deliberately not + part of this identity. + """ + + source = occurrence.source_message_id or f"entry:{occurrence.source_entry_id}" + # The hash is intentionally excluded here. A legacy snapshot may have + # indexed an occurrence before its inline bytes were decoded; a later + # canonical rebuild should be able to fill in that hash. If both sides do + # carry a hash, ``_merge_occurrence`` checks that they agree. + return (source, occurrence.ordinal) + + +def _merge_occurrence( + old: AttachmentOccurrence, + new: AttachmentOccurrence, +) -> AttachmentOccurrence: + if _logical_identity(old) != _logical_identity(new): + raise AttachmentManifestError( + f"attachment ID collision for {old.attachment_id}" + ) + if ( + old.sha256_ref is not None + and new.sha256_ref is not None + and old.sha256_ref != new.sha256_ref + ): + raise AttachmentManifestError( + f"attachment ID collision for {old.attachment_id}" + ) + # Prefer a usable material record over a degraded one, while retaining the + # oldest source location for deterministic ordering. + old_rank = ( + 2 + if old.material_state == MATERIAL_AVAILABLE + else 1 + if old.material_state == MATERIAL_INVALID + else 0 + ) + new_rank = ( + 2 + if new.material_state == MATERIAL_AVAILABLE + else 1 + if new.material_state == MATERIAL_INVALID + else 0 + ) + preferred = new if new_rank > old_rank else old + return replace( + preferred, + source_entry_id=( + old.source_entry_id + if old.source_entry_id is not None + else new.source_entry_id + ), + created_at=min(old.created_at, new.created_at), + name=(old.name if old.name != "attachment" else new.name), + mime=(old.mime if old.mime != "application/octet-stream" else new.mime), + ) + + +def merge_attachment_occurrences( + existing: Iterable[AttachmentOccurrence], + incoming: Iterable[AttachmentOccurrence], +) -> tuple[AttachmentOccurrence, ...]: + """Merge occurrence snapshots, de-duplicating exact logical repeats.""" + + by_id: dict[str, AttachmentOccurrence] = {} + order: list[str] = [] + for occurrence in (*tuple(existing), *tuple(incoming)): + if occurrence.attachment_id in by_id: + by_id[occurrence.attachment_id] = _merge_occurrence( + by_id[occurrence.attachment_id], occurrence + ) + else: + by_id[occurrence.attachment_id] = occurrence + order.append(occurrence.attachment_id) + if len(order) > _MAX_MANIFEST_OCCURRENCES: + raise AttachmentManifestError("attachment manifest is too large") + return tuple(by_id[attachment_id] for attachment_id in order) + + +@dataclass(frozen=True, slots=True) +class AttachmentManifest: + """Portable attachment occurrence snapshot for one session identity.""" + + session_id: str + session_key: str + occurrences: tuple[AttachmentOccurrence, ...] = () + covered_through_id: int = 0 + schema_version: int = ATTACHMENT_MANIFEST_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.covered_through_id < 0: + raise AttachmentManifestError("manifest coverage ID cannot be negative") + if self.schema_version != ATTACHMENT_MANIFEST_SCHEMA_VERSION: + raise AttachmentManifestError("unsupported attachment manifest schema") + # Validate duplicate IDs at construction time as well as during merge; + # callers often construct a snapshot directly from a migration. + merge_attachment_occurrences((), self.occurrences) + + def by_id(self, attachment_id: str) -> AttachmentOccurrence | None: + """Return one occurrence by exact logical ID.""" + + for occurrence in self.occurrences: + if occurrence.attachment_id == attachment_id: + return occurrence + return None + + def by_ids(self, attachment_ids: Iterable[str]) -> tuple[AttachmentOccurrence, ...]: + """Return occurrences in caller-supplied ID order, skipping misses.""" + + lookup = {occurrence.attachment_id: occurrence for occurrence in self.occurrences} + return tuple( + lookup[attachment_id] + for attachment_id in attachment_ids + if attachment_id in lookup + ) + + def merge( + self, + incoming: Iterable[AttachmentOccurrence], + *, + covered_through_id: int | None = None, + ) -> AttachmentManifest: + return replace( + self, + occurrences=merge_attachment_occurrences(self.occurrences, incoming), + covered_through_id=max( + self.covered_through_id, + covered_through_id if covered_through_id is not None else 0, + ), + ) + + def to_payload(self) -> dict[str, Any]: + """Return the JSON-compatible context-state payload.""" + + return { + "schema_version": self.schema_version, + "covered_through_id": self.covered_through_id, + "occurrences": [occurrence.to_payload() for occurrence in self.occurrences], + } + + @classmethod + def from_payload( + cls, + payload: object, + *, + session_id: str, + session_key: str, + ) -> AttachmentManifest: + if not isinstance(payload, Mapping): + raise AttachmentManifestError("attachment manifest payload must be an object") + schema_version = payload.get("schema_version", ATTACHMENT_MANIFEST_SCHEMA_VERSION) + if schema_version != ATTACHMENT_MANIFEST_SCHEMA_VERSION: + raise AttachmentManifestError("unsupported attachment manifest schema") + covered = _entry_id(payload.get("covered_through_id")) or 0 + raw_occurrences = payload.get("occurrences", []) + if not isinstance(raw_occurrences, (list, tuple)): + raise AttachmentManifestError("attachment manifest occurrences must be an array") + if len(raw_occurrences) > _MAX_MANIFEST_OCCURRENCES: + raise AttachmentManifestError("attachment manifest is too large") + parsed = tuple(AttachmentOccurrence.from_payload(item) for item in raw_occurrences) + return cls( + session_id=session_id, + session_key=canonicalize_session_key(session_key), + occurrences=merge_attachment_occurrences((), parsed), + covered_through_id=covered, + schema_version=ATTACHMENT_MANIFEST_SCHEMA_VERSION, + ) + + +def build_attachment_manifest( + entries: Iterable[object], + *, + session_id: str, + session_key: str, + covered_through_id: int | None = None, +) -> AttachmentManifest: + """Build a manifest from canonical active+archived transcript entries.""" + + occurrences = extract_attachment_occurrences(entries, session_id=session_id) + inferred_coverage = max( + (occurrence.source_entry_id or 0 for occurrence in occurrences), + default=0, + ) + return AttachmentManifest( + session_id=session_id, + session_key=canonicalize_session_key(session_key), + occurrences=merge_attachment_occurrences((), occurrences), + covered_through_id=max(inferred_coverage, covered_through_id or 0), + ) + + +def lookup_attachment_occurrence( + entries: Iterable[object], + attachment_id: str, + *, + session_id: str | None = None, +) -> AttachmentOccurrence | None: + """Look up an attachment in a canonical active/archive entry sequence. + + This pure helper is useful during lazy migration when a session has not + acquired a manifest snapshot yet. Callers normally pass the result of + ``SessionStorage.get_canonical_transcript`` so both active and archived + rows are covered in one deterministic scan. + """ + + occurrences = extract_attachment_occurrences(entries, session_id=session_id) + for occurrence in occurrences: + if occurrence.attachment_id == attachment_id: + return occurrence + return None + + +def manifest_context_state( + manifest: AttachmentManifest, + *, + created_at: int | None = None, +) -> SessionContextState: + """Create a portable context-state row for atomic compaction writes.""" + + return SessionContextState( + session_id=manifest.session_id, + session_key=canonicalize_session_key(manifest.session_key), + provider=ATTACHMENT_MANIFEST_PROVIDER, + model=None, + state_kind=ATTACHMENT_MANIFEST_STATE_KIND, + payload=manifest.to_payload(), + covered_through_id=manifest.covered_through_id, + created_at=created_at if created_at is not None else _now_ms(), + portable=True, + cacheable=True, + valid=True, + schema_version=ATTACHMENT_MANIFEST_SCHEMA_VERSION, + ) + + +def attachment_manifest_from_context_state( + state: SessionContextState, +) -> AttachmentManifest: + """Decode and validate one stored context-state row.""" + + if state.provider != ATTACHMENT_MANIFEST_PROVIDER: + raise AttachmentManifestError("context state provider is not portable") + if state.state_kind != ATTACHMENT_MANIFEST_STATE_KIND: + raise AttachmentManifestError("context state is not an attachment manifest") + return AttachmentManifest.from_payload( + state.payload, + session_id=state.session_id, + session_key=state.session_key, + ).merge((), covered_through_id=state.covered_through_id) + + +class AttachmentManifestStore: + """Persistence/query adapter backed by ``SessionContextState`` snapshots.""" + + def __init__(self, storage: AttachmentManifestStorage) -> None: + self._storage = storage + + async def load( + self, + session_key: str, + *, + session_id: str | None = None, + ) -> AttachmentManifest: + """Load the newest valid snapshot, tolerating a corrupt newest row.""" + + canonical_key = canonicalize_session_key(session_key) + states = await self._storage.get_context_states( + canonical_key, + provider=ATTACHMENT_MANIFEST_PROVIDER, + state_kind=ATTACHMENT_MANIFEST_STATE_KIND, + valid_only=True, + ) + # ``latest_context_state`` gives stable created_at/id ordering. Walk + # backwards if a partially written/old payload is malformed so one bad + # migration row does not hide a previous usable snapshot. + ordered = sorted( + states, + key=lambda state: ( + int(state.created_at or 0), + int(state.id or 0), + ), + ) + for state in reversed(ordered): + if session_id is not None and state.session_id != session_id: + continue + try: + manifest = attachment_manifest_from_context_state(state) + except AttachmentManifestError: + continue + return manifest + return AttachmentManifest( + session_id=session_id or "", + session_key=canonical_key, + occurrences=(), + ) + + async def save(self, manifest: AttachmentManifest) -> SessionContextState: + """Append a new immutable snapshot and return its stored row.""" + + return await self._storage.save_context_state(manifest_context_state(manifest)) + + async def rebuild( + self, + *, + session_id: str, + session_key: str, + entries: Iterable[object], + covered_through_id: int | None = None, + ) -> AttachmentManifest: + """Replace the logical snapshot with a canonical transcript rebuild.""" + + manifest = build_attachment_manifest( + entries, + session_id=session_id, + session_key=session_key, + covered_through_id=covered_through_id, + ) + await self.save(manifest) + return manifest + + async def rebuild_from_canonical( + self, + *, + session_id: str, + session_key: str, + covered_through_id: int | None = None, + ) -> AttachmentManifest: + """Rebuild and persist a snapshot from active plus archived rows.""" + + entries = await self._storage.get_canonical_transcript(session_id) + return await self.rebuild( + session_id=session_id, + session_key=session_key, + entries=entries, + covered_through_id=covered_through_id, + ) + + async def merge_entries( + self, + *, + session_id: str, + session_key: str, + entries: Iterable[object], + covered_through_id: int | None = None, + ) -> AttachmentManifest: + """Merge newly observed entries into the latest durable snapshot.""" + + current = await self.load(session_key, session_id=session_id) + incoming = extract_attachment_occurrences(entries, session_id=session_id) + merged = current.merge(incoming, covered_through_id=covered_through_id) + await self.save(merged) + return merged + + async def lookup( + self, + session_key: str, + attachment_id: str, + *, + session_id: str | None = None, + ) -> AttachmentOccurrence | None: + """Find one occurrence by exact ID in the latest valid snapshot.""" + + return (await self.load(session_key, session_id=session_id)).by_id(attachment_id) + + async def lookup_many( + self, + session_key: str, + attachment_ids: Iterable[str], + *, + session_id: str | None = None, + ) -> tuple[AttachmentOccurrence, ...]: + """Find multiple occurrences while preserving requested order.""" + + return (await self.load(session_key, session_id=session_id)).by_ids(attachment_ids) + + +__all__ = [ + "ATTACHMENT_MANIFEST_PROVIDER", + "ATTACHMENT_MANIFEST_SCHEMA_VERSION", + "ATTACHMENT_MANIFEST_STATE_KIND", + "MATERIAL_AVAILABLE", + "MATERIAL_INVALID", + "MATERIAL_MISSING", + "AttachmentManifest", + "AttachmentManifestError", + "AttachmentManifestStore", + "AttachmentOccurrence", + "attachment_manifest_from_context_state", + "build_attachment_manifest", + "deterministic_attachment_id", + "extract_attachment_occurrences", + "extract_attachment_occurrences_from_envelope", + "legacy_attachment_id", + "lookup_attachment_occurrence", + "manifest_context_state", + "merge_attachment_occurrences", + "normalize_attachment_mime", + "normalize_attachment_name", + "preserve_attachment_occurrence_ids", + "valid_attachment_id", + "valid_sha256", +] diff --git a/src/opensquilla/session/compaction.py b/src/opensquilla/session/compaction.py index 49f2821a4..c91bb4504 100644 --- a/src/opensquilla/session/compaction.py +++ b/src/opensquilla/session/compaction.py @@ -8,7 +8,7 @@ import json import time import uuid -from collections.abc import Awaitable, Callable, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Literal, cast @@ -34,6 +34,14 @@ derive_provider_request_correlation, ) from opensquilla.redaction import redact_error_text +from opensquilla.session.attachment_manifest import ( + extract_attachment_occurrences_from_envelope, + legacy_attachment_id, + normalize_attachment_mime, + normalize_attachment_name, + valid_attachment_id, + valid_sha256, +) from opensquilla.session.compaction_deployment import ( MAX_COMPACTION_LLM_CALLS, CompactionExecutionPlan, @@ -1128,7 +1136,47 @@ def _build_strict_identifier_instruction() -> str: ) -def _summarize_if_envelope(content: str) -> str: +def _summary_attachment_id( + attachment: dict[str, Any], + *, + session_id: str, + message_id: str, + ordinal: int, + derived_id: str | None = None, +) -> str: + """Return a stable, bounded ID for a compaction attachment descriptor. + + Compaction receives a flattened entry payload rather than the full session + object. Prefer the persisted occurrence ID; for legacy envelopes derive + the same deterministic namespace used by the attachment manifest. The + fallback intentionally does not inspect or emit inline bytes. + """ + + # ``derived_id`` comes from the manifest parser, which validates an + # explicit occurrence ID and deterministically replaces an invalid one. + # Prefer it so an arbitrary path/token cannot be smuggled into a summary + # through the attachment_id field. + if derived_id: + return derived_id + explicit = valid_attachment_id(attachment.get("attachment_id")) + if explicit is not None: + return explicit + raw_sha = attachment.get("sha256_ref") or attachment.get("sha256") + sha = valid_sha256(raw_sha) + return legacy_attachment_id( + session_id=session_id or "compaction", + message_id=message_id or "unknown", + index=max(0, ordinal), + sha256=sha, + ) + + +def _summarize_if_envelope( + content: str, + *, + session_id: str = "", + message_id: str = "", +) -> str: """Replace attachment-envelope JSON with a concise placeholder. User messages carrying images are persisted as @@ -1138,32 +1186,167 @@ def _summarize_if_envelope(content: str) -> str: Detect the envelope shape and return ``text`` plus a short attachment descriptor instead. Non-envelope strings pass through unchanged. """ - if not content.startswith('{"text":'): - return content try: parsed = json.loads(content) except (json.JSONDecodeError, ValueError): return content - if not isinstance(parsed, dict) or "text" not in parsed: + if not isinstance(parsed, dict): return content + atts = parsed.get("attachments") or [] text = parsed.get("text") if not isinstance(text, str): - return content - atts = parsed.get("attachments") or [] + # A malformed legacy envelope must still not expose attachment bytes + # or storage paths to the compactor. Preserve an empty narrative and + # render whatever valid attachment descriptors remain. + if not isinstance(atts, list) or not atts: + return content + text = "" if not isinstance(atts, list) or not atts: return text descs: list[str] = [] - for att in atts: + derived_ids: dict[int, str] = {} + try: + derived_ids = { + occurrence.ordinal: occurrence.attachment_id + for occurrence in extract_attachment_occurrences_from_envelope( + content, + session_id=session_id or "compaction", + source_message_id=message_id or "unknown", + ) + } + except (TypeError, ValueError): + derived_ids = {} + for ordinal, att in enumerate(atts): if not isinstance(att, dict): continue - name = att.get("name") or "image" - media = att.get("type") or "image/*" - descs.append(f"{name} ({media})") + raw_name = att.get("name") + if isinstance(raw_name, str): + # Persisted display names should already be basenames, but legacy + # envelopes sometimes stored a local path. A compaction summary + # needs a descriptor, never the host path. + raw_name = raw_name.replace("\\", "/").rsplit("/", 1)[-1] + name = normalize_attachment_name(raw_name, fallback="image") + media = normalize_attachment_mime( + att.get("mime") or att.get("type") or att.get("media_type") + ) + attachment_id = _summary_attachment_id( + att, + session_id=session_id, + message_id=message_id, + ordinal=ordinal, + derived_id=derived_ids.get(ordinal), + ) + descs.append(f"{name} ({media}; attachment_id={attachment_id})") if descs: return f"{text}\n[user attached: {', '.join(descs)}]" return text +_COMPACTION_IMAGE_MARKER = ( + "[image omitted from compaction input; original attachment remains in session history]" +) +_COMPACTION_IMAGE_BLOCK_TYPES = frozenset( + {"image", "image_url", "input_image", "output_image"} +) +_COMPACTION_IMAGE_PAYLOAD_KEYS = frozenset( + {"base64", "bytes", "data", "image_url", "path", "source", "url"} +) + + +def _is_known_image_mapping(value: Mapping[str, Any]) -> bool: + """Recognize persisted provider image blocks without inspecting prose.""" + + raw_type = value.get("type") + block_type = raw_type.strip().lower() if isinstance(raw_type, str) else "" + if block_type in _COMPACTION_IMAGE_BLOCK_TYPES or block_type.startswith("image/"): + return True + raw_mime = value.get("media_type") or value.get("mime") + mime = raw_mime.strip().lower() if isinstance(raw_mime, str) else "" + return mime.startswith("image/") and any( + key in value for key in _COMPACTION_IMAGE_PAYLOAD_KEYS + ) + + +def _project_compaction_images(value: Any) -> Any: + """Recursively replace known image blocks with a metadata-free marker. + + Tool results can contain provider content blocks at arbitrary depth. The + canonical transcript retains those blocks, while both compaction inputs + and durable-obligation extraction consume this detached projection. + """ + + if isinstance(value, Mapping): + if _is_known_image_mapping(value): + return {"type": "text", "text": _COMPACTION_IMAGE_MARKER} + return {key: _project_compaction_images(item) for key, item in value.items()} + if isinstance(value, list): + return [_project_compaction_images(item) for item in value] + if isinstance(value, tuple): + return tuple(_project_compaction_images(item) for item in value) + if isinstance(value, str) and value.lstrip().startswith(("{", "[")): + # OpenAI-compatible function arguments and some tool results persist + # structured content as a JSON string. Preserve the original spelling + # unless that decoded value actually contains a known image block. + try: + parsed = json.loads(value) + except (json.JSONDecodeError, RecursionError, TypeError, ValueError): + return value + projected = _project_compaction_images(parsed) + if projected != parsed: + return json.dumps(projected, ensure_ascii=False, sort_keys=True) + return value + + +def _attachment_safe_obligation_entries( + entries: Sequence[dict[str, Any]], +) -> list[dict[str, Any]]: + """Project attachment envelopes and image blocks before obligations. + + Obligation extraction deliberately scans raw prose for paths and opaque + identifiers. A persisted attachment envelope also contains storage-only + fields, while nested tool results may carry provider-native image blocks. + Scanning either raw value would incorrectly preserve media bytes, paths, + or path-shaped invalid IDs in the structured summary. Keep user text and + canonical occurrence IDs, but project known image blocks to a marker. + """ + + projected: list[dict[str, Any]] = [] + for entry in entries: + safe_entry = dict(entry) + if "tool_calls" in safe_entry: + safe_entry["tool_calls"] = _project_compaction_images( + safe_entry.get("tool_calls") + ) + content = str(entry.get("content") or "") + session_id = str(entry.get("session_id") or "compaction") + message_id = str(entry.get("message_id") or entry.get("id") or "unknown") + try: + occurrences = extract_attachment_occurrences_from_envelope( + content, + session_id=session_id, + source_message_id=message_id, + ) + except (TypeError, ValueError): + occurrences = () + if not occurrences: + projected.append(safe_entry) + continue + + try: + envelope = json.loads(content) + except (json.JSONDecodeError, TypeError, ValueError): + envelope = {} + text = envelope.get("text") if isinstance(envelope, dict) else "" + safe_parts = [text] if isinstance(text, str) and text else [] + safe_parts.extend( + f"[attachment reference: attachment_id={occurrence.attachment_id}]" + for occurrence in occurrences + ) + safe_entry["content"] = "\n".join(safe_parts) + projected.append(safe_entry) + return projected + + def _preview_text(text: str, max_chars: int = 240) -> str: if len(text) <= max_chars: return text @@ -1189,6 +1372,7 @@ def _summarize_tool_value(value: Any) -> str: def _summarize_tool_calls_for_llm(tool_calls: Any) -> str: + tool_calls = _project_compaction_images(tool_calls) if not isinstance(tool_calls, list) or not tool_calls: return "" lines = ["[tool payload summary]"] @@ -1268,7 +1452,11 @@ def _format_chunk_for_llm(chunk: list[dict[str, Any]]) -> str: lines: list[str] = [] for entry in chunk: role = entry.get("role", "unknown") - content = _summarize_if_envelope(str(entry.get("content") or "")) + content = _summarize_if_envelope( + str(entry.get("content") or ""), + session_id=str(entry.get("session_id") or ""), + message_id=str(entry.get("message_id") or entry.get("id") or ""), + ) rendered_parts = [f"[{role}]: {content}"] tool_summary = _summarize_tool_calls_for_llm(entry.get("tool_calls")) if tool_summary: @@ -1294,8 +1482,22 @@ def _summarize_chunk_fallback(chunk: list[dict[str, Any]], policy: str) -> str: lines.append(f"[Summary of {len(chunk)} messages]") for entry in chunk: role = entry.get("role", "unknown") - content = _summarize_if_envelope(str(entry.get("content") or "")) - preview = content[:200] + ("..." if len(content) > 200 else "") + content = _summarize_if_envelope( + str(entry.get("content") or ""), + session_id=str(entry.get("session_id") or ""), + message_id=str(entry.get("message_id") or entry.get("id") or ""), + ) + # Attachment descriptors are durable lookup handles, not expendable + # prose. Preview the user text while retaining the complete descriptor + # suffix even when the original prompt is long. + descriptor_index = content.rfind("\n[user attached:") + if descriptor_index >= 0 and content.endswith("]"): + preview = ( + _preview_text(content[:descriptor_index], 200) + + content[descriptor_index:] + ) + else: + preview = _preview_text(content, 200) lines.append(f" [{role}]: {preview}") tool_summary = _summarize_tool_calls_for_llm(entry.get("tool_calls")) if tool_summary: @@ -2195,7 +2397,7 @@ async def compact_context_new(request: CompactionRequest) -> CompactionResult: else: summary_source = "fallback" - obligation_entries = list(to_compact) + obligation_entries = _attachment_safe_obligation_entries(to_compact) if prev_summary: obligation_entries.insert( 0, diff --git a/src/opensquilla/session/compaction_state.py b/src/opensquilla/session/compaction_state.py index be406eb89..19305d282 100644 --- a/src/opensquilla/session/compaction_state.py +++ b/src/opensquilla/session/compaction_state.py @@ -100,7 +100,8 @@ class CompactionReport(BaseModel): _ARTIFACT_MARKERS = ("artifact", "generated artifact", "附件", "产物") _DECISION_PREFIXES = ("decision:", "rationale:", "reason:", "decided:", "决定:", "原因:") _IDENTIFIER_RE = re.compile( - r"\b(?:[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}" + r"(? SessionContextState | None: + """Build a portable attachment index without putting media in state. + + Compaction may run on databases created before the canonical archive was + complete. In that case retain the newest valid manifest and merge the + rows visible in this snapshot instead of replacing a fuller index with a + partial one. The returned row is inserted in the same rewrite transaction + as the summary by the caller. + """ + + prior: AttachmentManifest | None = None + ordered_states = sorted( + ( + state + for state in context_states + if getattr(state, "state_kind", "") == "attachment_manifest_v1" + and getattr(state, "provider", "") == "portable" + and bool(getattr(state, "valid", True)) + ), + key=lambda state: ( + int(getattr(state, "created_at", 0) or 0), + int(getattr(state, "id", 0) or 0), + ), + ) + for state in reversed(ordered_states): + try: + prior = attachment_manifest_from_context_state(state) + break + except (AttachmentManifestError, TypeError, ValueError): + continue + + # The canonical snapshot is authoritative. A collision or malformed + # manifest must abort compaction rather than archive media without the + # index needed to recover it later. + rebuilt = build_attachment_manifest( + entries, + session_id=node.session_id, + session_key=node.session_key, + ) + + if prior is None and (rebuilt is None or not rebuilt.occurrences): + return None + if prior is not None and rebuilt is not None: + manifest = prior.merge( + rebuilt.occurrences, + covered_through_id=max( + prior.covered_through_id, + rebuilt.covered_through_id, + ), + ) + else: + manifest = prior or rebuilt + if manifest is None or not manifest.occurrences: + return None + return manifest_context_state(manifest) + + _COMPACTION_SINGLEFLIGHT_LOCK = threading.Lock() _COMPACTION_SINGLEFLIGHTS: dict[ tuple[asyncio.AbstractEventLoop, int, _CompactionSingleflightKey], @@ -419,6 +490,7 @@ def _compaction_entry_payloads(entries: list[TranscriptEntry]) -> list[dict[str, payloads.append( { "id": entry.id, + "session_id": entry.session_id, "message_id": entry.message_id, "role": entry.role, "content": silent_reply.content or "", @@ -1837,8 +1909,21 @@ async def _branch_locked( forked = TranscriptEntry( session_id=child.session_id, session_key=new_session_key, + message_id=( + entry.message_id + if not is_prefix_fork + else str(uuid.uuid4()) + ), role=entry.role, - content=entry.content, + content=( + preserve_attachment_occurrence_ids( + entry.content, + session_id=parent.session_id, + source_message_id=entry.message_id, + ) + if is_prefix_fork and entry.role == "user" + else entry.content + ), tool_calls=entry.tool_calls, tool_call_id=entry.tool_call_id, reasoning_content=entry.reasoning_content, @@ -1991,7 +2076,15 @@ async def prepare_prefix_branch( session_id=child.session_id, session_key=new_session_key, role=entry.role, - content=entry.content, + content=( + preserve_attachment_occurrence_ids( + entry.content, + session_id=parent.session_id, + source_message_id=entry.message_id, + ) + if entry.role == "user" + else entry.content + ), tool_calls=entry.tool_calls, tool_call_id=entry.tool_call_id, reasoning_content=entry.reasoning_content, @@ -2676,14 +2769,36 @@ def _consume_late_checkpoint(done: asyncio.Task[Any]) -> None: return persisted async def get_canonical_transcript( - self, session_key: str, limit: int | None = None + self, + session_key: str, + limit: int | None = None, + *, + expected_session_id: str | None = None, + expected_session_epoch: int | None = None, ) -> list[TranscriptEntry]: """Return archived compacted rows plus the active transcript tail.""" session_key = canonicalize_session_key(session_key) node = await self._storage.get_session(session_key) if node is None: raise KeyError(f"Session not found: {session_key}") - return await self._storage.get_canonical_transcript(node.session_id, limit=limit) + _require_expected_session_owner( + node, + expected_session_id=expected_session_id, + expected_session_epoch=expected_session_epoch, + operation="canonical transcript read", + ) + entries = await self._storage.get_canonical_transcript(node.session_id, limit=limit) + if expected_session_id is not None or expected_session_epoch is not None: + current = await self._storage.get_session(session_key) + if current is None: + raise StaleEpochError("Session owner changed during canonical transcript read") + _require_expected_session_owner( + current, + expected_session_id=expected_session_id, + expected_session_epoch=expected_session_epoch, + operation="canonical transcript read", + ) + return entries async def get_canonical_transcript_page( self, @@ -2783,9 +2898,19 @@ async def mark_compaction_flush_receipt_status( status=status, ) - async def save_context_state(self, state: SessionContextState) -> SessionContextState: + async def save_context_state( + self, + state: SessionContextState, + *, + expected_session_id: str | None = None, + expected_session_epoch: int | None = None, + ) -> SessionContextState: """Persist portable or provider-specific context state.""" - return await self._storage.save_context_state(state) + return await self._storage.save_context_state( + state, + expected_session_id=expected_session_id, + expected_session_epoch=expected_session_epoch, + ) async def get_context_states( self, @@ -3111,6 +3236,8 @@ async def _compact_snapshot_with_result( ) -> CompactionResult: """Generate and atomically install one frozen compaction candidate.""" + import structlog as _structlog + result = await compact_context( CompactionRequest( session_id=node.session_id, @@ -3129,8 +3256,6 @@ async def _compact_snapshot_with_result( if result.removed_count == 0 and not result.replaced_previous_summary: return result if not result.summary: - import structlog as _structlog - _structlog.get_logger(__name__).warning( "session_compaction.empty_summary_not_persisted", session_key=session_key, @@ -3284,6 +3409,18 @@ async def _compact_snapshot_with_result( current_node, summary_record, ) + # Keep attachment identity/material state in the same atomic + # rewrite as the summary. The canonical archive is queried here + # (before the write transaction) so a compacted image can still be + # rehydrated after the active row is removed. + canonical_entries_for_manifest = ( + await self._storage.get_canonical_transcript(current_node.session_id) + ) + manifest_state = _merge_attachment_manifest_state( + node=current_node, + entries=canonical_entries_for_manifest, + context_states=current_context_states, + ) # Cancellation/deadline wins until this point. Once the atomic # SQLite rewrite starts, wait for its real outcome so a committed # summary can never be reported as cancelled. @@ -3301,7 +3438,12 @@ async def _compact_snapshot_with_result( node=current_node, summary=summary_record, entries=kept_entries, - context_states=[context_state] if context_state is not None else None, + context_states=[ + state + for state in (context_state, manifest_state) + if state is not None + ] + or None, archived_entries=removed_entries, expected_source_entries=current_entries, expected_source_preimage=preimage, @@ -3489,6 +3631,8 @@ async def persist_compaction_result( raw_removed_entries = [ { "id": entry.id, + "session_id": entry.session_id, + "message_id": entry.message_id, "role": entry.role, "content": entry.content or "", "tool_calls": entry.tool_calls, @@ -3508,7 +3652,9 @@ async def persist_compaction_result( else structured_summary.critical_carry_forward ) else: - obligations = extract_compaction_obligations(raw_removed_entries) + obligations = extract_compaction_obligations( + _attachment_safe_obligation_entries(raw_removed_entries) + ) structured_summary, coverage = build_structured_summary_from_text( summary, obligations, @@ -3572,6 +3718,15 @@ async def persist_compaction_result( node.compaction_count = (node.compaction_count or 0) + 1 node.updated_at = _now_ms() context_state = self._portable_structured_summary_state(node, summary_record) + canonical_entries_for_manifest = ( + await self._storage.get_canonical_transcript(node.session_id) + ) + existing_states = await self._storage.get_context_states(session_key) + manifest_state = _merge_attachment_manifest_state( + node=node, + entries=canonical_entries_for_manifest, + context_states=existing_states, + ) if deadline_config is not None: require_compaction_time(deadline_config, phase="committing") commit_started = time.monotonic() @@ -3588,7 +3743,12 @@ async def persist_compaction_result( node=node, summary=summary_record, entries=rewritten_entries, - context_states=[context_state] if context_state is not None else None, + context_states=[ + state + for state in (context_state, manifest_state) + if state is not None + ] + or None, archived_entries=removed_entries if summary_record is not None else None, expected_session_id=expected_session_id, expected_session_epoch=expected_session_epoch, diff --git a/src/opensquilla/session/storage.py b/src/opensquilla/session/storage.py index 32306b6d8..5053d7645 100644 --- a/src/opensquilla/session/storage.py +++ b/src/opensquilla/session/storage.py @@ -15486,15 +15486,41 @@ async def update_summary_flush_receipt_status_by_compaction( # ── SessionContextState CRUD ───────────────────────────────────────────── async def save_context_state( - self, state: SessionContextState + self, + state: SessionContextState, + *, + expected_session_id: str | None = None, + expected_session_epoch: int | None = None, ) -> SessionContextState: """Persist portable or provider-native context state for later replay.""" + _validate_optional_session_owner( + session_id=expected_session_id, + session_epoch=expected_session_epoch, + ) + if (expected_session_id is None) != (expected_session_epoch is None): + raise ValueError("context state write requires an exact session owner") + if expected_session_id is not None and state.session_id != expected_session_id: + raise ValueError("context state does not match the expected session owner") state.session_key = canonicalize_session_key(state.session_key) data = state.model_dump(exclude={"id"}) cols = list(data.keys()) placeholders = ", ".join("?" for _ in cols) values = [_serialize(data[c]) for c in cols] async with self._write_transaction("save_context_state") as conn: + if expected_session_id is not None: + assert expected_session_epoch is not None + if not await _matches_session_owner_on_conn( + conn, + session_key=state.session_key, + session_id=expected_session_id, + session_epoch=expected_session_epoch, + ): + await self._raise_stale_epoch( + conn, + session_key=state.session_key, + expected_epoch=expected_session_epoch, + expected_session_id=expected_session_id, + ) async with conn.execute( "INSERT INTO session_context_states " f"({', '.join(cols)}) VALUES ({placeholders})", diff --git a/tests/functional/test_gateway_attachment_history_e2e.py b/tests/functional/test_gateway_attachment_history_e2e.py index b925a64ba..70d747526 100644 --- a/tests/functional/test_gateway_attachment_history_e2e.py +++ b/tests/functional/test_gateway_attachment_history_e2e.py @@ -37,10 +37,6 @@ ) from opensquilla.gateway.websocket import SubscriptionManager, get_registry from opensquilla.provider import ChatConfig, DoneEvent, Message, ModelCapabilities -from opensquilla.provider.protocol import ( - IMAGE_INPUT_UNSUPPORTED_CODE, - IMAGE_INPUT_UNSUPPORTED_MESSAGE, -) from opensquilla.provider.types import ( ContentBlockImage, ContentBlockText, @@ -115,6 +111,8 @@ def override_model_with_fallback_chain( self, model: str, fallback_chain: list[object], # noqa: ARG002 + *, + preserve_existing_tail: bool = True, # noqa: ARG002 ) -> None: self.override_model(model) @@ -162,6 +160,9 @@ def resolve_vision_support( ) -> str: return "supported" if model_id == _VISION_MODEL else "unsupported" + def resolve_deployment_vision_support(self, model_id: str, **_kwargs: Any) -> str: + return "supported" if model_id == _VISION_MODEL else "unsupported" + class _EventSink: authenticated = True @@ -239,11 +240,10 @@ def _configure_gateway(tmp_path: Path) -> GatewayConfig: "model": _TEXT_MODEL, "supports_image": False, }, - "image_model": { + "c2": { "provider": _PROVIDER_ID, "model": _VISION_MODEL, "supports_image": True, - "image_only": True, }, } config.squilla_router.default_tier = "c1" @@ -394,6 +394,10 @@ def _inline_image_envelope(text: str, *payloads: bytes) -> str: @pytest.fixture async def _e2e_stack(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv("OPENSQUILLA_OPENROUTER_LIVE_PRICING", "0") + monkeypatch.setattr( + "opensquilla.provider.model_catalog.ModelCatalog.resolve_deployment_vision_support", + _FakeModelCatalog.resolve_deployment_vision_support, + ) config = _configure_gateway(tmp_path) store = UploadStore(marker_dir=tmp_path / "upload-markers") set_upload_store(store) @@ -467,6 +471,7 @@ async def _record_bootstrap_config(inp: Any) -> Any: "gate_provider": gate_provider, "manager": manager, "runner": runner, + "selector": selector, "sink": sink, "storage": storage, "store": store, @@ -482,7 +487,7 @@ async def _record_bootstrap_config(inp: Any) -> Any: @pytest.mark.asyncio -async def test_gateway_single_text_model_returns_structured_error_without_provider_call( +async def test_gateway_single_text_model_projects_marker_and_continues( _e2e_stack: dict[str, Any], ) -> None: config: GatewayConfig = _e2e_stack["config"] @@ -495,7 +500,7 @@ async def test_gateway_single_text_model_returns_structured_error_without_provid usage_sink: _UsageSink = _e2e_stack["usage_sink"] config.squilla_router.enabled = False key = "agent:main:single-text-model-image" - await manager.create(session_key=key, agent_id="main") + session = await manager.create(session_key=key, agent_id="main") subscription_manager.subscribe_messages(sink.conn_id, key) for index in range(6): await manager.append_message(key, "user", f"history-{index}:" + "u" * 5_000) @@ -514,23 +519,123 @@ async def test_gateway_single_text_model_returns_structured_error_without_provid sink=sink, message="请分析这张图片。", attachments=[_file_uuid_attachment(file_uuid)], - expected_error_code=IMAGE_INPUT_UNSUPPORTED_CODE, ) assert len(gate_provider.calls) == gate_calls_before - assert len(text_provider.calls) == text_calls_before + assert len(text_provider.calls) == text_calls_before + 1 assert len(vision_provider.calls) == vision_calls_before - assert len(usage_sink.started) == usage_started_before - assert len(usage_sink.finalized) == usage_finalized_before + assert len(usage_sink.started) == usage_started_before + 1 + assert len(usage_sink.finalized) == usage_finalized_before + 1 assert len(usage_sink.unknown) == usage_unknown_before - assert _event_payloads(sink, "session.event.text_delta") == [] - errors = _event_payloads(sink, "session.event.error") - assert errors[-1]["code"] == IMAGE_INPUT_UNSUPPORTED_CODE - assert errors[-1]["message"] == IMAGE_INPUT_UNSUPPORTED_MESSAGE - assert _event_payloads(sink, "session.event.done") == [] + sent_messages = text_provider.calls[-1]["messages"] + assert not any(_message_has_image(item) for item in sent_messages) + assert "图片未分析" in str(sent_messages) + assert _event_payloads(sink, "session.event.text_delta")[-1]["text"] == "text ok" + assert _event_payloads(sink, "session.event.error") == [] + assert _event_payloads(sink, "session.event.done") transcript = await manager.get_transcript(key) - assert transcript[-1].role == "system" - assert IMAGE_INPUT_UNSUPPORTED_MESSAGE in str(transcript[-1].content or "") + assert transcript[-1].role == "assistant" + assert transcript[-1].content == "text ok" + canonical = await manager.get_canonical_transcript(key) + image_envelope = next( + json.loads(str(entry.content)) + for entry in canonical + if '"attachments"' in str(entry.content or "") + ) + sha = image_envelope["attachments"][0]["sha256_ref"] + material_path = transcript_material_path( + Path(config.attachments.media_root or ""), + session.session_id, + sha, + ) + assert material_path.read_bytes() == _PNG_BYTES + + +@pytest.mark.asyncio +async def test_gateway_direct_model_switch_replays_canonical_history_image( + _e2e_stack: dict[str, Any], +) -> None: + config: GatewayConfig = _e2e_stack["config"] + manager: SessionManager = _e2e_stack["manager"] + selector: _RecordingSelector = _e2e_stack["selector"] + subscription_manager: SubscriptionManager = _e2e_stack["subscription_manager"] + sink: _EventSink = _e2e_stack["sink"] + text_provider: _RecordingProvider = _e2e_stack["text_provider"] + vision_provider: _RecordingProvider = _e2e_stack["vision_provider"] + config.squilla_router.enabled = False + key = "agent:main:direct-text-to-vision-switch" + await manager.create(session_key=key, agent_id="main") + subscription_manager.subscribe_messages(sink.conn_id, key) + + file_uuid = await _upload_png(_e2e_stack["app"]) + await _send_session_turn( + ctx=_e2e_stack["ctx"], + key=key, + sink=sink, + message="先保存这张图片。", + attachments=[_file_uuid_attachment(file_uuid)], + ) + assert "图片未分析" in str(text_provider.calls[-1]["messages"]) + + selector.model = _VISION_MODEL + config.llm.model = _VISION_MODEL + vision_calls_before = len(vision_provider.calls) + await _send_session_turn( + ctx=_e2e_stack["ctx"], + key=key, + sink=sink, + message="请重新分析上一张图片。", + ) + + assert len(vision_provider.calls) == vision_calls_before + 1 + sent_messages = vision_provider.calls[-1]["messages"] + historical_images = [ + block + for message in sent_messages[:-1] + for block in _message_image_blocks(message) + ] + assert len(historical_images) == 1 + assert base64.b64decode(historical_images[0].data, validate=True) == _PNG_BYTES + assert _event_payloads(sink, "session.event.error") == [] + + +@pytest.mark.asyncio +async def test_gateway_current_upload_does_not_replay_older_history_image( + _e2e_stack: dict[str, Any], +) -> None: + manager: SessionManager = _e2e_stack["manager"] + subscription_manager: SubscriptionManager = _e2e_stack["subscription_manager"] + sink: _EventSink = _e2e_stack["sink"] + vision_provider: _RecordingProvider = _e2e_stack["vision_provider"] + key = "agent:main:current-image-only" + await manager.create(session_key=key, agent_id="main") + subscription_manager.subscribe_messages(sink.conn_id, key) + + first_uuid = await _upload_png(_e2e_stack["app"]) + await _send_session_turn( + ctx=_e2e_stack["ctx"], + key=key, + sink=sink, + message="Describe the first image.", + attachments=[_file_uuid_attachment(first_uuid)], + ) + + second_uuid = await _upload_png(_e2e_stack["app"]) + await _send_session_turn( + ctx=_e2e_stack["ctx"], + key=key, + sink=sink, + message="Describe only this new image.", + attachments=[_file_uuid_attachment(second_uuid)], + ) + + sent_messages = vision_provider.calls[-1]["messages"] + image_blocks = [ + block + for message in sent_messages + for block in _message_image_blocks(message) + ] + assert len(image_blocks) == 1 @pytest.mark.asyncio @@ -731,7 +836,10 @@ async def _record_router_capacity( for message in sent_messages[:-1] if message.role == "user" and _message_has_image(message) ] - assert len(historical_users) == 1 + # A current upload does not request unrelated historical images. Router + # capacity replay may conservatively allow media, but execution filters + # the replay to the current occurrence IDs. + assert historical_users == [] decoded_images = [ base64.b64decode(block.data, validate=True) for message in sent_messages @@ -739,8 +847,8 @@ async def _record_router_capacity( ] assert payloads[0] not in decoded_images assert payloads[1] not in decoded_images - assert payloads[2] in decoded_images - assert payloads[3] in decoded_images + assert payloads[2] not in decoded_images + assert payloads[3] not in decoded_images assert _PNG_BYTES in decoded_images # The provider may receive typed image blocks, but legacy envelope/base64 @@ -832,6 +940,7 @@ async def test_historical_image_material_is_not_replayed_without_vision_support( provider=provider, config=AgentConfig( model_capabilities=ModelCapabilities(supports_vision=False), + model_vision_support="unsupported", preserve_historical_images=True, ), ) diff --git a/tests/functional/test_gateway_non_image_attachment_materialization_e2e.py b/tests/functional/test_gateway_non_image_attachment_materialization_e2e.py index d701b494e..b885fc81a 100644 --- a/tests/functional/test_gateway_non_image_attachment_materialization_e2e.py +++ b/tests/functional/test_gateway_non_image_attachment_materialization_e2e.py @@ -133,6 +133,8 @@ def override_model_with_fallback_chain( self, model: str, fallback_chain: list[object], # noqa: ARG002 + *, + preserve_existing_tail: bool = True, # noqa: ARG002 ) -> None: self.override_model(model) @@ -169,6 +171,9 @@ def get_capabilities( ) -> ModelCapabilities: return ModelCapabilities(supports_vision=model_id == _VISION_MODEL) + def resolve_deployment_vision_support(self, model_id: str, **_kwargs: Any) -> str: + return "supported" if model_id == _VISION_MODEL else "unsupported" + class _EventSink: authenticated = True @@ -230,11 +235,10 @@ def _configure_gateway(tmp_path: Path) -> GatewayConfig: "model": _TEXT_MODEL, "supports_image": False, }, - "image_model": { + "c2": { "provider": "openrouter", "model": _VISION_MODEL, "supports_image": True, - "image_only": True, }, } config.squilla_router.default_tier = "c1" @@ -361,6 +365,10 @@ def _message_has_image(message: Message) -> bool: @pytest.fixture async def _e2e_stack(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv("OPENSQUILLA_OPENROUTER_LIVE_PRICING", "0") + monkeypatch.setattr( + "opensquilla.provider.model_catalog.ModelCatalog.resolve_deployment_vision_support", + _FakeModelCatalog.resolve_deployment_vision_support, + ) monkeypatch.setattr(squilla_router_step, "_get_strategy", lambda _cfg: _TextTierStrategy()) config = _configure_gateway(tmp_path) store = UploadStore(marker_dir=tmp_path / "upload-markers") @@ -665,7 +673,7 @@ async def test_pdf_materialization_does_not_require_image_tier( _e2e_stack: dict[str, Any], ) -> None: config: GatewayConfig = _e2e_stack["config"] - config.squilla_router.tiers.pop("image_model", None) + config.squilla_router.tiers.pop("c2", None) manager: SessionManager = _e2e_stack["manager"] subscription_manager: SubscriptionManager = _e2e_stack["subscription_manager"] sink: _EventSink = _e2e_stack["sink"] diff --git a/tests/test_ci/test_windows_test_shards.py b/tests/test_ci/test_windows_test_shards.py index a76153559..37cb460d7 100644 --- a/tests/test_ci/test_windows_test_shards.py +++ b/tests/test_ci/test_windows_test_shards.py @@ -68,6 +68,10 @@ "tests/test_skills/test_meta_skill_creator_smoke_live.py", } RECENTLY_ADDED_ACTIVE_TESTS = { + "tests/test_engine/test_attachment_replay_ownership.py", + "tests/test_engine/test_router_configured_image_policy.py", + "tests/test_provider/test_image_projection.py", + "tests/test_session/test_attachment_manifest.py", "tests/contracts/test_approval_center_contract.py", "tests/test_gateway/test_chat_history_characterization.py", "tests/contracts/test_conversation_events_contract.py", diff --git a/tests/test_contracts/test_onboarding_catalog.py b/tests/test_contracts/test_onboarding_catalog.py index 0f2775784..8b0ee630d 100644 --- a/tests/test_contracts/test_onboarding_catalog.py +++ b/tests/test_contracts/test_onboarding_catalog.py @@ -95,12 +95,14 @@ ROUTER_MODE_KEYS = frozenset({"mode", "label", "description"}) ROUTER_PROFILE_KEYS = frozenset({"profileId", "providerId", "label", "tiers"}) ROUTER_TIER_PAYLOAD_KEYS = frozenset( - {"provider", "model", "description", "thinkingLevel", "supportsImage"} + {"provider", "model", "description", "thinkingLevel"} ) # Deliberate additive execution metadata. These keys are optional because only # tiers that opt into fusion need them; clients that predate the addition keep # receiving the frozen base shape for every other tier. -ROUTER_TIER_OPTIONAL_KEYS = frozenset({"ensembleEnabled", "ensembleSelectionMode"}) +ROUTER_TIER_OPTIONAL_KEYS = frozenset( + {"supportsImage", "ensembleEnabled", "ensembleSelectionMode"} +) def _assert_router_tier_payload_keys(tier: dict[str, object], context: object) -> None: diff --git a/tests/test_cross_provider_tiers.py b/tests/test_cross_provider_tiers.py index a74ff21bb..426e0ddc2 100644 --- a/tests/test_cross_provider_tiers.py +++ b/tests/test_cross_provider_tiers.py @@ -14,6 +14,7 @@ from opensquilla.engine.selector_override import ( apply_model_override, cross_provider_tier_config, + resolve_strict_router_fallback_chain, resolve_tier_provider_config, ) from opensquilla.gateway.config import GatewayConfig, LlmProviderProfile @@ -546,6 +547,77 @@ def test_apply_model_override_uses_tier_config() -> None: assert all(not cfg.replay_provider_state for cfg in selector.remaining_chain()) +def test_strict_cross_provider_override_keeps_resolved_router_chain( + monkeypatch, +) -> None: + monkeypatch.setattr("opensquilla.provider.selector._build_provider", lambda cfg: cfg) + cfg = _config_with_flag( + openai=LlmProviderProfile(api_key="oa-key"), + anthropic=LlmProviderProfile(api_key="an-key"), + ) + selector = ModelSelector( + SelectorConfig( + primary=ProviderConfig( + "openrouter", + "configured-primary", + api_key="or-key", + ) + ) + ) + metadata: dict[str, object] = { + "routing_applied": True, + "routing_source": "image_route", + "router_fallback_strict": True, + "routed_provider": "openai", + "routed_model": "configured-c0", + "router_fallback_chain": [ + {"tier": "c1", "model": "configured-c1"}, + { + "tier": "c2", + "provider": "anthropic", + "model": "configured-c2", + }, + { + "tier": "c3", + "provider": "openai", + "model": "configured-c3", + }, + ], + } + active_provider_id = selector.active_provider_id + tier_config = cross_provider_tier_config( + cfg, + metadata, + "configured-c0", + active_provider_id=active_provider_id, + ) + strict_chain = resolve_strict_router_fallback_chain( + cfg, + metadata, + active_provider_id=active_provider_id, + ) + + provider = apply_model_override( + selector, + "configured-c0", + turn_metadata=metadata, + realign_routed_model=False, + tier_provider_config=tier_config, + strict_router_fallback_chain=strict_chain, + ) + + assert provider is not None + assert [ + (item.provider, item.model) for item in selector.remaining_chain() + ] == [ + ("openai", "configured-c0"), + ("openrouter", "configured-c1"), + ("anthropic", "configured-c2"), + ("openai", "configured-c3"), + ] + assert all(not item.replay_provider_state for item in selector.remaining_chain()) + + def test_return_to_primary_after_foreign_native_state_disables_replay() -> None: cfg = _config_with_flag() selector = ModelSelector( diff --git a/tests/test_engine/goldens/routing_policy_parity_golden.json b/tests/test_engine/goldens/routing_policy_parity_golden.json index 1eb78b9b9..d2938b8a5 100644 --- a/tests/test_engine/goldens/routing_policy_parity_golden.json +++ b/tests/test_engine/goldens/routing_policy_parity_golden.json @@ -3522,17 +3522,49 @@ "message_len": 32, "message_tail": "please summarize this short note", "metadata": { - "applied_model": "dummy-vision-1", + "applied_model": "dummy-nano-1", "baseline_model": "dummy-base-model", + "image_input_mode": "native", + "image_input_reason": "configured_image_capable_tier", "image_route_reason": "current_turn", "large_context_capacity_required": true, "large_context_material_tokens": 8, "large_context_thinking_budget_tokens": 0, "rollout_phase": "observe", "route_max_history_turns": 1, - "routed_model": "dummy-vision-1", - "routed_tier": "image_model", - "router_fallback_chain": [], + "routed_model": "dummy-nano-1", + "routed_model_vision_support": "unknown", + "routed_tier": "c0", + "router_fallback_chain": [ + { + "model": "dummy-mini-1", + "tier": "c1", + "vision_support": "unknown" + }, + { + "model": "dummy-pro-1", + "tier": "c2", + "vision_support": "unknown" + }, + { + "model": "dummy-max-1", + "tier": "c3", + "vision_support": "unknown" + } + ], + "router_fallback_strict": true, + "router_image_configured_tiers": [ + "c0", + "c1", + "c2", + "c3" + ], + "router_image_tier_support": { + "c0": "unknown", + "c1": "unknown", + "c2": "unknown", + "c3": "unknown" + }, "router_tier_provider_role": "direct", "routing_applied": true, "routing_confidence": 1.0, @@ -3542,24 +3574,56 @@ "savings_pct": 0.0, "savings_routed_price_per_m": 3.0 }, - "model": "dummy-vision-1" + "model": "dummy-nano-1" }, "image_current_turn_bypass": { "error": null, "message_len": 32, "message_tail": "please summarize this short note", "metadata": { - "applied_model": "dummy-vision-1", + "applied_model": "dummy-nano-1", "baseline_model": "dummy-base-model", + "image_input_mode": "native", + "image_input_reason": "configured_image_capable_tier", "image_route_reason": "current_turn", "large_context_capacity_required": true, "large_context_material_tokens": 8, "large_context_thinking_budget_tokens": 0, "rollout_phase": "full", "route_max_history_turns": 1, - "routed_model": "dummy-vision-1", - "routed_tier": "image_model", - "router_fallback_chain": [], + "routed_model": "dummy-nano-1", + "routed_model_vision_support": "unknown", + "routed_tier": "c0", + "router_fallback_chain": [ + { + "model": "dummy-mini-1", + "tier": "c1", + "vision_support": "unknown" + }, + { + "model": "dummy-pro-1", + "tier": "c2", + "vision_support": "unknown" + }, + { + "model": "dummy-max-1", + "tier": "c3", + "vision_support": "unknown" + } + ], + "router_fallback_strict": true, + "router_image_configured_tiers": [ + "c0", + "c1", + "c2", + "c3" + ], + "router_image_tier_support": { + "c0": "unknown", + "c1": "unknown", + "c2": "unknown", + "c3": "unknown" + }, "router_tier_provider_role": "direct", "routing_applied": true, "routing_confidence": 1.0, @@ -3569,20 +3633,53 @@ "savings_pct": 0.0, "savings_routed_price_per_m": 3.0 }, - "model": "dummy-vision-1" + "model": "dummy-nano-1" }, "image_gate_history_bypass": { "error": null, "message_len": 32, "message_tail": "please summarize this short note", "metadata": { - "applied_model": "dummy-vision-1", + "applied_model": "dummy-nano-1", "baseline_model": "dummy-base-model", + "image_input_mode": "native", + "image_input_reason": "configured_image_capable_tier", "image_route_reason": "gate_history", "rollout_phase": "full", "route_max_history_turns": 8, - "routed_model": "dummy-vision-1", - "routed_tier": "image_model", + "routed_model": "dummy-nano-1", + "routed_model_vision_support": "unknown", + "routed_tier": "c0", + "router_fallback_chain": [ + { + "model": "dummy-mini-1", + "tier": "c1", + "vision_support": "unknown" + }, + { + "model": "dummy-pro-1", + "tier": "c2", + "vision_support": "unknown" + }, + { + "model": "dummy-max-1", + "tier": "c3", + "vision_support": "unknown" + } + ], + "router_fallback_strict": true, + "router_image_configured_tiers": [ + "c0", + "c1", + "c2", + "c3" + ], + "router_image_tier_support": { + "c0": "unknown", + "c1": "unknown", + "c2": "unknown", + "c3": "unknown" + }, "router_tier_provider_role": "direct", "router_vision_followup_needs_image": true, "routing_applied": true, @@ -3593,24 +3690,56 @@ "savings_pct": 0.0, "savings_routed_price_per_m": 3.0 }, - "model": "dummy-vision-1" + "model": "dummy-nano-1" }, "image_media_type_key_bypass": { "error": null, "message_len": 32, "message_tail": "please summarize this short note", "metadata": { - "applied_model": "dummy-vision-1", + "applied_model": "dummy-nano-1", "baseline_model": "dummy-base-model", + "image_input_mode": "native", + "image_input_reason": "configured_image_capable_tier", "image_route_reason": "current_turn", "large_context_capacity_required": true, "large_context_material_tokens": 8, "large_context_thinking_budget_tokens": 0, "rollout_phase": "full", "route_max_history_turns": 1, - "routed_model": "dummy-vision-1", - "routed_tier": "image_model", - "router_fallback_chain": [], + "routed_model": "dummy-nano-1", + "routed_model_vision_support": "unknown", + "routed_tier": "c0", + "router_fallback_chain": [ + { + "model": "dummy-mini-1", + "tier": "c1", + "vision_support": "unknown" + }, + { + "model": "dummy-pro-1", + "tier": "c2", + "vision_support": "unknown" + }, + { + "model": "dummy-max-1", + "tier": "c3", + "vision_support": "unknown" + } + ], + "router_fallback_strict": true, + "router_image_configured_tiers": [ + "c0", + "c1", + "c2", + "c3" + ], + "router_image_tier_support": { + "c0": "unknown", + "c1": "unknown", + "c2": "unknown", + "c3": "unknown" + }, "router_tier_provider_role": "direct", "routing_applied": true, "routing_confidence": 1.0, @@ -3620,21 +3749,66 @@ "savings_pct": 0.0, "savings_routed_price_per_m": 3.0 }, - "model": "dummy-vision-1" + "model": "dummy-nano-1" }, - "image_without_image_tier_errors": { + "image_without_legacy_image_tier_uses_c_ladder": { "error": null, "message_len": 32, "message_tail": "please summarize this short note", "metadata": { - "image_input_forced_rejection_reason": "router_image_route_unavailable", - "image_input_mode": "rejected", - "image_input_reason": "router_image_route_unavailable", + "applied_model": "dummy-nano-1", + "baseline_model": "dummy-base-model", + "image_input_mode": "native", + "image_input_reason": "configured_image_capable_tier", + "image_route_reason": "current_turn", "large_context_capacity_required": true, "large_context_material_tokens": 8, - "routing_turn_index": 0 + "large_context_thinking_budget_tokens": 0, + "rollout_phase": "full", + "route_max_history_turns": 1, + "routed_model": "dummy-nano-1", + "routed_model_vision_support": "unknown", + "routed_tier": "c0", + "router_fallback_chain": [ + { + "model": "dummy-mini-1", + "tier": "c1", + "vision_support": "unknown" + }, + { + "model": "dummy-pro-1", + "tier": "c2", + "vision_support": "unknown" + }, + { + "model": "dummy-max-1", + "tier": "c3", + "vision_support": "unknown" + } + ], + "router_fallback_strict": true, + "router_image_configured_tiers": [ + "c0", + "c1", + "c2", + "c3" + ], + "router_image_tier_support": { + "c0": "unknown", + "c1": "unknown", + "c2": "unknown", + "c3": "unknown" + }, + "router_tier_provider_role": "direct", + "routing_applied": true, + "routing_confidence": 1.0, + "routing_source": "image_route", + "routing_turn_index": 0, + "savings_max_price_per_m": 3.0, + "savings_pct": 0.0, + "savings_routed_price_per_m": 3.0 }, - "model": "dummy-base-model" + "model": "dummy-nano-1" }, "large_context_below_floor_boundary": { "error": null, diff --git a/tests/test_engine/test_agent_image_capability_guard.py b/tests/test_engine/test_agent_image_capability_guard.py index fff7ebe01..688b1f37a 100644 --- a/tests/test_engine/test_agent_image_capability_guard.py +++ b/tests/test_engine/test_agent_image_capability_guard.py @@ -6,14 +6,25 @@ import pytest -from opensquilla.engine import Agent, AgentConfig -from opensquilla.engine.types import DoneEvent, ErrorEvent, TextDeltaEvent -from opensquilla.provider import ChatConfig, Message, ModelCapabilities +from opensquilla.engine import Agent, AgentConfig, ToolResult +from opensquilla.engine.types import DoneEvent, ErrorEvent, TextDeltaEvent, ToolCall +from opensquilla.provider import ( + ChatConfig, + Message, + ModelCapabilities, + ToolDefinition, + ToolInputSchema, +) +from opensquilla.provider import DoneEvent as ProviderDoneEvent +from opensquilla.provider import ErrorEvent as ProviderErrorEvent +from opensquilla.provider import ToolUseEndEvent as ProviderToolUseEndEvent +from opensquilla.provider import ToolUseStartEvent as ProviderToolUseStartEvent from opensquilla.provider.protocol import ( - IMAGE_INPUT_UNSUPPORTED_CODE, + count_provider_image_blocks, validate_provider_chat_admission, ) -from opensquilla.provider.types import ContentBlockImage, ContentBlockToolResult +from opensquilla.provider.types import ContentBlockImage, ContentBlockText, ContentBlockToolResult +from opensquilla.tools.types import ToolContext class _RecordingProvider: @@ -42,6 +53,178 @@ async def _stream(self) -> AsyncIterator[Any]: yield ProviderDoneEvent(stop_reason="end_turn", input_tokens=3, output_tokens=2) +class _RejectImageOnceProvider(_RecordingProvider): + rejection_code = "image_input_unsupported" + rejection_message = "This model does not support image input." + + def chat( + self, + messages: list[Message], + tools: list[Any] | None = None, + config: ChatConfig | None = None, + ) -> AsyncIterator[Any]: + self.calls.append({"messages": messages, "tools": tools, "config": config}) + has_image = any( + isinstance(block, ContentBlockImage) + for message in messages + if isinstance(message.content, list) + for block in message.content + ) + return self._rejection_stream() if has_image else self._stream() + + async def _rejection_stream(self) -> AsyncIterator[Any]: + yield ProviderErrorEvent( + code=self.rejection_code, + message=self.rejection_message, + ) + + +class _RouterProbeProvider(_RejectImageOnceProvider): + provider_name = "router-probe" + + def __init__(self) -> None: + super().__init__() + self.models = ["configured-c0", "configured-c1", "configured-c2", "configured-c3"] + self.index = 0 + + @property + def model(self) -> str: + return self.models[self.index] + + def fallback_after_image_rejection(self, _reason: str) -> bool: + if self.index >= len(self.models) - 1: + return False + self.index += 1 + return True + + def chat( + self, + messages: list[Message], + tools: list[Any] | None = None, + config: ChatConfig | None = None, + ) -> AsyncIterator[Any]: + self.calls.append( + { + "model": self.model, + "messages": messages, + "tools": tools, + "config": config, + } + ) + has_image = any( + isinstance(block, ContentBlockImage) + for message in messages + if isinstance(message.content, list) + for block in message.content + ) + return self._rejection_stream() if has_image else self._stream() + + +class _TextThenRejectProvider(_RecordingProvider): + def chat( + self, + messages: list[Message], + tools: list[Any] | None = None, + config: ChatConfig | None = None, + ) -> AsyncIterator[Any]: + self.calls.append({"messages": messages, "tools": tools, "config": config}) + return self._text_then_reject_stream() + + async def _text_then_reject_stream(self) -> AsyncIterator[Any]: + from opensquilla.provider import TextDeltaEvent as ProviderTextDeltaEvent + + yield ProviderTextDeltaEvent(text="partial") + yield ProviderErrorEvent( + code="image_input_unsupported", + message="This model does not support image input.", + ) + + +class _ThinkingTextThenRejectProvider(_TextThenRejectProvider): + async def _text_then_reject_stream(self) -> AsyncIterator[Any]: + from opensquilla.provider import TextDeltaEvent as ProviderTextDeltaEvent + + yield ProviderTextDeltaEvent(text="FIRST-PARTIAL") + yield ProviderErrorEvent( + code="image_input_unsupported", + message="Image input is not supported while thinking is enabled.", + ) + + +class _PreflightRejectProvider(_RecordingProvider): + def __init__(self, *, code: str) -> None: + super().__init__() + self.code = code + + def validate_chat_admission( + self, + messages: list[Message], + _config: ChatConfig | None, + ) -> ProviderErrorEvent | None: + if any( + isinstance(block, ContentBlockImage) + for message in messages + if isinstance(message.content, list) + for block in message.content + ): + return ProviderErrorEvent( + code=self.code, + message="This deployment does not support image input.", + ) + return None + + +class _ToolThenRejectImageProvider(_RecordingProvider): + def chat( + self, + messages: list[Message], + tools: list[Any] | None = None, + config: ChatConfig | None = None, + ) -> AsyncIterator[Any]: + self.calls.append({"messages": messages, "tools": tools, "config": config}) + if len(self.calls) == 1: + return self._tool_stream() + return self._rejection_stream() + + async def _tool_stream(self) -> AsyncIterator[Any]: + yield ProviderToolUseStartEvent(tool_use_id="tool-1", tool_name="observe") + yield ProviderToolUseEndEvent( + tool_use_id="tool-1", + tool_name="observe", + arguments={}, + ) + yield ProviderDoneEvent(stop_reason="tool_use", input_tokens=1, output_tokens=1) + + async def _rejection_stream(self) -> AsyncIterator[Any]: + yield ProviderErrorEvent( + code="image_input_unsupported", + message="This model does not support image input.", + ) + + +class _ToolThenEmptyImageRejectProvider(_ToolThenRejectImageProvider): + async def _rejection_stream(self) -> AsyncIterator[Any]: + yield ProviderErrorEvent( + code="empty_response", + message="This model does not support image input.", + ) + + +class _ToolMediaProbeProvider(_ToolThenRejectImageProvider): + def chat( + self, + messages: list[Message], + tools: list[Any] | None = None, + config: ChatConfig | None = None, + ) -> AsyncIterator[Any]: + self.calls.append({"messages": messages, "tools": tools, "config": config}) + if len(self.calls) == 1: + return self._tool_stream() + if count_provider_image_blocks(messages): + return self._rejection_stream() + return self._stream() + + def _image_message() -> Message: return Message( role="user", @@ -63,7 +246,7 @@ def test_provider_admission_without_config_preserves_unknown_capability() -> Non @pytest.mark.asyncio -async def test_non_vision_model_returns_structured_error_without_provider_call() -> None: +async def test_non_vision_model_receives_marker_and_continues() -> None: provider = _RecordingProvider() config = AgentConfig( model_id="text-only-model", @@ -80,12 +263,24 @@ async def test_non_vision_model_returns_structured_error_without_provider_call() ) ] - assert provider.calls == [] - assert not any(isinstance(event, TextDeltaEvent) for event in events) - assert not any(isinstance(event, DoneEvent) for event in events) - error = next(event for event in events if isinstance(event, ErrorEvent)) - assert error.code == IMAGE_INPUT_UNSUPPORTED_CODE - assert config.metadata["image_input_mode"] == "rejected" + assert len(provider.calls) == 1 + sent = provider.calls[0]["messages"] + assert not any( + isinstance(block, ContentBlockImage) + for message in sent + if isinstance(message.content, list) + for block in message.content + ) + assert any( + isinstance(block, ContentBlockText) and "图片未分析" in block.text + for message in sent + if isinstance(message.content, list) + for block in message.content + ) + assert any(isinstance(event, TextDeltaEvent) for event in events) + assert any(isinstance(event, DoneEvent) for event in events) + assert not any(isinstance(event, ErrorEvent) for event in events) + assert config.metadata["image_input_mode"] == "marker" assert config.metadata["image_input_reason"] == "model_vision_unsupported" assert config.metadata["image_input_count"] == 1 assert config.metadata["image_input_stage"] == "primary" @@ -147,8 +342,422 @@ async def test_unknown_model_capability_defers_to_provider() -> None: assert len(provider.calls) == 1 +@pytest.mark.parametrize( + ("rejection_code", "rejection_message"), + [ + ("image_input_unsupported", "This model does not support image input."), + ("404", "No endpoints found that support image input."), + ], +) +@pytest.mark.asyncio +async def test_unknown_model_retries_same_model_with_failure_marker( + rejection_code: str, + rejection_message: str, +) -> None: + provider = _RejectImageOnceProvider() + provider.provider_name = "openrouter" + provider.rejection_code = rejection_code + provider.rejection_message = rejection_message + original = _image_message() + agent = Agent( + provider=provider, + config=AgentConfig( + max_iterations=1, + model_id="custom-model", + model_vision_support="unknown", + ), + ) + + events = [ + event + async for event in agent.run_turn( + "Describe this image.", + extra_messages=[original], + ) + ] + + assert len(provider.calls) == 2 + assert any( + isinstance(block, ContentBlockImage) + for message in provider.calls[0]["messages"] + if isinstance(message.content, list) + for block in message.content + ) + assert not any( + isinstance(block, ContentBlockImage) + for message in provider.calls[1]["messages"] + if isinstance(message.content, list) + for block in message.content + ) + assert any( + isinstance(block, ContentBlockText) and "图片分析失败" in block.text + for message in provider.calls[1]["messages"] + if isinstance(message.content, list) + for block in message.content + ) + assert isinstance(original.content, list) + assert isinstance(original.content[0], ContentBlockImage) + assert any(isinstance(event, DoneEvent) for event in events) + assert not any(isinstance(event, ErrorEvent) for event in events) + + +@pytest.mark.parametrize("error_code", ["unsupported_image", "bad_request"]) @pytest.mark.asyncio -async def test_tool_result_image_uses_the_same_admission_guard() -> None: +async def test_precise_preflight_rejection_retries_with_marker( + error_code: str, +) -> None: + provider = _PreflightRejectProvider(code=error_code) + agent = Agent( + provider=provider, + config=AgentConfig( + max_iterations=1, + model_id="custom-model", + model_vision_support="unknown", + ), + ) + + events = [ + event + async for event in agent.run_turn( + "Describe this image.", + extra_messages=[_image_message()], + ) + ] + + # Preflight prevents the native physical call; the same configured model + # then receives exactly one marker-projected request. + assert len(provider.calls) == 1 + assert not any( + isinstance(block, ContentBlockImage) + for message in provider.calls[0]["messages"] + if isinstance(message.content, list) + for block in message.content + ) + assert any(isinstance(event, DoneEvent) for event in events) + assert not any(isinstance(event, ErrorEvent) for event in events) + + +@pytest.mark.parametrize( + ("rejection_code", "rejection_message"), + [ + ("image_input_unsupported", "This model does not support image input."), + ("404", "No endpoints found that support image input."), + ], +) +@pytest.mark.asyncio +async def test_router_exhausts_four_configured_models_before_direct_marker( + rejection_code: str, + rejection_message: str, +) -> None: + provider = _RouterProbeProvider() + provider.provider_name = "openrouter" + provider.rejection_code = rejection_code + provider.rejection_message = rejection_message + original = _image_message() + agent = Agent( + provider=provider, + config=AgentConfig( + max_iterations=1, + model_id="configured-c0", + model_vision_support="unknown", + metadata={ + "routing_source": "image_route", + "router_fallback_strict": True, + "image_input_mode": "native", + }, + ), + ) + + events = [ + event + async for event in agent.run_turn( + "Describe the image.", + extra_messages=[original], + ) + ] + + assert [call["model"] for call in provider.calls] == [ + "configured-c0", + "configured-c1", + "configured-c2", + "configured-c3", + "configured-c3", + ] + assert all( + any( + isinstance(block, ContentBlockImage) + for message in call["messages"] + if isinstance(message.content, list) + for block in message.content + ) + for call in provider.calls[:4] + ) + assert not any( + isinstance(block, ContentBlockImage) + for message in provider.calls[-1]["messages"] + if isinstance(message.content, list) + for block in message.content + ) + assert any(isinstance(event, DoneEvent) for event in events) + + +@pytest.mark.asyncio +async def test_image_marker_retry_is_suppressed_after_visible_output() -> None: + provider = _TextThenRejectProvider() + agent = Agent( + provider=provider, + config=AgentConfig( + max_iterations=1, + model_id="custom-model", + model_vision_support="unknown", + ), + ) + + events = [ + event + async for event in agent.run_turn( + "Describe this image.", + extra_messages=[_image_message()], + ) + ] + + assert len(provider.calls) == 1 + assert any(isinstance(event, TextDeltaEvent) for event in events) + assert any(isinstance(event, ErrorEvent) for event in events) + + +@pytest.mark.asyncio +async def test_thinking_fallback_cannot_bypass_image_retry_barrier() -> None: + provider = _ThinkingTextThenRejectProvider() + agent = Agent( + provider=provider, + config=AgentConfig( + max_iterations=1, + model_id="custom-model", + model_vision_support="unknown", + thinking=True, + ), + ) + + events = [ + event + async for event in agent.run_turn( + "Describe this image.", + extra_messages=[_image_message()], + ) + ] + + assert len(provider.calls) == 1 + assert [event.text for event in events if isinstance(event, TextDeltaEvent)] == [ + "FIRST-PARTIAL" + ] + assert any(isinstance(event, ErrorEvent) for event in events) + + +@pytest.mark.asyncio +async def test_image_marker_retry_is_suppressed_after_prior_tool_execution() -> None: + provider = _ToolThenRejectImageProvider() + executions: list[str] = [] + + async def _tool_handler(call: ToolCall) -> ToolResult: + executions.append(call.tool_use_id) + return ToolResult( + tool_use_id=call.tool_use_id, + tool_name=call.tool_name, + content="observed", + ) + + agent = Agent( + provider=provider, + config=AgentConfig( + max_iterations=2, + model_id="custom-model", + model_vision_support="unknown", + ), + tool_definitions=[ + ToolDefinition( + name="observe", + description="Observe once.", + input_schema=ToolInputSchema(properties={}, required=[]), + ) + ], + tool_handler=_tool_handler, + ) + + events = [ + event + async for event in agent.run_turn( + "Inspect this image with the tool.", + extra_messages=[_image_message()], + ) + ] + + assert executions == ["tool-1"] + assert len(provider.calls) == 2 + assert any(isinstance(event, ErrorEvent) for event in events) + + +@pytest.mark.asyncio +async def test_precise_image_rejection_cannot_use_generic_post_tool_retry() -> None: + provider = _ToolThenEmptyImageRejectProvider() + executions: list[str] = [] + + async def _tool_handler(call: ToolCall) -> ToolResult: + executions.append(call.tool_use_id) + return ToolResult( + tool_use_id=call.tool_use_id, + tool_name=call.tool_name, + content="observed", + ) + + agent = Agent( + provider=provider, + config=AgentConfig( + max_iterations=2, + max_provider_retries=2, + model_id="custom-model", + model_vision_support="unknown", + ), + tool_definitions=[ + ToolDefinition( + name="observe", + description="Observe once.", + input_schema=ToolInputSchema(properties={}, required=[]), + ) + ], + tool_handler=_tool_handler, + ) + + events = [ + event + async for event in agent.run_turn( + "Inspect this image with the tool.", + extra_messages=[_image_message()], + ) + ] + + assert executions == ["tool-1"] + assert len(provider.calls) == 2 + assert any(isinstance(event, ErrorEvent) for event in events) + + +@pytest.mark.asyncio +async def test_unknown_tool_image_is_projected_before_post_tool_request() -> None: + provider = _ToolMediaProbeProvider() + tool_context = ToolContext() + screenshot = base64.b64encode(b"tool screenshot").decode("ascii") + + async def _tool_handler(call: ToolCall) -> ToolResult: + tool_context.tool_result_media[call.tool_use_id] = [ + {"mime": "image/png", "data": screenshot} + ] + return ToolResult( + tool_use_id=call.tool_use_id, + tool_name=call.tool_name, + content="captured", + ) + + agent = Agent( + provider=provider, + config=AgentConfig( + max_iterations=2, + model_id="unknown-vision-model", + model_vision_support="unknown", + ), + tool_definitions=[ + ToolDefinition( + name="observe", + description="Capture once.", + input_schema=ToolInputSchema(properties={}, required=[]), + ) + ], + tool_handler=_tool_handler, + tool_context=tool_context, + ) + + events = [event async for event in agent.run_turn("Capture and explain.")] + + assert len(provider.calls) == 2 + assert count_provider_image_blocks(provider.calls[1]["messages"]) == 0 + assert any( + isinstance(block, ContentBlockText) and "图片未分析" in block.text + for message in provider.calls[1]["messages"] + if isinstance(message.content, list) + for block in message.content + ) + assert any(isinstance(event, DoneEvent) for event in events) + assert not any(isinstance(event, ErrorEvent) for event in events) + assert ( + agent.config.metadata["image_input_reason"] + == "image_probe_unsafe_after_irreversible_effect" + ) + + +@pytest.mark.asyncio +async def test_text_projection_does_not_destroy_image_for_later_vision_turn() -> None: + provider = _RecordingProvider() + config = AgentConfig( + max_iterations=1, + model_id="text-model", + model_capabilities=ModelCapabilities(supports_vision=False), + model_vision_support="unsupported", + preserve_historical_images=False, + ) + agent = Agent(provider=provider, config=config) + canonical_image = _image_message() + agent.set_history( + [ + canonical_image, + Message(role="assistant", content="I could not inspect it yet."), + ] + ) + + first_events = [event async for event in agent.run_turn("Keep it for later.")] + assert any(isinstance(event, DoneEvent) for event in first_events) + assert not any( + isinstance(block, ContentBlockImage) + for message in provider.calls[0]["messages"] + if isinstance(message.content, list) + for block in message.content + ) + assert any( + isinstance(block, ContentBlockText) + and "历史图片本回合未重新读取" in block.text + for message in provider.calls[0]["messages"] + if isinstance(message.content, list) + for block in message.content + ) + assert any( + isinstance(block, ContentBlockImage) + for message in agent.history_snapshot() + if isinstance(message.content, list) + for block in message.content + ) + assert not any( + isinstance(block, ContentBlockText) + and "历史图片本回合未重新读取" in block.text + for message in agent.history_snapshot() + if isinstance(message.content, list) + for block in message.content + ) + + config.model_id = "vision-model" + config.model_capabilities = ModelCapabilities(supports_vision=True) + config.model_vision_support = "supported" + config.preserve_historical_images = True + second_events = [event async for event in agent.run_turn("Now inspect it.")] + + assert any(isinstance(event, DoneEvent) for event in second_events) + assert any( + isinstance(block, ContentBlockImage) + for message in provider.calls[1]["messages"] + if isinstance(message.content, list) + for block in message.content + ) + + +@pytest.mark.asyncio +async def test_tool_result_image_uses_the_same_marker_projection() -> None: provider = _RecordingProvider() agent = Agent( provider=provider, @@ -167,22 +776,23 @@ async def test_tool_result_image_uses_the_same_admission_guard() -> None: ], ) - events = [ - event - async for event in agent.run_turn( - "Inspect the tool result.", - extra_messages=[tool_result_message], - ) - ] + projected, result = agent._project_image_input_for_provider( + [tool_result_message], + force_marker=True, + ) - assert provider.calls == [] - assert [event.code for event in events if isinstance(event, ErrorEvent)] == [ - IMAGE_INPUT_UNSUPPORTED_CODE - ] + assert result.input_image_count == 1 + assert result.output_image_count == 0 + nested = projected[0].content[0] + assert isinstance(nested, ContentBlockToolResult) + assert isinstance(nested.content, list) + assert isinstance(nested.content[0], ContentBlockText) + assert "图片未分析" in nested.content[0].text + assert isinstance(tool_result_message.content[0].content[0], ContentBlockImage) @pytest.mark.asyncio -async def test_forced_router_rejection_does_not_require_a_current_turn_image() -> None: +async def test_forced_router_marker_does_not_require_a_current_turn_image() -> None: provider = _RecordingProvider() agent = Agent( provider=provider, @@ -197,9 +807,8 @@ async def test_forced_router_rejection_does_not_require_a_current_turn_image() - events = [event async for event in agent.run_turn("Inspect the previous image.")] - assert provider.calls == [] - assert [event.code for event in events if isinstance(event, ErrorEvent)] == [ - IMAGE_INPUT_UNSUPPORTED_CODE - ] + assert len(provider.calls) == 1 + assert any(isinstance(event, DoneEvent) for event in events) + assert not any(isinstance(event, ErrorEvent) for event in events) assert agent.config.metadata["image_input_reason"] == "router_image_route_unavailable" assert agent.config.metadata["image_input_count"] == 0 diff --git a/tests/test_engine/test_agent_message_count_recovery.py b/tests/test_engine/test_agent_message_count_recovery.py index 8f4192cbc..ca3acbc16 100644 --- a/tests/test_engine/test_agent_message_count_recovery.py +++ b/tests/test_engine/test_agent_message_count_recovery.py @@ -17,6 +17,7 @@ from opensquilla.gateway.config import GatewayConfig from opensquilla.provider import ( ChatConfig, + ContentBlockImage, ContentBlockToolResult, ContentBlockToolUse, Message, @@ -635,6 +636,54 @@ async def test_message_limit_recovery_retries_once_below_headroom_without_rewrit ) +@pytest.mark.asyncio +async def test_message_limit_recovery_preserves_referenced_and_uploaded_images( + monkeypatch: pytest.MonkeyPatch, +) -> None: + compact_requests: list[Any] = [] + _install_exact_compactor(monkeypatch, compact_requests) + provider = _ExactMessageLimitProvider([100, None]) + agent = Agent( + provider=provider, + config=AgentConfig( + max_provider_retries=0, + flush_enabled=False, + model_vision_support="supported", + model_capabilities=ModelCapabilities(supports_vision=True), + ), + ) + agent.set_history(_plain_history()) + recovered_image = ContentBlockImage( + data="b2xkLWltYWdl", media_type="image/png", attachment_id="att_recovered", + ) + current_image = ContentBlockImage( + data="bmV3LWltYWdl", media_type="image/png", attachment_id="att_current", + ) + agent.set_request_image_context([Message(role="user", content=[recovered_image])]) + + events = [ + event async for event in agent.run_turn( + "Compare the images.", + extra_messages=[Message(role="user", content=[current_image])], + ) + ] + + assert len(provider.calls) == 2 + assert len(compact_requests) == 1 + assert not any(isinstance(event, ErrorEvent) for event in events) + for messages in provider.calls: + images = [ + block + for message in messages + if isinstance(message.content, list) + for block in message.content + if isinstance(block, ContentBlockImage) + ] + assert images == [recovered_image, current_image] + assert not any("b2xkLWltYWdl" in str(entry) for entry in compact_requests[0].entries) + assert agent._request_image_context == [] + + @pytest.mark.asyncio async def test_message_limit_cut_never_splits_parallel_tool_group( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_engine/test_artifact_execution_policy.py b/tests/test_engine/test_artifact_execution_policy.py index 46490473d..35c52f4a0 100644 --- a/tests/test_engine/test_artifact_execution_policy.py +++ b/tests/test_engine/test_artifact_execution_policy.py @@ -507,6 +507,9 @@ def __init__(self) -> None: def set_history(self, history: list[Any]) -> None: self.history = history + def set_request_image_context(self, messages: list[Any]) -> None: + assert messages == [] + restricted_agent = _HistoryCapture() restricted_summary = await runner._load_history( restricted_agent, diff --git a/tests/test_engine/test_attachment_messages.py b/tests/test_engine/test_attachment_messages.py index e2ab7ec6a..893a98757 100644 --- a/tests/test_engine/test_attachment_messages.py +++ b/tests/test_engine/test_attachment_messages.py @@ -27,6 +27,7 @@ ContentBlockImage, ContentBlockText, ) +from opensquilla.session.attachment_manifest import legacy_attachment_id def _b64(payload: bytes) -> str: @@ -217,6 +218,95 @@ def test_historical_inline_image_envelope_can_replay_for_vision() -> None: assert image_blocks[0].data == _b64(b"\x89PNG\r\n\x1a\n") +def test_forked_legacy_historical_image_keeps_allowed_attachment_id() -> None: + payload = b"legacy-fork-image" + message_id = "message-legacy-fork-image" + content = json.dumps( + { + "text": "continue from this legacy image", + "attachments": [ + { + "type": "image/png", + "data": _b64(payload), + "name": "legacy.png", + } + ], + } + ) + parent_attachment_id = legacy_attachment_id( + session_id="parent-session", + message_id=message_id, + index=0, + sha256=hashlib.sha256(payload).hexdigest(), + ) + + out = TurnRunner._maybe_unpack_attachments( + content, + preserve_image_attachments=True, + allowed_image_attachment_ids=frozenset({parent_attachment_id}), + session_id="child-session", + source_message_id=message_id, + ) + + assert isinstance(out, list) + image_blocks = [block for block in out if isinstance(block, ContentBlockImage)] + assert len(image_blocks) == 1 + assert image_blocks[0].data == _b64(payload) + assert image_blocks[0].attachment_id == parent_attachment_id + assert any( + isinstance(block, ContentBlockText) + and block.text == f"[historical image attachment_id={parent_attachment_id}]" + for block in out + ) + + +def test_explicit_multi_image_replay_exposes_stable_id_to_image_mapping() -> None: + first_id = "att_abcdefgh" + second_id = "att_ijklmnop" + first_payload = b"first-image" + second_payload = b"second-image" + content = json.dumps( + { + "text": "Compare the referenced images.", + "attachments": [ + { + "attachment_id": first_id, + "type": "image/png", + "data": _b64(first_payload), + "name": "first.png", + }, + { + "attachment_id": second_id, + "type": "image/png", + "data": _b64(second_payload), + "name": "second.png", + }, + ], + } + ) + + out = TurnRunner._maybe_unpack_attachments( + content, + preserve_image_attachments=True, + # Reference order is intentionally the reverse of transcript order; + # adjacent labels make the mapping unambiguous on the provider wire. + allowed_image_attachment_ids=frozenset((second_id, first_id)), + ) + + assert isinstance(out, list) + mapped_blocks = [ + (out[index].text, out[index + 1].data) + for index in range(len(out) - 1) + if isinstance(out[index], ContentBlockText) + and out[index].text.startswith("[historical image attachment_id=") + and isinstance(out[index + 1], ContentBlockImage) + ] + assert mapped_blocks == [ + (f"[historical image attachment_id={first_id}]", _b64(first_payload)), + (f"[historical image attachment_id={second_id}]", _b64(second_payload)), + ] + + def test_historical_image_ref_envelope_can_replay_for_vision(tmp_path: Path) -> None: payload = b"\x89PNG\r\n\x1a\n" sha = hashlib.sha256(payload).hexdigest() diff --git a/tests/test_engine/test_attachment_replay_ownership.py b/tests/test_engine/test_attachment_replay_ownership.py new file mode 100644 index 000000000..90de8b361 --- /dev/null +++ b/tests/test_engine/test_attachment_replay_ownership.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import json +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import pytest +import pytest_asyncio + +from opensquilla.engine.runtime import TurnRunner +from opensquilla.gateway.config import GatewayConfig +from opensquilla.session.attachment_manifest import build_attachment_manifest +from opensquilla.session.manager import SessionManager +from opensquilla.session.models import SessionIntent +from opensquilla.session.storage import SessionStorage, StaleEpochError + + +@pytest_asyncio.fixture +async def replay_session( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> AsyncIterator[tuple[TurnRunner, SessionManager, SessionManager, Any, Any, str]]: + monkeypatch.setenv("OPENSQUILLA_SESSION_ARCHIVE_DIR", str(tmp_path / "archives")) + database = str(tmp_path / "attachment-owners.db") + storage = SessionStorage(database) + other_storage = SessionStorage(database) + await storage.connect() + await other_storage.connect() + try: + manager = SessionManager(storage, inject_time_prefix=False) + other_manager = SessionManager(other_storage, inject_time_prefix=False) + node = await manager.create("agent:main:attachment-owner") + entry = await manager.append_message( + node.session_key, + "user", + json.dumps({ + "text": "Inspect this image.", + "attachments": [{"type": "image/png", "name": "sample.png", "data": "aW1hZ2U="}], + }), + ) + manifest = build_attachment_manifest( + [entry], session_id=node.session_id, session_key=node.session_key + ) + runner = TurnRunner( + provider_selector=MagicMock(), + session_manager=manager, + config=GatewayConfig(), + ) + yield runner, manager, other_manager, node, entry, manifest.occurrences[0].attachment_id + finally: + await other_storage.close() + await storage.close() + + +async def test_attachment_helpers_use_admitted_owner_without_resolving_key( + replay_session: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + runner, manager, _, node, entry, attachment_id = replay_session + + async def unexpected_resolution(*args: Any, **kwargs: Any) -> str: + raise AssertionError("exact-owner replay must not resolve the key again") + + monkeypatch.setattr(runner, "_resolve_session_id_for_log", unexpected_resolution) + owner = {"expected_session_id": node.session_id, "expected_session_epoch": node.epoch} + assert await runner._validated_image_attachment_ids( + node.session_key, [attachment_id], **owner + ) == (attachment_id,) + await runner._persist_attachment_manifest_best_effort(node.session_key, [entry], **owner) + states = await manager.get_context_states(node.session_key, **owner) + assert len(states) == 1 + assert states[0].session_id == node.session_id + + +@pytest.mark.parametrize("operation", ["canonical", "validate", "persist"]) +@pytest.mark.parametrize("replacement", ["reset", "epoch"]) +async def test_attachment_helpers_reject_retired_owner( + replay_session: Any, operation: str, replacement: str +) -> None: + runner, manager, other_manager, node, entry, attachment_id = replay_session + if replacement == "reset": + await other_manager.apply_intent(node.session_key, SessionIntent.RESET_SAME_KEY) + else: + await other_manager.storage.increment_epoch(node.session_key) + owner = {"expected_session_id": node.session_id, "expected_session_epoch": node.epoch} + with pytest.raises(StaleEpochError): + if operation == "canonical": + await runner._canonical_transcript_for_attachment_replay( + node.session_key, [entry], **owner + ) + elif operation == "validate": + await runner._validated_image_attachment_ids( + node.session_key, [attachment_id], **owner + ) + else: + await runner._persist_attachment_manifest_best_effort( + node.session_key, [entry], **owner + ) + assert await manager.get_context_states(node.session_key, valid_only=False) == [] + + +async def test_canonical_attachment_read_rejects_reset_after_storage_read( + replay_session: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + runner, manager, other_manager, node, entry, _ = replay_session + original_read = manager.storage.get_canonical_transcript + + async def reset_after_read(*args: Any, **kwargs: Any) -> Any: + result = await original_read(*args, **kwargs) + await other_manager.apply_intent(node.session_key, SessionIntent.RESET_SAME_KEY) + await other_manager.append_message(node.session_key, "user", "Replacement input.") + return result + + monkeypatch.setattr(manager.storage, "get_canonical_transcript", reset_after_read) + with pytest.raises(StaleEpochError, match="canonical transcript read"): + await runner._canonical_transcript_for_attachment_replay( + node.session_key, + [entry], + expected_session_id=node.session_id, + expected_session_epoch=node.epoch, + ) + assert [row.content for row in await manager.get_transcript(node.session_key)] == [ + "Replacement input." + ] + + +@pytest.mark.parametrize("replacement", ["reset", "epoch"]) +async def test_manifest_save_rechecks_owner_inside_write_transaction( + replay_session: Any, monkeypatch: pytest.MonkeyPatch, replacement: str +) -> None: + runner, manager, other_manager, node, entry, _ = replay_session + original_transaction = manager.storage._write_transaction + save_attempted = False + + @asynccontextmanager + async def replace_before_transaction( + operation: str, **kwargs: Any + ) -> AsyncIterator[Any]: + nonlocal save_attempted + if operation == "save_context_state": + save_attempted = True + if replacement == "reset": + await other_manager.apply_intent(node.session_key, SessionIntent.RESET_SAME_KEY) + else: + await other_manager.storage.increment_epoch(node.session_key) + async with original_transaction(operation, **kwargs) as connection: + yield connection + + monkeypatch.setattr(manager.storage, "_write_transaction", replace_before_transaction) + with pytest.raises(StaleEpochError): + await runner._persist_attachment_manifest_best_effort( + node.session_key, + [entry], + expected_session_id=node.session_id, + expected_session_epoch=node.epoch, + ) + assert save_attempted is True + assert await manager.get_context_states(node.session_key, valid_only=False) == [] + + +@pytest.mark.parametrize("replacement", ["reset", "epoch"]) +async def test_bound_attachment_owner_failure_stops_pipeline( + replay_session: Any, monkeypatch: pytest.MonkeyPatch, replacement: str +) -> None: + runner, _, other_manager, node, entry, _ = replay_session + if replacement == "reset": + await other_manager.apply_intent(node.session_key, SessionIntent.RESET_SAME_KEY) + else: + await other_manager.storage.increment_epoch(node.session_key) + pipeline_entered = False + + async def unexpected_pipeline(*args: Any, **kwargs: Any) -> Any: + nonlocal pipeline_entered + pipeline_entered = True + raise AssertionError("A retired bound prompt must not reach pipeline steps") + + monkeypatch.setattr("opensquilla.engine.pipeline.run_pipeline", unexpected_pipeline) + with pytest.raises(StaleEpochError): + await runner._run_pipeline( + "Inspect the attachment again.", + node.session_key, + MagicMock(), + None, + [], + "system", + [], + bound_user_message_id=entry.message_id, + expected_session_id=node.session_id, + expected_session_epoch=node.epoch, + ) + assert pipeline_entered is False diff --git a/tests/test_engine/test_capability_gate.py b/tests/test_engine/test_capability_gate.py index 3f675ef6f..850588daa 100644 --- a/tests/test_engine/test_capability_gate.py +++ b/tests/test_engine/test_capability_gate.py @@ -382,20 +382,36 @@ def facts_catalog() -> Iterator[ModelCatalog]: set_shared_catalog(None) +def _facts_context(provider: str = "mainprov") -> TurnContext: + config = GatewayConfig(llm={"provider": provider}) + config.squilla_router.cross_provider_tiers = True + return TurnContext( + message="", + session_key="capability-facts", + config=config, + provider=None, + model="", + tool_defs=[], + system_prompt="", + ) + + def test_facts_definite_signals_from_catalog(facts_catalog: ModelCatalog) -> None: tiers = { "c0": {"model": "dummy-small-live-1"}, "c1": {"model": "dummy-mini-unknown-1"}, "c2": {"model": "dummy-vis-live-1"}, } - facts = _tier_capability_facts(tiers, ["c0", "c1", "c2"], "mainprov") + facts = _tier_capability_facts( + _facts_context("openrouter"), tiers, ["c0", "c1", "c2"], "openrouter" + ) assert facts["c0"] == TierCapability(supports_vision=False, context_window=8_000) assert facts["c2"] == TierCapability(supports_vision=True, context_window=200_000) def test_facts_synthesized_entry_gives_no_signal(facts_catalog: ModelCatalog) -> None: facts = _tier_capability_facts( - {"c1": {"model": "dummy-mini-unknown-1"}}, ["c1"], "mainprov" + _facts_context(), {"c1": {"model": "dummy-mini-unknown-1"}}, ["c1"], "mainprov" ) assert facts["c1"] == TierCapability(supports_vision=None, context_window=None) @@ -407,17 +423,16 @@ def test_facts_user_override_window_counts_as_definite(facts_catalog: ModelCatal {"mainprov/dummy-pinned-window-1": {"context_window": 64_000}} ) facts = _tier_capability_facts( - {"c1": {"model": "dummy-pinned-window-1"}}, ["c1"], "mainprov" + _facts_context(), {"c1": {"model": "dummy-pinned-window-1"}}, ["c1"], "mainprov" ) assert facts["c1"].context_window == 64_000 -def test_facts_anthropic_flag_gated_caps_stay_unknown(facts_catalog: ModelCatalog) -> None: - # The user-override layer claims vision for this model. Under a normal - # provider that is a definite signal; under anthropic the capabilities - # are flag-gated to the empty ModelCapabilities() for one release, so - # the gate must treat the same claim as unknown. +def test_facts_keep_capability_evidence_provider_scoped(facts_catalog: ModelCatalog) -> None: + # Explicit evidence for one provider cannot authorize a different + # provider's deployment with the same model identifier. overridden = _tier_capability_facts( + _facts_context(), {"c2": {"model": "dummy-anthropic-x", "provider": "anthroprov"}}, ["c2"], "mainprov", @@ -425,6 +440,7 @@ def test_facts_anthropic_flag_gated_caps_stay_unknown(facts_catalog: ModelCatalo assert overridden["c2"].supports_vision is True anthropic_facts = _tier_capability_facts( + _facts_context(), {"c2": {"model": "dummy-anthropic-x", "provider": "anthropic"}}, ["c2"], "mainprov", @@ -435,7 +451,7 @@ def test_facts_anthropic_flag_gated_caps_stay_unknown(facts_catalog: ModelCatalo def test_facts_blank_model_gives_no_signal(facts_catalog: ModelCatalog) -> None: - facts = _tier_capability_facts({"c0": {"model": ""}}, ["c0"], "mainprov") + facts = _tier_capability_facts(_facts_context(), {"c0": {"model": ""}}, ["c0"], "mainprov") assert facts["c0"] == TierCapability() diff --git a/tests/test_engine/test_historical_image_followup.py b/tests/test_engine/test_historical_image_followup.py index 0f4877644..38be69737 100644 --- a/tests/test_engine/test_historical_image_followup.py +++ b/tests/test_engine/test_historical_image_followup.py @@ -18,7 +18,16 @@ from opensquilla.engine.steps.vision_followup_gate import apply_vision_followup_gate from opensquilla.gateway.config import GatewayConfig from opensquilla.provider import ChatConfig, DoneEvent, Message, ModelCapabilities, TextDeltaEvent -from opensquilla.provider.types import ContentBlockImage +from opensquilla.provider.types import ContentBlockImage, ContentBlockText +from opensquilla.session.attachment_manifest import ( + ATTACHMENT_MANIFEST_PROVIDER, + ATTACHMENT_MANIFEST_STATE_KIND, + attachment_manifest_from_context_state, + build_attachment_manifest, + manifest_context_state, +) +from opensquilla.session.manager import SessionManager +from opensquilla.session.storage import SessionStorage @dataclass @@ -69,6 +78,18 @@ async def get_context_states(self, session_key: str) -> list[Any]: # noqa: ARG0 return [] +class _CanonicalSessionManager(_FakeSessionManager): + def __init__(self) -> None: + super().__init__() + self._canonical: dict[str, list[_TranscriptEntry]] = {} + + async def get_canonical_transcript( + self, + session_key: str, + ) -> list[_TranscriptEntry]: + return list(self._canonical.get(session_key, self._transcripts.get(session_key, []))) + + class _CapturingProvider: provider_name = "fake" @@ -137,12 +158,694 @@ def _inline_image_envelope(text: str, payload: bytes = b"\x89PNG\r\n\x1a\n") -> ) +def _inline_image_envelope_many(text: str, *payloads: bytes) -> str: + return json.dumps( + { + "text": text, + "attachments": [ + { + "type": "image/png", + "name": f"image-{index}.png", + "data": _b64(payload), + } + for index, payload in enumerate(payloads) + ], + } + ) + + def _message_has_image(message: Message) -> bool: return isinstance(message.content, list) and any( isinstance(block, ContentBlockImage) for block in message.content ) +def _message_has_marker(message: Message, marker: str) -> bool: + return isinstance(message.content, list) and any( + isinstance(block, ContentBlockText) and marker in block.text + for block in message.content + ) or isinstance(message.content, str) and marker in message.content + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("vision_support", "expects_image"), + [("supported", True), ("unknown", True), ("unsupported", False)], +) +async def test_bound_image_message_reprojects_after_model_switch( + vision_support: str, + expects_image: bool, +) -> None: + manager = _FakeSessionManager() + key = f"agent:main:bound-image-switch-{vision_support}" + config = GatewayConfig(llm={"provider": "openrouter"}) + config.squilla_router.vision_history_lookback_turns = 0 + runner = TurnRunner(provider_selector=MagicMock(), session_manager=manager, config=config) + await manager.create(key) + envelope = _inline_image_envelope("Describe this image.") + await manager.append_message( + key, + "user", + envelope, + message_id="bound-image-message", + ) + + provider = _CapturingProvider() + agent = Agent( + provider=provider, + config=AgentConfig( + max_iterations=1, + model_id=f"configured-{vision_support}", + model_vision_support=vision_support, + metadata={"attachment_count": 0}, + ), + ) + await runner._load_history( + agent, + key, + bound_user_message_id="bound-image-message", + ) + events = [event async for event in agent.run_turn("Describe this image.")] + + assert any(event.kind == "done" for event in events) + sent = provider.calls[0]["messages"] + assert any(_message_has_image(message) for message in sent) is expects_image + assert any(_message_has_marker(message, "图片") for message in sent) is ( + not expects_image + ) + assert any( + _message_has_marker(message, "Image replay context for this request") + for message in sent + ) is expects_image + assert manager._transcripts[key][0].content == envelope + + +@pytest.mark.asyncio +@pytest.mark.parametrize("vision_support", ["supported", "unknown", "unsupported"]) +async def test_bound_plain_text_message_does_not_replay_unrelated_archived_image( + vision_support: str, +) -> None: + manager = _CanonicalSessionManager() + key = f"agent:main:bound-plain-text-{vision_support}" + config = GatewayConfig(llm={"provider": "openrouter"}) + config.squilla_router.vision_history_lookback_turns = 3 + runner = TurnRunner(provider_selector=MagicMock(), session_manager=manager, config=config) + await manager.create(key) + archived_image = _TranscriptEntry( + role="user", + content=_inline_image_envelope("Archived unrelated image."), + message_id="archived-image-message", + ) + archived_answer = _TranscriptEntry( + role="assistant", + content="Archived answer.", + message_id="archived-answer-message", + ) + current_user = _TranscriptEntry( + role="user", + content="Answer this unrelated text question.", + message_id="bound-plain-text-message", + ) + manager._transcripts[key] = [current_user] + manager._canonical[key] = [archived_image, archived_answer, current_user] + + provider = _CapturingProvider() + agent = Agent( + provider=provider, + config=AgentConfig( + max_iterations=1, + model_id=f"configured-{vision_support}", + model_vision_support=vision_support, + metadata={"attachment_count": 0}, + ), + ) + await runner._load_history( + agent, + key, + bound_user_message_id="bound-plain-text-message", + ) + events = [event async for event in agent.run_turn(current_user.content)] + + assert any(event.kind == "done" for event in events) + sent = provider.calls[0]["messages"] + assert not any(_message_has_image(message) for message in sent) + assert "Archived unrelated image." not in str(sent) + assert "historical attachment omitted" not in str(sent) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("archived", [False, True]) +@pytest.mark.parametrize("vision_support", ["supported", "unknown", "unsupported"]) +async def test_explicit_images_survive_a_full_history_window( + archived: bool, + vision_support: str, +) -> None: + manager = _CanonicalSessionManager() + key = "agent:main:referenced-image-window" + config = GatewayConfig(llm={"provider": "openrouter"}) + config.squilla_router.vision_history_lookback_turns = 0 + runner = TurnRunner(provider_selector=MagicMock(), session_manager=manager, config=config) + node = await manager.create(key) + image_entries = [ + _TranscriptEntry( + role="user", + content=_inline_image_envelope(f"Old instruction {index}.", bytes([index])), + message_id=f"historical-image-{index}", + ) + for index in range(3) + ] + tail = [ + _TranscriptEntry(role="user", content="Recent question.", message_id="recent-user"), + _TranscriptEntry(role="assistant", content="Recent answer.", message_id="recent-answer"), + _TranscriptEntry(role="user", content="Compare the selected images.", message_id="current"), + _TranscriptEntry(role="user", content="Queued future input.", message_id="queued"), + ] + manager._canonical[key] = [*image_entries, *tail] + manager._transcripts[key] = tail if archived else [*image_entries, *tail] + manifest = build_attachment_manifest( + image_entries, session_id=node.session_id, session_key=key, + ) + requested_ids = [item.attachment_id for item in manifest.occurrences[:2]] + provider = _CapturingProvider() + agent = Agent( + provider=provider, + config=AgentConfig( + max_iterations=1, + max_history_turns=1, + model_id="configured-model", + model_vision_support=vision_support, + metadata={"image_intent_attachment_ids": requested_ids}, + ), + ) + + await runner._load_history(agent, key, bound_user_message_id="current") + # A second load replaces request context instead of accumulating images. + await runner._load_history(agent, key, bound_user_message_id="current") + admission_messages = agent._assemble_compaction_consumer_request( + replay_summary="Earlier conversation.", + kept_entries=[], + active_user_message=tail[2].content, + active_user_in_history=False, + bound_user_message_id=None, + attachment_messages=None, + runtime_context_message=Message(role="user", content="[Runtime context for this turn]"), + ) + assert admission_messages is not None + assert all(attachment_id in str(admission_messages) for attachment_id in requested_ids) + + events = [event async for event in agent.run_turn(tail[2].content)] + + assert any(event.kind == "done" for event in events) + sent = provider.calls[0]["messages"] + image_blocks = [ + block + for message in sent + if isinstance(message.content, list) + for block in message.content + if isinstance(block, ContentBlockImage) + ] + assert [block.data for block in image_blocks] == ( + [] if vision_support == "unsupported" else [_b64(bytes([0])), _b64(bytes([1]))] + ) + assert all(attachment_id in str(sent) for attachment_id in requested_ids) + assert manifest.occurrences[2].attachment_id not in str(sent) + assert "Recent question." in str(sent) + assert "Recent answer." in str(sent) + assert "Old instruction" not in str(sent) + assert "Queued future input." not in str(sent) + if vision_support == "unsupported": + assert "图片" in str(sent) + assert manager._canonical[key][0] is image_entries[0] + assert "attachment_id" not in image_entries[0].content + + agent.clear_history() + async for _ in agent.run_turn("Answer without historical input."): + pass + assert not any(_message_has_image(message) for message in provider.calls[1]["messages"]) + assert not any(attachment_id in str(provider.calls[1]) for attachment_id in requested_ids) + + +@pytest.mark.asyncio +async def test_explicit_attachment_id_rehydrates_image_from_compacted_archive() -> None: + manager = _CanonicalSessionManager() + key = "agent:main:compacted-image-id-replay" + config = GatewayConfig(llm={"provider": "openrouter"}) + config.squilla_router.vision_history_lookback_turns = 0 + runner = TurnRunner(provider_selector=MagicMock(), session_manager=manager, config=config) + node = await manager.create(key) + archived_image = _TranscriptEntry( + role="user", + content=_inline_image_envelope("Archived image."), + message_id="archived-image-message", + ) + archived_answer = _TranscriptEntry( + role="assistant", + content="Earlier textual analysis.", + message_id="archived-answer-message", + ) + active_user = _TranscriptEntry( + role="user", + content="Analyze attachment att-id again.", + message_id="current-message", + ) + manager._transcripts[key] = [active_user] + manager._canonical[key] = [archived_image, archived_answer, active_user] + manifest = build_attachment_manifest( + manager._canonical[key], + session_id=node.session_id, + session_key=key, + ) + attachment_id = manifest.occurrences[0].attachment_id + + provider = _CapturingProvider() + agent = Agent( + provider=provider, + config=AgentConfig( + max_iterations=1, + model_id="configured-vision", + model_vision_support="supported", + metadata={"image_attachment_ids": [attachment_id]}, + ), + ) + await runner._load_history(agent, key) + events = [event async for event in agent.run_turn("Analyze it again.")] + + assert any(event.kind == "done" for event in events) + sent = provider.calls[0]["messages"] + assert any(_message_has_image(message) for message in sent) + assert manager._canonical[key][0] is archived_image + + +@pytest.mark.asyncio +async def test_current_image_and_one_archived_image_id_are_merged_and_exact() -> None: + manager = _CanonicalSessionManager() + key = "agent:main:current-plus-one-archived-image" + config = GatewayConfig(llm={"provider": "openrouter"}) + config.squilla_router.vision_history_lookback_turns = 0 + runner = TurnRunner(provider_selector=MagicMock(), session_manager=manager, config=config) + node = await manager.create(key) + archived_first = b"archived-first" + archived_second = b"archived-second" + current_payload = b"current-image" + archived_image = _TranscriptEntry( + role="user", + content=_inline_image_envelope_many( + "Two archived images.", + archived_first, + archived_second, + ), + message_id="archived-two-image-message", + ) + archived_answer = _TranscriptEntry( + role="assistant", + content="Earlier answer.", + message_id="archived-two-image-answer", + ) + current_user = _TranscriptEntry( + role="user", + content=_inline_image_envelope("Compare these images.", current_payload), + message_id="current-image-message", + ) + manager._transcripts[key] = [current_user] + manager._canonical[key] = [archived_image, archived_answer, current_user] + manifest = build_attachment_manifest( + manager._canonical[key], + session_id=node.session_id, + session_key=key, + ) + archived_second_id = next( + occurrence.attachment_id + for occurrence in manifest.occurrences + if occurrence.source_message_id == archived_image.message_id + and occurrence.ordinal == 1 + ) + current_id = next( + occurrence.attachment_id + for occurrence in manifest.occurrences + if occurrence.source_message_id == current_user.message_id + ) + + provider = _CapturingProvider() + agent = Agent( + provider=provider, + config=AgentConfig( + max_iterations=1, + model_id="configured-vision", + model_vision_support="supported", + metadata={ + "attachment_count": 1, + "image_attachment_ids": [current_id], + "image_intent_attachment_ids": [archived_second_id], + }, + ), + ) + await runner._load_history(agent, key) + current_message = Message( + role="user", + content=[ + ContentBlockText(text="Compare these images."), + ContentBlockImage( + media_type="image/png", + data=_b64(current_payload), + attachment_id=current_id, + ), + ], + ) + events = [ + event + async for event in agent.run_turn("", extra_messages=[current_message]) + ] + + assert any(event.kind == "done" for event in events) + image_payloads = [ + block.data + for message in provider.calls[0]["messages"] + if isinstance(message.content, list) + for block in message.content + if isinstance(block, ContentBlockImage) + ] + assert image_payloads == [_b64(archived_second), _b64(current_payload)] + assert _b64(archived_first) not in image_payloads + + +@pytest.mark.asyncio +async def test_persisted_compacted_archive_rehydrates_explicit_attachment_id() -> None: + storage = SessionStorage(":memory:") + await storage.connect() + try: + manager = SessionManager(storage, inject_time_prefix=False) + key = "agent:main:persisted-compacted-image-id-replay" + node = await manager.create(key) + payload = b"\x89PNG\r\n\x1a\npersisted-archive" + image_entry = await manager.append_message( + key, + "user", + _inline_image_envelope("Archived image.", payload), + message_id="persisted-archived-image", + ) + current_entry = await manager.append_message( + key, + "user", + "Analyze the archived attachment again.", + message_id="persisted-current-message", + ) + manifest = build_attachment_manifest( + [image_entry, current_entry], + session_id=node.session_id, + session_key=key, + ) + attachment_id = manifest.occurrences[0].attachment_id + source = await manager.capture_compaction_source( + key, + boundary_message_id=current_entry.message_id, + ) + installed = await manager.persist_compaction_result( + key, + f"Archived image attachment_id={attachment_id}", + [{"role": "user", "content": current_entry.content}], + compaction_id="cmp-persisted-image", + removed_count=1, + source_entries=source.entries, + source_preimage=source.preimage, + source_boundary_message_id=source.boundary_message_id, + source_boundary_entry_id=source.boundary_entry_id, + ) + assert installed is True + assert [entry.message_id for entry in await manager.get_transcript(key)] == [ + current_entry.message_id + ] + assert [ + entry.message_id for entry in await manager.get_canonical_transcript(key) + ] == [image_entry.message_id, current_entry.message_id] + + config = GatewayConfig(llm={"provider": "openrouter"}) + config.squilla_router.vision_history_lookback_turns = 0 + runner = TurnRunner( + provider_selector=MagicMock(), + session_manager=manager, + config=config, + ) + provider = _CapturingProvider() + current_prompt = f"Analyze attachment_id={attachment_id} again." + turn, returned_provider = await runner._run_pipeline( + current_prompt, + key, + provider, + None, + [], + "system", + [], + semantic_message=current_prompt, + ) + assert returned_provider is provider + assert turn.metadata["image_intent_attachment_ids"] == [attachment_id] + assert turn.metadata["router_vision_followup_needs_image"] is True + agent = Agent( + provider=provider, + config=AgentConfig( + max_iterations=1, + model_id="configured-vision", + model_vision_support="supported", + metadata=turn.metadata, + ), + ) + + await runner._load_history(agent, key) + events = [event async for event in agent.run_turn(current_prompt)] + + assert any(event.kind == "done" for event in events) + image_blocks = [ + block + for message in provider.calls[0]["messages"] + if isinstance(message.content, list) + for block in message.content + if isinstance(block, ContentBlockImage) + ] + assert [block.data for block in image_blocks] == [_b64(payload)] + finally: + await storage.close() + + +@pytest.mark.asyncio +async def test_full_fork_rehydrates_parent_legacy_attachment_id() -> None: + storage = SessionStorage(":memory:") + await storage.connect() + try: + manager = SessionManager(storage, inject_time_prefix=False) + parent_key = "agent:main:legacy-image-parent" + parent = await manager.create(parent_key) + payload = b"\x89PNG\r\n\x1a\nlegacy-full-fork" + await manager.append_message( + parent_key, + "user", + _inline_image_envelope("Legacy archived image.", payload), + message_id="legacy-full-fork-image", + ) + await manager.append_message(parent_key, "assistant", "Earlier analysis.") + active_tail = await manager.append_message( + parent_key, + "user", + "Keep this active tail.", + message_id="legacy-full-fork-tail", + ) + parent_manifest = build_attachment_manifest( + await manager.get_canonical_transcript(parent_key), + session_id=parent.session_id, + session_key=parent_key, + ) + attachment_id = parent_manifest.occurrences[0].attachment_id + source = await manager.capture_compaction_source( + parent_key, + boundary_message_id=active_tail.message_id, + ) + assert await manager.persist_compaction_result( + parent_key, + f"Legacy image attachment_id={attachment_id}", + [{"role": "user", "content": active_tail.content}], + compaction_id="cmp-legacy-full-fork-image", + removed_count=2, + source_entries=source.entries, + source_preimage=source.preimage, + source_boundary_message_id=source.boundary_message_id, + source_boundary_entry_id=source.boundary_entry_id, + ) + + child_key = "agent:main:legacy-image-child" + child = await manager.branch(parent_key, child_key, fork_transcript=True) + prompt = f"Analyze attachment_id={attachment_id} again." + current = await manager.append_message( + child_key, + "user", + prompt, + message_id="legacy-full-fork-current", + ) + child_manifest = build_attachment_manifest( + await manager.get_canonical_transcript(child_key), + session_id=child.session_id, + session_key=child_key, + ) + assert child_manifest.by_id(attachment_id) is not None + + config = GatewayConfig(llm={"provider": "openrouter"}) + config.squilla_router.vision_history_lookback_turns = 0 + runner = TurnRunner( + provider_selector=MagicMock(), + session_manager=manager, + config=config, + ) + provider = _CapturingProvider() + agent = Agent( + provider=provider, + config=AgentConfig( + max_iterations=1, + model_id="configured-vision", + model_vision_support="supported", + metadata={ + "attachment_count": 0, + "image_intent_attachment_ids": [attachment_id], + }, + ), + ) + await runner._load_history( + agent, + child_key, + bound_user_message_id=current.message_id, + ) + events = [event async for event in agent.run_turn(prompt)] + + assert any(event.kind == "done" for event in events) + sent_images = [ + block.data + for message in provider.calls[0]["messages"] + if isinstance(message.content, list) + for block in message.content + if isinstance(block, ContentBlockImage) + ] + assert sent_images == [_b64(payload)] + assert any( + attachment_id in str(message.content) + for message in provider.calls[0]["messages"] + ) + finally: + await storage.close() + + +@pytest.mark.asyncio +async def test_lazy_manifest_backfill_monotonically_merges_active_subset() -> None: + storage = SessionStorage(":memory:") + await storage.connect() + try: + manager = SessionManager(storage, inject_time_prefix=False) + key = "agent:main:manifest-active-subset-merge" + node = await manager.create(key) + archived = await manager.append_message( + key, + "user", + _inline_image_envelope("Archived A.", b"image-a"), + message_id="manifest-image-a", + ) + active = await manager.append_message( + key, + "user", + _inline_image_envelope("Active B.", b"image-b"), + message_id="manifest-image-b", + ) + archived_manifest = build_attachment_manifest( + [archived], + session_id=node.session_id, + session_key=key, + ) + await manager.save_context_state(manifest_context_state(archived_manifest)) + runner = TurnRunner( + provider_selector=MagicMock(), + session_manager=manager, + config=GatewayConfig(llm={"provider": "openrouter"}), + ) + + await runner._persist_attachment_manifest_best_effort(key, [active]) + states = await manager.get_context_states( + key, + provider=ATTACHMENT_MANIFEST_PROVIDER, + state_kind=ATTACHMENT_MANIFEST_STATE_KIND, + ) + latest = max(states, key=lambda state: (state.created_at, state.id or 0)) + merged = attachment_manifest_from_context_state(latest) + assert [item.source_message_id for item in merged.occurrences] == [ + "manifest-image-a", + "manifest-image-b", + ] + + state_count = len(states) + await runner._persist_attachment_manifest_best_effort(key, [active]) + states = await manager.get_context_states( + key, + provider=ATTACHMENT_MANIFEST_PROVIDER, + state_kind=ATTACHMENT_MANIFEST_STATE_KIND, + ) + assert len(states) == state_count + finally: + await storage.close() + + +@pytest.mark.asyncio +async def test_non_image_attachment_id_does_not_force_vision_route() -> None: + storage = SessionStorage(":memory:") + await storage.connect() + try: + manager = SessionManager(storage, inject_time_prefix=False) + key = "agent:main:non-image-attachment-reference" + node = await manager.create(key) + document_entry = await manager.append_message( + key, + "user", + json.dumps( + { + "text": "Saved document.", + "attachments": [ + { + "type": "application/pdf", + "name": "notes.pdf", + "data": _b64(b"synthetic-pdf"), + } + ], + } + ), + message_id="manifest-document", + ) + manifest = build_attachment_manifest( + [document_entry], + session_id=node.session_id, + session_key=key, + ) + document_id = manifest.occurrences[0].attachment_id + config = GatewayConfig(llm={"provider": "openrouter"}) + config.squilla_router.enabled = False + runner = TurnRunner( + provider_selector=MagicMock(), + session_manager=manager, + config=config, + ) + provider = _CapturingProvider() + prompt = f"Summarize attachment_id={document_id}." + + turn, returned_provider = await runner._run_pipeline( + prompt, + key, + provider, + None, + [], + "system", + [], + semantic_message=prompt, + ) + + assert returned_provider is provider + assert "image_intent_attachment_ids" not in turn.metadata + assert turn.metadata.get("router_vision_followup_needs_image") is not True + finally: + await storage.close() + + @pytest.mark.asyncio async def test_image_followup_routes_vision_and_replays_inline_history_image() -> None: manager = _FakeSessionManager() @@ -510,8 +1213,18 @@ async def test_text_model_history_keeps_image_as_marker_not_provider_image() -> key = "agent:main:image-followup-text-model" config = GatewayConfig(llm={"provider": "openrouter"}) runner = TurnRunner(provider_selector=MagicMock(), session_manager=manager, config=config) - await manager.create(key) - await manager.append_message(key, "user", _inline_image_envelope("Describe this image.")) + node = await manager.create(key) + image_entry = await manager.append_message( + key, + "user", + _inline_image_envelope("Describe this image."), + message_id="legacy-image-message", + ) + manifest = build_attachment_manifest( + [image_entry], + session_id=node.session_id, + session_key=key, + ) await manager.append_message(key, "assistant", "It shows a small test image.") await manager.append_message(key, "user", "Continue as text.") @@ -530,6 +1243,7 @@ async def test_text_model_history_keeps_image_as_marker_not_provider_image() -> sent_messages = provider.calls[0]["messages"] assert not any(_message_has_image(message) for message in sent_messages) assert "historical attachment omitted" in str(sent_messages[0].content) + assert manifest.occurrences[0].attachment_id in str(sent_messages[0].content) @pytest.mark.asyncio diff --git a/tests/test_engine/test_history_silent_reply.py b/tests/test_engine/test_history_silent_reply.py index 13efd2acf..dc40d0eb2 100644 --- a/tests/test_engine/test_history_silent_reply.py +++ b/tests/test_engine/test_history_silent_reply.py @@ -153,6 +153,7 @@ async def test_turn_runner_load_history_passes_goal_provenance_to_sanitizer() -> preserve_historical_images=False, ), set_history=MagicMock(), + set_request_image_context=MagicMock(), ) await runner._load_history(agent, "agent:main:test", trim_last_user=False) diff --git a/tests/test_engine/test_preflight_compaction.py b/tests/test_engine/test_preflight_compaction.py index 083ffd400..f6c5ae056 100644 --- a/tests/test_engine/test_preflight_compaction.py +++ b/tests/test_engine/test_preflight_compaction.py @@ -686,6 +686,9 @@ def __init__(self) -> None: def set_history(self, history: list[Any]) -> None: self.history = history + def set_request_image_context(self, messages: list[Any]) -> None: + assert messages == [] + agent = _HistoryCapture() summary_context = await runner._load_history( agent, @@ -1179,6 +1182,9 @@ def __init__(self) -> None: def set_history(self, history: list[Any]) -> None: self.history = history + def set_request_image_context(self, messages: list[Any]) -> None: + assert messages == [] + agent = _HistoryCapture() summary_context = await runner._load_history(agent, session_key, trim_last_user=False) @@ -1338,6 +1344,9 @@ def __init__(self) -> None: def set_history(self, history: list[Any]) -> None: self.history = history + def set_request_image_context(self, messages: list[Any]) -> None: + assert messages == [] + agent = _HistoryCapture() summary_context = await runner._load_history(agent, session_key, trim_last_user=False) @@ -1394,6 +1403,9 @@ def __init__(self) -> None: def set_history(self, history: list[Any]) -> None: self.history = history + def set_request_image_context(self, messages: list[Any]) -> None: + assert messages == [] + agent = _HistoryCapture() summary_context = await runner._load_history(agent, session_key, trim_last_user=False) assert summary_context is None diff --git a/tests/test_engine/test_route_plan.py b/tests/test_engine/test_route_plan.py index 056d9374d..24346c807 100644 --- a/tests/test_engine/test_route_plan.py +++ b/tests/test_engine/test_route_plan.py @@ -248,11 +248,11 @@ def test_route_plan_freezes_text_candidates_aliases_ensemble_and_winner() -> Non assert event.router_tier_snapshot == snapshot -def test_route_plan_freezes_only_executable_image_candidates() -> None: +def test_route_plan_freezes_configured_image_candidates_without_legacy_tier() -> None: turn = _turn() turn.metadata.update( { - "routed_tier": "image_model", + "routed_tier": "c0", "routed_provider": "image-provider", "routed_model": "image/winner", "routing_source": "image_route", @@ -305,15 +305,69 @@ def test_route_plan_freezes_only_executable_image_candidates() -> None: "tiers": [ { "tier": "c0", - "provider": "vision-provider", - "model": "vision/fallback", + "provider": "image-provider", + "model": "image/winner", "execution_kind": "single_model", }, { - "tier": "image_model", - "provider": "image-provider", - "model": "image/winner", + "tier": "c1", + "provider": "text-provider", + "model": "text/only", "execution_kind": "single_model", }, ], } + + +def test_route_plan_image_marker_snapshot_keeps_four_configured_text_tiers() -> None: + turn = _turn() + turn.metadata.update( + routed_tier="c1", + routed_model="text/actual-winner", + routing_source="image_route", + image_input_mode="marker", + ) + turn.config = SimpleNamespace( + squilla_router=SimpleNamespace( + tiers={ + **{ + f"c{index}": { + "model": f"text/configured-{index}", + "supports_image": False, + } + for index in range(4) + }, + "image_model": { + "model": "legacy/unused-vision", + "supports_image": True, + "image_only": True, + }, + } + ), + llm_ensemble=SimpleNamespace(enabled=False), + ) + + plan = pin_route_plan( + turn, + turn_id="turn-snapshot-image-marker", + provider="provider-a", + model="text/actual-winner", + context_window=32_000, + capabilities=ModelCapabilities(supports_vision=False), + effective_thinking=False, + ) + + assert plan is not None and plan.router_tier_snapshot is not None + snapshot = plan.router_tier_snapshot.as_dict() + assert snapshot["request_kind"] == "image" + assert [entry["tier"] for entry in snapshot["tiers"]] == ["c0", "c1", "c2", "c3"] + assert [entry["model"] for entry in snapshot["tiers"]] == [ + "text/configured-0", + "text/actual-winner", + "text/configured-2", + "text/configured-3", + ] + assert turn.metadata["route_plan"]["router_tier_snapshot"] == snapshot + event = build_router_decision_event(turn) + assert event is not None + assert event.router_tier_snapshot == snapshot diff --git a/tests/test_engine/test_router_configured_image_policy.py b/tests/test_engine/test_router_configured_image_policy.py new file mode 100644 index 000000000..f8af0f191 --- /dev/null +++ b/tests/test_engine/test_router_configured_image_policy.py @@ -0,0 +1,364 @@ +"""Router image policy: only the configured c0-c3 ladder may execute.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from opensquilla.engine.pipeline import TurnContext +from opensquilla.engine.runtime import TurnRunner +from opensquilla.engine.steps.squilla_router import ( + _tier_deployment_vision_support, + apply_squilla_router, + finalize_squilla_router_capacity, +) +from opensquilla.gateway.config import GatewayConfig +from opensquilla.provider.model_catalog import ModelCatalog + + +def _catalog_evidence( + monkeypatch: pytest.MonkeyPatch, + *, + supported: tuple[str, ...] = (), + unsupported: tuple[str, ...] = (), +) -> ModelCatalog: + catalog = ModelCatalog() + catalog._populate_from_data( + [ + { + "id": model, + "architecture": {"input_modalities": modalities}, + } + for models, modalities in ( + (supported, ["text", "image"]), + (unsupported, ["text"]), + ) + for model in models + ] + ) + monkeypatch.setattr("opensquilla.engine.steps.squilla_router.shared_catalog", lambda: catalog) + return catalog + + +def _context( + tiers: dict[str, dict[str, object]], + *, + message: str = "Describe the image.", +) -> TurnContext: + config = GatewayConfig(llm={"provider": "openrouter"}) + config.squilla_router.tiers = tiers + return TurnContext( + message=message, + session_key="router-configured-image-policy", + config=config, + provider=None, + model=config.llm.model, + tool_defs=[], + system_prompt="system", + attachments=[{"type": "image", "mime_type": "image/png"}], + ) + + +@pytest.mark.asyncio +async def test_image_model_is_not_an_implicit_router_deployment( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _catalog_evidence(monkeypatch, supported=("configured/vision",)) + ctx = _context( + { + "c1": {"model": "configured/vision", "supports_image": True}, + "image_model": { + "model": "unconfigured/dedicated-vision", + "supports_image": True, + "image_only": True, + }, + } + ) + + routed = await apply_squilla_router(ctx) + + assert routed.metadata["routed_tier"] == "c1" + assert routed.model == "configured/vision" + assert routed.metadata["image_input_mode"] == "native" + assert routed.metadata["routed_model_vision_support"] == "supported" + assert all(entry["tier"] != "image_model" for entry in routed.metadata["router_fallback_chain"]) + + +@pytest.mark.asyncio +async def test_all_catalog_text_only_c_tiers_use_direct_marker_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _catalog_evidence(monkeypatch, unsupported=tuple(f"configured/c{index}" for index in range(4))) + ctx = _context( + { + "c0": {"model": "configured/c0", "supports_image": False}, + "c1": {"model": "configured/c1", "supports_image": False}, + "c2": {"model": "configured/c2", "supports_image": False}, + "c3": {"model": "configured/c3", "supports_image": False}, + "image_model": { + "model": "unconfigured/dedicated-vision", + "supports_image": True, + "image_only": True, + }, + } + ) + + routed = await apply_squilla_router(ctx) + + assert routed.metadata["routed_tier"] == "c1" + assert routed.model == "configured/c1" + assert routed.metadata["routing_source"] == "image_route" + assert routed.metadata["image_input_mode"] == "marker" + assert routed.metadata["image_input_projection_required"] is True + assert routed.metadata["router_image_capability_exhausted"] is True + assert "image_input_forced_rejection_reason" not in routed.metadata + assert routed.metadata["router_fallback_chain"] == [] + assert routed.metadata["router_fallback_strict"] is True + assert routed.metadata["routed_model_vision_support"] == "unsupported" + + +@pytest.mark.asyncio +async def test_omitted_support_is_probeable_but_image_model_is_still_ignored( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _Catalog: + def resolve_deployment_vision_support(self, _model: str, **_: object) -> str: + return "unknown" + + monkeypatch.setattr( + "opensquilla.engine.steps.squilla_router.shared_catalog", + lambda: _Catalog(), + ) + ctx = _context( + { + "c0": {"model": "configured/probeable"}, + "image_model": { + "model": "unconfigured/dedicated-vision", + "supports_image": True, + "image_only": True, + }, + } + ) + + routed = await apply_squilla_router(ctx) + + assert routed.metadata["routed_tier"] == "c0" + assert routed.metadata["image_input_mode"] == "native" + assert routed.metadata["router_image_tier_support"] == {"c0": "unknown"} + assert routed.metadata["routed_model_vision_support"] == "unknown" + + +@pytest.mark.asyncio +async def test_image_fallback_chain_carries_each_configured_tier_support( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _catalog_evidence( + monkeypatch, + supported=("configured/c0", "configured/c1"), + unsupported=("configured/c2",), + ) + ctx = _context( + { + "c0": {"model": "configured/c0", "supports_image": True}, + "c1": {"model": "configured/c1", "supports_image": True}, + "c2": {"model": "configured/c2", "supports_image": False}, + } + ) + + routed = await apply_squilla_router(ctx) + + assert routed.metadata["routed_tier"] == "c0" + assert routed.metadata["router_fallback_chain"] == [ + { + "tier": "c1", + "model": "configured/c1", + "vision_support": "supported", + } + ] + + +@pytest.mark.asyncio +async def test_ensemble_c3_is_text_only_and_lower_configured_vision_tier_wins( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _catalog_evidence(monkeypatch, supported=("configured/vision", "configured/ensemble-draft")) + ctx = _context( + { + "c0": {"model": "configured/vision", "supports_image": True}, + "c3": { + "model": "configured/ensemble-draft", + "supports_image": True, + "ensemble_enabled": True, + }, + } + ) + + routed = await apply_squilla_router(ctx) + + assert routed.metadata["routed_tier"] == "c0" + assert routed.metadata["image_input_mode"] == "native" + + +@pytest.mark.asyncio +async def test_structural_edit_image_route_and_fallbacks_obey_c3_execution_floor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _catalog_evidence(monkeypatch, supported=tuple(f"configured/c{index}" for index in range(4))) + monkeypatch.setattr( + "opensquilla.engine.steps.squilla_router.model_has_request_capacity", + lambda **_: True, + ) + ctx = _context( + { + "c0": {"model": "configured/c0", "supports_image": True}, + "c1": {"model": "configured/c1", "supports_image": True}, + "c2": {"model": "configured/c2", "supports_image": True}, + "c3": {"model": "configured/c3", "supports_image": True}, + } + ) + ctx.metadata.update( + { + "artifact_format": "html", + "artifact_operation_class": "structural_edit", + } + ) + + routed = await apply_squilla_router(ctx) + + assert routed.metadata["routed_tier"] == "c3" + assert routed.model == "configured/c3" + assert routed.metadata["image_input_mode"] == "native" + assert routed.metadata["router_fallback_chain"] == [] + + finalized = await finalize_squilla_router_capacity(routed) + + assert finalized.metadata["routed_tier"] == "c3" + assert finalized.model == "configured/c3" + assert finalized.metadata["router_fallback_chain"] == [] + + +@pytest.mark.asyncio +async def test_image_shortcut_reselects_active_provider_when_mismatch_is_vetoed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _catalog_evidence(monkeypatch, supported=("foreign/vision", "configured/vision")) + monkeypatch.setattr( + "opensquilla.engine.steps.squilla_router.model_has_request_capacity", + lambda **_: True, + ) + ctx = _context( + { + "c0": { + "provider": "foreign", + "model": "foreign/vision", + "supports_image": True, + }, + "c1": { + "provider": "openrouter", + "model": "configured/vision", + "supports_image": True, + }, + } + ) + ctx.config.squilla_router.tier_provider_mismatch = "veto" + ctx.config.llm_ensemble.enabled = True + + routed = await apply_squilla_router(ctx) + + assert routed.metadata["routed_tier"] == "c1" + assert routed.model == "configured/vision" + assert routed.metadata["provider_mismatch_veto_applied"] is True + assert routed.metadata["provider_mismatch_veto_from_tier"] == "c0" + assert routed.metadata["provider_mismatch_veto_to_tier"] == "c1" + assert routed.metadata["router_tier_provider_role"] == "direct" + assert "router_tier_provider_mismatch" not in routed.metadata + + finalized = await finalize_squilla_router_capacity(routed) + + assert finalized.metadata["routed_tier"] == "c1" + assert finalized.metadata["router_tier_provider_role"] == "direct" + + +@pytest.mark.parametrize("legacy_flag", [True, False, None]) +@pytest.mark.parametrize("support", ["supported", "unsupported", "unknown"]) +async def test_deployment_evidence_ignores_legacy_tier_switch( + monkeypatch: pytest.MonkeyPatch, + legacy_flag: bool | None, + support: str, +) -> None: + _catalog_evidence( + monkeypatch, + supported=("configured/c1",) if support == "supported" else (), + unsupported=("configured/c1",) if support == "unsupported" else (), + ) + raw: dict[str, object] = {"model": "configured/c1"} + if legacy_flag is not None: + raw["supports_image"] = legacy_flag + ctx = _context({"c1": raw}) + + routed = await apply_squilla_router(ctx) + + assert routed.model == "configured/c1" + assert routed.metadata["router_image_tier_support"] == {"c1": support} + assert routed.metadata["routed_model_vision_support"] == support + assert routed.metadata["image_input_mode"] == ( + "marker" if support == "unsupported" else "native" + ) + assert ctx.config.squilla_router.tiers["c1"] == raw + + +@pytest.mark.parametrize("cross_provider", [False, True]) +def test_tier_capability_lookup_uses_physical_deployment_authority( + monkeypatch: pytest.MonkeyPatch, cross_provider: bool +) -> None: + calls: list[tuple[str, dict[str, object]]] = [] + + class _Catalog: + def resolve_deployment_vision_support(self, model: str, **kwargs: object) -> str: + calls.append((model, kwargs)) + return "unknown" + + monkeypatch.setattr( + "opensquilla.engine.steps.squilla_router.shared_catalog", lambda: _Catalog() + ) + raw = {"model": "configured/c1", "provider": "other-provider", "supports_image": True} + ctx = _context({"c1": raw}) + ctx.config.squilla_router.cross_provider_tiers = cross_provider + ctx.config.llm.api_key = "synthetic-key" + ctx.config.llm.base_url = "https://synthetic.invalid/api" + + assert _tier_deployment_vision_support(ctx, raw) == "unknown" + assert calls == [ + ( + "configured/c1", + { + "provider": "other-provider" if cross_provider else "openrouter", + "api_key": "" if cross_provider else "synthetic-key", + "base_url": "" if cross_provider else "https://synthetic.invalid/api", + "proxy": "", + }, + ) + ] + + +def test_image_history_capacity_includes_configured_marker_fallback_providers() -> None: + ctx = _context( + { + "c0": {"provider": "anthropic", "model": "configured/unknown", "supports_image": False}, + "c1": {"provider": "ollama", "model": "configured/text", "supports_image": False}, + "c2": {"provider": "empty-provider", "model": ""}, + "c3": {"provider": "hidden-provider", "model": "hidden", "image_only": True}, + "image_model": { + "provider": "legacy-provider", + "model": "legacy", + "supports_image": True, + }, + } + ) + ctx.config.squilla_router.cross_provider_tiers = True + ctx.metadata["image_route_reason"] = "current_turn" + + assert TurnRunner._route_capacity_provider_kinds( + ctx, initial_provider_config=SimpleNamespace(provider="openrouter") + ) == frozenset({"openrouter", "anthropic", "ollama"}) diff --git a/tests/test_engine/test_router_control.py b/tests/test_engine/test_router_control.py index 671bcac98..93f864322 100644 --- a/tests/test_engine/test_router_control.py +++ b/tests/test_engine/test_router_control.py @@ -300,6 +300,13 @@ async def test_large_attachment_foreign_hold_veto_fails_closed(monkeypatch) -> N @pytest.mark.asyncio async def test_image_attachments_bypass_text_hold(monkeypatch) -> None: + class _TextCatalog: + def resolve_deployment_vision_support(self, _model: str, **_kwargs: object) -> str: + return "unsupported" + + monkeypatch.setattr( + "opensquilla.engine.steps.squilla_router.shared_catalog", lambda: _TextCatalog() + ) cfg = _router_cfg(_router_tier_profile_defaults("openrouter")) target = resolve_router_control_target(cfg, "tier:c3") store = RouterControlHoldStore() @@ -325,7 +332,9 @@ def fail_strategy(_cfg: object) -> object: assert out.metadata["routing_source"] == "image_route" assert out.metadata.get("router_control_hold_applied") is not True - assert out.model == "moonshotai/kimi-k2.6" + assert out.model == cfg.tiers["c1"]["model"] + assert out.metadata["image_input_mode"] == "marker" + assert out.metadata["router_image_capability_exhausted"] is True def test_prompt_block_contains_canonical_targets_not_aliases() -> None: diff --git a/tests/test_engine/test_routing_policy_parity.py b/tests/test_engine/test_routing_policy_parity.py index 7d1e3717e..e6297b909 100644 --- a/tests/test_engine/test_routing_policy_parity.py +++ b/tests/test_engine/test_routing_policy_parity.py @@ -24,16 +24,12 @@ goldens therefore include the sanitized floor tier and omit fallback models below it; the remaining corpus still pins the original extraction parity. -The image-bypass cases were intentionally recaptured for the #1195 reopen -after attachment capacity became mandatory for every attachment turn. Those -entries include the capacity-required marker, material estimate, thinking -reserve, and capacity-filtered fallback chain. Non-attachment cases retain -their prior byte-identical behavior. - -The missing image-tier case was intentionally recaptured after Router image -admission began returning a structured rejection instead of raising a runtime -configuration exception. Its golden pins the stable rejection metadata while -the later provider admission layer owns the user-facing error event. +The image-bypass cases were intentionally recaptured for the configured-only +multimodal policy. Router now evaluates only c0-c3, treats omitted capability +as probeable, prefers proven support, and never executes the legacy +``image_model`` row. Their goldens pin the strict configured fallback chain +and image projection metadata; non-attachment cases retain their prior +byte-identical behavior. Classifier outputs are injected through a fake strategy: the corpus never loads the LightGBM/ONNX bundle, touches the network, or needs credentials. @@ -260,7 +256,7 @@ def build_corpus() -> list[Case]: ) cases.append( Case( - name="image_without_image_tier_errors", + name="image_without_legacy_image_tier_uses_c_ladder", tiers={k: v for k, v in synthetic_tiers().items() if k != "image_model"}, attachments=[{"type": "image/png"}], classify_expected=False, diff --git a/tests/test_engine/test_runtime_artifacts.py b/tests/test_engine/test_runtime_artifacts.py index 60030cf9b..591c0a9d1 100644 --- a/tests/test_engine/test_runtime_artifacts.py +++ b/tests/test_engine/test_runtime_artifacts.py @@ -1207,6 +1207,9 @@ def __init__(self) -> None: def set_history(self, history) -> None: self.history = history + def set_request_image_context(self, messages) -> None: + assert messages == [] + history_capture = _HistoryCapture() await runner._load_history(agent=history_capture, session_key=session_key) assert "[generated artifact omitted: runtime.txt (text/plain)]" in str( diff --git a/tests/test_engine/test_selector_fallback_routed_model.py b/tests/test_engine/test_selector_fallback_routed_model.py index 21a3eaa8f..607c17a16 100644 --- a/tests/test_engine/test_selector_fallback_routed_model.py +++ b/tests/test_engine/test_selector_fallback_routed_model.py @@ -70,6 +70,151 @@ async def chat(self, messages, tools=None, config=None): yield DoneEvent(model="cheap/fallback") +def test_strict_router_chain_fails_closed_for_legacy_selector_hook() -> None: + class _LegacySelector: + def __init__(self) -> None: + self.resolved = False + + def override_model_with_fallback_chain( + self, + model: str, + chain: list[object], + ) -> None: + del model, chain + + def override_model(self, model: str) -> None: + del model + + def resolve(self) -> object: + self.resolved = True + return object() + + selector = _LegacySelector() + + with pytest.raises(RuntimeError, match="strict router fallback isolation"): + apply_model_override( + selector, + "configured-c0", + turn_metadata={ + "routing_applied": True, + "router_fallback_strict": True, + "router_fallback_chain": [], + }, + realign_routed_model=False, + ) + + assert selector.resolved is False + + +def test_strict_cross_provider_chain_fails_closed_without_provider_chain_hook() -> None: + class _LegacySelector: + def __init__(self) -> None: + self.overridden = False + self.resolved = False + + def override_provider_config( + self, + config: object, + *, + preserve_existing_tail: bool = True, + ) -> None: + del config, preserve_existing_tail + self.overridden = True + + def resolve(self) -> object: + self.resolved = True + return object() + + selector = _LegacySelector() + primary = SimpleNamespace(provider="anthropic", model="configured-c0") + configured_fallback = SimpleNamespace( + provider="openrouter", + model="configured-c1", + ) + + with pytest.raises(RuntimeError, match="strict router fallback isolation"): + apply_model_override( + selector, + primary.model, + turn_metadata={ + "routing_applied": True, + "router_fallback_strict": True, + "router_fallback_chain": [ + { + "provider": configured_fallback.provider, + "model": configured_fallback.model, + } + ], + }, + realign_routed_model=False, + tier_provider_config=primary, + strict_router_fallback_chain=[configured_fallback], + ) + + assert selector.overridden is False + assert selector.resolved is False + + +def test_blocked_strict_route_replaces_opaque_selector_tail_before_resolve() -> None: + primary = SimpleNamespace(provider="openrouter", model="configured-primary") + configured_fallback = SimpleNamespace(provider="openrouter", model="configured-c1") + plugin_fallback = SimpleNamespace(provider="plugin", model="unconfigured-plugin") + + class _StrictSelector: + active_provider_id = "openrouter" + + def __init__(self) -> None: + self.current_config = primary + self.chain = [primary, plugin_fallback] + self.preserve_existing_tail: bool | None = None + + def override_model_with_fallback_chain( + self, + model: str, + chain: list[object], + *, + preserve_existing_tail: bool = True, + ) -> None: + assert model == primary.model + assert chain == [configured_fallback] + self.preserve_existing_tail = preserve_existing_tail + self.chain = [primary, configured_fallback] + + def resolve(self) -> object: + return "configured-provider" + + def remaining_chain(self) -> list[object]: + return list(self.chain) + + selector = _StrictSelector() + metadata: dict[str, Any] = { + "routing_applied": True, + "routed_provider": "foreign", + "routed_model": "foreign-model", + "routed_provider_blocked": "cross_provider_tiers_disabled", + "router_fallback_strict": True, + "router_fallback_chain": [ + {"provider": "openrouter", "model": configured_fallback.model} + ], + } + + provider = apply_model_override( + selector, + "foreign-model", + turn_metadata=metadata, + realign_routed_model=False, + strict_router_fallback_chain=[configured_fallback], + ) + + assert provider == "configured-provider" + assert selector.preserve_existing_tail is False + assert plugin_fallback not in selector.chain + assert metadata["selector_execution_chain"] == [ + {"provider": "openrouter", "model": "configured-primary"}, + {"provider": "openrouter", "model": "configured-c1"}, + ] + + async def test_fallback_realigns_only_when_provider_call_starts() -> None: metadata: dict[str, object] = { "routed_model": "expensive/model", @@ -105,6 +250,192 @@ async def test_fallback_realigns_only_when_provider_call_starts() -> None: assert metadata["savings_routed_price_per_m"] == 0.0 +async def test_precise_image_rejection_bypasses_generic_selector_fallback() -> None: + class _Primary: + provider_name = "openai" + + def __init__(self) -> None: + self.calls: list[list[Message]] = [] + + async def chat(self, messages, tools=None, config=None): + del tools, config + self.calls.append(messages) + has_image = any( + isinstance(block, ContentBlockImage) + for message in messages + if isinstance(message.content, list) + for block in message.content + ) + if has_image: + yield ErrorEvent( + code="400", + message="This model does not support image input.", + ) + return + yield TextDeltaEvent(text="marker handled by configured primary") + yield DoneEvent(model="configured-primary") + + class _UnconfiguredFallback(_SuccessfulProvider): + def __init__(self) -> None: + self.calls = 0 + + async def chat(self, messages, tools=None, config=None): + del messages, tools, config + self.calls += 1 + yield TextDeltaEvent(text="must not run") + yield DoneEvent(model="unconfigured-model") + + class _Selector: + active_provider_id = "openai" + current_config = SimpleNamespace(provider="openai", model="configured-primary") + + def __init__(self, fallback: _UnconfiguredFallback) -> None: + self.fallback = fallback + self.generic_fallback_calls = 0 + + def next_fallback_after_failure(self, _exc: Exception) -> object: + self.generic_fallback_calls += 1 + self.current_config = SimpleNamespace( + provider="unconfigured-provider", + model="unconfigured-model", + ) + return self.fallback + + primary = _Primary() + unconfigured = _UnconfiguredFallback() + selector = _Selector(unconfigured) + wrapper = _SelectorFallbackProvider(primary, selector) + agent = Agent( + provider=wrapper, + config=AgentConfig( + max_iterations=1, + model_id="configured-primary", + provider_id="openai", + model_vision_support="unknown", + ), + ) + image_message = Message( + role="user", + content=[ContentBlockImage(media_type="image/png", data="c3ludGhldGlj")], + ) + + events = [ + event + async for event in agent.run_turn( + "Describe it.", + extra_messages=[image_message], + ) + ] + + assert len(primary.calls) == 2 + assert selector.generic_fallback_calls == 0 + assert unconfigured.calls == 0 + assert any(isinstance(event, EngineDoneEvent) for event in events) + + +async def test_router_image_rejection_uses_only_strict_configured_probe() -> None: + primary_config = SimpleNamespace(provider="openai", model="configured-c0") + fallback_config = SimpleNamespace(provider="openai", model="configured-c1") + + class _Primary: + provider_name = "openai" + + def __init__(self) -> None: + self.calls = 0 + + async def chat(self, messages, tools=None, config=None): + del messages, tools, config + self.calls += 1 + yield ErrorEvent( + code="400", + message="This model does not support image input.", + ) + + class _ConfiguredFallback: + provider_name = "openai" + + def __init__(self) -> None: + self.calls: list[list[Message]] = [] + + async def chat(self, messages, tools=None, config=None): + del tools, config + self.calls.append(messages) + yield TextDeltaEvent(text="configured vision probe succeeded") + yield DoneEvent(model="configured-c1") + + class _Selector: + def __init__(self, fallback: _ConfiguredFallback) -> None: + self.current_config = primary_config + self.fallback = fallback + self.image_probe_calls = 0 + self.generic_fallback_calls = 0 + + @property + def active_provider_id(self) -> str: + return str(self.current_config.provider) + + def next_fallback_matching(self, *, predicate): + assert predicate(fallback_config) + self.image_probe_calls += 1 + self.current_config = fallback_config + return self.fallback + + def next_fallback_after_failure(self, _exc: Exception) -> object: + self.generic_fallback_calls += 1 + raise AssertionError("generic fallback must not own image rejection") + + metadata: dict[str, Any] = { + "routing_source": "image_route", + "router_fallback_strict": True, + "image_input_mode": "native", + } + primary = _Primary() + configured_fallback = _ConfiguredFallback() + selector = _Selector(configured_fallback) + wrapper = _SelectorFallbackProvider( + primary, + selector, + turn_metadata=metadata, + ) + wrapper.configure_fallback_deployment_vision_support( + [(fallback_config, "unknown")] + ) + agent = Agent( + provider=wrapper, + config=AgentConfig( + max_iterations=1, + model_id="configured-c0", + provider_id="openai", + model_vision_support="unknown", + metadata=metadata, + ), + ) + image_message = Message( + role="user", + content=[ContentBlockImage(media_type="image/png", data="c3ludGhldGlj")], + ) + + events = [ + event + async for event in agent.run_turn( + "Describe it.", + extra_messages=[image_message], + ) + ] + + assert primary.calls == 1 + assert selector.image_probe_calls == 1 + assert selector.generic_fallback_calls == 0 + assert len(configured_fallback.calls) == 1 + assert any( + isinstance(block, ContentBlockImage) + for message in configured_fallback.calls[0] + if isinstance(message.content, list) + for block in message.content + ) + assert any(isinstance(event, EngineDoneEvent) for event in events) + + def test_fallback_to_same_model_keeps_savings() -> None: metadata: dict[str, object] = {"routed_model": "same/model", "savings_pct": 7.0} wrapper = _SelectorFallbackProvider( @@ -1616,7 +1947,7 @@ def next_fallback_after_failure(self, _exc: Exception) -> _Provider: @pytest.mark.parametrize("fallback_vision_support", ["unsupported", "unknown"]) -async def test_image_request_does_not_call_text_only_fallback( +async def test_image_request_projects_or_probes_configured_fallback( monkeypatch: Any, fallback_vision_support: str, ) -> None: @@ -1640,18 +1971,19 @@ class _Provider: def __init__(self, *, fails: bool) -> None: self.fails = fails self.calls = 0 + self.messages: list[list[Message]] = [] async def chat(self, messages, tools=None, config=None): - del messages, tools, config + del tools, config + self.messages.append(messages) self.calls += 1 if self.fails: yield ErrorEvent( - message="rate limited", - code="429", - retry_after_s=901.0, + message="provider unavailable", + code="503", ) return - yield TextDeltaEvent(text="fallback must not run") + yield TextDeltaEvent(text="configured fallback reply") yield DoneEvent(model="text-fallback") class _Selector: @@ -1713,31 +2045,43 @@ def next_fallback_after_failure(self, _exc: Exception) -> _Provider: ] assert selector.primary.calls == 1 - assert selector.fallback.calls == 0 - assert not any( + assert selector.fallback.calls == 1 + assert any( isinstance(event, ProviderActivityEvent) - and event.phase in {"retry_wait", "fallback"} + and event.phase == "fallback" for event in events ) - assert [event.code for event in events if isinstance(event, ErrorEvent)] == [ - IMAGE_INPUT_UNSUPPORTED_CODE + assert not any(isinstance(event, ErrorEvent) for event in events) + assert [event.text for event in events if isinstance(event, TextDeltaEvent)] == [ + "configured fallback reply" ] - assert not any(isinstance(event, TextDeltaEvent) for event in events) - assert not any(isinstance(event, DoneEvent) for event in events) - assert metadata["image_input_mode"] == "rejected" + assert any(isinstance(event, DoneEvent) for event in events) + fallback_has_image = any( + isinstance(block, ContentBlockImage) + for sent_message in selector.fallback.messages[0] + if isinstance(sent_message.content, list) + for block in sent_message.content + ) + assert fallback_has_image is (fallback_vision_support == "unknown") + if fallback_vision_support == "unsupported": + assert "图片未分析" in str(selector.fallback.messages[0]) + assert metadata["image_input_mode"] == ( + "native" if fallback_vision_support == "unknown" else "marker" + ) assert metadata["image_input_reason"] == ( - "capability_unknown" + "capability_probe" if fallback_vision_support == "unknown" else "model_vision_unsupported" ) assert metadata["image_input_stage"] == "fallback" - assert metadata["routed_model"] == "vision-primary" - assert metadata["executed_model"] == "vision-primary" - assert "router_fallback_hops" not in metadata - assert "router_fallback_reason" not in metadata - assert metadata["savings_pct"] == 17.0 + assert metadata["routed_model"] == "text-fallback" + assert metadata["executed_model"] == "text-fallback" + assert metadata["router_fallback_hops"] == 1 + assert metadata["router_fallback_reason"] == "selector_fallback" + assert metadata["savings_pct"] == 0.0 assert [leg["model"] for leg in metadata["execution_legs"]] == [ - "vision-primary" + "vision-primary", + "text-fallback", ] @@ -1890,9 +2234,9 @@ def next_fallback_after_failure(self, _exc: Exception) -> _Provider: assert done_events[-1].input_tokens == 3 assert done_events[-1].output_tokens == 1024 assert done_events[-1].reasoning_tokens == 1023 - assert "image_input_mode" not in metadata - assert "image_input_reason" not in metadata - assert "image_input_stage" not in metadata + assert metadata["image_input_mode"] == "native" + assert metadata["image_input_reason"] == "model_vision_unsupported" + assert metadata["image_input_stage"] == "primary" assert metadata["routed_model"] == "vision-primary" assert metadata["executed_model"] == "vision-primary" assert "router_fallback_hops" not in metadata diff --git a/tests/test_engine/test_t3_upgrade_compaction.py b/tests/test_engine/test_t3_upgrade_compaction.py index 4b74dbb65..2addd5413 100644 --- a/tests/test_engine/test_t3_upgrade_compaction.py +++ b/tests/test_engine/test_t3_upgrade_compaction.py @@ -854,6 +854,9 @@ def __init__(self) -> None: def set_history(self, history: list[Any]) -> None: self.history = history + def set_request_image_context(self, messages: list[Any]) -> None: + assert messages == [] + agent = _HistoryCapture() summary_context = await runner._load_history(agent, session_key, trim_last_user=False) diff --git a/tests/test_engine/test_turn_prompt_binding.py b/tests/test_engine/test_turn_prompt_binding.py index d0e29d9f9..3da0dff6c 100644 --- a/tests/test_engine/test_turn_prompt_binding.py +++ b/tests/test_engine/test_turn_prompt_binding.py @@ -706,13 +706,18 @@ async def test_router_capacity_accepts_retained_missing_reason_marker() -> None: preserve_image_attachments=True, ) - marker = "[historical attachment omitted: lost.png (image/png)]" + marker = "[historical attachment omitted: lost.png (image/png);" marker_count = sum( message.content.count(marker) for message in replay.messages if isinstance(message.content, str) ) assert marker_count == 1 + assert any( + "历史图片不可用" in message.content and "原图已保留" not in message.content + for message in replay.messages + if isinstance(message.content, str) + ) assert replay.estimate_complete is True assert context["history_capacity_message_count"] == 2 assert context["history_capacity_estimate_complete"] is True diff --git a/tests/test_engine/test_usage_event_accounting.py b/tests/test_engine/test_usage_event_accounting.py index 4fd6166da..94d140b73 100644 --- a/tests/test_engine/test_usage_event_accounting.py +++ b/tests/test_engine/test_usage_event_accounting.py @@ -268,6 +268,16 @@ class _PhysicalLegProvider(_SequenceProvider): def __init__(self, name: str, events: list[Any]) -> None: super().__init__([events]) self.provider_name = name + self.message_calls: list[list[Message]] = [] + + def chat( + self, + messages: list[Message], + tools: list[Any] | None = None, + config: ChatConfig | None = None, + ) -> AsyncIterator[Any]: + self.message_calls.append(messages) + return super().chat(messages, tools=tools, config=config) class _CorrelationCapturingPhysicalLegProvider(_PhysicalLegProvider): @@ -380,7 +390,7 @@ def write(self, kind: str, payload: dict[str, Any]) -> None: @pytest.mark.asyncio -async def test_selector_preflight_rejects_ensemble_image_before_usage_or_fallback() -> None: +async def test_selector_projects_ensemble_image_and_accounts_fallback_call() -> None: sink = _RecordingSink() fallback = _PhysicalLegProvider( "anthropic", @@ -395,12 +405,19 @@ async def test_selector_preflight_rejects_ensemble_image_before_usage_or_fallbac with bind_usage_accounting_scope(scope): events = [event async for event in wrapper.chat([_image_message()])] - assert [getattr(event, "code", "") for event in events] == [ - "ensemble_multimodal_unsupported" - ] - assert fallback.calls == 0 - assert sink.started == [] - assert sink.finalized == [] + assert not any(isinstance(event, ProviderError) for event in events) + assert any(isinstance(event, ProviderDone) for event in events) + assert fallback.calls == 1 + assert len(fallback.message_calls) == 1 + assert not any( + isinstance(block, ContentBlockImage) + for message in fallback.message_calls[0] + if isinstance(message.content, list) + for block in message.content + ) + assert "图片未分析" in str(fallback.message_calls[0]) + assert len(sink.started) == 1 + assert len(sink.finalized) == 1 assert sink.unknown == [] @@ -429,7 +446,7 @@ async def test_selector_does_not_project_usage_accounting_error_as_provider_fail @pytest.mark.asyncio @pytest.mark.parametrize("image_location", ["current", "history"]) @pytest.mark.parametrize("wrapped_by_selector", [False, True]) -async def test_agent_preflight_rejects_ensemble_image_before_call_accounting( +async def test_agent_projects_ensemble_image_and_accounts_physical_call( image_location: str, wrapped_by_selector: bool, ) -> None: @@ -484,21 +501,27 @@ async def test_agent_preflight_rejects_ensemble_image_before_call_accounting( ) ] - errors = [event for event in events if isinstance(event, ErrorEvent)] - assert [error.code for error in errors] == ["ensemble_multimodal_unsupported"] - assert fallback.calls == 0 - assert sink.started == [] - assert sink.finalized == [] + assert not any(isinstance(event, ErrorEvent) for event in events) + assert any(isinstance(event, EngineDoneEvent) for event in events) + assert fallback.calls == 1 + assert len(fallback.message_calls) == 1 + assert not any( + isinstance(block, ContentBlockImage) + for sent_message in fallback.message_calls[0] + if isinstance(sent_message.content, list) + for block in sent_message.content + ) + assert "图片未分析" in str(fallback.message_calls[0]) + assert len(sink.started) == 1 + assert len(sink.finalized) == 1 assert sink.unknown == [] - assert tracker.rows == [] - assert observer_calls == [] + assert len(tracker.rows) == 1 + assert len(observer_calls) == 1 assert "router_fallback_hops" not in turn_metadata - assert not any(record["kind"] == "llm_request" for record in turn_log.records) - [decision] = [ - record for record in turn_log.records if record["kind"] == "turn_policy_decision" - ] - assert decision["payload"]["code"] == "ensemble_multimodal_unsupported" - assert "messages" not in decision["payload"] + kinds = [record["kind"] for record in turn_log.records] + assert "image_input_projection" in kinds + assert "llm_request" in kinds + assert "llm_response" in kinds @pytest.mark.asyncio diff --git a/tests/test_engine/turn_runner/test_agent_bootstrap_stage_unit.py b/tests/test_engine/turn_runner/test_agent_bootstrap_stage_unit.py index fb3b61435..707f2b5d8 100644 --- a/tests/test_engine/turn_runner/test_agent_bootstrap_stage_unit.py +++ b/tests/test_engine/turn_runner/test_agent_bootstrap_stage_unit.py @@ -626,6 +626,80 @@ async def test_bootstrap_installs_known_fallback_limits_on_provider_wrapper() -> ] == 8_192 +@pytest.mark.asyncio +@pytest.mark.parametrize("declared_fallback_provider", ["provider-b", "foreign-provider"]) +async def test_router_vision_declarations_override_stale_catalog_for_each_leg( + declared_fallback_provider: str, +) -> None: + primary = _ResolvedCatalog( + max_tokens=16_384, + context_window=128_000, + capabilities=ModelCapabilities(supports_vision=False), + vision_support="unsupported", + ) + fallback = SimpleNamespace( + provider="provider-b", + model="fallback/model", + api_key="fallback-key", + base_url="", + proxy="", + ) + + class _Catalog: + def lookup(self, model_id: str, provider: str = "") -> _ResolvedCatalog: + if (provider, model_id) == ("provider-a", "primary/model"): + return primary + assert model_id == "fallback/model" + assert provider in {"provider-b", "foreign-provider"} + return replace(primary, max_tokens=8_192, context_window=64_000) + + class _Provider: + vision_entries: list[tuple[Any, str]] = [] + + def fallback_deployment_configs(self) -> tuple[Any, ...]: + return (fallback,) + + def configure_fallback_deployment_limits(self, _limits: Any) -> None: + return None + + def configure_fallback_deployment_vision_support( + self, + entries: list[tuple[Any, str]], + ) -> None: + self.vision_entries = entries + + def configure_fallback_limits(self, _limits: Any) -> None: + return None + + provider = _Provider() + turn = _make_turn( + metadata={ + "routed_provider": "provider-a", + "routed_model": "primary/model", + "routed_model_vision_support": "supported", + "router_fallback_chain": [ + { + "provider": declared_fallback_provider, + "model": "fallback/model", + "vision_support": "supported", + } + ], + } + ) + + out = await _make_stage(catalog=_Catalog()).run( + _make_input( + provider=provider, + turn=turn, + resolved_model="primary/model", + active_provider_id="provider-a", + ) + ) + + assert out.output.agent_config.model_vision_support == "supported" + assert provider.vision_entries == [(fallback, "supported")] + + @pytest.mark.asyncio @pytest.mark.parametrize("fallback_supports_tools", [True, False]) async def test_fallback_capability_does_not_downgrade_active_model_verification( diff --git a/tests/test_gateway/test_rpc_model_routing.py b/tests/test_gateway/test_rpc_model_routing.py index 5a91e13a3..49ee0f72c 100644 --- a/tests/test_gateway/test_rpc_model_routing.py +++ b/tests/test_gateway/test_rpc_model_routing.py @@ -377,7 +377,7 @@ def resolve_deployment_vision_support(self, *args: Any, **kwargs: Any) -> str: snapshot = model_routing_snapshot(config) assert snapshot["image_input"] == { - "admission": "blocked", + "admission": "allowed", "reason": "model_vision_unsupported", } assert "api_key" not in str(snapshot) @@ -448,7 +448,7 @@ def resolve_deployment_vision_support( }, "ensemble": { "image_input": { - "admission": "blocked", + "admission": "allowed", "reason": "ensemble_mode_unsupported", } }, @@ -523,7 +523,7 @@ def project(config: Any, mode: str) -> dict[str, Any]: "unknown", } assert capabilities["ensemble"]["image_input"] == { - "admission": "blocked", + "admission": "allowed", "reason": "ensemble_mode_unsupported", } @@ -536,19 +536,19 @@ def project(config: Any, mode: str) -> dict[str, Any]: "supported", { "admission": "allowed", - "reason": "router_image_route_available", + "reason": "router_image_route_unavailable", }, ), ( - {"image_model": {"model": "text-only-model", "supports_image": True}}, + {"c0": {"model": "text-only-model"}}, "unsupported", { - "admission": "blocked", - "reason": "model_vision_unsupported", + "admission": "allowed", + "reason": "router_image_route_unavailable", }, ), ( - {"image_model": {"model": "unlisted-model", "supports_image": True}}, + {"c0": {"model": "unlisted-model"}}, "unknown", { "admission": "unknown", @@ -556,13 +556,32 @@ def project(config: Any, mode: str) -> dict[str, Any]: }, ), ( - {"image_model": {"model": "", "supports_image": True}}, - "supported", + {"c0": {"model": "declared-vision", "supports_image": True}}, + "unsupported", { - "admission": "blocked", + "admission": "allowed", "reason": "router_image_route_unavailable", }, ), + ( + {"c0": {"model": "text-only-model", "supports_image": False}}, + "supported", + { + "admission": "allowed", + "reason": "router_image_route_available", + }, + ), + ( + { + "c0": {"model": "unknown-model"}, + "c1": {"model": "declared-vision", "supports_image": True}, + }, + "unknown", + { + "admission": "unknown", + "reason": "capability_unknown", + }, + ), ], ) def test_model_routing_snapshot_applies_image_route_in_observe( @@ -595,6 +614,56 @@ def resolve_deployment_vision_support(self, *args: Any, **kwargs: Any) -> str: assert snapshot["image_input"] == expected +def test_model_routing_snapshot_preserves_unknown_direct_image_admission( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _Catalog: + def resolve_deployment_vision_support(self, *args: Any, **kwargs: Any) -> str: + del args, kwargs + return "unknown" + + monkeypatch.setattr( + "opensquilla.provider.model_catalog.shared_catalog", + lambda: _Catalog(), + ) + config = GatewayConfig( + llm={"provider": "custom", "model": "unlisted"}, + squilla_router={"enabled": False, "rollout_phase": "observe"}, + llm_ensemble={"enabled": False}, + ) + + assert model_routing_snapshot(config)["image_input"] == { + "admission": "unknown", + "reason": "capability_unknown", + } + + +@pytest.mark.parametrize( + ("selection_mode", "router_enabled"), + [ + ("static_openrouter_b5", False), + ("static_tokenrhythm_b5", False), + ("router_dynamic", True), + ], +) +def test_model_routing_snapshot_allows_ensemble_images_for_marker_degradation( + selection_mode: str, + router_enabled: bool, +) -> None: + config = GatewayConfig( + squilla_router={"enabled": router_enabled, "rollout_phase": "full"}, + llm_ensemble={"enabled": True, "selection_mode": selection_mode}, + ) + + snapshot = model_routing_snapshot(config) + + assert snapshot["mode"] == "ensemble" + assert snapshot["image_input"] == { + "admission": "allowed", + "reason": "ensemble_mode_unsupported", + } + + @pytest.mark.parametrize( ("selection_mode", "router_enabled"), [ @@ -972,16 +1041,17 @@ def resolve_deployment_vision_support( "enabled": False, "rollout_phase": "observe", "tiers": { - "c1": {"model": "router-text", "supports_image": True}, + "c1": {"model": "router-text"}, }, }, ) ctx, events = _routing_event_ctx(config, monkeypatch) before = model_routing_public_snapshot(config) assert before["mode"] == "ensemble" - assert before["capabilities_by_mode"]["router"]["image_input"][ - "admission" - ] == "blocked" + assert before["capabilities_by_mode"]["router"]["image_input"] == { + "admission": "allowed", + "reason": "router_image_route_unavailable", + } response = await _handle_config_patch( {"patches": {"squilla_router.tiers.c1.model": "router-vision"}}, diff --git a/tests/test_gateway/test_rpc_onboarding.py b/tests/test_gateway/test_rpc_onboarding.py index b6602dc32..2a9fe060f 100644 --- a/tests/test_gateway/test_rpc_onboarding.py +++ b/tests/test_gateway/test_rpc_onboarding.py @@ -253,6 +253,8 @@ async def test_router_configure_accepts_tier_overrides_without_rebinding_direct_ persisted = tomllib.loads((tmp_path / "c.toml").read_text()) assert persisted["llm"]["model"] == "gpt-5.4-mini" assert persisted["squilla_router"]["tiers"]["c2"]["model"] == "gpt-5.5-custom" + assert "supports_image" not in persisted["squilla_router"]["tiers"]["c2"] + assert "supports_image" not in ctx.config.squilla_router.tiers["c2"] assert persisted["squilla_router"]["tiers"]["image_model"]["supports_image"] is True diff --git a/tests/test_gateway/test_rpc_workbench_resources.py b/tests/test_gateway/test_rpc_workbench_resources.py index 37e99342e..6d0dee80d 100644 --- a/tests/test_gateway/test_rpc_workbench_resources.py +++ b/tests/test_gateway/test_rpc_workbench_resources.py @@ -29,6 +29,7 @@ from opensquilla.gateway.rpc import RpcContext, RpcUnavailableError, get_dispatcher from opensquilla.gateway.scopes import METHOD_SCOPES, READ_SCOPE, WRITE_SCOPE from opensquilla.gateway.transcripts import build_transcript_attachment_envelope +from opensquilla.session.attachment_manifest import legacy_attachment_id from opensquilla.session.manager import SessionManager from opensquilla.session.models import TranscriptEntry from opensquilla.session.storage import SessionStorage @@ -1240,7 +1241,8 @@ async def test_historical_attachment_ids_are_stable_per_message_occurrence( ) -> None: env = resource_env html = b"

historical

" - for message_id, name in (("legacy-one", "one.html"), ("legacy-two", "two.html")): + message_specs = (("legacy-one", "one.html"), ("legacy-two", "two.html")) + for message_id, name in message_specs: attachment = { "type": "text/html", "data": base64.b64encode(html).decode("ascii"), @@ -1274,9 +1276,28 @@ async def test_historical_attachment_ids_are_stable_per_message_occurrence( first_ids = [item["resource"]["id"] for item in first.payload["resources"]] second_ids = [item["resource"]["id"] for item in second.payload["resources"]] assert first_ids == second_ids + assert first_ids == [ + legacy_attachment_id( + session_id=env.session.session_id, + message_id=message_id, + index=0, + sha256=hashlib.sha256(html).hexdigest(), + ) + for message_id, _name in message_specs + ] assert len(set(first_ids)) == 2 assert all(item.startswith("att_legacy_") for item in first_ids) + child_key = "agent:main:webchat:workbench-resources-legacy-fork" + await env.manager.branch(SESSION_KEY, child_key, fork_transcript=True) + forked = await _dispatch( + env, + "workbench.resources.list", + {"sessionKey": child_key, "types": ["attachment"]}, + ) + assert forked.error is None, forked.error + assert [item["resource"]["id"] for item in forked.payload["resources"]] == first_ids + @pytest.mark.asyncio async def test_import_is_session_scoped_and_recovers_reserved_candidate_after_crash( diff --git a/tests/test_live_provider_profile_gateway_e2e.py b/tests/test_live_provider_profile_gateway_e2e.py index c829d1963..7c6030894 100644 --- a/tests/test_live_provider_profile_gateway_e2e.py +++ b/tests/test_live_provider_profile_gateway_e2e.py @@ -756,7 +756,7 @@ def test_attachment_capacity_seed_uses_only_isolated_session_state(tmp_path: Pat assert session_state == (0, "router") -def test_attachment_capacity_config_is_single_call_and_includes_image_tier( +def test_attachment_capacity_config_is_single_call_with_configured_vision_c2( tmp_path: Path, ) -> None: config_path = tmp_path / "gateway.toml" @@ -785,14 +785,18 @@ def test_attachment_capacity_config_is_single_call_and_includes_image_tier( assert data["agent_runtime_timeout_seconds"] == 75.0 assert data["task_runtime"]["turn_hard_deadline_s"] == 75.0 assert data["naming"]["enabled"] is False - assert data["squilla_router"]["tiers"]["image_model"] == tiers["image_model"] - assert data["squilla_router"]["tiers"]["image_model"]["model"] == "kimi-k2.6" - assert all(tiers[slot]["supports_image"] is False for slot in e2e.TEXT_PROFILE_SLOTS) + assert "image_model" not in data["squilla_router"]["tiers"] + assert data["squilla_router"]["tiers"]["c2"] == tiers["c2"] + assert tiers["c2"]["model"] == "kimi-k2.6" + assert tiers["c2"]["image_only"] is False + assert all("supports_image" not in tiers[slot] for slot in ("c0", "c1", "c2", "c3")) assert ( data["models"]["tokenrhythm"]["deepseek-v4-pro-0813"]["context_window"] == e2e.ATTACHMENT_CAPACITY_BASE_CONTEXT_WINDOW_TOKENS ) assert data["models"]["tokenrhythm"]["kimi-k2.6"]["supports_vision"] is True + for slot in ("c0", "c1", "c3"): + assert data["models"]["tokenrhythm"][tiers[slot]["model"]]["supports_vision"] is False def test_attachment_capacity_runner_reaches_provider_through_real_gateway( @@ -866,6 +870,8 @@ def do_POST(self) -> None: # noqa: N802 - stdlib handler contract assert case["usage"]["physical_response_count"] == 1 assert case["usage"]["compaction_count"] == 0 assert case["usage"]["provider_proof_fits"] is True + assert case["usage"]["provider_proof_media_blocks"] == 3 + assert case["usage"]["route_max_history_turns"] == 1 @pytest.mark.parametrize( @@ -938,7 +944,8 @@ def do_POST(self) -> None: # noqa: N802 - stdlib handler contract @pytest.mark.parametrize( ("mode", "expected_failure", "expected_response_count"), [ - ("empty", "implementation", 0), + # An empty SSE body has no terminal evidence and raises incomplete_stream. + ("empty", "transport", 0), ("truncated", "implementation", 0), ("marker_missing", "implementation", 1), ("timeout", "transport", 0), @@ -1174,7 +1181,7 @@ def _attachment_capacity_evidence_records( { "step_name": "apply_squilla_router", "routing_source": "image_route", - "routed_tier": "image_model", + "routed_tier": "c2", } ], } diff --git a/tests/test_model_router_behavior.py b/tests/test_model_router_behavior.py index 4fb0593bd..099be2b65 100644 --- a/tests/test_model_router_behavior.py +++ b/tests/test_model_router_behavior.py @@ -1263,7 +1263,7 @@ def __init__(self, **kwargs) -> None: @pytest.mark.asyncio -async def test_image_input_routes_directly_to_vision_model_without_prompt_injection( +async def test_catalog_text_only_ladder_projects_image_without_prompt_injection( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr( @@ -1275,23 +1275,38 @@ async def test_image_input_routes_directly_to_vision_model_without_prompt_inject "What is in this screenshot?", attachments=[{"type": "image", "mime_type": "image/png"}], ) + catalog = ModelCatalog() + catalog._populate_from_data( + [ + {"id": tier["model"], "architecture": {"input_modalities": ["text"]}} + for name, tier in ctx.config.squilla_router.tiers.items() + if name in {"c0", "c1", "c2", "c3"} + ] + ) + monkeypatch.setattr("opensquilla.provider.model_catalog._shared_catalog", catalog) routed = await apply_squilla_router(ctx) - assert routed.model == "moonshotai/kimi-k2.6" - assert routed.metadata["routed_tier"] == "image_model" - assert routed.metadata["routed_model"] == "moonshotai/kimi-k2.6" + assert routed.model == ctx.config.squilla_router.tiers["c1"]["model"] + assert routed.metadata["routed_tier"] == "c1" + assert routed.metadata["routed_model"] == routed.model assert routed.metadata["routing_applied"] is True assert routed.metadata["routing_confidence"] == 1.0 assert routed.metadata["routing_source"] == "image_route" + assert routed.metadata["image_input_mode"] == "marker" + assert routed.metadata["image_input_projection_required"] is True + assert routed.metadata["router_fallback_strict"] is True + assert routed.metadata["router_fallback_chain"] == [] assert routed.metadata["route_max_history_turns"] == 1 assert routed.metadata["thinking_requested"] is True - assert routed.metadata["thinking_level"] == "medium" + assert routed.metadata["thinking_level"] == ctx.config.squilla_router.tiers["c1"][ + "thinking_level" + ] assert "[RESPONSE_POLICY:" not in routed.message @pytest.mark.asyncio -async def test_tokenrhythm_default_image_route_skips_kimi_code_c2( +async def test_tokenrhythm_default_image_route_uses_configured_catalog_vision( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr( @@ -1301,7 +1316,7 @@ async def test_tokenrhythm_default_image_route_skips_kimi_code_c2( ) config = GatewayConfig() assert config.squilla_router.tiers["c2"]["model"] == "kimi-k2.7-code" - assert config.squilla_router.tiers["c2"]["supports_image"] is False + config.squilla_router.tiers["c2"]["supports_image"] = False ctx = TurnContext( message="What is in this screenshot?", session_key="test-tokenrhythm-image", @@ -1315,8 +1330,11 @@ async def test_tokenrhythm_default_image_route_skips_kimi_code_c2( routed = await apply_squilla_router(ctx) - assert routed.metadata["routed_tier"] == "image_model" - assert routed.model == "kimi-k2.6" + assert routed.metadata["routed_tier"] == "c2" + assert routed.model == config.squilla_router.tiers["c2"]["model"] + assert routed.metadata["image_input_mode"] == "native" + assert routed.metadata["routed_model_vision_support"] == "supported" + assert routed.metadata["router_fallback_chain"] == [] @pytest.mark.asyncio @@ -1328,7 +1346,7 @@ async def test_tokenrhythm_default_image_route_skips_kimi_code_c2( ], ids=["shared", "legacy"], ) -async def test_c3_fusion_prefers_dedicated_image_model( +async def test_c3_fusion_ignores_legacy_image_model_and_uses_configured_c0( monkeypatch: pytest.MonkeyPatch, fusion_config: dict[str, object], ) -> None: @@ -1360,8 +1378,12 @@ async def test_c3_fusion_prefers_dedicated_image_model( routed = await apply_squilla_router(ctx) - assert routed.metadata["routed_tier"] == "image_model" - assert routed.model == "vision/dedicated" + assert routed.metadata["routed_tier"] == "c0" + assert routed.model == "vision/fast" + assert all( + entry["tier"] != "image_model" + for entry in routed.metadata["router_fallback_chain"] + ) @pytest.mark.asyncio @@ -1405,7 +1427,7 @@ async def test_c3_fusion_uses_another_non_c3_image_tier_without_dedicated_model( @pytest.mark.asyncio -async def test_c3_fusion_rejects_image_input_when_no_independent_image_tier_is_available( +async def test_c3_fusion_with_no_independent_image_tier_uses_marker( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr( @@ -1427,13 +1449,15 @@ async def test_c3_fusion_rejects_image_input_when_no_independent_image_tier_is_a result = await apply_squilla_router(ctx) - assert result.metadata["image_input_forced_rejection_reason"] == ( - "router_image_route_unavailable" - ) + assert result.metadata["routed_tier"] == "c3" + assert result.metadata["image_input_mode"] == "marker" + assert result.metadata["image_input_projection_required"] is True + assert result.metadata["router_image_capability_exhausted"] is True + assert result.metadata["router_fallback_chain"] == [] @pytest.mark.asyncio -async def test_global_fusion_rejects_c3_as_the_only_image_fallback( +async def test_global_fusion_with_c3_only_uses_marker( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr( @@ -1457,9 +1481,10 @@ async def test_global_fusion_rejects_c3_as_the_only_image_fallback( result = await apply_squilla_router(ctx) - assert result.metadata["image_input_forced_rejection_reason"] == ( - "router_image_route_unavailable" - ) + assert result.metadata["routed_tier"] == "c3" + assert result.metadata["image_input_mode"] == "marker" + assert result.metadata["router_image_capability_exhausted"] is True + assert result.metadata["router_fallback_chain"] == [] @pytest.mark.asyncio @@ -1490,7 +1515,7 @@ async def test_single_c3_remains_available_for_image_routing( @pytest.mark.asyncio -async def test_image_route_prefers_dedicated_tier_over_declaration_order( +async def test_image_route_ignores_legacy_dedicated_tier( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr( @@ -1503,7 +1528,7 @@ async def test_image_route_prefers_dedicated_tier_over_declaration_order( attachments=[{"type": "image/png", "data": "abc"}], ) ctx.config.squilla_router.tiers = { - "vision_primary": { + "c0": { "model": "vision/primary", "supports_image": True, }, @@ -1516,12 +1541,13 @@ async def test_image_route_prefers_dedicated_tier_over_declaration_order( routed = await apply_squilla_router(ctx) - assert routed.metadata["routed_tier"] == "image_model" - assert routed.model == "vision/dedicated" + assert routed.metadata["routed_tier"] == "c0" + assert routed.model == "vision/primary" + assert routed.metadata["router_image_configured_tiers"] == ["c0"] @pytest.mark.asyncio -async def test_image_route_falls_back_when_dedicated_tier_has_blank_model( +async def test_image_route_uses_c_tier_when_legacy_image_model_is_blank( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr( @@ -1534,7 +1560,7 @@ async def test_image_route_falls_back_when_dedicated_tier_has_blank_model( attachments=[{"type": "image/png", "data": "abc"}], ) ctx.config.squilla_router.tiers = { - "vision_primary": { + "c0": { "model": "vision/primary", "supports_image": True, }, @@ -1547,12 +1573,12 @@ async def test_image_route_falls_back_when_dedicated_tier_has_blank_model( routed = await apply_squilla_router(ctx) - assert routed.metadata["routed_tier"] == "vision_primary" + assert routed.metadata["routed_tier"] == "c0" assert routed.model == "vision/primary" @pytest.mark.asyncio -async def test_image_route_rejects_when_every_image_tier_has_blank_model( +async def test_image_route_with_no_configured_c_tier_degrades_to_marker( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr( @@ -1575,9 +1601,12 @@ async def test_image_route_rejects_when_every_image_tier_has_blank_model( result = await apply_squilla_router(ctx) - assert result.metadata["image_input_forced_rejection_reason"] == ( - "router_image_route_unavailable" - ) + assert result.metadata["image_input_mode"] == "marker" + assert result.metadata["image_input_projection_required"] is True + assert result.metadata["router_image_capability_exhausted"] is True + assert result.metadata["routing_applied"] is True + assert result.metadata["router_fallback_strict"] is True + assert result.metadata["router_fallback_chain"] == [] @pytest.mark.asyncio @@ -1608,23 +1637,16 @@ async def test_large_image_attachment_bypass_honors_capacity_floor( attachments=[{"type": "image", "mime_type": "image/png"}], ) ctx.config.squilla_router.tiers = { - "vision_small": { + "c0": { "provider": "foreign", "model": "vision-small", "supports_image": True, - "image_only": True, - "thinking_level": "off", - }, - "vision_large": { - "provider": "openrouter", - "model": "vision-large", - "supports_image": True, - "image_only": True, "thinking_level": "off", }, "c3": { "provider": "openrouter", "model": "vision-large", + "supports_image": True, "thinking_level": "off", }, } @@ -1634,7 +1656,7 @@ async def test_large_image_attachment_bypass_honors_capacity_floor( routed = await apply_squilla_router(ctx) assert routed.metadata["large_context_floor_min_tier"] == "c3" - assert routed.metadata["routed_tier"] == "vision_large" + assert routed.metadata["routed_tier"] == "c3" assert routed.model == "vision-large" assert routed.metadata["router_fallback_chain"] == [] assert "router_tier_provider_mismatch" not in routed.metadata @@ -1736,22 +1758,22 @@ async def test_image_route_uses_first_configured_image_tier_without_random_choic attachments=[{"type": "image/png", "data": "abc"}], ) ctx.config.squilla_router.tiers = { - "vision_primary": { + "c0": { "model": "vision/primary", "supports_image": True, "thinking_level": "low", }, - "vision_backup": { + "c2": { "model": "vision/backup", "supports_image": True, "thinking_level": "high", }, - "c1": {"model": "text/model"}, + "c1": {"model": "text/model", "supports_image": False}, } routed = await apply_squilla_router(ctx) - assert routed.metadata["routed_tier"] == "vision_primary" + assert routed.metadata["routed_tier"] == "c0" assert routed.metadata["routed_model"] == "vision/primary" assert routed.model == "vision/primary" @@ -1772,13 +1794,12 @@ async def test_image_route_records_tier_provider_for_cross_provider_execution( ctx.config.llm.provider = "anthropic" ctx.config.squilla_router.cross_provider_tiers = True ctx.config.squilla_router.tiers = { - "image_model": { + "c0": { "model": "vision/model-1", "provider": "openai", "supports_image": True, - "image_only": True, }, - "c1": {"model": "text/model-1"}, + "c1": {"model": "text/model-1", "supports_image": False}, } routed = await apply_squilla_router(ctx) @@ -1804,13 +1825,12 @@ async def test_image_route_flags_tier_provider_mismatch_when_cross_provider_disa ctx.config.llm.provider = "anthropic" ctx.config.squilla_router.cross_provider_tiers = False ctx.config.squilla_router.tiers = { - "image_model": { + "c0": { "model": "vision/model-1", "provider": "openai", "supports_image": True, - "image_only": True, }, - "c1": {"model": "text/model-1"}, + "c1": {"model": "text/model-1", "supports_image": False}, } routed = await apply_squilla_router(ctx) @@ -1837,13 +1857,16 @@ async def test_global_fixed_lineup_keeps_image_provider_direct( ctx.config.llm_ensemble.selection_mode = "static_openrouter_b5" ctx.config.squilla_router.cross_provider_tiers = False ctx.config.squilla_router.tiers = { - "image_model": { + "c0": { "model": "vision/model-1", "provider": "openai", "supports_image": True, - "image_only": True, }, - "c1": {"model": "text/model-1", "provider": "openai"}, + "c1": { + "model": "text/model-1", + "provider": "openai", + "supports_image": False, + }, } routed = await apply_squilla_router(ctx) @@ -1869,11 +1892,15 @@ async def test_gate_needs_image_routes_followup_to_vision_model( ctx.metadata["router_turns_since_last_image"] = 1 ctx.metadata["router_vision_followup_gate_decision"] = "needs_image" ctx.metadata["router_vision_followup_needs_image"] = True + ctx.config.squilla_router.tiers = { + "c0": {"model": "configured/vision", "supports_image": True}, + "c1": {"model": "configured/text", "supports_image": False}, + } routed = await apply_squilla_router(ctx) - assert routed.model == "moonshotai/kimi-k2.6" - assert routed.metadata["routed_tier"] == "image_model" + assert routed.model == "configured/vision" + assert routed.metadata["routed_tier"] == "c0" assert routed.metadata["routing_source"] == "image_route" assert routed.metadata["image_route_reason"] == "gate_history" assert routed.metadata["route_max_history_turns"] == 8 @@ -1926,7 +1953,7 @@ async def test_recent_historical_image_without_sticky_uses_text_router( @pytest.mark.asyncio -async def test_image_attachment_without_image_tier_fails_locally( +async def test_image_attachment_without_multimodal_c_tier_uses_marker( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr( @@ -1941,12 +1968,22 @@ async def test_image_attachment_without_image_tier_fails_locally( attachments=[{"type": "image", "mime_type": "image/png"}], ) ctx.config.squilla_router.tiers["image_model"]["supports_image"] = False + catalog = ModelCatalog() + catalog._populate_from_data( + [ + {"id": tier["model"], "architecture": {"input_modalities": ["text"]}} + for name, tier in ctx.config.squilla_router.tiers.items() + if name in {"c0", "c1", "c2", "c3"} + ] + ) + monkeypatch.setattr("opensquilla.provider.model_catalog._shared_catalog", catalog) result = await apply_squilla_router(ctx) - assert result.metadata["image_input_forced_rejection_reason"] == ( - "router_image_route_unavailable" - ) + assert result.metadata["routed_tier"] == "c1" + assert result.metadata["image_input_mode"] == "marker" + assert result.metadata["image_input_projection_required"] is True + assert result.metadata["router_fallback_chain"] == [] @pytest.mark.asyncio @@ -1967,11 +2004,12 @@ async def test_caption_less_image_attachment_still_routes_to_vision_tier( routed = await apply_squilla_router(ctx) assert routed.metadata["routing_source"] == "image_route" - assert routed.metadata["routed_tier"] == "image_model" + assert routed.metadata["routed_tier"] == "c3" + assert routed.metadata["image_input_mode"] == "native" @pytest.mark.asyncio -async def test_caption_less_image_attachment_without_image_tier_fails_locally( +async def test_caption_less_image_without_multimodal_c_tier_uses_marker( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr( @@ -1986,12 +2024,21 @@ async def test_caption_less_image_attachment_without_image_tier_fails_locally( attachments=[{"type": "image", "mime_type": "image/png"}], ) ctx.config.squilla_router.tiers["image_model"]["supports_image"] = False + catalog = ModelCatalog() + catalog._populate_from_data( + [ + {"id": tier["model"], "architecture": {"input_modalities": ["text"]}} + for name, tier in ctx.config.squilla_router.tiers.items() + if name in {"c0", "c1", "c2", "c3"} + ] + ) + monkeypatch.setattr("opensquilla.provider.model_catalog._shared_catalog", catalog) result = await apply_squilla_router(ctx) - assert result.metadata["image_input_forced_rejection_reason"] == ( - "router_image_route_unavailable" - ) + assert result.metadata["routed_tier"] == "c1" + assert result.metadata["image_input_mode"] == "marker" + assert result.metadata["router_fallback_chain"] == [] @pytest.mark.asyncio diff --git a/tests/test_onboarding/test_mutations.py b/tests/test_onboarding/test_mutations.py index 91c80fc37..b5048eebe 100644 --- a/tests/test_onboarding/test_mutations.py +++ b/tests/test_onboarding/test_mutations.py @@ -9,6 +9,7 @@ from opensquilla.onboarding.mutations import ( LlmProfileActivationError, MutationResult, + _tiers_equal_after_canonical_normalization, list_channel_entries, remove_channel, set_channel_enabled, @@ -1660,6 +1661,21 @@ def test_upsert_router_persists_explicit_single_model_over_the_recommended_c3_de assert res.public_payload["mode"] == "custom" +@pytest.mark.parametrize("legacy_field", ["supports_image", "supportsImage"]) +@pytest.mark.parametrize("legacy_value", [True, False]) +def test_retired_image_switch_does_not_customize_a_preset( + legacy_field: str, legacy_value: bool, +): + preset = {"c2": {"provider": "synthetic", "model": "configured-model"}} + saved = {"c2": {**preset["c2"], legacy_field: legacy_value}} + + assert _tiers_equal_after_canonical_normalization(saved, preset) + assert saved["c2"][legacy_field] is legacy_value + assert not _tiers_equal_after_canonical_normalization( + {"c2": {**saved["c2"], "model": "different-model"}}, preset + ) + + def test_upsert_router_forces_image_model_role_invariants(): cfg = GatewayConfig(llm={"provider": "openrouter", "model": "z-ai/glm-5.1"}) @@ -1682,6 +1698,45 @@ def test_upsert_router_forces_image_model_role_invariants(): assert image_tier["image_only"] is True +def test_upsert_router_warns_and_round_trips_legacy_image_model(): + cfg = GatewayConfig(llm={"provider": "openrouter", "model": "configured/text-model"}) + tiers = { + name: {"provider": "openrouter", "model": f"configured/{name}", "supports_image": False} + for name in ("c0", "c1", "c2", "c3") + } + tiers["image_model"] = { + "provider": "openai", + "model": "saved/vision-model", + "description": "Saved legacy image setting", + "thinking_level": "high", + "supports_image": True, + "image_only": True, + } + cfg.squilla_router.tiers = tiers + cfg.squilla_router.tier_profile = None + + saved = upsert_router(cfg, mode="custom") + assert len(saved.warnings) == 1 + assert "image_model" in saved.warnings[0] + assert "not used for image input" in saved.warnings[0] + assert "c0-c3" in saved.warnings[0] + assert saved.config.squilla_router.tiers["image_model"] == tiers["image_model"] + assert saved.public_payload["tiers"]["image_model"] == tiers["image_model"] + assert all( + saved.config.squilla_router.tiers[name]["model"] == tiers[name]["model"] + for name in ("c0", "c1", "c2", "c3") + ) + + reloaded = GatewayConfig.model_validate(saved.config.to_toml_dict()) + resaved = upsert_router(reloaded, mode="custom") + assert resaved.config.squilla_router.tiers["image_model"] == tiers["image_model"] + assert resaved.warnings == saved.warnings + + disabled = upsert_router(resaved.config, mode="disabled") + assert disabled.warnings == [] + assert disabled.config.squilla_router.tiers["image_model"] == tiers["image_model"] + + def test_upsert_router_can_disable(): cfg = GatewayConfig(llm={"provider": "openrouter", "model": "deepseek/x"}) @@ -1832,8 +1887,10 @@ def test_upsert_router_custom_accepts_explicit_tiers_for_synthesized_presets(): "description": ( "groq balanced route (synthesized default; no curated per-tier model ladder)." ), - "supports_image": False, } + # An omitted declaration remains probeable instead of becoming a false + # capability claim for an unknown custom deployment. + assert "supports_image" not in res.config.squilla_router.tiers["c1"] # Router tier selection is independent from the direct/fallback model. assert res.config.llm.model == "m" diff --git a/tests/test_provider/test_image_projection.py b/tests/test_provider/test_image_projection.py new file mode 100644 index 000000000..2b31a1faa --- /dev/null +++ b/tests/test_provider/test_image_projection.py @@ -0,0 +1,425 @@ +from __future__ import annotations + +import pytest + +from opensquilla.provider import ( + ChatConfig, + ContentBlockImage, + ContentBlockText, + ContentBlockToolResult, + ErrorEvent, + ImageFailureKind, + ImageMarkerState, + ImageProjectionMode, + ImageProjectionPolicy, + Message, + VisionSupportEvidence, + VisionSupportSource, + assert_text_only_messages, + bind_image_attachment_ids, + classify_image_failure, + classify_image_input_error, + count_image_blocks, + image_marker, + normalize_vision_support, + project_messages, + projection_mode_for_support, +) + + +def _image(payload: str = "c3ludGhldGlj") -> ContentBlockImage: + return ContentBlockImage(media_type="image/png", data=payload) + + +def test_native_projection_deep_copies_without_mutating_canonical_messages() -> None: + canonical = [ + Message( + role="user", + content=[ + ContentBlockText(text="look"), + _image(), + ContentBlockToolResult( + tool_use_id="tool-1", + content=[ContentBlockText(text="result"), _image("b"), {"x": [1]}], + ), + ], + ) + ] + + result = project_messages( + canonical, + vision_support="supported", + attachment_ids=("att-1", "att-2"), + ) + + assert result.mode is ImageProjectionMode.NATIVE + assert result.input_image_count == 2 + assert result.output_image_count == 2 + assert result.marker_count == 0 + assert result.messages is not canonical + assert result.messages[0] is not canonical[0] + assert result.messages[0].content is not canonical[0].content + assert count_image_blocks(canonical) == 2 + assert count_image_blocks(result.messages) == 2 + + # Mutating the provider view must not mutate the canonical transcript. + assert isinstance(result.messages[0].content, list) + first_text = result.messages[0].content[0] + assert isinstance(first_text, ContentBlockText) + first_text.text = "changed in request" + assert isinstance(canonical[0].content, list) + assert isinstance(canonical[0].content[0], ContentBlockText) + assert canonical[0].content[0].text == "look" + + +def test_marker_projection_recursively_replaces_typed_and_mapping_images() -> None: + canonical = [ + Message( + role="user", + content=[ + ContentBlockText(text="inspect"), + _image(), + ContentBlockToolResult( + tool_use_id="tool-1", + content=[ + {"type": "image", "source_type": "base64", "data": "a"}, + ContentBlockToolResult( + tool_use_id="nested", + content=[_image("b")], + ), + ], + ), + ], + ) + ] + + result = project_messages( + canonical, + mode="marker", + attachment_ids=("att-1", "att-2", "att-3"), + marker_states={"att-2": ImageMarkerState.ANALYSIS_FAILED}, + ) + + assert result.mode is ImageProjectionMode.MARKER + assert result.input_image_count == 3 + assert result.output_image_count == 0 + assert result.marker_count == 3 + assert len(result.decisions) == 3 + assert all(decision.marker for decision in result.decisions) + assert all("att-" in (decision.marker or "") for decision in result.decisions) + assert any("图片未分析" in (decision.marker or "") for decision in result.decisions) + assert any("图片分析失败" in (decision.marker or "") for decision in result.decisions) + assert_text_only_messages(result.messages) + + # The original graph remains image-bearing and unchanged. + assert count_image_blocks(canonical) == 3 + assert isinstance(canonical[0].content, list) + assert isinstance(canonical[0].content[1], ContentBlockImage) + + +def test_bound_image_ids_do_not_shift_across_history_current_or_nested_images() -> None: + history_id = "att_" + "a" * 16 + current_id = "att_" + "b" * 16 + canonical = [ + Message( + role="user", + content=[ + ContentBlockToolResult( + tool_use_id="nested", + content=[_image("tool-image")], + ), + ContentBlockImage( + media_type="image/png", + data="history-image", + attachment_id=history_id, + ), + ], + ), + Message(role="user", content=[_image("current-image")]), + ] + canonical[1:] = bind_image_attachment_ids(canonical[1:], [current_id]) + + result = project_messages( + canonical, + mode="marker", + # Runtime metadata may contain only the current bound-envelope ID. + attachment_ids=(current_id,), + ) + + assert [decision.attachment_id for decision in result.decisions] == [ + None, + history_id, + current_id, + ] + rendered = str(result.messages) + assert rendered.count(history_id) == 1 + assert rendered.count(current_id) == 1 + assert current_id not in canonical[0].model_dump_json() + assert current_id not in canonical[1].model_dump_json() + + +def test_tool_use_arguments_are_not_mistaken_for_content_images() -> None: + message = Message( + role="assistant", + content=[ + { + "type": "tool_use", + "id": "call-1", + "name": "fake", + "input": {"type": "image", "data": "not-a-content-block"}, + } + ], + ) + + result = project_messages(message_list := [message], mode="marker") + + assert result.input_image_count == 0 + assert result.output_image_count == 0 + assert result.marker_count == 0 + assert result.messages[0].content == message_list[0].content + + +def test_policy_surrogate_is_bounded_to_the_matching_attachment() -> None: + messages = [Message(role="user", content=[_image("a"), _image("b")])] + policy = ImageProjectionPolicy( + mode=ImageProjectionMode.SURROGATE, + attachment_ids=("att-a", "att-b"), + surrogate_by_attachment_id={"att-b": "a small blue square"}, + ) + + result = project_messages(messages, policy=policy) + + assert result.output_image_count == 0 + assert result.marker_count == 2 + assert isinstance(result.messages[0].content, list) + assert isinstance(result.messages[0].content[0], ContentBlockText) + assert "图片未分析" in result.messages[0].content[0].text + assert isinstance(result.messages[0].content[1], ContentBlockText) + assert "图片派生描述" in result.messages[0].content[1].text + + +def test_capability_normalization_preserves_omitted_vs_explicit_false() -> None: + assert normalize_vision_support(False) == "unsupported" + assert normalize_vision_support(None) == "unknown" + assert normalize_vision_support(False, field_present=False) == "unknown" + assert projection_mode_for_support("supported") is ImageProjectionMode.NATIVE + assert projection_mode_for_support("unknown") is ImageProjectionMode.NATIVE + assert projection_mode_for_support("unsupported") is ImageProjectionMode.MARKER + assert ( + projection_mode_for_support("supported", force_text_only=True) + is ImageProjectionMode.MARKER + ) + + evidence = VisionSupportEvidence( + status=False, + source=VisionSupportSource.USER_CONFIG, + deployment="openai:test", + ) + assert evidence.status == "unsupported" + assert evidence.rejects_images + assert evidence.deployment == "openai:test" + + +def test_marker_state_text_is_truthful_and_id_is_sanitized() -> None: + not_read = image_marker(ImageMarkerState.NOT_REREAD, attachment_id="att/unsafe\n1") + failed = image_marker("failed", attachment_id="att-2") + unavailable = image_marker("missing", attachment_id="att-3") + + assert "历史图片本回合未重新读取" in not_read + assert "att_unsafe_1" in not_read + assert "图片分析失败" in failed + assert "历史图片不可用" in unavailable + assert not_read.endswith("]") + assert failed.endswith("]") + + +def test_marker_preserves_maximum_length_manifest_id() -> None: + attachment_id = "att_" + "a" * 160 + + assert attachment_id in image_marker(attachment_id=attachment_id) + + +def test_image_error_classifier_only_caches_precise_unsupported_evidence() -> None: + assert ( + classify_image_input_error( + ErrorEvent(code="image_input_unsupported", message="vision unavailable") + ) + is ImageFailureKind.UNSUPPORTED_INPUT + ) + assert ( + classify_image_input_error( + ErrorEvent( + code="image_input_unsupported", + message="This model does not support the supplied image format.", + ) + ) + is ImageFailureKind.UNSUPPORTED_INPUT + ) + assert ( + classify_image_input_error( + {"code": "bad_request", "message": "this model does not support image input"} + ) + is ImageFailureKind.UNSUPPORTED_INPUT + ) + assert ( + classify_image_input_error( + {"status_code": 429, "code": "rate_limit", "message": "try later"}, + provider_name="openai", + ) + is ImageFailureKind.RATE_LIMITED + ) + assert ( + classify_image_input_error( + {"status_code": 401, "code": "unauthorized", "message": "bad key"}, + provider_name="openai", + ) + is ImageFailureKind.AUTHENTICATION + ) + assert ( + classify_image_input_error( + {"code": "invalid_image", "message": "image decode failed"} + ) + is ImageFailureKind.INVALID_MEDIA + ) + # Generic unsupported feature text does not prove image capability. + assert ( + classify_image_input_error( + {"code": "unsupported_feature", "message": "feature unsupported"}, + provider_name="openai", + ) + is ImageFailureKind.UNKNOWN + ) + + classified = classify_image_failure( + ErrorEvent(code="image_input_unsupported", message="images unsupported") + ) + assert classified.is_unsupported + assert classified.caches_unsupported + assert classified.retry_without_image + + +def test_image_error_classifier_prioritizes_media_and_provider_failures() -> None: + assert ( + classify_image_input_error( + { + "status_code": 400, + "code": "invalid_image", + "message": "image unable decoded", + }, + provider_name="openai", + ) + is ImageFailureKind.INVALID_MEDIA + ) + + for status_code, expected in ( + (401, ImageFailureKind.AUTHENTICATION), + (429, ImageFailureKind.RATE_LIMITED), + (503, ImageFailureKind.TRANSIENT), + ): + assert ( + classify_image_input_error( + { + "status_code": status_code, + "code": "image_input_unsupported", + "message": "This model does not support image input.", + }, + provider_name="openai", + ) + is expected + ) + + # Real provider adapters often expose the HTTP status only through the + # event's string ``code`` field. + assert ( + classify_image_input_error( + ErrorEvent( + code=str(status_code), + message="This model does not support image input.", + ), + provider_name="openai", + ) + is expected + ) + + invalid = classify_image_failure( + { + "status_code": 400, + "code": "invalid_image", + "message": "image decode failed", + }, + provider_name="openai", + ) + assert not invalid.is_unsupported + assert not invalid.caches_unsupported + assert not invalid.retry_without_image + + +def test_image_endpoint_404_is_a_capability_rejection() -> None: + for error in ( + ErrorEvent( + code="404", + message="No endpoints found that support image input.", + ), + { + "status_code": 404, + "code": "not_found", + "message": "No endpoints found that support image inputs.", + }, + ): + failure = classify_image_failure(error, provider_name="openrouter") + assert failure.kind is ImageFailureKind.UNSUPPORTED_INPUT + assert failure.retry_without_image + assert failure.caches_unsupported + + +@pytest.mark.parametrize( + "message", + [ + "No endpoints found for configured/text-model.", + "No endpoints found that support tool use.", + "The requested image model does not exist.", + ], +) +def test_non_image_capability_404_remains_model_not_found(message: str) -> None: + failure = classify_image_failure( + ErrorEvent(code="404", message=message), + provider_name="openrouter", + ) + assert failure.kind is ImageFailureKind.MODEL_NOT_FOUND + assert not failure.retry_without_image + assert not failure.caches_unsupported + + +@pytest.mark.parametrize( + ("status", "expected"), + [ + (401, ImageFailureKind.AUTHENTICATION), + (403, ImageFailureKind.AUTHENTICATION), + (402, ImageFailureKind.INSUFFICIENT_CREDITS), + (429, ImageFailureKind.RATE_LIMITED), + (500, ImageFailureKind.TRANSIENT), + (503, ImageFailureKind.TRANSIENT), + (504, ImageFailureKind.TRANSIENT), + ], +) +def test_image_endpoint_wording_does_not_override_provider_failures( + status: int, + expected: ImageFailureKind, +) -> None: + failure = classify_image_failure( + ErrorEvent( + code=str(status), + message="No endpoints found that support image input.", + ), + provider_name="openrouter", + ) + assert failure.kind is expected + assert not failure.retry_without_image + assert not failure.caches_unsupported + + +def test_chat_config_remains_importable_with_projection_types() -> None: + # Sanity check that importing the additive module does not create a provider + # package cycle or alter the existing ChatConfig contract. + config = ChatConfig() + assert config.model_vision_support == "unknown" diff --git a/tests/test_provider/test_preset_registry.py b/tests/test_provider/test_preset_registry.py index 985571339..a02f72246 100644 --- a/tests/test_provider/test_preset_registry.py +++ b/tests/test_provider/test_preset_registry.py @@ -221,7 +221,10 @@ def test_synthesized_presets_bind_all_text_tiers_to_provider_default() -> None: else: assert entry["model"] assert entry["description"] - assert entry["supports_image"] is False + # Synthesized rows do not carry authoritative capability evidence. + # Omission remains probeable; only an operator-authored false may + # be treated as a definitive negative declaration. + assert "supports_image" not in entry def test_curated_synthesized_presets_pin_live_verified_ladders() -> None: diff --git a/tests/test_provider_ensemble.py b/tests/test_provider_ensemble.py index 0c8796ad7..0b280f5d3 100644 --- a/tests/test_provider_ensemble.py +++ b/tests/test_provider_ensemble.py @@ -59,6 +59,7 @@ ensemble_runtime_status, ) from opensquilla.provider.failures import ProviderFailureKind +from opensquilla.provider.image_projection import count_image_blocks from opensquilla.provider.request_proof import project_final_request_payload from opensquilla.provider.selector import ProviderConfig from opensquilla.provider.types import ( @@ -1223,14 +1224,21 @@ def _ensemble_for_validation( ], ids=["base64", "historical-url", "mixed", "typed-tool-result"], ) -async def test_ensemble_rejects_typed_images_before_starting_any_leg( +async def test_ensemble_projects_typed_images_before_starting_any_leg( monkeypatch: pytest.MonkeyPatch, messages: list[Message], ) -> None: + original_snapshot = [ + message.model_dump(mode="json", exclude_none=True) for message in messages + ] registry = _FakeRegistry( { - "p1": _FakePlan([DoneEvent(model="p1")]), - "agg": _FakePlan([DoneEvent(model="agg")]), + "p1": _FakePlan( + [TextDeltaEvent(text="draft"), DoneEvent(model="p1")] + ), + "agg": _FakePlan( + [TextDeltaEvent(text="final"), DoneEvent(model="agg")] + ), } ) monkeypatch.setattr("opensquilla.provider.ensemble._build_provider", registry.provider_for) @@ -1238,19 +1246,51 @@ async def test_ensemble_rejects_typed_images_before_starting_any_leg( events = [event async for event in provider.chat(messages)] - assert len(events) == 1 - assert isinstance(events[0], ErrorEvent) - assert events[0].code == "ensemble_multimodal_unsupported" - assert events[0].message == ( - "Ensemble does not support image input yet. " - "Switch to a single-model routing mode and try again." + assert [call["model"] for call in registry.calls] == ["p1", "agg"] + assert all(count_image_blocks(call["messages"]) == 0 for call in registry.calls) + assert "图片未分析" in str(registry.calls[0]["messages"]) + assert count_image_blocks(messages) == 1 + assert [ + message.model_dump(mode="json", exclude_none=True) for message in messages + ] == original_snapshot + assert not any(isinstance(event, ErrorEvent) for event in events) + assert any(isinstance(event, DoneEvent) for event in events) + + +@pytest.mark.asyncio +async def test_ensemble_gives_each_physical_member_a_fresh_text_only_copy( + monkeypatch: pytest.MonkeyPatch, +) -> None: + registry = _FakeRegistry( + { + "p1": _FakePlan([TextDeltaEvent(text="one"), DoneEvent(model="p1")]), + "p2": _FakePlan([TextDeltaEvent(text="two"), DoneEvent(model="p2")]), + "agg": _FakePlan([TextDeltaEvent(text="final"), DoneEvent(model="agg")]), + } ) - assert registry.calls == [] + monkeypatch.setattr("opensquilla.provider.ensemble._build_provider", registry.provider_for) + provider = _ensemble_for_validation(proposers=[_member("p1"), _member("p2")]) + messages = [ + Message( + role="user", + content=[ContentBlockImage(media_type="image/png", data="aW1hZ2U=")], + ) + ] + + events = [event async for event in provider.chat(messages)] + + calls = {call["model"]: call for call in registry.calls} + assert set(calls) == {"p1", "p2", "agg"} + assert calls["p1"]["messages"] is not calls["p2"]["messages"] + assert calls["p1"]["messages"][0] is not calls["p2"]["messages"][0] + assert all(count_image_blocks(call["messages"]) == 0 for call in calls.values()) + assert count_image_blocks(messages) == 1 + assert any(isinstance(event, DoneEvent) for event in events) @pytest.mark.asyncio @pytest.mark.parametrize("all_failed_policy", ["error", "fallback_single"]) -async def test_ensemble_image_validation_precedes_empty_lineup_fallback( +async def test_ensemble_image_projection_preserves_empty_lineup_policy( all_failed_policy: Literal["error", "fallback_single"], ) -> None: registry = _FakeRegistry( @@ -1276,10 +1316,16 @@ async def test_ensemble_image_validation_precedes_empty_lineup_fallback( events = [event async for event in provider.chat(messages)] - assert [getattr(event, "code", "") for event in events] == [ - "ensemble_multimodal_unsupported" - ] - assert registry.calls == [] + if all_failed_policy == "error": + assert [getattr(event, "code", "") for event in events] == [ + "ensemble_no_proposers" + ] + assert registry.calls == [] + else: + assert [call["model"] for call in registry.calls] == ["fallback"] + assert count_image_blocks(registry.calls[0]["messages"]) == 0 + assert "图片未分析" in str(registry.calls[0]["messages"]) + assert not any(isinstance(event, ErrorEvent) for event in events) def test_ensemble_image_validation_does_not_guess_untyped_or_document_content() -> None: diff --git a/tests/test_provider_model_catalog.py b/tests/test_provider_model_catalog.py index 16bd68495..a22aeb294 100644 --- a/tests/test_provider_model_catalog.py +++ b/tests/test_provider_model_catalog.py @@ -381,6 +381,46 @@ def test_vision_support_distinguishes_live_evidence_from_synthesized_default() - ) == "unknown" +@pytest.mark.parametrize( + ("architecture", "expected"), + [ + ({}, "unknown"), + ({"input_modalities": None}, "unknown"), + ({"input_modalities": []}, "unknown"), + ({"input_modalities": "text"}, "unknown"), + ({"input_modalities": [None]}, "unknown"), + ({"input_modalities": [""]}, "unknown"), + ({"input_modalities": ["text"]}, "unsupported"), + ({"input_modalities": ["text", "image"]}, "supported"), + ({"input_modalities": [" TEXT ", " IMAGE "]}, "supported"), + ({"output_modalities": ["image"]}, "unknown"), + ], +) +def test_live_vision_requires_explicit_input_modality_evidence( + architecture: dict, expected: str, +) -> None: + catalog = ModelCatalog() + catalog._populate_from_data( + [{"id": "synthetic/capability-check", "architecture": architecture}] + ) + + assert catalog.resolve_deployment_vision_support( + "synthetic/capability-check", provider="openrouter" + ) == expected + + +def test_missing_live_vision_does_not_mask_known_catalog_input_capability() -> None: + catalog = ModelCatalog() + catalog._populate_from_data([{"id": "synthetic/catalog-vision"}]) + with patch( + "opensquilla.provider.model_catalog._snapshot_layer_fields", + return_value={"supports_vision": True}, + ): + assert catalog.resolve_vision_support( + "synthetic/catalog-vision", provider_name="openrouter" + ) == "supported" + + @pytest.mark.parametrize( ("reasoning_format", "supports_reasoning"), [("deepseek", True), ("none", False)], diff --git a/tests/test_provider_selector.py b/tests/test_provider_selector.py index a0a9d0227..1935d066a 100644 --- a/tests/test_provider_selector.py +++ b/tests/test_provider_selector.py @@ -513,6 +513,48 @@ def test_strict_router_fallback_chain_discards_configured_lower_tail(monkeypatch ] +def test_strict_router_fallback_chain_keeps_resolved_cross_provider_config( + monkeypatch, +) -> None: + monkeypatch.setattr("opensquilla.provider.selector._build_provider", lambda cfg: cfg) + selector = ModelSelector( + SelectorConfig( + primary=ProviderConfig( + provider="openrouter", + model="configured/c0", + api_key="primary-key", + ), + fallbacks=[ + ProviderConfig( + provider="plugin-provider", + model="outside-router-chain", + api_key="plugin-key", + ) + ], + ) + ) + configured_foreign = ProviderConfig( + provider="anthropic", + model="configured/c1", + api_key="foreign-key", + replay_provider_state=True, + ) + + selector.override_model_with_fallback_chain( + "configured/c0", + [configured_foreign], + preserve_existing_tail=False, + ) + + chain = selector.remaining_chain() + assert [(cfg.provider, cfg.model) for cfg in chain] == [ + ("openrouter", "configured/c0"), + ("anthropic", "configured/c1"), + ] + assert chain[1].api_key == "foreign-key" + assert chain[1].replay_provider_state is False + + def test_strict_empty_router_fallback_chain_removes_every_lower_model(monkeypatch) -> None: monkeypatch.setattr("opensquilla.provider.selector._build_provider", lambda cfg: cfg) selector = ModelSelector( @@ -541,6 +583,52 @@ def test_strict_empty_router_fallback_chain_removes_every_lower_model(monkeypatc assert [cfg.model for cfg in selector.remaining_chain()] == [HIGH_TIER_MODEL] +def test_strict_router_fallback_chain_ignores_plugin_replacement(monkeypatch) -> None: + plugin_calls = 0 + + class _Plugin: + def failover_hook(self, primary_failure: Exception) -> list[ProviderConfig]: + nonlocal plugin_calls + del primary_failure + plugin_calls += 1 + return [ + ProviderConfig( + provider="openrouter", + model="outside-router-chain", + api_key="plugin-key", + ) + ] + + monkeypatch.setattr("opensquilla.provider.selector._build_provider", lambda cfg: cfg) + selector = ModelSelector( + SelectorConfig( + primary=ProviderConfig( + provider="openrouter", + model="configured-c0", + api_key="test-key", + ) + ), + plugin=_Plugin(), + ) + selector.override_model_with_fallback_chain( + "configured-c0", + [ + { + "tier": "c1", + "provider": "openrouter", + "model": "configured-c1", + } + ], + preserve_existing_tail=False, + ) + + fallback = selector.next_fallback_after_failure(RuntimeError("rate limited")) + + assert fallback.model == "configured-c1" + assert plugin_calls == 0 + assert [cfg.model for cfg in selector.remaining_chain()] == ["configured-c1"] + + @pytest.mark.parametrize( "router_chain, expected_models", [ diff --git a/tests/test_router_tier_contract.py b/tests/test_router_tier_contract.py index 61a81d3c1..60fca9cc7 100644 --- a/tests/test_router_tier_contract.py +++ b/tests/test_router_tier_contract.py @@ -501,9 +501,7 @@ def test_global_fixed_lineup_suppresses_provider_switch_conflicts_and_warnings() shared_selection_mode="static_openrouter_b5", ensemble_globally_enabled=True, ) - assert len(warnings) == 1 - assert "image_model" in warnings[0] - assert "openai" in warnings[0] + assert warnings == [] # --------------------------------------------------------------------------- @@ -1623,10 +1621,73 @@ def test_upsert_router_surfaces_cross_provider_warning() -> None: assert any("cross-provider" in w.lower() for w in res.warnings) -def test_upsert_router_no_warning_for_matching_tiers() -> None: +def test_upsert_router_no_cross_provider_warning_for_matching_tiers() -> None: cfg = GatewayConfig() res = upsert_router(cfg, mode="recommended") - assert res.warnings == [] + assert len(res.warnings) == 1 + assert "legacy image_model" in res.warnings[0] + assert "preserved for compatibility" in res.warnings[0] + assert "not used for image input" in res.warnings[0] + assert not any("cross-provider" in warning.lower() for warning in res.warnings) + + +def test_upsert_router_model_override_keeps_omitted_vision_support_unknown() -> None: + cfg = GatewayConfig(llm={"provider": "openai", "model": "gpt-5.4-mini"}) + + res = upsert_router( + cfg, + mode="recommended", + tiers={"c2": {"provider": "openai", "model": "operator/custom-model"}}, + ) + + tier = res.config.squilla_router.tiers["c2"] + assert tier["model"] == "operator/custom-model" + assert "supports_image" not in tier + assert res.config.squilla_router.preset_binding == "custom" + + persisted = res.config.to_toml_dict() + persisted_tier = persisted["squilla_router"]["tiers"]["c2"] + assert "supports_image" not in persisted_tier + reloaded = GatewayConfig(**persisted) + assert "supports_image" not in reloaded.squilla_router.tiers["c2"] + + +def test_upsert_router_explicit_false_vision_support_remains_authoritative() -> None: + cfg = GatewayConfig(llm={"provider": "openai", "model": "gpt-5.4-mini"}) + + res = upsert_router( + cfg, + mode="recommended", + tiers={ + "c2": { + "provider": "openai", + "model": "operator/custom-model", + "supportsImage": False, + } + }, + ) + + assert res.config.squilla_router.tiers["c2"]["supports_image"] is False + persisted = res.config.to_toml_dict() + assert persisted["squilla_router"]["tiers"]["c2"]["supports_image"] is False + reloaded = GatewayConfig(**persisted) + assert reloaded.squilla_router.tiers["c2"]["supports_image"] is False + + +def test_synthesized_managed_preset_does_not_generate_negative_vision_claims() -> None: + cfg = GatewayConfig( + llm={"provider": "groq", "model": "llama-3.3-70b-versatile"} + ) + + res = upsert_router(cfg, mode="recommended") + + for tier_name in ("c0", "c1", "c2", "c3"): + tier = res.config.squilla_router.tiers[tier_name] + assert tier["model"] == "llama-3.3-70b-versatile" + assert "supports_image" not in tier + persisted = res.config.to_toml_dict() + for tier_name in ("c0", "c1", "c2", "c3"): + assert "supports_image" not in persisted["squilla_router"]["tiers"][tier_name] def test_upsert_router_redacts_secret_like_tier_fields() -> None: diff --git a/tests/test_session/test_attachment_manifest.py b/tests/test_session/test_attachment_manifest.py new file mode 100644 index 000000000..04cbd7a66 --- /dev/null +++ b/tests/test_session/test_attachment_manifest.py @@ -0,0 +1,415 @@ +"""Tests for the portable attachment occurrence manifest.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +from pathlib import Path + +import pytest + +from opensquilla.session.attachment_manifest import ( + ATTACHMENT_MANIFEST_STATE_KIND, + MATERIAL_AVAILABLE, + MATERIAL_INVALID, + MATERIAL_MISSING, + AttachmentManifest, + AttachmentManifestError, + AttachmentManifestStore, + attachment_manifest_from_context_state, + build_attachment_manifest, + extract_attachment_occurrences, + extract_attachment_occurrences_from_envelope, + legacy_attachment_id, + manifest_context_state, + merge_attachment_occurrences, + normalize_attachment_name, + preserve_attachment_occurrence_ids, +) +from opensquilla.session.models import SessionNode, TranscriptEntry +from opensquilla.session.storage import SessionStorage + + +def _b64(payload: bytes) -> str: + return base64.b64encode(payload).decode("ascii") + + +def _entry( + message_id: str, + content: object, + *, + entry_id: int | None = None, + created_at: int = 100, + session_id: str = "session-a", +) -> TranscriptEntry: + encoded = content if isinstance(content, str) else json.dumps(content) + return TranscriptEntry( + id=entry_id, + session_id=session_id, + session_key="agent:main:webchat:manifest", + message_id=message_id, + role="user", + content=encoded, + created_at=created_at, + ) + + +def test_legacy_attachment_id_matches_stable_algorithm() -> None: + first = legacy_attachment_id( + session_id="session-a", + message_id="message-1", + index=0, + sha256="a" * 64, + ) + forked = legacy_attachment_id( + session_id="session-b", + message_id="message-1", + index=0, + sha256="a" * 64, + ) + different_index = legacy_attachment_id( + session_id="session-a", + message_id="message-1", + index=1, + sha256="a" * 64, + ) + different_message = legacy_attachment_id( + session_id="session-a", + message_id="message-2", + index=0, + sha256="a" * 64, + ) + different_hash = legacy_attachment_id( + session_id="session-a", + message_id="message-1", + index=0, + sha256="b" * 64, + ) + digest = hashlib.sha256( + ("message-1\0" + "0" + "\0" + "a" * 64).encode("utf-8") + ).digest()[:18] + expected = "att_legacy_" + base64.urlsafe_b64encode(digest).decode("ascii").rstrip("=") + assert first == expected + assert first == forked + assert len({first, different_index, different_message, different_hash}) == 4 + assert first.startswith("att_legacy_") + + +@pytest.mark.parametrize( + "content", + [ + None, + "ordinary message", + "{malformed JSON", + '{"text": "no attachment"}', + '{"text": "invalid list", "attachments": "not an array"}', + '{"text": "already bound", "attachments": ' + '[{"attachment_id": "att_existing_123", "type": "image/png", "data": "cG5n"}]}', + ], +) +def test_preserve_occurrence_ids_keeps_nonlegacy_serialization(content: str | None) -> None: + assert preserve_attachment_occurrence_ids( + content, + session_id="parent-session", + source_message_id="parent-message", + ) == content + + +@pytest.mark.parametrize( + ("raw_name", "expected"), + [ + ("/private/tmp/uploads/diagram.png", "diagram.png"), + (r"C:\Temp\uploads\diagram.png", "diagram.png"), + ("/private/tmp/uploads/", "attachment"), + ], +) +def test_normalize_attachment_name_keeps_only_basename( + raw_name: str, + expected: str, +) -> None: + assert normalize_attachment_name(raw_name) == expected + + +def test_extracts_inline_ref_and_missing_occurrences_without_bytes_in_payload() -> None: + payload = b"image-bytes" + sha = hashlib.sha256(payload).hexdigest() + entries = [ + _entry( + "message-1", + { + "text": "look", + "attachments": [ + { + "type": "image/png; charset=binary", + "name": " diagram\n.png ", + "data": _b64(payload), + }, + { + "attachment_id": "att_explicit_123456", + "mime": "image/jpeg", + "name": "stored.jpg", + "sha256_ref": sha, + "size": len(payload), + }, + { + "name": "gone.png", + "mime": "image/png", + "missing_reason": "material was pruned", + }, + ], + }, + entry_id=7, + ) + ] + + occurrences = extract_attachment_occurrences(entries, session_id="session-a") + + assert len(occurrences) == 3 + inline, stored, missing = occurrences + assert inline.material_state == MATERIAL_AVAILABLE + assert inline.sha256_ref == sha + assert inline.name == "diagram .png" + assert inline.mime == "image/png" + assert inline.size == len(payload) + assert stored.attachment_id == "att_explicit_123456" + assert stored.material_state == MATERIAL_AVAILABLE + assert missing.material_state == MATERIAL_MISSING + assert missing.missing_reason == "material was pruned" + assert all("data" not in occurrence.to_payload() for occurrence in occurrences) + + +def test_invalid_material_is_retained_and_legacy_id_is_deterministic() -> None: + envelope = { + "attachments": [ + { + "name": "broken.png", + "type": "image/png", + "data": "not-base64", + "size": 10, + } + ] + } + first = extract_attachment_occurrences_from_envelope( + envelope, + session_id="session-a", + source_message_id="message-1", + source_entry_id=3, + )[0] + second = extract_attachment_occurrences_from_envelope( + envelope, + session_id="session-a", + source_message_id="message-1", + source_entry_id=3, + )[0] + assert first.material_state == MATERIAL_INVALID + assert first.attachment_id == second.attachment_id + assert first.sha256_ref is None + assert first.missing_reason == "invalid inline attachment data" + + +def test_size_or_hash_mismatch_is_invalid() -> None: + payload = b"payload" + wrong_sha = "0" * 64 + occurrences = extract_attachment_occurrences_from_envelope( + { + "attachments": [ + { + "type": "image/png", + "data": _b64(payload), + "sha256_ref": wrong_sha, + "size": len(payload) + 1, + } + ] + }, + session_id="s", + source_message_id="m", + ) + assert occurrences[0].material_state == MATERIAL_INVALID + assert occurrences[0].missing_reason == "attachment size mismatch" + + +def test_non_user_rows_are_not_indexed_by_default() -> None: + content = {"attachments": [{"type": "image/png", "data": _b64(b"x")}]} + entries = [ + _entry("assistant-message", content), + _entry("user-message", content), + ] + entries[0].role = "assistant" + assert len(extract_attachment_occurrences(entries, session_id="s")) == 1 + assert len( + extract_attachment_occurrences( + entries, + session_id="s", + include_roles=("assistant",), + ) + ) == 1 + + +def test_manifest_merge_deduplicates_forked_physical_entry_ids() -> None: + content = {"attachments": [{"type": "image/png", "data": _b64(b"x")}]} + original = extract_attachment_occurrences( + [_entry("message-1", content, entry_id=10)], session_id="session-a" + ) + forked = extract_attachment_occurrences( + [_entry("message-1", content, entry_id=900)], session_id="session-a" + ) + merged = merge_attachment_occurrences(original, forked) + assert len(merged) == 1 + assert merged[0].source_entry_id == 10 + + +def test_manifest_rejects_same_id_for_different_logical_occurrence() -> None: + first = extract_attachment_occurrences_from_envelope( + { + "attachments": [ + { + "attachment_id": "att_explicit_123456", + "type": "image/png", + "data": _b64(b"a"), + } + ] + }, + session_id="s", + source_message_id="m1", + ) + second = extract_attachment_occurrences_from_envelope( + { + "attachments": [ + { + "attachment_id": "att_explicit_123456", + "type": "image/png", + "data": _b64(b"b"), + } + ] + }, + session_id="s", + source_message_id="m2", + ) + with pytest.raises(AttachmentManifestError, match="collision"): + merge_attachment_occurrences(first, second) + + +def test_manifest_payload_roundtrip_is_metadata_only() -> None: + entries = [ + _entry( + "message-1", + {"attachments": [{"type": "image/png", "data": _b64(b"x")}]}, + entry_id=12, + ) + ] + manifest = build_attachment_manifest( + entries, + session_id="session-a", + session_key="webchat:default", + ) + payload = manifest.to_payload() + decoded = AttachmentManifest.from_payload( + payload, + session_id="session-a", + session_key="webchat:default", + ) + assert decoded == manifest + serialized = json.dumps(payload) + assert "data" not in serialized + assert "path" not in serialized + assert decoded.session_key == "agent:main:webchat:default" + + +@pytest.mark.parametrize( + "invalid_id", + ["", "att_short", "../../private/image.png", "att_invalid/slash_123"], +) +def test_manifest_payload_rejects_invalid_attachment_id(invalid_id: str) -> None: + payload = { + "schema_version": 1, + "covered_through_id": 1, + "occurrences": [ + { + "attachment_id": invalid_id, + "source_message_id": "message-1", + "ordinal": 0, + "material_state": MATERIAL_MISSING, + } + ], + } + + with pytest.raises(AttachmentManifestError, match="occurrence ID is invalid"): + AttachmentManifest.from_payload( + payload, + session_id="session-a", + session_key="webchat:default", + ) + + +@pytest.mark.asyncio +async def test_manifest_store_persists_and_reloads_across_storage_restart( + tmp_path: Path, +) -> None: + db_path = tmp_path / "manifest.db" + key = "agent:main:webchat:manifest-store" + session_id = "manifest-session" + storage = SessionStorage(str(db_path)) + await storage.connect() + try: + await storage.upsert_session( + SessionNode( + session_key=key, + session_id=session_id, + agent_id="main", + created_at=1, + updated_at=1, + ) + ) + entry = _entry( + "message-1", + {"attachments": [{"type": "image/png", "data": _b64(b"x")}]}, + entry_id=8, + session_id=session_id, + ) + await storage.append_transcript_entry(entry) + store = AttachmentManifestStore(storage) + manifest = await store.rebuild( + session_id=session_id, + session_key=key, + entries=await storage.get_canonical_transcript(session_id), + ) + found = await store.lookup( + key, + manifest.occurrences[0].attachment_id, + session_id=session_id, + ) + assert found == manifest.occurrences[0] + states = await storage.get_context_states( + key, + provider="portable", + state_kind=ATTACHMENT_MANIFEST_STATE_KIND, + ) + assert len(states) == 1 + finally: + await storage.close() + + restarted = SessionStorage(str(db_path)) + await restarted.connect() + try: + store = AttachmentManifestStore(restarted) + loaded = await store.load(key, session_id=session_id) + assert loaded.occurrences == manifest.occurrences + assert await store.lookup(key, "att_missing_123456", session_id=session_id) is None + finally: + await restarted.close() + + +def test_manifest_context_state_factory_and_decoder() -> None: + manifest = AttachmentManifest( + session_id="s", + session_key="agent:main:webchat:m", + occurrences=(), + covered_through_id=22, + ) + state = manifest_context_state(manifest, created_at=123) + assert state.state_kind == ATTACHMENT_MANIFEST_STATE_KIND + assert state.portable is True + assert state.cacheable is True + assert state.covered_through_id == 22 + assert attachment_manifest_from_context_state(state) == manifest diff --git a/tests/test_session/test_compaction.py b/tests/test_session/test_compaction.py index 8f84bb433..f2b57ae81 100644 --- a/tests/test_session/test_compaction.py +++ b/tests/test_session/test_compaction.py @@ -1,21 +1,28 @@ """Tests for context window compaction logic.""" import asyncio +import base64 import json import pytest from opensquilla.provider.types import ProviderRequestCorrelation +from opensquilla.session.attachment_manifest import ( + extract_attachment_occurrences_from_envelope, +) from opensquilla.session.compaction import ( CompactionConfig, CompactionRequest, _api_round_groups, + _format_chunk_for_llm, + _summarize_chunk_fallback, arm_compaction_deadline, await_compaction_phase, build_compaction_config_from_provider, call_compaction_llm, compact_context, compaction_remaining_seconds, + compaction_replay_summary, estimate_entries_model_replay_chars, estimate_entry_model_replay_tokens, estimate_entry_replay_tokens, @@ -38,6 +45,270 @@ def _make_entries(n: int, tokens_each: int = 100) -> list[dict]: ] +def test_compaction_attachment_descriptor_is_safe_stable_and_not_truncated() -> None: + image_data = base64.b64encode(b"image-bytes").decode("ascii") + content = json.dumps( + { + # Deliberately put attachments first and pretty-print the object: + # legacy detection must not depend on a compact ``{"text":`` prefix. + "attachments": [ + { + "attachment_id": "/private/tmp/not-an-occurrence-id.png", + "path": "/private/tmp/material/image.png", + "name": "/private/tmp/upload/image.png", + "type": "image/png", + "data": image_data, + } + ], + "text": "long prompt " + "x" * 500, + }, + indent=2, + ) + [occurrence] = extract_attachment_occurrences_from_envelope( + content, + session_id="session-image", + source_message_id="message-image", + ) + entry = { + "id": 1, + "session_id": "session-image", + "message_id": "message-image", + "role": "user", + "content": content, + } + + llm_input = _format_chunk_for_llm([entry]) + fallback = _summarize_chunk_fallback([entry], "strict") + + for rendered in (llm_input, fallback): + assert occurrence.attachment_id in rendered + assert "image.png (image/png" in rendered + assert image_data not in rendered + assert "/private/tmp" not in rendered + assert '"path"' not in rendered + assert "not-an-occurrence-id" not in rendered + + +@pytest.mark.asyncio +async def test_durable_attachment_summary_backfills_id_without_media_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + image_data = base64.b64encode(b"durable-image-bytes").decode("ascii") + content = json.dumps( + { + "attachments": [ + { + "attachment_id": "/private/tmp/not-an-occurrence-id.png", + "path": "/private/tmp/material/image.png", + "name": "/private/tmp/upload/image.png", + "type": "image/png", + "data": image_data, + } + ], + "text": "Inspect the archived image.", + }, + indent=2, + ) + [occurrence] = extract_attachment_occurrences_from_envelope( + content, + session_id="session-image", + source_message_id="message-image", + ) + entries = [ + { + "id": 1, + "session_id": "session-image", + "message_id": "message-image", + "role": "user", + "content": content, + "token_count": 5, + }, + { + "id": 2, + "session_id": "session-image", + "message_id": "message-answer", + "role": "assistant", + "content": "Earlier answer.", + "token_count": 5, + }, + { + "id": 3, + "session_id": "session-image", + "message_id": "message-current", + "role": "user", + "content": "Continue.", + "token_count": 5, + }, + { + "id": 4, + "session_id": "session-image", + "message_id": "message-current-answer", + "role": "assistant", + "content": "Current answer.", + "token_count": 5, + }, + ] + + async def summary_without_attachment(**kwargs): # noqa: ANN003 + del kwargs + return "Safe summary without an attachment reference." + + monkeypatch.setattr( + "opensquilla.session.compaction.call_compaction_llm", + summary_without_attachment, + ) + result = await compact_context( + CompactionRequest( + session_id="session-image", + entries=entries, + context_window_tokens=2_000, + config=CompactionConfig( + model="test/model", + api_key="test-key", + safety_margin=1.0, + protected_recent_messages=2, + ), + forced_prefix_cut=2, + trigger="message_count", + ) + ) + + assert result.removed_count == 2 + assert result.summary_payload is not None + assert result.summary_payload["files_and_artifacts"] == [] + assert occurrence.attachment_id in result.summary_payload["important_identifiers"] + serialized_payload = json.dumps(result.summary_payload, sort_keys=True) + replay = compaction_replay_summary(result) + for rendered in (serialized_payload, replay): + assert occurrence.attachment_id in rendered + assert image_data not in rendered + assert "/private/tmp" not in rendered + assert "not-an-occurrence-id" not in rendered + + +@pytest.mark.asyncio +async def test_nested_tool_result_images_are_projected_out_of_compaction( + monkeypatch: pytest.MonkeyPatch, +) -> None: + image_data = base64.b64encode(b"nested-tool-image-bytes").decode("ascii") + image_path = "/private/tmp/tool-results/private-image.png" + nested_result = { + "items": [ + { + "type": "image", + "path": image_path, + "source": { + "type": "base64", + "media_type": "image/png", + "data": image_data, + }, + }, + { + "wrapper": { + "type": "base64", + "media_type": "image/jpeg", + "path": image_path, + "data": image_data, + } + }, + ] + } + tool_calls = [ + { + "type": "tool_use", + "id": "tool-image-result-1", + "name": "inspect_image", + "input": json.dumps({"payload": nested_result}), + }, + { + "type": "tool_result", + "tool_use_id": "tool-image-result-1", + "result": json.dumps(nested_result), + }, + ] + entries = [ + { + "id": 1, + "session_id": "session-tool-image", + "message_id": "message-request", + "role": "user", + "content": "Inspect the tool image.", + "token_count": 5, + }, + { + "id": 2, + "session_id": "session-tool-image", + "message_id": "message-tool-result", + "role": "assistant", + "content": "Tool completed.", + "tool_calls": tool_calls, + "token_count": 5, + }, + { + "id": 3, + "session_id": "session-tool-image", + "message_id": "message-follow-up", + "role": "user", + "content": "Continue without replaying bytes.", + "token_count": 5, + }, + { + "id": 4, + "session_id": "session-tool-image", + "message_id": "message-answer", + "role": "assistant", + "content": "Latest answer.", + "token_count": 5, + }, + ] + captured_chunks: list[str] = [] + + async def safe_summary(**kwargs): # noqa: ANN003 + captured_chunks.append(kwargs["chunk_text"]) + return "The tool returned an image for inspection." + + monkeypatch.setattr( + "opensquilla.session.compaction.call_compaction_llm", + safe_summary, + ) + + llm_projection = _format_chunk_for_llm(entries[:2]) + fallback_projection = _summarize_chunk_fallback(entries[:2], "strict") + result = await compact_context( + CompactionRequest( + session_id="session-tool-image", + entries=entries, + context_window_tokens=2_000, + config=CompactionConfig( + model="test/model", + api_key="test-key", + safety_margin=1.0, + protected_recent_messages=2, + ), + forced_prefix_cut=2, + trigger="message_count", + ) + ) + + assert captured_chunks + assert result.removed_count == 2 + assert result.summary_payload is not None + serialized_payload = json.dumps(result.summary_payload, sort_keys=True) + replay = compaction_replay_summary(result) + for rendered in ( + llm_projection, + fallback_projection, + *captured_chunks, + serialized_payload, + replay, + ): + assert "image omitted from compaction input" in rendered + assert image_data not in rendered + assert image_path not in rendered + assert "/private/tmp" not in rendered + assert tool_calls[1]["result"] == json.dumps(nested_result) + + def test_api_round_groups_keep_user_role_tool_result_with_its_call() -> None: active_user = {"role": "user", "content": "inspect the file"} tool_call = { diff --git a/tests/test_session/test_manager.py b/tests/test_session/test_manager.py index e469a6041..a60a6eea3 100644 --- a/tests/test_session/test_manager.py +++ b/tests/test_session/test_manager.py @@ -15,6 +15,11 @@ import pytest_asyncio from opensquilla.session import manager as session_manager_module +from opensquilla.session.attachment_manifest import ( + ATTACHMENT_MANIFEST_STATE_KIND, + attachment_manifest_from_context_state, + build_attachment_manifest, +) from opensquilla.session.compaction import CompactionConfig, CompactionResult from opensquilla.session.context_view import ( build_compaction_context_records, @@ -1915,6 +1920,329 @@ async def test_branch_fork_transcript_copies_compacted_archive(manager): assert child_page.canonical_complete is True +@pytest.mark.asyncio +@pytest.mark.parametrize("fork_mode", ["before_message", "through_turn", "prepared"]) +@pytest.mark.parametrize("archived", [False, True]) +async def test_prefix_forks_preserve_legacy_attachment_ids_with_new_message_identity( + manager, + fork_mode: str, + archived: bool, +) -> None: + parent = await manager.create("agent:main:prefix-attachment-parent") + image_content = json.dumps( + { + "text": "inspect the attachments", + "attachments": [ + {"type": "image/png", "name": "first.png", "data": "cG5n"}, + {"type": "image/png", "name": "duplicate.png", "data": "cG5n"}, + { + "attachment_id": "att_existing_prefix_123", + "type": "image/png", + "sha256_ref": "a" * 64, + "name": "stored.png", + }, + ], + } + ) + image_entry = TranscriptEntry( + session_id=parent.session_id, + session_key=parent.session_key, + role="user", + content=image_content, + turn_context={"turn_id": "prefix-attachment-turn"}, + ) + await manager._storage.append_transcript_entry(image_entry) + parent_manifest = build_attachment_manifest( + [image_entry], + session_id=parent.session_id, + session_key=parent.session_key, + ) + attachment_ids = [item.attachment_id for item in parent_manifest.occurrences] + assert len(set(attachment_ids)) == 3 + reference_text = "Attachment references: " + ", ".join(attachment_ids) + answer_entry = TranscriptEntry( + session_id=parent.session_id, + session_key=parent.session_key, + role="assistant", + content=reference_text, + turn_context={"turn_id": "prefix-attachment-turn"}, + ) + await manager._storage.append_transcript_entry(answer_entry) + future = await manager.append_message(parent.session_key, "user", "later request") + await manager._storage.create_agent_task( + AgentTaskRecord( + task_id="prefix-attachment-turn", + session_key=parent.session_key, + status=AgentTaskStatus.SUCCEEDED, + ) + ) + if archived: + assert await manager.persist_compaction_result( + parent.session_key, + reference_text, + [{"role": "user", "content": future.content}], + compaction_id="cmp-prefix-attachment-parent", + ) + + child_key = "agent:main:prefix-attachment-child" + if fork_mode == "prepared": + plan = await manager.prepare_prefix_branch( + parent.session_key, + child_key, + fork_before_message_id=future.message_id, + ) + child = plan.node + await manager._storage.upsert_session(child) + for entry in plan.initial_transcript_entries: + await manager._storage.append_transcript_entry(entry) + else: + options = ( + {"fork_before_message_id": future.message_id} + if fork_mode == "before_message" + else {"fork_through_turn_id": "prefix-attachment-turn"} + ) + child = await manager.branch( + parent.session_key, + child_key, + fork_transcript=True, + **options, + ) + + child_entries = await manager.get_canonical_transcript(child.session_key) + assert len(child_entries) == 2 + assert {entry.message_id for entry in child_entries}.isdisjoint( + {image_entry.message_id, answer_entry.message_id, future.message_id} + ) + assert child_entries[1].content == reference_text + copied_envelope = json.loads(child_entries[0].content) + original_envelope = json.loads(image_content) + for original, copied, attachment_id in zip( + original_envelope["attachments"], + copied_envelope["attachments"], + attachment_ids, + strict=True, + ): + assert copied == {**original, "attachment_id": attachment_id} + parent_entries = await manager.get_canonical_transcript(parent.session_key) + assert next( + entry.content for entry in parent_entries if entry.message_id == image_entry.message_id + ) == image_content + child_manifest = build_attachment_manifest( + child_entries, + session_id=child.session_id, + session_key=child.session_key, + ) + assert [item.attachment_id for item in child_manifest.occurrences] == attachment_ids + assert all( + item.source_message_id == child_entries[0].message_id + for item in child_manifest.occurrences + ) + + child_tail = await manager.append_message(child.session_key, "user", "child continuation") + assert await manager.persist_compaction_result( + child.session_key, + reference_text, + [{"role": "user", "content": child_tail.content}], + compaction_id="cmp-prefix-attachment-child", + ) + states = await manager.get_context_states( + child.session_key, + provider="portable", + state_kind=ATTACHMENT_MANIFEST_STATE_KIND, + ) + compacted_manifest = attachment_manifest_from_context_state( + max(states, key=lambda state: (state.created_at, state.id or 0)) + ) + assert [item.attachment_id for item in compacted_manifest.occurrences] == attachment_ids + nested = await manager.branch( + child.session_key, + "agent:main:prefix-attachment-nested", + fork_transcript=True, + fork_before_message_id=child_tail.message_id, + ) + nested_entries = await manager.get_canonical_transcript(nested.session_key) + nested_manifest = build_attachment_manifest( + nested_entries, + session_id=nested.session_id, + session_key=nested.session_key, + ) + assert [item.attachment_id for item in nested_manifest.occurrences] == attachment_ids + assert nested_entries[0].message_id != child_entries[0].message_id + + +@pytest.mark.asyncio +async def test_full_fork_preserves_attachment_message_id_for_manifest_rebuild( + manager, +) -> None: + parent = await manager.create("agent:main:attachment-fork-parent") + await manager.append_message(parent.session_key, "user", "old request") + await manager.append_message(parent.session_key, "assistant", "old answer") + image_content = json.dumps( + { + "text": "inspect this image", + "attachments": [ + { + "attachment_id": "att_full_fork_image_123", + "type": "image/png", + "name": "fork.png", + "data": "Zm9yay1pbWFnZQ==", + } + ], + } + ) + parent_image = await manager.append_message( + parent.session_key, + "user", + image_content, + message_id="message-full-fork-image", + ) + assert await manager.persist_compaction_result( + parent.session_key, + "older parent context", + [{"role": "user", "content": image_content}], + compaction_id="cmp-parent-attachment-fork", + ) + parent_states = await manager.get_context_states(parent.session_key) + parent_manifest_state = next( + state + for state in parent_states + if state.state_kind == ATTACHMENT_MANIFEST_STATE_KIND + ) + [parent_occurrence] = attachment_manifest_from_context_state( + parent_manifest_state + ).occurrences + assert parent_occurrence.source_message_id == parent_image.message_id + + child = await manager.branch( + parent.session_key, + "agent:main:attachment-fork-child", + fork_transcript=True, + ) + [child_image] = await manager.get_transcript(child.session_key) + assert child_image.message_id == parent_image.message_id + await manager.append_message(child.session_key, "assistant", "child latest") + + assert await manager.persist_compaction_result( + child.session_key, + "child image context", + [{"role": "assistant", "content": "child latest"}], + compaction_id="cmp-child-attachment-fork", + ) + child_states = await manager.get_context_states(child.session_key) + child_manifest_state = next( + state + for state in child_states + if state.state_kind == ATTACHMENT_MANIFEST_STATE_KIND + ) + [child_occurrence] = attachment_manifest_from_context_state( + child_manifest_state + ).occurrences + assert child_occurrence.attachment_id == "att_full_fork_image_123" + assert child_occurrence.source_message_id == parent_image.message_id + + +@pytest.mark.asyncio +async def test_full_fork_preserves_legacy_attachment_id_from_compacted_archive( + manager, +) -> None: + parent = await manager.create("agent:main:legacy-attachment-fork-parent") + image_content = json.dumps( + { + "text": "inspect this legacy image", + "attachments": [ + { + "type": "image/png", + "name": "legacy-fork.png", + "data": "bGVnYWN5LWZvcms=", + } + ], + } + ) + parent_image = await manager.append_message( + parent.session_key, + "user", + image_content, + message_id="message-legacy-full-fork-image", + ) + await manager.append_message(parent.session_key, "assistant", "old answer") + await manager.append_message(parent.session_key, "user", "active tail") + parent_manifest = build_attachment_manifest( + await manager.get_canonical_transcript(parent.session_key), + session_id=parent.session_id, + session_key=parent.session_key, + ) + [parent_occurrence] = parent_manifest.occurrences + + assert await manager.persist_compaction_result( + parent.session_key, + f"legacy image attachment_id={parent_occurrence.attachment_id}", + [{"role": "user", "content": "active tail"}], + compaction_id="cmp-parent-legacy-attachment-fork", + ) + child = await manager.branch( + parent.session_key, + "agent:main:legacy-attachment-fork-child", + fork_transcript=True, + ) + + child_canonical = await manager.get_canonical_transcript(child.session_key) + child_image = next( + entry for entry in child_canonical if entry.message_id == parent_image.message_id + ) + assert child_image.message_id == parent_image.message_id + child_manifest = build_attachment_manifest( + child_canonical, + session_id=child.session_id, + session_key=child.session_key, + ) + + assert [item.attachment_id for item in child_manifest.occurrences] == [ + parent_occurrence.attachment_id + ] + child_states = await manager.get_context_states( + child.session_key, + provider="portable", + state_kind=ATTACHMENT_MANIFEST_STATE_KIND, + ) + copied_manifest = attachment_manifest_from_context_state( + max(child_states, key=lambda state: (state.created_at, state.id or 0)) + ) + assert copied_manifest.by_id(parent_occurrence.attachment_id) is not None + + await manager.append_message(child.session_key, "assistant", "child answer") + assert await manager.persist_compaction_result( + child.session_key, + f"legacy image attachment_id={parent_occurrence.attachment_id}", + [{"role": "assistant", "content": "child answer"}], + compaction_id="cmp-child-legacy-attachment-fork", + ) + compacted_child_states = await manager.get_context_states( + child.session_key, + provider="portable", + state_kind=ATTACHMENT_MANIFEST_STATE_KIND, + ) + compacted_child_manifest = attachment_manifest_from_context_state( + max( + compacted_child_states, + key=lambda state: (state.created_at, state.id or 0), + ) + ) + assert [ + ( + item.attachment_id, + item.source_message_id, + item.ordinal, + ) + for item in compacted_child_manifest.occurrences + ] == [ + ( + parent_occurrence.attachment_id, + parent_image.message_id, + 0, + ) + ] + + @pytest.mark.asyncio async def test_full_branch_preserves_incomplete_parent_compaction_evidence(manager): parent = await manager.create("agent:main:main") @@ -3511,7 +3839,26 @@ async def test_persist_compaction_result_rewrite_failure_keeps_session_state_ato monkeypatch: pytest.MonkeyPatch, ): node = await manager.create("agent:main:main") - for index in range(4): + await manager.append_message( + node.session_key, + "user", + json.dumps( + { + "text": "image before failed rewrite", + "attachments": [ + { + "attachment_id": "att_atomic_rollback_123", + "type": "image/png", + "name": "rollback.png", + "data": "aW1hZ2U=", + } + ], + } + ), + message_id="message-atomic-image", + token_count=5, + ) + for index in range(1, 4): await manager.append_message("agent:main:main", "user", f"msg {index}", token_count=5) original_transcript = await manager.get_transcript("agent:main:main") original_canonical_transcript = await manager.get_canonical_transcript("agent:main:main") @@ -3744,6 +4091,120 @@ async def test_persist_compaction_result_stores_summary_out_of_band(manager): assert states[0].payload["compaction_id"] == "cmp_inline_1" +@pytest.mark.asyncio +async def test_compaction_atomically_persists_attachment_manifest_with_summary( + manager, +) -> None: + node = await manager.create("agent:main:attachment-compaction") + image_data = "aW1hZ2UtYnl0ZXM=" + await manager.append_message( + node.session_key, + "user", + json.dumps( + { + "text": "inspect this image", + "attachments": [ + { + "attachment_id": "att_compaction_image_123", + "path": "/private/tmp/material/image.png", + "type": "image/png", + "name": "diagram.png", + "data": image_data, + } + ], + } + ), + message_id="message-image", + ) + await manager.append_message(node.session_key, "assistant", "old answer") + await manager.append_message(node.session_key, "user", "follow up") + await manager.append_message(node.session_key, "assistant", "latest reply") + + await manager.persist_compaction_result( + node.session_key, + "image discussion summary", + [{"role": "assistant", "content": "latest reply"}], + compaction_id="cmp-image", + ) + + active = await manager.get_transcript(node.session_key) + canonical = await manager.get_canonical_transcript(node.session_key) + states = await manager.get_context_states(node.session_key) + states_by_kind = {state.state_kind: state for state in states} + + assert [entry.content for entry in active] == ["latest reply"] + assert any(image_data in entry.content for entry in canonical) + assert "structured_summary_v1" in states_by_kind + assert ATTACHMENT_MANIFEST_STATE_KIND in states_by_kind + manifest = attachment_manifest_from_context_state( + states_by_kind[ATTACHMENT_MANIFEST_STATE_KIND] + ) + [occurrence] = manifest.occurrences + assert occurrence.attachment_id == "att_compaction_image_123" + assert occurrence.source_message_id == "message-image" + summaries = await manager.get_summaries(node.session_key) + assert len(summaries) == 1 + assert summaries[0].summary_payload is not None + assert summaries[0].summary_payload["files_and_artifacts"] == [] + assert occurrence.attachment_id in summaries[0].summary_payload[ + "important_identifiers" + ] + serialized_summary = json.dumps(summaries[0].summary_payload, sort_keys=True) + assert "/private/tmp" not in serialized_summary + assert image_data not in serialized_summary + serialized_manifest = json.dumps( + states_by_kind[ATTACHMENT_MANIFEST_STATE_KIND].payload, + sort_keys=True, + ) + assert image_data not in serialized_manifest + assert '"data"' not in serialized_manifest + assert '"path"' not in serialized_manifest + + +@pytest.mark.asyncio +async def test_attachment_manifest_build_failure_aborts_compaction_atomically( + manager, + monkeypatch: pytest.MonkeyPatch, +) -> None: + node = await manager.create("agent:main:attachment-compaction-failure") + for index in range(4): + await manager.append_message( + node.session_key, + "user" if index % 2 == 0 else "assistant", + f"message {index}", + ) + original_transcript = await manager.get_transcript(node.session_key) + original_canonical = await manager.get_canonical_transcript(node.session_key) + original_node = await manager.get_session(node.session_key) + + def fail_manifest(*args, **kwargs): # noqa: ANN002, ANN003 + del args, kwargs + raise ValueError("manifest collision") + + monkeypatch.setattr( + session_manager_module, + "_merge_attachment_manifest_state", + fail_manifest, + ) + + with pytest.raises(ValueError, match="manifest collision"): + await manager.persist_compaction_result( + node.session_key, + "summary must not commit", + [{"role": "assistant", "content": "message 3"}], + compaction_id="cmp-manifest-failure", + ) + + assert await manager.get_transcript(node.session_key) == original_transcript + assert await manager.get_canonical_transcript(node.session_key) == original_canonical + assert await manager.get_summaries(node.session_key) == [] + assert await manager.get_context_states(node.session_key) == [] + current_node = await manager.get_session(node.session_key) + assert current_node is not None + assert original_node is not None + assert current_node.compaction_count == original_node.compaction_count + + @pytest.mark.asyncio async def test_persist_compaction_result_preserves_structured_tail_metadata(manager): node = await manager.create("agent:main:structured-tail")