diff --git a/.github/agents/experimental/experiment-designer.agent.md b/.github/agents/experimental/experiment-designer.agent.md index ab60c30b0..a8ca5e075 100644 --- a/.github/agents/experimental/experiment-designer.agent.md +++ b/.github/agents/experimental/experiment-designer.agent.md @@ -1,11 +1,6 @@ --- name: Experiment Designer description: "Conversational coach that guides users through designing a Minimum Viable Experiment (MVE) with structured hypothesis formation, vetting, and experiment planning - Brought to you by microsoft/hve-core" -handoffs: - - label: "Compact" - agent: Experiment Designer - send: true - prompt: "/compact Make sure summarization includes that all state is managed through the .copilot-tracking folder files, be sure to include file paths for all of the current Tracking Artifacts. Be sure to include the user's problem statement, hypotheses, and vetting results. Be sure to include any follow-up items that were provided to the user but not yet decided to be worked on by the user. Be sure to include the user's specific original requirements and requests. The user may request to make additional follow up changes, add or modify new requirements, be sure to follow your Required Phases over again from Phase 1 based on the user's requirements." --- # Experiment Designer diff --git a/.github/agents/experimental/pptx.agent.md b/.github/agents/experimental/pptx.agent.md index c6575e96e..71cd8ff9a 100644 --- a/.github/agents/experimental/pptx.agent.md +++ b/.github/agents/experimental/pptx.agent.md @@ -5,11 +5,6 @@ disable-model-invocation: true agents: - Researcher Subagent - PowerPoint Subagent -handoffs: - - label: "Compact" - agent: PowerPoint Builder - send: true - prompt: "/compact Make sure summarization includes that all state is managed through the .copilot-tracking folder files, be sure to include file paths for all of the current Tracking Artifacts. Be sure to include any current analysis log artifacts. Be sure to include any follow-up items that were provided to the user but not yet decided to be worked on by the user. Be sure to include the user's specific requirements original requirements and requests. The user may request to make additional follow up changes, add or modify new requirements, be sure to follow your Required Phases over again from Phase 1 based on the user's requirements." --- # PowerPoint Builder diff --git a/.github/agents/hve-core/prompt-builder.agent.md b/.github/agents/hve-core/prompt-builder.agent.md index e4fb8f67d..4023ac63c 100644 --- a/.github/agents/hve-core/prompt-builder.agent.md +++ b/.github/agents/hve-core/prompt-builder.agent.md @@ -8,10 +8,6 @@ agents: - Prompt Updater - Researcher Subagent handoffs: - - label: "Compact" - agent: Prompt Builder - send: true - prompt: "/compact Make sure summarization includes that all state is managed through the .copilot-tracking folder files, be sure to include file paths for all of the current Tracking Artifacts. Be sure to include any current analysis log artifacts. Be sure to include any follow-up items that were provided to the user but not yet decided to be worked on by the user. Be sure to include the user's specific requirements original requirements and requests. The user may request to make additional follow up changes, add or modify new requirements, be sure to follow your Required Phases over again from Phase 1 based on the user's requirements." - label: "💡 Update/Create" agent: Prompt Builder prompt: "/prompt-build" @@ -62,6 +58,16 @@ Cross-run continuity: Subagents can read and reference files from prior sandbox * When using the `runSubagent` tool, select the named agent directly and provide the required inputs listed for that phase. * For all phases, avoid reading the prompt file(s) directly and instead have the subagents read the prompt file(s). +### Model Selection for Subagents + +Apply cost-first model selection: use a fast model for tasks that do not write or design prompts. + +* Researcher Subagent: specify `model: "Claude Haiku 4.5 (copilot)"` (read-only research). +* Prompt Evaluator: specify `model: "Claude Haiku 4.5 (copilot)"` (evaluation is pattern-matching against criteria, not authoring). +* Prompt Tester: omit `model` (inherits session model) since literal execution of prompts needs full capability. +* Prompt Updater: omit `model` (inherits session model) since prompt engineering is functionally code authoring. +* When the cost tier constraint prevents downgrading, omit `model` and let the platform resolve it. + ## Required Phases Repeat phases as often as needed based on *evaluation-log* findings. diff --git a/.github/agents/hve-core/rpi-agent.agent.md b/.github/agents/hve-core/rpi-agent.agent.md index db5b3b67c..f6ad761cb 100644 --- a/.github/agents/hve-core/rpi-agent.agent.md +++ b/.github/agents/hve-core/rpi-agent.agent.md @@ -7,10 +7,6 @@ agents: - Researcher Subagent - Phase Implementor handoffs: - - label: Compact - agent: RPI Agent - prompt: "/compact Make sure summarization includes that all state is managed through the .copilot-tracking folder files, always include file paths for all of the Tracking Artifacts for this session. Indicate percent complete for each of the artifacts. Include the last Phase before compaction, steps of phase completed, in-progress step of phase, remaining steps of phase. Be sure to include executive details for each of the `Phase 4: Review` most recent findings. Must include all of the most recent `Phase 5: Discover` follow up work items and their order with complete and consistent details." - send: true - label: "1️⃣" agent: RPI Agent prompt: "/rpi continue=1" diff --git a/.github/agents/hve-core/subagents/implementation-validator.agent.md b/.github/agents/hve-core/subagents/implementation-validator.agent.md index bcff5f27f..467a00d54 100644 --- a/.github/agents/hve-core/subagents/implementation-validator.agent.md +++ b/.github/agents/hve-core/subagents/implementation-validator.agent.md @@ -5,6 +5,9 @@ user-invocable: false tools: - read - search +model: + - Claude Haiku 4.5 (copilot) + - GPT-5.4 mini (copilot) --- # Implementation Validator diff --git a/.github/agents/hve-core/subagents/plan-validator.agent.md b/.github/agents/hve-core/subagents/plan-validator.agent.md index 16422c1fb..4a67dc843 100644 --- a/.github/agents/hve-core/subagents/plan-validator.agent.md +++ b/.github/agents/hve-core/subagents/plan-validator.agent.md @@ -2,6 +2,9 @@ name: Plan Validator description: 'Validates implementation plans against research documents, updating the Planning Log Discrepancy Log section with severity-graded findings - Brought to you by microsoft/hve-core' user-invocable: false +model: + - Claude Haiku 4.5 (copilot) + - GPT-5.4 mini (copilot) --- # Plan Validator diff --git a/.github/agents/hve-core/subagents/prompt-evaluator.agent.md b/.github/agents/hve-core/subagents/prompt-evaluator.agent.md index 30f8fcf77..a65fe31ad 100644 --- a/.github/agents/hve-core/subagents/prompt-evaluator.agent.md +++ b/.github/agents/hve-core/subagents/prompt-evaluator.agent.md @@ -2,6 +2,9 @@ name: Prompt Evaluator description: 'Evaluates prompt execution results against Prompt Quality Criteria with severity-graded findings and categorized remediation guidance' user-invocable: false +model: + - Claude Haiku 4.5 (copilot) + - GPT-5.4 mini (copilot) --- # Prompt Evaluator diff --git a/.github/agents/hve-core/subagents/researcher-subagent.agent.md b/.github/agents/hve-core/subagents/researcher-subagent.agent.md index b6b6554c0..d94ae0165 100644 --- a/.github/agents/hve-core/subagents/researcher-subagent.agent.md +++ b/.github/agents/hve-core/subagents/researcher-subagent.agent.md @@ -2,6 +2,9 @@ name: Researcher Subagent description: 'Research subagent using search tools, read tools, fetch web page, github repo, and mcp tools' user-invocable: false +model: + - Claude Haiku 4.5 (copilot) + - GPT-5.4 mini (copilot) --- # Researcher Subagent diff --git a/.github/agents/hve-core/subagents/rpi-validator.agent.md b/.github/agents/hve-core/subagents/rpi-validator.agent.md index b77616383..e4c5d8ed2 100644 --- a/.github/agents/hve-core/subagents/rpi-validator.agent.md +++ b/.github/agents/hve-core/subagents/rpi-validator.agent.md @@ -2,6 +2,9 @@ name: RPI Validator description: 'Validates a Changes Log against the Implementation Plan, Planning Log, and Research Documents for a specific plan phase - Brought to you by microsoft/hve-core' user-invocable: false +model: + - Claude Haiku 4.5 (copilot) + - GPT-5.4 mini (copilot) --- # RPI Validator diff --git a/.github/agents/hve-core/task-challenger.agent.md b/.github/agents/hve-core/task-challenger.agent.md index 97f9cb5da..c90fdcd2b 100644 --- a/.github/agents/hve-core/task-challenger.agent.md +++ b/.github/agents/hve-core/task-challenger.agent.md @@ -4,10 +4,6 @@ description: 'Adversarial questioning agent that interrogates implementations wi disable-model-invocation: true tools: [read, search, edit/createFile, edit/editFiles, execute/runInTerminal, execute/getTerminalOutput] handoffs: - - label: "Compact" - agent: Task Challenger - send: true - prompt: "/compact Preserve the current challenge session state. Include the confirmed scope, all questions asked with the user's complete answers, any probe questions and responses, and all items marked unresolved. The challenge tracking document in .copilot-tracking/challenges/ contains the session record: reference the most recent document by date. When resuming, continue from the last question asked." - label: "🔬 Research Questions" agent: Task Researcher prompt: "/task-research Find and read the most recent challenge tracking document in .copilot-tracking/challenges/ (most recent by date prefix) for the Q&A log and unresolved items: these define the research scope." diff --git a/.github/agents/hve-core/task-implementor.agent.md b/.github/agents/hve-core/task-implementor.agent.md index 388d76831..65a08413e 100644 --- a/.github/agents/hve-core/task-implementor.agent.md +++ b/.github/agents/hve-core/task-implementor.agent.md @@ -6,10 +6,6 @@ agents: - Phase Implementor - Researcher Subagent handoffs: - - label: "Compact" - agent: Task Implementor - send: true - prompt: "/compact Make sure summarization includes that all state is managed through the .copilot-tracking folder files, and be sure to include that the next agent instructions will be Task Reviewer and the user will switch to it when they are done with Task Implementation" - label: "✅ Review" agent: Task Reviewer prompt: /task-review @@ -56,6 +52,15 @@ The researcher-subagent returns deep research findings: subagent research docume Subagents can run in parallel when investigating independent topics or executing independent phases. +### Model Selection for Subagents + +Apply cost-first model selection: use a fast model for tasks that do not write code, and inherit the session model for code generation. + +* Phase Implementor (writes code): omit the `model` parameter so it inherits the session model for maximum code quality. +* Researcher Subagent (read-only research): specify `model: "Claude Haiku 4.5 (copilot)"` to reduce cost. +* If a research task requires deep code-level analysis: omit `model` to inherit the session model. +* When the cost tier constraint prevents downgrading below the session model, omit `model` and let the platform resolve it. + ## Required Artifacts | Artifact | Path Pattern | Required | diff --git a/.github/agents/hve-core/task-planner.agent.md b/.github/agents/hve-core/task-planner.agent.md index 2de2dd648..12eb9e54c 100644 --- a/.github/agents/hve-core/task-planner.agent.md +++ b/.github/agents/hve-core/task-planner.agent.md @@ -6,10 +6,6 @@ agents: - Researcher Subagent - Plan Validator handoffs: - - label: "Compact" - agent: Task Planner - send: true - prompt: "/compact make sure summarization includes that all state is managed through the .copilot-tracking folder files, and be sure to include that the next agent instructions will be Task Implementor and the user will switch to it when they are done with Task Planner" - label: "⚡ Implement" agent: Task Implementor prompt: /task-implement @@ -58,6 +54,15 @@ Run `Plan Validator` using `runSubagent` or `task`, providing these inputs: Subagents can run in parallel when investigating independent topics or validating independent concerns. +### Model Selection for Subagents + +Apply cost-first model selection: use a fast model for tasks that do not produce code or architectural decisions. + +* Researcher Subagent (read-only research): specify `model: "Claude Haiku 4.5 (copilot)"` to reduce cost. +* Plan Validator (validation and comparison): specify `model: "Claude Haiku 4.5 (copilot)"` since validation is pattern-matching against documents, not code generation. +* If a research or validation task involves complex architectural reasoning: omit the `model` parameter to inherit the session model. +* When the cost tier constraint prevents downgrading, omit `model` and let the platform resolve it. + ## File Locations Planning files reside in `.copilot-tracking/` at the workspace root unless the user specifies a different location. diff --git a/.github/agents/hve-core/task-researcher.agent.md b/.github/agents/hve-core/task-researcher.agent.md index 3739a0338..85eb53f24 100644 --- a/.github/agents/hve-core/task-researcher.agent.md +++ b/.github/agents/hve-core/task-researcher.agent.md @@ -5,10 +5,6 @@ disable-model-invocation: true agents: - Researcher Subagent handoffs: - - label: "Compact" - agent: Task Researcher - send: true - prompt: "/compact make sure summarization includes that all state is managed through the .copilot-tracking folder files, and be sure to include that the next agent instructions will be Task Planner and the user will switch to it when they are done with Task Researcher" - label: "📋 Create Plan" agent: Task Planner prompt: /task-plan @@ -50,6 +46,14 @@ Run `Researcher Subagent` with `runSubagent` or `task`, and parallelize calls wh Subagents can run in parallel when investigating independent topics or sources. +### Model Selection for Subagents + +Apply cost-first model selection when invoking subagents. Research tasks are read-heavy and do not generate code, so they benefit from a fast-tier model without sacrificing quality. + +* Research subagent calls: specify `model: "Claude Haiku 4.5 (copilot)"` on the `runSubagent` invocation to reduce cost. +* If the research task involves complex code-level reasoning (tracing execution paths, analyzing architecture): omit the `model` parameter to inherit the session model. +* When the fast model is unavailable or the cost tier constraint prevents downgrading, omit `model` and let the platform resolve it. + ## File Locations Research files reside in `.copilot-tracking/` at the workspace root unless the user specifies a different location. diff --git a/.github/agents/hve-core/task-reviewer.agent.md b/.github/agents/hve-core/task-reviewer.agent.md index b1675147a..40bcb9474 100644 --- a/.github/agents/hve-core/task-reviewer.agent.md +++ b/.github/agents/hve-core/task-reviewer.agent.md @@ -7,10 +7,6 @@ agents: - Researcher Subagent - Implementation Validator handoffs: - - label: "Compact" - agent: Task Reviewer - send: true - prompt: "/compact Make sure summarization includes that all state is managed through the .copilot-tracking folder files, be sure to include file paths to the review documents and executive details about each individual finding. Be sure to include that the next agent instructions will be one-of Task Researcher for deeper research on the chosen findings to address, Task Planner to go right into planning based off of the chosen findings from the review document, or right back into implementation addressing the chosen findings from the review document. The user will switch to the agent instructions when they are done with Task Review." - label: "🔬 Research More" agent: Task Researcher prompt: /task-research @@ -106,6 +102,15 @@ Read the validation files produced by each `RPI Validator` run. Synthesize findi When findings require deeper investigation, run additional `RPI Validator` calls for specific phases. Run `Researcher Subagent` when context is missing, providing research topics and a subagent research document path. +#### Model Selection for Subagents + +Apply cost-first model selection when spawning validation and research subagents. + +* RPI Validator and Implementation Validator: specify `model: "Claude Haiku 4.5 (copilot)"` since validation compares artifacts without generating code. +* Researcher Subagent: specify `model: "Claude Haiku 4.5 (copilot)"` for read-only research. +* If validation requires complex code reasoning or architectural judgment: omit `model` to inherit the session model. +* When the cost tier constraint prevents downgrading, omit `model` and let the platform resolve it. + Proceed to Phase 3 when RPI validation is complete. ### Phase 3: Quality Validation diff --git a/.github/agents/rai-planning/rai-planner.agent.md b/.github/agents/rai-planning/rai-planner.agent.md index 07ef14e19..2c5e74dfd 100644 --- a/.github/agents/rai-planning/rai-planner.agent.md +++ b/.github/agents/rai-planning/rai-planner.agent.md @@ -8,10 +8,6 @@ handoffs: agent: Security Planner prompt: /security-capture send: true - - label: "Compact" - agent: RAI Planner - send: true - prompt: "/compact Make sure summarization includes that all state is managed through .copilot-tracking/rai-plans/ folder files, and be sure to include the current phase, entry mode, and project slug" tools: - read - edit/createFile diff --git a/.github/agents/security/security-planner.agent.md b/.github/agents/security/security-planner.agent.md index e74d110ff..a69cf11d2 100644 --- a/.github/agents/security/security-planner.agent.md +++ b/.github/agents/security/security-planner.agent.md @@ -14,10 +14,6 @@ tools: - web - agent handoffs: - - label: "Compact" - agent: Security Planner - send: true - prompt: "/compact make sure summarization includes that all state is managed through the .copilot-tracking folder files, and be sure to include the current security planning phase and project slug" - label: "RAI Planner" agent: RAI Planner prompt: /rai-plan-from-security-plan diff --git a/.github/agents/security/security-reviewer.agent.md b/.github/agents/security/security-reviewer.agent.md index 4296a492c..f38e10148 100644 --- a/.github/agents/security/security-reviewer.agent.md +++ b/.github/agents/security/security-reviewer.agent.md @@ -107,6 +107,16 @@ Skill resolution: Read the applicable security skill (e.g., `owasp-top-10`, `owa | Report Generator | `.github/agents/**/report-generator.agent.md` | Collates all verified findings and generates the final vulnerability report. | | Skill Assessor | `.github/agents/**/skill-assessor.agent.md` | Assesses a single skill against the codebase, returning structured findings. | +### Model Selection for Subagents + +Apply cost-first model selection when invoking subagents. Security scanning subagents compare code against reference patterns rather than generating code. + +* Codebase Profiler: specify `model: "Claude Haiku 4.5 (copilot)"` (read-only scanning and classification). +* Skill Assessor: specify `model: "Claude Haiku 4.5 (copilot)"` (pattern matching against vulnerability references). +* Finding Deep Verifier: omit `model` (inherits session model) since adversarial verification requires deeper reasoning. +* Report Generator: specify `model: "Claude Haiku 4.5 (copilot)"` (collation and formatting, not analysis). +* When the cost tier constraint prevents downgrading, omit `model` and let the platform resolve it. + ### Available Skills * owasp-agentic diff --git a/.github/agents/security/sssc-planner.agent.md b/.github/agents/security/sssc-planner.agent.md index 346388e0e..ef881d980 100644 --- a/.github/agents/security/sssc-planner.agent.md +++ b/.github/agents/security/sssc-planner.agent.md @@ -8,10 +8,6 @@ description: >- agents: - Researcher Subagent handoffs: - - label: "Compact" - agent: SSSC Planner - send: true - prompt: "/compact Make sure summarization includes that all state is managed through .copilot-tracking/sssc-plans/ folder files, and be sure to include the current phase, entry mode, and project slug" - label: "Security Planner" agent: Security Planner prompt: /security-capture diff --git a/.github/agents/security/subagents/codebase-profiler.agent.md b/.github/agents/security/subagents/codebase-profiler.agent.md index ff684b740..296e428f3 100644 --- a/.github/agents/security/subagents/codebase-profiler.agent.md +++ b/.github/agents/security/subagents/codebase-profiler.agent.md @@ -8,6 +8,9 @@ tools: - search/textSearch - read/readFile user-invocable: false +model: + - Claude Haiku 4.5 (copilot) + - GPT-5.4 mini (copilot) --- # Codebase Profiler diff --git a/.github/agents/security/subagents/report-generator.agent.md b/.github/agents/security/subagents/report-generator.agent.md index 63e0872b5..091f3813c 100644 --- a/.github/agents/security/subagents/report-generator.agent.md +++ b/.github/agents/security/subagents/report-generator.agent.md @@ -7,6 +7,9 @@ tools: - search/fileSearch - read/readFile user-invocable: false +model: + - Claude Haiku 4.5 (copilot) + - GPT-5.4 mini (copilot) --- # Report Generator diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 9758222e0..a65e9a7d7 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -52,7 +52,7 @@ Scripts are organized by function: * Collections (`scripts/collections/`) - Collection validation and shared helper modules. * Extension (`scripts/extension/`) - Extension packaging and preparation. -* Linting (`scripts/linting/`) - Markdown validation, link checking, frontmatter validation, and PowerShell analysis. +* Linting (`scripts/linting/`) - Markdown validation, link checking, frontmatter validation, model reference validation, and PowerShell analysis. * Security (`scripts/security/`) - Dependency pinning validation, SHA staleness checks, and action version consistency. * Library (`scripts/lib/`) - Shared utilities such as verified downloads. * Plugins (`scripts/plugins/`) - Plugin generation and marketplace validation. @@ -207,7 +207,9 @@ Agents should use npm scripts for all validation: * `npm run lint:version-consistency` - Action version consistency * `npm run lint:marketplace` - Marketplace validation * `npm run lint:py` - Python linting via ruff -* `npm run lint:all` - Run all linters (chains `format:tables`, `lint:md`, `lint:ps`, `lint:yaml`, `lint:links`, `lint:frontmatter`, `lint:collections-metadata`, `lint:marketplace`, `lint:version-consistency`, `lint:permissions`, `lint:dependency-pinning`, `lint:py`, and `validate:skills`) +* `npm run lint:models` - Model reference validation against catalog +* `npm run lint:models:refresh` - Refresh model catalog from upstream documentation +* `npm run lint:all` - Run all linters (chains `format:tables`, `lint:md`, `lint:ps`, `lint:yaml`, `lint:links`, `lint:frontmatter`, `lint:collections-metadata`, `lint:marketplace`, `lint:version-consistency`, `lint:permissions`, `lint:dependency-pinning`, `lint:py`, `validate:skills`, `lint:ai-artifacts`, and `lint:models`) * `npm run validate:copyright` - Copyright header validation * `npm run validate:skills` - Skill structure validation * `npm run spell-check` - Spelling validation diff --git a/.github/instructions/hve-core/prompt-builder.instructions.md b/.github/instructions/hve-core/prompt-builder.instructions.md index e97bd058d..fb052a0b5 100644 --- a/.github/instructions/hve-core/prompt-builder.instructions.md +++ b/.github/instructions/hve-core/prompt-builder.instructions.md @@ -412,7 +412,7 @@ Optional fields available by file type: * `disable-model-invocation:` - Boolean. Set to `true` to prevent Copilot from automatically invoking the agent. Use for agents that run subagents, agents that cause side effects (git operations, backlog management, deployments), or agents that should only run when explicitly requested. Defaults to `false` when omitted. * `agent:` - Agent delegation for prompt files and handoffs. Use the human-readable name from the agent's `name:` frontmatter (for example, `Prompt Builder`). * `argument-hint:` - Hint text for prompt picker display. -* `model:` - Model specification. Accepts any valid model identifier string (for example, `gpt-4o`, `claude-sonnet-4`). When omitted, the default model is used. +* `model:` - Model specification. For **agent files**, accepts a single model identifier string (for example, `claude-sonnet-4`) or a prioritized array of model identifiers; when an array is specified, the system tries each model in order until an available one is found. For **prompt files**, accepts a single model identifier string only. When omitted, the currently selected model in the model picker is used. * `license:` - SPDX license identifier for skill content (for example, `MIT`, `CC-BY-SA-4.0`). Defaults to the repository license when omitted. Use for skills that incorporate third-party content under a specific license. * `metadata:` - Object containing provenance and versioning metadata for skills. Recognized fields include `authors`, `spec_version`, `framework_revision`, `last_updated`, `skill_based_on`, and `content_based_on`. diff --git a/.github/instructions/rai-planning/rai-identity.instructions.md b/.github/instructions/rai-planning/rai-identity.instructions.md index a0a9d209f..29ea412ac 100644 --- a/.github/instructions/rai-planning/rai-identity.instructions.md +++ b/.github/instructions/rai-planning/rai-identity.instructions.md @@ -336,7 +336,6 @@ Display the disclaimer blockquote and attribution notices to the user at the beg Re-display the disclaimer blockquote to the user at every session exit point. Exit points include: * **Phase 6 completion**: After presenting the final review summary and before creating any backlog work items. -* **Compact handoff**: Before compacting the conversation context. * **Error exit**: When an unrecoverable error terminates the session early. * **User-initiated exit**: When the user ends the session before completing all phases. diff --git a/.github/prompts/github/github-add-issue.prompt.md b/.github/prompts/github/github-add-issue.prompt.md index b9c6583de..b8beeb428 100644 --- a/.github/prompts/github/github-add-issue.prompt.md +++ b/.github/prompts/github/github-add-issue.prompt.md @@ -2,6 +2,7 @@ description: 'Create a GitHub issue using discovered repository templates and conversational field collection' agent: GitHub Backlog Manager argument-hint: "[templateName=...] [title=...] [labels=...]" +model: Claude Haiku 4.5 (copilot) --- # Add GitHub Issue diff --git a/.github/prompts/github/github-discover-issues.prompt.md b/.github/prompts/github/github-discover-issues.prompt.md index 1c4e7847b..af0cf12ce 100644 --- a/.github/prompts/github/github-discover-issues.prompt.md +++ b/.github/prompts/github/github-discover-issues.prompt.md @@ -2,6 +2,7 @@ description: 'Discover GitHub issues through user-centric queries, artifact-driven analysis, or search-based exploration and produce planning files for review' agent: GitHub Backlog Manager argument-hint: "documents=... [milestone=...] [searchTerms=...]" +model: Claude Haiku 4.5 (copilot) --- # Discover GitHub Issues diff --git a/.github/prompts/github/github-triage-issues.prompt.md b/.github/prompts/github/github-triage-issues.prompt.md index 2b37a13db..7fad3a643 100644 --- a/.github/prompts/github/github-triage-issues.prompt.md +++ b/.github/prompts/github/github-triage-issues.prompt.md @@ -1,6 +1,7 @@ --- description: 'Triage GitHub issues not yet triaged with automated label suggestions, milestone assignment, and duplicate detection' agent: GitHub Backlog Manager +model: Claude Haiku 4.5 (copilot) --- # Triage GitHub Issues diff --git a/.github/prompts/hve-core/checkpoint.prompt.md b/.github/prompts/hve-core/checkpoint.prompt.md index db12f8f40..69d13db83 100644 --- a/.github/prompts/hve-core/checkpoint.prompt.md +++ b/.github/prompts/hve-core/checkpoint.prompt.md @@ -2,6 +2,7 @@ description: "Save or restore conversation context using memory files - Brought to you by microsoft/hve-core" agent: Memory argument-hint: "[mode={save|continue|incremental}] [description=...]" +model: Claude Haiku 4.5 (copilot) --- # Checkpoint diff --git a/.github/prompts/hve-core/git-commit-message.prompt.md b/.github/prompts/hve-core/git-commit-message.prompt.md index 6db69294a..40e00f5eb 100644 --- a/.github/prompts/hve-core/git-commit-message.prompt.md +++ b/.github/prompts/hve-core/git-commit-message.prompt.md @@ -1,6 +1,7 @@ --- agent: 'agent' description: 'Generates a commit message following the commit-message.instructions.md rules based on all changes in the branch' +model: Claude Haiku 4.5 (copilot) --- # Generate Commit Message diff --git a/.github/prompts/hve-core/git-commit.prompt.md b/.github/prompts/hve-core/git-commit.prompt.md index 7a9ec29fe..aed6ff12f 100644 --- a/.github/prompts/hve-core/git-commit.prompt.md +++ b/.github/prompts/hve-core/git-commit.prompt.md @@ -1,6 +1,7 @@ --- agent: 'agent' description: 'Stages all changes, generates a conventional commit message, shows it to the user, and commits using only git add/commit' +model: Claude Haiku 4.5 (copilot) --- # Stage, Generate, and Commit diff --git a/.github/prompts/hve-core/git-setup.prompt.md b/.github/prompts/hve-core/git-setup.prompt.md index 9e8b12332..590c943ea 100644 --- a/.github/prompts/hve-core/git-setup.prompt.md +++ b/.github/prompts/hve-core/git-setup.prompt.md @@ -1,6 +1,7 @@ --- agent: 'agent' description: 'Interactive, verification-first Git configuration assistant (non-destructive)' +model: Claude Haiku 4.5 (copilot) --- # Git Environment Setup (Verification-First) diff --git a/.github/workflows/model-validation.yml b/.github/workflows/model-validation.yml new file mode 100644 index 000000000..d817cdb84 --- /dev/null +++ b/.github/workflows/model-validation.yml @@ -0,0 +1,86 @@ +name: Model Reference Validation + +on: + schedule: + # Weekly scan: Wednesdays at 08:00 UTC + - cron: '0 8 * * 3' + pull_request: + paths: + - '.github/agents/**/*.agent.md' + - '.github/prompts/**/*.prompt.md' + - 'scripts/linting/model-catalog.json' + - 'scripts/linting/Test-ModelReferences.ps1' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + validate-models: + name: Validate Model References + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install PowerShell-Yaml + shell: pwsh + run: | + if (-not (Get-Module -ListAvailable -Name PowerShell-Yaml | Where-Object { $_.Version -eq '0.4.7' })) { + Install-Module -Name PowerShell-Yaml -RequiredVersion 0.4.7 -Force -Scope CurrentUser + } + + - name: Refresh model catalog from GitHub docs + shell: pwsh + run: | + & scripts/linting/Update-ModelCatalog.ps1 -CatalogPath scripts/linting/model-catalog.json + + - name: Detect catalog drift from committed version + shell: bash + run: | + if ! git diff --quiet scripts/linting/model-catalog.json; then + echo "::warning::model-catalog.json differs from committed version after upstream refresh. Run 'npm run lint:models:refresh' locally and commit the updated catalog." + echo "### Catalog drift detected" >> "$GITHUB_STEP_SUMMARY" + git diff --stat scripts/linting/model-catalog.json >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Validate model references against catalog + id: validate + shell: pwsh + run: | + New-Item -ItemType Directory -Force -Path logs | Out-Null + & scripts/linting/Test-ModelReferences.ps1 -OutputPath logs/model-validation-results.json + + - name: Upload results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: model-validation-results + path: | + logs/model-validation-results.json + retention-days: 30 + if-no-files-found: ignore + + - name: Report catalog drift + if: always() + shell: pwsh + run: | + $catalog = Get-Content -Path scripts/linting/model-catalog.json -Raw | ConvertFrom-Json + $retiring = @($catalog.models | Where-Object { $_.status -eq 'retiring' }) + if ($retiring.Count -gt 0) { + Write-Host "Models marked as retiring:" -ForegroundColor Yellow + foreach ($m in $retiring) { + Write-Host " - $($m.name) (retiring: $($m.retiredDate))" -ForegroundColor Yellow + } + echo "::warning::$($retiring.Count) model(s) are marked as retiring. Update agent/prompt references." + } else { + Write-Host "No models are retiring. All references current." -ForegroundColor Green + } diff --git a/docs/contributing/ai-artifacts-common.md b/docs/contributing/ai-artifacts-common.md index a083dccec..4d44d0add 100644 --- a/docs/contributing/ai-artifacts-common.md +++ b/docs/contributing/ai-artifacts-common.md @@ -79,19 +79,65 @@ Focus on agents that: All AI artifacts (agents, instructions, prompts) **MUST** target the **latest available models** from Anthropic and OpenAI only. +The model catalog (`scripts/linting/model-catalog.json`) contains the full list of models available in GitHub Copilot for validation purposes, but not all cataloged models are accepted for use in hve-core artifacts, per above. + ### Accepted Models -| Provider | Models | -|-----------|-------------------------------------------------------------| -| Anthropic | Latest Claude models (e.g., Claude Sonnet 4, Claude Opus 4) | -| OpenAI | Latest GPT models (e.g., GPT-5, 5.1-COdEX) | +| Provider | Models | +|-----------|-----------------------------------------------------------------| +| Anthropic | Latest Claude models (e.g., Claude Sonnet 4.6, Claude Opus 4.6) | +| OpenAI | Latest GPT models (e.g., GPT-5.4, GPT-5.3-Codex) | ### Not Accepted -* ❌ Older model versions (e.g., GPT-4o, Claude 4) * ❌ Models from other providers +* ❌ Older model versions not in the catalog (e.g., GPT-4o, Claude 3.5) * ❌ Custom or fine-tuned models -* ❌ Deprecated model versions +* ❌ Deprecated or retired model versions + +### Model Name Format + +Model references in frontmatter use the VS Code display name with vendor suffix: + +```yaml +# Single model +model: Claude Haiku 4.5 (copilot) + +# Prioritized fallback array (system tries each in order) +model: + - Claude Haiku 4.5 (copilot) + - GPT-5.4 mini (copilot) +``` + +The `(copilot)` suffix is required. Run `npm run lint:models` to validate all model references against the catalog. + +### Model Selection (Optional) + +The `model` frontmatter property is **optional**. When omitted, the agent or prompt inherits the user's session model (whatever is selected in the VS Code model picker). + +Use explicit model selection for cost optimization: + +| Tier | Multiplier | Use When | Example Models | +|----------|-------------|---------------------------------------------------------|--------------------------------| +| Fast | 0.25x–0.33x | Read-only research, mechanical file ops, classification | Claude Haiku 4.5, GPT-5.4 mini | +| Standard | 1x | Code generation, architecture, complex synthesis | Claude Sonnet 4.6, GPT-5.4 | +| Premium | 3x–15x | Vision-capable tasks, complex architectural decisions | Claude Opus 4.6, GPT-5.5 | + +### Cost Tier Constraint + +The `model` property is a **preference hint**, not a hard constraint. VS Code never fails a prompt or agent invocation due to model unavailability. When a specified model is unavailable or exceeds the cost tier of the parent model, VS Code falls back through the array entries in order, then to the session model (model picker selection). + +VS Code enforces that subagent models cannot exceed the cost tier of the parent model. If the user selects Sonnet (standard) in the model picker, subagents can use Haiku (fast) but not Opus (premium). Fallback arrays provide resilience when the preferred model is unavailable or exceeds the cost tier. A single-model string is equally safe: it falls back to the session model when the specified model cannot be used. + +### Model Catalog Validation + +Model references are validated against `scripts/linting/model-catalog.json` by the `lint:models` script. A scheduled GitHub Actions workflow (`model-validation.yml`) runs weekly to detect catalog drift and retiring models. + +To refresh the catalog from upstream documentation: + +```bash +npm run lint:models:refresh +``` ### Rationale @@ -99,6 +145,7 @@ All AI artifacts (agents, instructions, prompts) **MUST** target the **latest av 2. Maintenance burden: supporting multiple model versions creates testing and compatibility overhead 3. Performance: latest models provide superior reasoning, accuracy, and efficiency 4. Future-proofing: older models will be deprecated and removed from service +5. Cost optimization: fast-tier models reduce consumption for tasks that do not require premium reasoning ## Collections @@ -689,6 +736,9 @@ npm run spell-check # Validate all links npm run lint:md-links +# Validate model references in agent/prompt frontmatter +npm run lint:models + # PowerShell analysis (if applicable) npm run lint:ps diff --git a/docs/contributing/custom-agents.md b/docs/contributing/custom-agents.md index e9f34a22d..39e71c714 100644 --- a/docs/contributing/custom-agents.md +++ b/docs/contributing/custom-agents.md @@ -93,13 +93,31 @@ Focus on agents that: ### Model Version Requirements -All agents **MUST** target the **latest available models** from **Anthropic and OpenAI only**. +All agents **MUST** target the **latest available models** from Anthropic and OpenAI only. The model catalog (`scripts/linting/model-catalog.json`) contains the full list of models available in GitHub Copilot, but hve-core restricts usage to Anthropic and OpenAI. -Accepted: Latest Claude models (e.g., Claude Sonnet 4, Claude Opus 4) and latest GPT models (e.g., GPT-5.1, o1) +Accepted: Latest Claude and GPT models with `(copilot)` suffix (e.g., `Claude Sonnet 4.6 (copilot)`, `GPT-5.4 (copilot)`) -Not Accepted: Older model versions (e.g., GPT-3.5, GPT-4.1, Claude 2), models from other providers, custom/fine-tuned models +Not Accepted: Models from other providers, older model versions not in the catalog, custom/fine-tuned models, deprecated versions -Rationale: Latest models provide superior capabilities, reduce maintenance burden, and ensure future compatibility. Older model versions will be deprecated. +### Model Selection for Subagents + +The `model` frontmatter property is **optional**. When omitted, the agent inherits the parent conversation model. Use explicit model selection for cost optimization on subagents that perform read-only or validation tasks: + +```yaml +# Subagent that does research (read-only) — use fast-tier model +model: + - Claude Haiku 4.5 (copilot) + - GPT-5.4 mini (copilot) +``` + +```yaml +# Subagent that writes code — omit model to inherit session model +# (no model property) +``` + +Parent agents can also pass `model` dynamically on `runSubagent` calls via instructions in the agent body. The cost tier constraint means subagent models cannot exceed the parent model's tier. + +Run `npm run lint:models` to validate model references against the catalog. ## File Structure Requirements diff --git a/docs/contributing/prompts.md b/docs/contributing/prompts.md index 045084473..0f93b5197 100644 --- a/docs/contributing/prompts.md +++ b/docs/contributing/prompts.md @@ -89,14 +89,20 @@ Prompt files MUST: | Style | Keep hints concise; lead with required arguments | | Example | `"project=... [type={Epic\|Feature\|UserStory\|Bug\|Task}] [title=...]"` | -**`model`** (string) - -| Property | Value | -|----------|-----------------------------------------------------------------------------------| -| Purpose | Specifies a preferred AI model for prompt invocation | -| Format | Model identifier string | -| Style | Use the model's canonical identifier; omit if the workspace default is acceptable | -| Example | `gpt-4o` | +**`model`** (string or array of strings) + +| Property | Value | +|----------|------------------------------------------------------------------------------------------------| +| Purpose | Specifies a preferred AI model for prompt invocation (cost optimization) | +| Format | Model display name with `(copilot)` suffix, or prioritized array for fallback | +| Style | Use names from `scripts/linting/model-catalog.json`; omit if the session default is acceptable | +| Example | `Claude Haiku 4.5 (copilot)` | + +Use `model` on prompts that perform mechanical operations (git commits, issue creation, file I/O) rather than complex reasoning or code generation. + +The `model` property is a **preference hint**, not a hard requirement. When the specified model is unavailable or exceeds the user's session model cost tier, VS Code falls back through the array (if specified) then to the session model. A single-model string is safe: it never causes failure. Fallback arrays add resilience when cost-tier constraints may make the primary model unavailable. + +Run `npm run lint:models` to validate model references against the catalog. **`disable-model-invocation`** (boolean) diff --git a/docs/rpi/context-engineering.md b/docs/rpi/context-engineering.md index d8b7bb722..5aea057f5 100644 --- a/docs/rpi/context-engineering.md +++ b/docs/rpi/context-engineering.md @@ -88,11 +88,12 @@ The `/task-*` prompts attempt to auto-discover recent artifacts in `.copilot-tra `/compact` takes a different approach. Instead of removing conversation history entirely, it summarizes the history into a condensed form that preserves key context while reducing the token count. +`/compact` remains available as a typed command but is no longer offered as an agent handoff button. It was removed from agent handoffs because Autopilot mode could trigger compaction loops that degraded context unpredictably. + When to use `/compact`: * Mid-phase, when a conversation grows long but you need to continue the current task * When you want to retain awareness of prior decisions without carrying the full token weight -* When handoff buttons between phases embed transition context into the summary prompt When to use `/clear` instead: @@ -100,13 +101,16 @@ When to use `/clear` instead: * When switching to a different task entirely * When agent behavior has visibly degraded +For session persistence across phases, use the Memory Agent (`/checkpoint`) instead of relying on `/compact`. The Memory Agent writes structured state to disk, making context recovery deterministic rather than dependent on summarization quality. + The tradeoff is precision. `/compact` summaries lose detail because the model decides what to keep and what to discard. Critical nuances from earlier in the conversation may not survive the summarization. -| Command | Effect | Use When | -|------------|------------------------------------|--------------------------------------| -| `/clear` | Removes all conversation history | Between phases, switching tasks | -| `/compact` | Summarizes history, reduces tokens | Mid-phase, conversation growing long | -| New chat | Fresh conversation, new context | Starting unrelated work | +| Command | Effect | Use When | +|---------------|------------------------------------|--------------------------------------| +| `/clear` | Removes all conversation history | Between phases, switching tasks | +| `/compact` | Summarizes history, reduces tokens | Mid-phase, conversation growing long | +| `/checkpoint` | Persists state to disk | Between sessions, preserving context | +| New chat | Fresh conversation, new context | Starting unrelated work | ## The rpi-agent Difference diff --git a/package.json b/package.json index 1133610f5..c0afc20dc 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,9 @@ "audit:npm": "audit-ci --config audit-ci.json", "rai:sign": "pwsh -NoProfile -File scripts/security/Sign-PlannerArtifacts.ps1", "lint:ai-artifacts": "pwsh -NoProfile -Command \"& './scripts/linting/Validate-PlannerArtifacts.ps1' -FailOnMissing\"", - "lint:all": "npm run format:tables && npm run lint:md && npm run lint:ps && npm run lint:yaml && npm run lint:links && npm run lint:frontmatter && npm run lint:collections-metadata && npm run lint:marketplace && npm run lint:version-consistency && npm run lint:permissions && npm run lint:dependency-pinning && npm run lint:ps-module-pins && npm run lint:py && npm run validate:skills && npm run lint:ai-artifacts", + "lint:models": "pwsh -NoProfile -File scripts/linting/Test-ModelReferences.ps1 -OutputPath logs/model-validation-results.json", + "lint:models:refresh": "pwsh -NoProfile -File scripts/linting/Update-ModelCatalog.ps1", + "lint:all": "npm run format:tables && npm run lint:md && npm run lint:ps && npm run lint:yaml && npm run lint:links && npm run lint:frontmatter && npm run lint:collections-metadata && npm run lint:marketplace && npm run lint:version-consistency && npm run lint:permissions && npm run lint:dependency-pinning && npm run lint:ps-module-pins && npm run lint:py && npm run validate:skills && npm run lint:ai-artifacts && npm run lint:models", "format:tables": "markdown-table-formatter \"**/*.md\"", "extension:prepare": "pwsh ./scripts/extension/Prepare-Extension.ps1", "extension:prepare:prerelease": "pwsh ./scripts/extension/Prepare-Extension.ps1 -Channel PreRelease", diff --git a/scripts/linting/Test-ModelReferences.ps1 b/scripts/linting/Test-ModelReferences.ps1 new file mode 100644 index 000000000..3133345f0 --- /dev/null +++ b/scripts/linting/Test-ModelReferences.ps1 @@ -0,0 +1,317 @@ +#!/usr/bin/env pwsh +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT + +#Requires -Version 7.0 + +<# +.SYNOPSIS + Validates model references in agent and prompt files against the model catalog. + +.DESCRIPTION + Scans all .agent.md and .prompt.md files for model frontmatter references and + validates them against scripts/linting/model-catalog.json. Reports unrecognized + models and models with retiring status. + +.PARAMETER OutputPath + Path for the JSON results file. + +.PARAMETER CatalogPath + Path to the model catalog JSON file. + +.EXAMPLE + ./Test-ModelReferences.ps1 + +.EXAMPLE + ./Test-ModelReferences.ps1 -OutputPath logs/model-validation-results.json +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $false)] + [string]$OutputPath = 'logs/model-validation-results.json', + + [Parameter(Mandatory = $false)] + [string]$CatalogPath = 'scripts/linting/model-catalog.json' +) + +$ErrorActionPreference = 'Stop' + +$gitRoot = git rev-parse --show-toplevel 2>$null +$RepoRoot = if ($gitRoot) { $gitRoot } else { (Join-Path $PSScriptRoot '..' '..' | Resolve-Path).Path } + +Import-Module PowerShell-Yaml -ErrorAction Stop + +#region Functions + +function Get-FrontmatterFromFile { + <# + .SYNOPSIS + Extracts YAML frontmatter from a markdown file. + + .PARAMETER FilePath + Path to the markdown file. + + .OUTPUTS + [hashtable] Parsed frontmatter or $null if no frontmatter found. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$FilePath + ) + + $content = Get-Content -Path $FilePath -Raw -ErrorAction Stop + if ($content -match '(?s)^---\r?\n(.*?)\r?\n---(\r?\n|\z)') { + $yamlBlock = $Matches[1] + try { + return ConvertFrom-Yaml -Yaml $yamlBlock + } + catch { + Write-Warning "Failed to parse YAML in $FilePath : $_" + return $null + } + } + return $null +} + +function Get-ModelReferences { + <# + .SYNOPSIS + Extracts model references from frontmatter. + + .PARAMETER Frontmatter + Parsed frontmatter hashtable. + + .OUTPUTS + [string[]] Array of model name strings. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [hashtable]$Frontmatter + ) + + $modelValue = $Frontmatter['model'] + if ($null -eq $modelValue) { + return @() + } + + if ($modelValue -is [System.Collections.IEnumerable] -and $modelValue -isnot [string]) { + return @($modelValue | ForEach-Object { $_.ToString() }) + } + + return @($modelValue.ToString()) +} + +#endregion Functions + +#region Main + +function Invoke-ModelReferenceValidation { + <# + .SYNOPSIS + Runs model reference validation and returns structured results. + + .PARAMETER CatalogPath + Path to the model catalog JSON file. + + .PARAMETER ScanPath + Root path to scan for agent and prompt files. + + .OUTPUTS + [hashtable] Validation results with counts, file results, warnings, and errors. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$CatalogPath, + + [Parameter(Mandatory = $false)] + [string]$ScanPath = '.github' + ) + + if (-not (Test-Path -Path $CatalogPath)) { + throw "Model catalog not found at: $CatalogPath" + } + + $catalog = Get-Content -Path $CatalogPath -Raw | ConvertFrom-Json + $validModelNames = @($catalog.models | ForEach-Object { $_.name }) + $retiringModels = @($catalog.models | Where-Object { $_.status -eq 'retiring' } | ForEach-Object { $_.name }) + + # Provider allowlist — controls which providers are permitted in references. + # To allow additional providers, add them to providerAllowlist in model-catalog.json. + $providerAllowlist = @() + if ($catalog.providerAllowlist) { + $providerAllowlist = @($catalog.providerAllowlist) + } + $providerLookup = @{} + foreach ($m in $catalog.models) { + if ($m.provider) { $providerLookup[$m.name] = $m.provider } + } + + # Find all agent and prompt files + $agentFiles = Get-ChildItem -Path $ScanPath -Recurse -Filter '*.agent.md' -ErrorAction SilentlyContinue + $promptFiles = Get-ChildItem -Path $ScanPath -Recurse -Filter '*.prompt.md' -ErrorAction SilentlyContinue + $allFiles = @($agentFiles) + @($promptFiles) | Where-Object { $null -ne $_ } + + $results = @() + $warnings = @() + $errors = @() + $totalReferences = 0 + $validReferences = 0 + $invalidReferences = 0 + $retiringReferences = 0 + $filesWithModels = 0 + + foreach ($file in $allFiles) { + $relativePath = $file.FullName -replace [regex]::Escape($RepoRoot + [System.IO.Path]::DirectorySeparatorChar), '' + $relativePath = $relativePath.Replace('\', '/') + + $frontmatter = Get-FrontmatterFromFile -FilePath $file.FullName + if ($null -eq $frontmatter) { + continue + } + + $models = Get-ModelReferences -Frontmatter $frontmatter + if ($models.Count -eq 0) { + continue + } + + $filesWithModels++ + $fileStatus = 'valid' + $fileModels = @() + + foreach ($modelName in $models) { + $totalReferences++ + $fileModels += $modelName + + if ($modelName -notin $validModelNames) { + $invalidReferences++ + $fileStatus = 'invalid' + $errors += @{ + file = $relativePath + model = $modelName + message = "Unrecognized model: '$modelName' not found in catalog" + } + } + elseif ($modelName -in $retiringModels) { + $retiringReferences++ + $validReferences++ + if ($fileStatus -ne 'invalid') { $fileStatus = 'warning' } + $warnings += @{ + file = $relativePath + model = $modelName + message = "Model '$modelName' is marked as retiring in the catalog" + } + } + elseif ($providerAllowlist.Count -gt 0 -and $providerLookup.ContainsKey($modelName) -and + $providerLookup[$modelName] -notin $providerAllowlist) { + $invalidReferences++ + $fileStatus = 'invalid' + $provider = $providerLookup[$modelName] + $errors += @{ + file = $relativePath + model = $modelName + message = "Provider '$provider' is not in the allowed providers list ($($providerAllowlist -join ', '))" + } + } + else { + $validReferences++ + } + } + + $results += @{ + file = $relativePath + models = $fileModels + status = $fileStatus + } + } + + return @{ + timestamp = (Get-Date -Format 'o') + catalogLastUpdated = $catalog.lastUpdated + totalFiles = $allFiles.Count + filesWithModels = $filesWithModels + totalReferences = $totalReferences + validReferences = $validReferences + invalidReferences = $invalidReferences + retiringReferences = $retiringReferences + results = $results + warnings = $warnings + errors = $errors + } +} + +function Write-ModelReferenceOutput { + <# + .SYNOPSIS + Writes validation results to a JSON file and outputs a summary. + + .PARAMETER ValidationResult + Hashtable from Invoke-ModelReferenceValidation. + + .PARAMETER OutputPath + Path for the JSON results file. + + .OUTPUTS + [int] Exit code: 1 if invalid references found, 0 otherwise. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [hashtable]$ValidationResult, + + [Parameter(Mandatory = $true)] + [string]$OutputPath + ) + + # Ensure output directory exists + $outputDir = Split-Path -Path $OutputPath -Parent + if ($outputDir -and -not (Test-Path -Path $outputDir)) { + New-Item -ItemType Directory -Path $outputDir -Force | Out-Null + } + + $ValidationResult | ConvertTo-Json -Depth 5 | Set-Content -Path $OutputPath -Encoding utf8 + + # Summary output + Write-Host "Model Reference Validation Results:" -ForegroundColor Cyan + Write-Host " Total files scanned: $($ValidationResult.totalFiles)" + Write-Host " Files with model references: $($ValidationResult.filesWithModels)" + Write-Host " Total model references: $($ValidationResult.totalReferences)" + Write-Host " Valid references: $($ValidationResult.validReferences)" -ForegroundColor Green + if ($ValidationResult.retiringReferences -gt 0) { + Write-Host " Retiring references: $($ValidationResult.retiringReferences)" -ForegroundColor Yellow + } + if ($ValidationResult.invalidReferences -gt 0) { + Write-Host " Invalid references: $($ValidationResult.invalidReferences)" -ForegroundColor Red + foreach ($err in $ValidationResult.errors) { + Write-Host " ERROR: $($err.file) - $($err.message)" -ForegroundColor Red + } + } + foreach ($warn in $ValidationResult.warnings) { + Write-Host " WARNING: $($warn.file) - $($warn.message)" -ForegroundColor Yellow + } + + Write-Host "`nResults written to: $OutputPath" + + if ($ValidationResult.invalidReferences -gt 0) { + return 1 + } + return 0 +} + +# Only run main logic when executed directly (not dot-sourced for testing) +if ($MyInvocation.InvocationName -ne '.') { + # Validate catalog exists + if (-not (Test-Path -Path $CatalogPath)) { + Write-Error "Model catalog not found at: $CatalogPath" + exit 1 + } + + $output = Invoke-ModelReferenceValidation -CatalogPath $CatalogPath + $exitCode = Write-ModelReferenceOutput -ValidationResult $output -OutputPath $OutputPath + exit $exitCode +} + +#endregion Main diff --git a/scripts/linting/Update-ModelCatalog.ps1 b/scripts/linting/Update-ModelCatalog.ps1 new file mode 100644 index 000000000..d5ac46318 --- /dev/null +++ b/scripts/linting/Update-ModelCatalog.ps1 @@ -0,0 +1,419 @@ +#!/usr/bin/env pwsh +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT + +#Requires -Version 7.0 + +<# +.SYNOPSIS + Refreshes the model catalog by fetching current models from GitHub docs data. + +.DESCRIPTION + Fetches structured YAML data files from the github/docs repository that define + Copilot model names, release status, and multipliers. Merges these into the + local model-catalog.json. Reports additions, removals, and multiplier changes. + +.PARAMETER CatalogPath + Path to the model catalog JSON file to update. + +.PARAMETER DryRun + When specified, reports changes without modifying the catalog file. + +.PARAMETER BaseUrl + Base URL for raw YAML data files in the github/docs repository. + +.EXAMPLE + ./Update-ModelCatalog.ps1 + +.EXAMPLE + ./Update-ModelCatalog.ps1 -DryRun + +.NOTES + Data files are structured YAML from github/docs and are more stable than + rendered page scraping. If the file paths change, update BaseUrl or the + file names in the script. + + Upstream is fetched from github/docs@main without a pinned SHA or tag. + This means results are non-deterministic across runs — upstream additions + or removals can appear between invocations. The CI workflow detects drift + between the refreshed catalog and the committed version, so staleness is + surfaced rather than silently accepted. +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $false)] + [string]$CatalogPath = 'scripts/linting/model-catalog.json', + + [Parameter(Mandatory = $false)] + [switch]$DryRun, + + [Parameter(Mandatory = $false)] + [string]$BaseUrl = 'https://raw.githubusercontent.com/github/docs/main/data/tables/copilot' +) + +$ErrorActionPreference = 'Stop' + +Import-Module PowerShell-Yaml -ErrorAction Stop + +#region Functions + +function Get-RemoteYaml { + <# + .SYNOPSIS + Fetches and parses a remote YAML file. + + .PARAMETER Url + URL to fetch. + + .OUTPUTS + Parsed YAML content. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Url + ) + + $response = Invoke-WebRequest -Uri $Url -UseBasicParsing -TimeoutSec 30 -ErrorAction Stop + return ConvertFrom-Yaml -Yaml $response.Content -AllDocuments +} + +function Get-ModelProvider { + <# + .SYNOPSIS + Derives the provider name from a model display name using prefix matching. + + .DESCRIPTION + Maps model names to their provider using known prefix patterns. Models not + matching any known prefix are classified as 'Unknown'. To support additional + providers in the future, add a new entry to the $providerPatterns array below. + + .PARAMETER ModelName + The model display name (without the "(copilot)" suffix). + + .OUTPUTS + [string] Provider name. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$ModelName + ) + + # Provider prefix patterns — add new providers here as they become available. + # Order matters: first match wins. + $providerPatterns = @( + @{ Pattern = '^Claude'; Provider = 'Anthropic' } + @{ Pattern = '^GPT-|^o\d'; Provider = 'OpenAI' } + @{ Pattern = '^Gemini'; Provider = 'Google' } + @{ Pattern = '^Grok'; Provider = 'xAI' } + ) + + foreach ($entry in $providerPatterns) { + if ($ModelName -match $entry.Pattern) { + return $entry.Provider + } + } + + return 'Unknown' +} + +function Merge-ModelData { + <# + .SYNOPSIS + Merges model release status and multiplier data into catalog entries. + + .PARAMETER ReleaseStatus + Array of model release status objects from model-release-status.yml. + + .PARAMETER Multipliers + Array of model multiplier objects from model-multipliers.yml. + + .OUTPUTS + [hashtable[]] Array of merged model catalog entries. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [object[]]$ReleaseStatus, + + [Parameter(Mandatory = $true)] + [object[]]$Multipliers + ) + + $multiplierLookup = @{} + foreach ($m in $Multipliers) { + $multiplierLookup[$m.name] = $m + } + + $models = @() + foreach ($model in $ReleaseStatus) { + $name = $model.name + $status = if ($model.release_status -eq 'GA') { 'ga' } else { 'preview' } + + # Look up multiplier (use paid multiplier as canonical) + $multiplier = 1.0 + if ($multiplierLookup.ContainsKey($name)) { + $mData = $multiplierLookup[$name] + $paidVal = $mData.multiplier_paid + if ($paidVal -is [string] -and $paidVal -eq 'Not applicable') { + $multiplier = 0 + } + elseif ($null -ne $paidVal) { + $multiplier = [double]$paidVal + } + } + + # Determine tier from multiplier + $tier = if ($multiplier -eq 0) { 'free' } + elseif ($multiplier -le 0.33) { 'fast' } + elseif ($multiplier -le 1) { 'standard' } + elseif ($multiplier -le 5) { 'premium' } + else { 'ultra' } + + $models += @{ + name = "$name (copilot)" + tier = $tier + multiplier = $multiplier + status = $status + provider = Get-ModelProvider -ModelName $name + } + } + + return $models +} + +function Compare-Catalogs { + <# + .SYNOPSIS + Compares current catalog models against newly discovered models. + + .PARAMETER Current + Array of current catalog model objects. + + .PARAMETER Discovered + Array of newly discovered model objects. + + .OUTPUTS + [hashtable] With added, removed, and changed arrays. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [object[]]$Current, + + [Parameter(Mandatory = $true)] + [object[]]$Discovered + ) + + $currentNames = @($Current | ForEach-Object { $_.name }) + $discoveredNames = @($Discovered | ForEach-Object { $_.name }) + + $added = @($Discovered | Where-Object { $_.name -notin $currentNames }) + $removed = @($Current | Where-Object { $_.name -notin $discoveredNames }) + + $changed = @() + foreach ($disc in $Discovered) { + $curr = $Current | Where-Object { $_.name -eq $disc.name } + if ($curr -and $curr.multiplier -ne $disc.multiplier) { + $changed += @{ + name = $disc.name + oldMultiplier = $curr.multiplier + newMultiplier = $disc.multiplier + } + } + } + + return @{ + added = $added + removed = $removed + changed = $changed + } +} + +#endregion Functions + +#region Orchestration + +function Invoke-ModelCatalogUpdate { + <# + .SYNOPSIS + Orchestrates catalog update from fetched model data. + + .PARAMETER ReleaseStatus + Array of model release status objects. + + .PARAMETER Multipliers + Array of model multiplier objects. + + .PARAMETER CatalogPath + Path to the catalog JSON file. + + .PARAMETER DryRun + When true, reports changes without writing to disk. + + .OUTPUTS + [hashtable] With status ('unchanged', 'updated', 'created', 'dryrun'), diff, and finalModels. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [object[]]$ReleaseStatus, + + [Parameter(Mandatory = $true)] + [object[]]$Multipliers, + + [Parameter(Mandatory = $true)] + [string]$CatalogPath, + + [Parameter(Mandatory = $false)] + [switch]$DryRun + ) + + $discoveredModels = Merge-ModelData -ReleaseStatus $ReleaseStatus -Multipliers $Multipliers + Write-Host " Discovered $($discoveredModels.Count) models from docs" -ForegroundColor Green + + $diff = $null + $finalModels = $null + + # Load current catalog if it exists + if (Test-Path -Path $CatalogPath) { + $currentCatalog = Get-Content -Path $CatalogPath -Raw | ConvertFrom-Json + $currentModels = @($currentCatalog.models) + + $diff = Compare-Catalogs -Current $currentModels -Discovered $discoveredModels + + if ($diff.added.Count -gt 0) { + Write-Host "`n Added models:" -ForegroundColor Green + foreach ($m in $diff.added) { Write-Host " + $($m['name']) (tier: $($m['tier']), multiplier: $($m['multiplier']))" -ForegroundColor Green } + } + if ($diff.removed.Count -gt 0) { + Write-Host "`n Removed models (marking as retiring):" -ForegroundColor Yellow + foreach ($m in $diff.removed) { Write-Host " - $($m.name)" -ForegroundColor Yellow } + } + if ($diff.changed.Count -gt 0) { + Write-Host "`n Multiplier changes:" -ForegroundColor Cyan + foreach ($c in $diff.changed) { Write-Host " ~ $($c.name): $($c.oldMultiplier) -> $($c.newMultiplier)" -ForegroundColor Cyan } + } + + if ($diff.added.Count -eq 0 -and $diff.removed.Count -eq 0 -and $diff.changed.Count -eq 0) { + Write-Host "`n No changes detected. Catalog is current." -ForegroundColor Green + if (-not $DryRun) { + $currentCatalog.lastUpdated = (Get-Date -Format 'yyyy-MM-dd') + $currentCatalog | ConvertTo-Json -Depth 5 | Set-Content -Path $CatalogPath -Encoding utf8 + } + return @{ status = 'unchanged'; diff = $diff; finalModels = $currentModels } + } + + # Mark removed models as retiring instead of deleting + $finalModels = @() + foreach ($curr in $currentModels) { + if ($curr.name -in @($diff.removed | ForEach-Object { $_.name })) { + $retiring = [PSCustomObject]@{ + name = $curr.name + tier = $curr.tier + multiplier = $curr.multiplier + status = 'retiring' + retiredDate = (Get-Date).AddDays(60).ToString('yyyy-MM-dd') + } + $finalModels += $retiring + } + else { + # Update multiplier if changed + $change = $diff.changed | Where-Object { $_.name -eq $curr.name } + if ($change) { + $curr.multiplier = $change.newMultiplier + } + $finalModels += $curr + } + } + # Add new models + $finalModels += $diff.added + } + else { + Write-Host " No existing catalog found. Creating new catalog." -ForegroundColor Yellow + $finalModels = $discoveredModels + } + + if ($DryRun) { + Write-Host "`n [DRY RUN] No changes written to disk." -ForegroundColor Yellow + return @{ status = 'dryrun'; diff = $diff; finalModels = $finalModels } + } + + # Write updated catalog + # providerAllowlist controls which providers are permitted in agent/prompt model + # references. To allow additional providers, add them to this array. + $allowlist = @('Anthropic', 'OpenAI') + if (Test-Path -Path $CatalogPath) { + $existingCatalog = Get-Content -Path $CatalogPath -Raw | ConvertFrom-Json + if ($existingCatalog.providerAllowlist) { + $allowlist = @($existingCatalog.providerAllowlist) + } + } + + $newCatalog = @{ + '$schema' = './schemas/model-catalog.schema.json' + lastUpdated = (Get-Date -Format 'yyyy-MM-dd') + source = 'https://docs.github.com/en/copilot/reference/ai-models/supported-models' + providerAllowlist = $allowlist + models = $finalModels + } + + $outputDir = Split-Path -Path $CatalogPath -Parent + if ($outputDir -and -not (Test-Path -Path $outputDir)) { + New-Item -ItemType Directory -Path $outputDir -Force | Out-Null + } + + $newCatalog | ConvertTo-Json -Depth 5 | Set-Content -Path $CatalogPath -Encoding utf8 + Write-Host "`n Catalog updated: $CatalogPath" -ForegroundColor Green + Write-Host " Total models: $($finalModels.Count)" -ForegroundColor Green + + $resultStatus = if ($null -eq $diff) { 'created' } else { 'updated' } + return @{ status = $resultStatus; diff = $diff; finalModels = $finalModels } +} + +#endregion Orchestration + +#region Main + +# Only run main logic when executed directly +if ($MyInvocation.InvocationName -ne '.') { + Write-Host "Fetching model data from github/docs YAML sources..." -ForegroundColor Cyan + + try { + $releaseStatusUrl = "$BaseUrl/model-release-status.yml" + $multipliersUrl = "$BaseUrl/model-multipliers.yml" + + Write-Host " Fetching: $releaseStatusUrl" + $releaseStatus = Get-RemoteYaml -Url $releaseStatusUrl + + Write-Host " Fetching: $multipliersUrl" + $multipliers = Get-RemoteYaml -Url $multipliersUrl + } + catch { + Write-Warning "Failed to fetch source data: $_" + Write-Warning "Model catalog not updated. Check network or source URLs." + exit 1 + } + + if (-not $releaseStatus -or $releaseStatus.Count -eq 0) { + Write-Warning "No models found in release status data. Source format may have changed." + exit 1 + } + + $updateParams = @{ + ReleaseStatus = $releaseStatus + Multipliers = $multipliers + CatalogPath = $CatalogPath + } + if ($DryRun) { $updateParams['DryRun'] = $true } + + $result = Invoke-ModelCatalogUpdate @updateParams + if ($result.status -eq 'unchanged' -or $result.status -eq 'dryrun') { + exit 0 + } + exit 0 +} + +#endregion Main diff --git a/scripts/linting/model-catalog.json b/scripts/linting/model-catalog.json new file mode 100644 index 000000000..fa99c91e3 --- /dev/null +++ b/scripts/linting/model-catalog.json @@ -0,0 +1,180 @@ +{ + "models": [ + { + "name": "Claude Haiku 4.5 (copilot)", + "tier": "fast", + "multiplier": 0.33, + "status": "ga", + "provider": "Anthropic" + }, + { + "name": "Claude Sonnet 4 (copilot)", + "tier": "standard", + "multiplier": 1, + "status": "ga", + "provider": "Anthropic" + }, + { + "name": "Claude Sonnet 4.5 (copilot)", + "tier": "standard", + "multiplier": 1, + "status": "ga", + "provider": "Anthropic" + }, + { + "name": "Claude Sonnet 4.6 (copilot)", + "tier": "standard", + "multiplier": 1, + "status": "ga", + "provider": "Anthropic" + }, + { + "name": "Claude Opus 4.5 (copilot)", + "tier": "premium", + "multiplier": 3, + "status": "ga", + "provider": "Anthropic" + }, + { + "name": "Claude Opus 4.6 (copilot)", + "tier": "premium", + "multiplier": 3, + "status": "ga", + "provider": "Anthropic" + }, + { + "name": "Claude Opus 4.7 (copilot)", + "tier": "premium", + "multiplier": 15, + "status": "ga", + "provider": "Anthropic" + }, + { + "name": "GPT-4.1 (copilot)", + "tier": "free", + "multiplier": 0, + "status": "ga", + "provider": "OpenAI" + }, + { + "name": "GPT-5 mini (copilot)", + "tier": "free", + "multiplier": 0, + "status": "ga", + "provider": "OpenAI" + }, + { + "name": "GPT-5.2 (copilot)", + "tier": "standard", + "multiplier": 1, + "status": "ga", + "provider": "OpenAI" + }, + { + "name": "GPT-5.2-Codex (copilot)", + "tier": "standard", + "multiplier": 1, + "status": "ga", + "provider": "OpenAI" + }, + { + "name": "GPT-5.3-Codex (copilot)", + "tier": "standard", + "multiplier": 1, + "status": "ga", + "provider": "OpenAI" + }, + { + "name": "GPT-5.4 (copilot)", + "tier": "standard", + "multiplier": 1, + "status": "ga", + "provider": "OpenAI" + }, + { + "name": "GPT-5.4 mini (copilot)", + "tier": "fast", + "multiplier": 0.33, + "status": "ga", + "provider": "OpenAI" + }, + { + "name": "GPT-5.5 (copilot)", + "tier": "premium", + "multiplier": 7.5, + "status": "ga", + "provider": "OpenAI" + }, + { + "name": "Gemini 2.5 Pro (copilot)", + "tier": "standard", + "multiplier": 1, + "status": "ga", + "provider": "Google" + }, + { + "status": "retiring", + "tier": "fast", + "multiplier": 0.33, + "name": "Gemini 3 Flash (Preview) (copilot)", + "retiredDate": "2026-07-05", + "provider": "Google" + }, + { + "name": "Gemini 3.1 Pro (copilot)", + "tier": "standard", + "multiplier": 1, + "status": "preview", + "provider": "Google" + }, + { + "name": "Grok Code Fast 1 (copilot)", + "tier": "fast", + "multiplier": 0.25, + "status": "ga", + "provider": "xAI" + }, + { + "status": "ga", + "tier": "fast", + "multiplier": 0.25, + "name": "GPT-5.4 nano (copilot)", + "provider": "OpenAI" + }, + { + "status": "preview", + "tier": "ultra", + "multiplier": 30.0, + "name": "Claude Opus 4.6 (fast mode) (preview) (copilot)", + "provider": "Anthropic" + }, + { + "status": "preview", + "tier": "fast", + "multiplier": 0.33, + "name": "Gemini 3 Flash (copilot)", + "provider": "Google" + }, + { + "status": "preview", + "tier": "free", + "multiplier": 0.0, + "name": "Raptor mini (copilot)", + "provider": "Unknown" + }, + { + "status": "preview", + "tier": "free", + "multiplier": 0, + "name": "Goldeneye (copilot)", + "provider": "Unknown" + } + ], + "providerAllowlist": [ + "Anthropic", + "OpenAI" + ], + "$schema": "./schemas/model-catalog.schema.json", + "lastUpdated": "2026-05-06", + "source": "https://docs.github.com/en/copilot/reference/ai-models/supported-models" +} diff --git a/scripts/linting/schemas/model-catalog.schema.json b/scripts/linting/schemas/model-catalog.schema.json new file mode 100644 index 000000000..b38fac6f4 --- /dev/null +++ b/scripts/linting/schemas/model-catalog.schema.json @@ -0,0 +1,90 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "model-catalog.schema.json", + "title": "Model Catalog", + "description": "Schema for the Copilot model catalog used by model reference validation", + "type": "object", + "required": [ + "lastUpdated", + "source", + "models", + "providerAllowlist" + ], + "properties": { + "$schema": { + "type": "string" + }, + "providerAllowlist": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Providers permitted for use in agent/prompt model references. Add entries here to allow additional providers in the future." + }, + "lastUpdated": { + "type": "string", + "format": "date", + "description": "ISO 8601 date when the catalog was last updated" + }, + "source": { + "type": "string", + "format": "uri", + "description": "URL of the authoritative model list" + }, + "models": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name", + "tier", + "multiplier", + "status", + "provider" + ], + "properties": { + "name": { + "type": "string", + "description": "VS Code display name with vendor suffix" + }, + "tier": { + "type": "string", + "enum": [ + "free", + "fast", + "standard", + "premium", + "ultra" + ], + "description": "Cost tier classification" + }, + "multiplier": { + "type": "number", + "minimum": 0, + "description": "Request multiplier relative to standard tier" + }, + "status": { + "type": "string", + "enum": [ + "ga", + "preview", + "retiring" + ], + "description": "Availability status" + }, + "retiredDate": { + "type": "string", + "format": "date", + "description": "Date when model will be retired (only for retiring status)" + }, + "provider": { + "type": "string", + "description": "Model provider name (e.g., Anthropic, OpenAI, Google, xAI)" + } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false +} diff --git a/scripts/linting/schemas/prompt-frontmatter.schema.json b/scripts/linting/schemas/prompt-frontmatter.schema.json index 2331b4ea9..0a3937e32 100644 --- a/scripts/linting/schemas/prompt-frontmatter.schema.json +++ b/scripts/linting/schemas/prompt-frontmatter.schema.json @@ -4,7 +4,9 @@ "title": "Prompt File Frontmatter Schema", "description": "Frontmatter schema for .prompt.md files per VS Code Copilot specification", "type": "object", - "required": ["description"], + "required": [ + "description" + ], "properties": { "description": { "type": "string", @@ -37,4 +39,4 @@ } }, "additionalProperties": true -} \ No newline at end of file +} diff --git a/scripts/tests/linting/Test-ModelReferences.Tests.ps1 b/scripts/tests/linting/Test-ModelReferences.Tests.ps1 new file mode 100644 index 000000000..d6198a695 --- /dev/null +++ b/scripts/tests/linting/Test-ModelReferences.Tests.ps1 @@ -0,0 +1,635 @@ +#Requires -Modules Pester +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT + +BeforeAll { + $script:ScriptPath = Join-Path $PSScriptRoot '../../linting/Test-ModelReferences.ps1' + . $script:ScriptPath + + # Suppress Write-Host output during tests + Mock Write-Host {} + + # Create temp directory for test fixtures + $script:TempDir = Join-Path ([System.IO.Path]::GetTempPath()) "ModelRefTests_$([guid]::NewGuid().ToString('N'))" + New-Item -ItemType Directory -Path $script:TempDir -Force | Out-Null +} + +AfterAll { + if (Test-Path $script:TempDir) { + Remove-Item -Path $script:TempDir -Recurse -Force -ErrorAction SilentlyContinue + } +} + +#region Get-FrontmatterFromFile Tests + +Describe 'Get-FrontmatterFromFile' -Tag 'Unit' { + BeforeAll { + $script:FixtureDir = Join-Path $script:TempDir 'frontmatter' + New-Item -ItemType Directory -Path $script:FixtureDir -Force | Out-Null + } + + Context 'when file has valid frontmatter' { + It 'Returns parsed hashtable with string model' { + $filePath = Join-Path $script:FixtureDir 'valid-string.agent.md' + @" +--- +name: Test Agent +model: Claude Haiku 4.5 (copilot) +--- + +# Content +"@ | Set-Content -Path $filePath -Encoding utf8 + + $result = Get-FrontmatterFromFile -FilePath $filePath + $result | Should -Not -BeNullOrEmpty + $result['name'] | Should -Be 'Test Agent' + $result['model'] | Should -Be 'Claude Haiku 4.5 (copilot)' + } + + It 'Returns parsed hashtable with array model' { + $filePath = Join-Path $script:FixtureDir 'valid-array.agent.md' + @" +--- +name: Test Agent +model: + - Claude Haiku 4.5 (copilot) + - GPT-5.4 mini (copilot) +--- + +# Content +"@ | Set-Content -Path $filePath -Encoding utf8 + + $result = Get-FrontmatterFromFile -FilePath $filePath + $result | Should -Not -BeNullOrEmpty + $result['model'] | Should -HaveCount 2 + } + + It 'Handles frontmatter with additional properties' { + $filePath = Join-Path $script:FixtureDir 'extra-props.prompt.md' + @" +--- +description: 'A test prompt' +agent: 'agent' +model: GPT-5.4 mini (copilot) +--- + +# Content +"@ | Set-Content -Path $filePath -Encoding utf8 + + $result = Get-FrontmatterFromFile -FilePath $filePath + $result['description'] | Should -Be 'A test prompt' + $result['agent'] | Should -Be 'agent' + $result['model'] | Should -Be 'GPT-5.4 mini (copilot)' + } + } + + Context 'when file has no frontmatter' { + It 'Returns null for file without frontmatter' { + $filePath = Join-Path $script:FixtureDir 'no-frontmatter.agent.md' + @" +# Just a heading + +No frontmatter here. +"@ | Set-Content -Path $filePath -Encoding utf8 + + $result = Get-FrontmatterFromFile -FilePath $filePath + $result | Should -BeNullOrEmpty + } + + It 'Returns null for empty file' { + $filePath = Join-Path $script:FixtureDir 'empty.agent.md' + '' | Set-Content -Path $filePath -Encoding utf8 + + $result = Get-FrontmatterFromFile -FilePath $filePath + $result | Should -BeNullOrEmpty + } + } + + Context 'when file has invalid YAML' { + It 'Returns null and writes warning for malformed YAML' { + $filePath = Join-Path $script:FixtureDir 'bad-yaml.agent.md' + @" +--- +name: [unclosed bracket +model: bad: yaml: here +--- + +# Content +"@ | Set-Content -Path $filePath -Encoding utf8 + + $result = Get-FrontmatterFromFile -FilePath $filePath + $result | Should -BeNullOrEmpty + } + } +} + +#endregion + +#region Get-ModelReferences Tests + +Describe 'Get-ModelReferences' -Tag 'Unit' { + Context 'when frontmatter has no model property' { + It 'Returns empty array' { + $frontmatter = @{ name = 'Test Agent'; description = 'No model here' } + $result = Get-ModelReferences -Frontmatter $frontmatter + $result | Should -HaveCount 0 + } + } + + Context 'when model is a string' { + It 'Returns single-element array' { + $frontmatter = @{ model = 'Claude Haiku 4.5 (copilot)' } + $result = @(Get-ModelReferences -Frontmatter $frontmatter) + $result | Should -HaveCount 1 + $result[0] | Should -Be 'Claude Haiku 4.5 (copilot)' + } + } + + Context 'when model is an array' { + It 'Returns all model names' { + $frontmatter = @{ model = @('Claude Haiku 4.5 (copilot)', 'GPT-5.4 mini (copilot)', 'Gemini 3 Flash (Preview) (copilot)') } + $result = Get-ModelReferences -Frontmatter $frontmatter + $result | Should -HaveCount 3 + $result[0] | Should -Be 'Claude Haiku 4.5 (copilot)' + $result[1] | Should -Be 'GPT-5.4 mini (copilot)' + $result[2] | Should -Be 'Gemini 3 Flash (Preview) (copilot)' + } + + It 'Returns single-element array for single-item list' { + $frontmatter = @{ model = @('GPT-5.4 mini (copilot)') } + $result = @(Get-ModelReferences -Frontmatter $frontmatter) + $result | Should -HaveCount 1 + $result[0] | Should -Be 'GPT-5.4 mini (copilot)' + } + } + + Context 'when model value is null' { + It 'Returns empty array' { + $frontmatter = @{ model = $null } + $result = Get-ModelReferences -Frontmatter $frontmatter + $result | Should -HaveCount 0 + } + } +} + +#endregion + +#region Invoke-ModelReferenceValidation Tests + +Describe 'Invoke-ModelReferenceValidation' -Tag 'Unit' { + BeforeAll { + $script:ValidationDir = Join-Path $script:TempDir 'validation' + New-Item -ItemType Directory -Path $script:ValidationDir -Force | Out-Null + + # Create a minimal catalog + $script:CatalogDir = Join-Path $script:TempDir 'catalog' + New-Item -ItemType Directory -Path $script:CatalogDir -Force | Out-Null + $script:TestCatalogPath = Join-Path $script:CatalogDir 'model-catalog.json' + } + + Context 'when catalog does not exist' { + It 'Throws error for missing catalog' { + $nonexistentPath = Join-Path $script:CatalogDir 'nonexistent.json' + { Invoke-ModelReferenceValidation -CatalogPath $nonexistentPath -ScanPath $script:ValidationDir } | Should -Throw '*not found*' + } + } + + Context 'when all models are valid' { + BeforeAll { + $script:ValidDir = Join-Path $script:ValidationDir 'valid-models' + New-Item -ItemType Directory -Path $script:ValidDir -Force | Out-Null + + # Create catalog with known models + $catalog = @{ + lastUpdated = '2026-05-06' + source = 'https://example.com' + models = @( + @{ name = 'Claude Haiku 4.5 (copilot)'; tier = 'fast'; multiplier = 0.33; status = 'ga' } + @{ name = 'GPT-5.4 mini (copilot)'; tier = 'fast'; multiplier = 0.33; status = 'ga' } + @{ name = 'Gemini 3 Flash (Preview) (copilot)'; tier = 'fast'; multiplier = 0.33; status = 'preview' } + ) + } + $catalog | ConvertTo-Json -Depth 5 | Set-Content -Path $script:TestCatalogPath -Encoding utf8 + + # Create agent file with valid model array + @" +--- +name: Valid Agent +model: + - Claude Haiku 4.5 (copilot) + - GPT-5.4 mini (copilot) +--- + +# Agent +"@ | Set-Content -Path (Join-Path $script:ValidDir 'test.agent.md') -Encoding utf8 + + # Create prompt file with valid single model + @" +--- +description: Valid Prompt +model: Gemini 3 Flash (Preview) (copilot) +--- + +# Prompt +"@ | Set-Content -Path (Join-Path $script:ValidDir 'test.prompt.md') -Encoding utf8 + + $script:ValidResult = Invoke-ModelReferenceValidation -CatalogPath $script:TestCatalogPath -ScanPath $script:ValidDir + } + + It 'Reports zero invalid references' { + $script:ValidResult.invalidReferences | Should -Be 0 + } + + It 'Reports correct total references' { + $script:ValidResult.totalReferences | Should -Be 3 + } + + It 'Reports correct valid references' { + $script:ValidResult.validReferences | Should -Be 3 + } + + It 'Reports correct files with models count' { + $script:ValidResult.filesWithModels | Should -Be 2 + } + + It 'Reports correct total file count' { + $script:ValidResult.totalFiles | Should -Be 2 + } + + It 'Returns empty errors array' { + $script:ValidResult.errors | Should -HaveCount 0 + } + + It 'Returns results for each file with models' { + $script:ValidResult.results | Should -HaveCount 2 + } + + It 'Marks all results as valid status' { + $script:ValidResult.results | ForEach-Object { $_.status | Should -Be 'valid' } + } + + It 'Includes catalog last updated date' { + $script:ValidResult.catalogLastUpdated | Should -Be '2026-05-06' + } + } + + Context 'when models are invalid' { + BeforeAll { + $script:InvalidDir = Join-Path $script:ValidationDir 'invalid-models' + New-Item -ItemType Directory -Path $script:InvalidDir -Force | Out-Null + + # Create agent file with invalid model + @" +--- +name: Bad Agent +model: + - Claude Haiku 4.5 (copilot) + - Nonexistent Model (copilot) +--- + +# Agent +"@ | Set-Content -Path (Join-Path $script:InvalidDir 'bad.agent.md') -Encoding utf8 + + $script:InvalidResult = Invoke-ModelReferenceValidation -CatalogPath $script:TestCatalogPath -ScanPath $script:InvalidDir + } + + It 'Reports one invalid reference' { + $script:InvalidResult.invalidReferences | Should -Be 1 + } + + It 'Reports one valid reference' { + $script:InvalidResult.validReferences | Should -Be 1 + } + + It 'Contains error with model name' { + $script:InvalidResult.errors | Should -HaveCount 1 + $script:InvalidResult.errors[0].model | Should -Be 'Nonexistent Model (copilot)' + } + + It 'Contains descriptive error message' { + $script:InvalidResult.errors[0].message | Should -Match 'Unrecognized model' + } + + It 'Marks file result as invalid' { + $script:InvalidResult.results[0].status | Should -Be 'invalid' + } + } + + Context 'when models are retiring' { + BeforeAll { + $script:RetiringDir = Join-Path $script:ValidationDir 'retiring-models' + New-Item -ItemType Directory -Path $script:RetiringDir -Force | Out-Null + + # Create catalog with a retiring model + $retiringCatalog = @{ + lastUpdated = '2026-05-06' + source = 'https://example.com' + models = @( + @{ name = 'Claude Haiku 4.5 (copilot)'; tier = 'fast'; multiplier = 0.33; status = 'ga' } + @{ name = 'Old Model (copilot)'; tier = 'fast'; multiplier = 0.33; status = 'retiring'; retiredDate = '2026-06-01' } + ) + } + $script:RetiringCatalogPath = Join-Path $script:CatalogDir 'retiring-catalog.json' + $retiringCatalog | ConvertTo-Json -Depth 5 | Set-Content -Path $script:RetiringCatalogPath -Encoding utf8 + + # Create agent file with retiring model + @" +--- +name: Retiring Agent +model: + - Old Model (copilot) + - Claude Haiku 4.5 (copilot) +--- + +# Agent +"@ | Set-Content -Path (Join-Path $script:RetiringDir 'retiring.agent.md') -Encoding utf8 + + $script:RetiringResult = Invoke-ModelReferenceValidation -CatalogPath $script:RetiringCatalogPath -ScanPath $script:RetiringDir + } + + It 'Reports one retiring reference' { + $script:RetiringResult.retiringReferences | Should -Be 1 + } + + It 'Reports zero invalid references' { + $script:RetiringResult.invalidReferences | Should -Be 0 + } + + It 'Counts retiring model as valid' { + $script:RetiringResult.validReferences | Should -Be 2 + } + + It 'Contains warning with retiring model name' { + $script:RetiringResult.warnings | Should -HaveCount 1 + $script:RetiringResult.warnings[0].model | Should -Be 'Old Model (copilot)' + } + + It 'Contains descriptive warning message' { + $script:RetiringResult.warnings[0].message | Should -Match 'retiring' + } + + It 'Marks file result as warning status' { + $script:RetiringResult.results[0].status | Should -Be 'warning' + } + } + + Context 'when file has both invalid and retiring models' { + BeforeAll { + $script:MixedDir = Join-Path $script:ValidationDir 'mixed-models' + New-Item -ItemType Directory -Path $script:MixedDir -Force | Out-Null + + # Create agent file with both invalid and retiring model + @" +--- +name: Mixed Agent +model: + - Old Model (copilot) + - Fake Model (copilot) +--- + +# Agent +"@ | Set-Content -Path (Join-Path $script:MixedDir 'mixed.agent.md') -Encoding utf8 + + $script:MixedResult = Invoke-ModelReferenceValidation -CatalogPath $script:RetiringCatalogPath -ScanPath $script:MixedDir + } + + It 'Marks file as invalid when both retiring and invalid present' { + $script:MixedResult.results[0].status | Should -Be 'invalid' + } + + It 'Reports one invalid and one retiring' { + $script:MixedResult.invalidReferences | Should -Be 1 + $script:MixedResult.retiringReferences | Should -Be 1 + } + } + + Context 'when files have no model property' { + BeforeAll { + $script:NoModelDir = Join-Path $script:ValidationDir 'no-model' + New-Item -ItemType Directory -Path $script:NoModelDir -Force | Out-Null + + @" +--- +name: No Model Agent +description: Agent without model +--- + +# Agent +"@ | Set-Content -Path (Join-Path $script:NoModelDir 'no-model.agent.md') -Encoding utf8 + + $script:NoModelResult = Invoke-ModelReferenceValidation -CatalogPath $script:TestCatalogPath -ScanPath $script:NoModelDir + } + + It 'Reports zero files with models' { + $script:NoModelResult.filesWithModels | Should -Be 0 + } + + It 'Reports zero total references' { + $script:NoModelResult.totalReferences | Should -Be 0 + } + + It 'Returns empty results array' { + $script:NoModelResult.results | Should -HaveCount 0 + } + + It 'Counts the file in total files' { + $script:NoModelResult.totalFiles | Should -Be 1 + } + } + + Context 'when scan path has no matching files' { + BeforeAll { + $script:EmptyDir = Join-Path $script:ValidationDir 'empty' + New-Item -ItemType Directory -Path $script:EmptyDir -Force | Out-Null + + $script:EmptyResult = Invoke-ModelReferenceValidation -CatalogPath $script:TestCatalogPath -ScanPath $script:EmptyDir + } + + It 'Reports zero total files' { + $script:EmptyResult.totalFiles | Should -Be 0 + } + + It 'Reports zero references' { + $script:EmptyResult.totalReferences | Should -Be 0 + } + } + + Context 'when files have no frontmatter' { + BeforeAll { + $script:NoFmDir = Join-Path $script:ValidationDir 'no-frontmatter' + New-Item -ItemType Directory -Path $script:NoFmDir -Force | Out-Null + + @" +# No Frontmatter Agent + +Just content, no YAML. +"@ | Set-Content -Path (Join-Path $script:NoFmDir 'bare.agent.md') -Encoding utf8 + + $script:NoFmResult = Invoke-ModelReferenceValidation -CatalogPath $script:TestCatalogPath -ScanPath $script:NoFmDir + } + + It 'Counts file in total but not in models' { + $script:NoFmResult.totalFiles | Should -Be 1 + $script:NoFmResult.filesWithModels | Should -Be 0 + } + } +} + +#endregion + +#region Write-ModelReferenceOutput Tests + +Describe 'Write-ModelReferenceOutput' -Tag 'Unit' { + BeforeAll { + $script:OutputDir = Join-Path $script:TempDir 'output' + New-Item -ItemType Directory -Path $script:OutputDir -Force | Out-Null + } + + Context 'when validation has no errors' { + BeforeAll { + $script:CleanOutputPath = Join-Path $script:OutputDir 'clean-results.json' + $script:CleanValidation = @{ + timestamp = '2026-05-06T00:00:00Z' + catalogLastUpdated = '2026-05-06' + totalFiles = 5 + filesWithModels = 3 + totalReferences = 4 + validReferences = 4 + invalidReferences = 0 + retiringReferences = 0 + results = @( + @{ file = 'test.agent.md'; models = @('Model A (copilot)'); status = 'valid' } + ) + warnings = @() + errors = @() + } + + $script:CleanExitCode = Write-ModelReferenceOutput -ValidationResult $script:CleanValidation -OutputPath $script:CleanOutputPath + } + + It 'Returns exit code 0' { + $script:CleanExitCode | Should -Be 0 + } + + It 'Writes JSON output file' { + Test-Path $script:CleanOutputPath | Should -BeTrue + } + + It 'Written JSON contains correct totalFiles' { + $written = Get-Content $script:CleanOutputPath -Raw | ConvertFrom-Json + $written.totalFiles | Should -Be 5 + } + + It 'Written JSON contains correct validReferences' { + $written = Get-Content $script:CleanOutputPath -Raw | ConvertFrom-Json + $written.validReferences | Should -Be 4 + } + } + + Context 'when validation has invalid references' { + BeforeAll { + $script:ErrorOutputPath = Join-Path $script:OutputDir 'error-results.json' + $script:ErrorValidation = @{ + timestamp = '2026-05-06T00:00:00Z' + catalogLastUpdated = '2026-05-06' + totalFiles = 2 + filesWithModels = 1 + totalReferences = 2 + validReferences = 1 + invalidReferences = 1 + retiringReferences = 0 + results = @( + @{ file = 'bad.agent.md'; models = @('Model A (copilot)', 'Fake (copilot)'); status = 'invalid' } + ) + warnings = @() + errors = @( + @{ file = 'bad.agent.md'; model = 'Fake (copilot)'; message = "Unrecognized model: 'Fake (copilot)'" } + ) + } + + $script:ErrorExitCode = Write-ModelReferenceOutput -ValidationResult $script:ErrorValidation -OutputPath $script:ErrorOutputPath + } + + It 'Returns exit code 1' { + $script:ErrorExitCode | Should -Be 1 + } + + It 'Writes JSON output file' { + Test-Path $script:ErrorOutputPath | Should -BeTrue + } + + It 'Written JSON contains error details' { + $written = Get-Content $script:ErrorOutputPath -Raw | ConvertFrom-Json + $written.errors | Should -HaveCount 1 + } + } + + Context 'when validation has retiring references' { + BeforeAll { + $script:WarnOutputPath = Join-Path $script:OutputDir 'warn-results.json' + $script:WarnValidation = @{ + timestamp = '2026-05-06T00:00:00Z' + catalogLastUpdated = '2026-05-06' + totalFiles = 1 + filesWithModels = 1 + totalReferences = 1 + validReferences = 1 + invalidReferences = 0 + retiringReferences = 1 + results = @( + @{ file = 'old.agent.md'; models = @('Old Model (copilot)'); status = 'warning' } + ) + warnings = @( + @{ file = 'old.agent.md'; model = 'Old Model (copilot)'; message = "Model 'Old Model (copilot)' is retiring" } + ) + errors = @() + } + + $script:WarnExitCode = Write-ModelReferenceOutput -ValidationResult $script:WarnValidation -OutputPath $script:WarnOutputPath + } + + It 'Returns exit code 0 for warnings only' { + $script:WarnExitCode | Should -Be 0 + } + + It 'Writes JSON with warning details' { + $written = Get-Content $script:WarnOutputPath -Raw | ConvertFrom-Json + $written.warnings | Should -HaveCount 1 + } + } + + Context 'when output directory does not exist' { + BeforeAll { + $script:NestedOutputPath = Join-Path $script:OutputDir 'nested/deep/results.json' + if (Test-Path (Split-Path $script:NestedOutputPath -Parent)) { + Remove-Item (Split-Path $script:NestedOutputPath -Parent) -Recurse -Force + } + + $script:NestedValidation = @{ + timestamp = '2026-05-06T00:00:00Z' + catalogLastUpdated = '2026-05-06' + totalFiles = 0 + filesWithModels = 0 + totalReferences = 0 + validReferences = 0 + invalidReferences = 0 + retiringReferences = 0 + results = @() + warnings = @() + errors = @() + } + + $script:NestedExitCode = Write-ModelReferenceOutput -ValidationResult $script:NestedValidation -OutputPath $script:NestedOutputPath + } + + It 'Creates the directory and writes file' { + Test-Path $script:NestedOutputPath | Should -BeTrue + } + + It 'Returns exit code 0' { + $script:NestedExitCode | Should -Be 0 + } + } +} + +#endregion diff --git a/scripts/tests/linting/Test-UpdateModelCatalog.Tests.ps1 b/scripts/tests/linting/Test-UpdateModelCatalog.Tests.ps1 new file mode 100644 index 000000000..99f05c7bc --- /dev/null +++ b/scripts/tests/linting/Test-UpdateModelCatalog.Tests.ps1 @@ -0,0 +1,651 @@ +#Requires -Modules Pester +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT + +BeforeAll { + $script:ScriptPath = Join-Path $PSScriptRoot '../../linting/Update-ModelCatalog.ps1' + . $script:ScriptPath + + # Suppress Write-Host output during tests + Mock Write-Host {} + Mock Write-Warning {} +} + +#region Get-RemoteYaml Tests + +Describe 'Get-RemoteYaml' -Tag 'Unit' { + Context 'when URL returns valid YAML' { + It 'Parses YAML content into objects' { + $yamlContent = @" +- name: Claude Sonnet 4 + release_status: GA +- name: GPT-5 mini + release_status: Preview +"@ + Mock Invoke-WebRequest { + [PSCustomObject]@{ Content = $yamlContent } + } + + $result = Get-RemoteYaml -Url 'https://example.com/test.yml' + $result | Should -HaveCount 2 + $result[0].name | Should -Be 'Claude Sonnet 4' + $result[1].release_status | Should -Be 'Preview' + } + } + + Context 'when URL request fails' { + It 'Throws an error' { + Mock Invoke-WebRequest { throw 'Network error' } + + { Get-RemoteYaml -Url 'https://example.com/fail.yml' } | Should -Throw + } + } + + Context 'when YAML is empty' { + It 'Returns null or empty' { + Mock Invoke-WebRequest { + [PSCustomObject]@{ Content = '' } + } + + $result = Get-RemoteYaml -Url 'https://example.com/empty.yml' + $result | Should -BeNullOrEmpty + } + } +} + +#endregion Get-RemoteYaml Tests + +#region Merge-ModelData Tests + +Describe 'Merge-ModelData' -Tag 'Unit' { + Context 'when given matching release status and multiplier data' { + BeforeAll { + $script:ReleaseStatus = @( + @{ name = 'Claude Sonnet 4'; release_status = 'GA' } + @{ name = 'GPT-5 mini'; release_status = 'Preview' } + ) + $script:Multipliers = @( + @{ name = 'Claude Sonnet 4'; multiplier_paid = 1 } + @{ name = 'GPT-5 mini'; multiplier_paid = 0 } + ) + $script:Result = @(Merge-ModelData -ReleaseStatus $script:ReleaseStatus -Multipliers $script:Multipliers) + } + + It 'Returns an entry for each model in release status' { + $script:Result | Should -HaveCount 2 + } + + It 'Appends (copilot) suffix to model names' { + $script:Result[0].name | Should -Be 'Claude Sonnet 4 (copilot)' + $script:Result[1].name | Should -Be 'GPT-5 mini (copilot)' + } + + It 'Maps GA release status to ga' { + $script:Result[0].status | Should -Be 'ga' + } + + It 'Maps non-GA release status to preview' { + $script:Result[1].status | Should -Be 'preview' + } + + It 'Sets correct multiplier values' { + $script:Result[0].multiplier | Should -Be 1 + $script:Result[1].multiplier | Should -Be 0 + } + } + + Context 'tier classification from multiplier values' { + It 'Assigns free tier for multiplier 0' { + $release = @(@{ name = 'Free Model'; release_status = 'GA' }) + $mult = @(@{ name = 'Free Model'; multiplier_paid = 0 }) + $result = @(Merge-ModelData -ReleaseStatus $release -Multipliers $mult) + $result[0].tier | Should -Be 'free' + } + + It 'Assigns fast tier for multiplier 0.25' { + $release = @(@{ name = 'Fast Model'; release_status = 'GA' }) + $mult = @(@{ name = 'Fast Model'; multiplier_paid = 0.25 }) + $result = @(Merge-ModelData -ReleaseStatus $release -Multipliers $mult) + $result[0].tier | Should -Be 'fast' + } + + It 'Assigns fast tier for multiplier 0.33' { + $release = @(@{ name = 'Fast Edge'; release_status = 'GA' }) + $mult = @(@{ name = 'Fast Edge'; multiplier_paid = 0.33 }) + $result = @(Merge-ModelData -ReleaseStatus $release -Multipliers $mult) + $result[0].tier | Should -Be 'fast' + } + + It 'Assigns standard tier for multiplier 0.5' { + $release = @(@{ name = 'Standard Low'; release_status = 'GA' }) + $mult = @(@{ name = 'Standard Low'; multiplier_paid = 0.5 }) + $result = @(Merge-ModelData -ReleaseStatus $release -Multipliers $mult) + $result[0].tier | Should -Be 'standard' + } + + It 'Assigns standard tier for multiplier 1' { + $release = @(@{ name = 'Standard Model'; release_status = 'GA' }) + $mult = @(@{ name = 'Standard Model'; multiplier_paid = 1 }) + $result = @(Merge-ModelData -ReleaseStatus $release -Multipliers $mult) + $result[0].tier | Should -Be 'standard' + } + + It 'Assigns premium tier for multiplier 3' { + $release = @(@{ name = 'Premium Model'; release_status = 'GA' }) + $mult = @(@{ name = 'Premium Model'; multiplier_paid = 3 }) + $result = @(Merge-ModelData -ReleaseStatus $release -Multipliers $mult) + $result[0].tier | Should -Be 'premium' + } + + It 'Assigns premium tier for multiplier 5' { + $release = @(@{ name = 'Premium Edge'; release_status = 'GA' }) + $mult = @(@{ name = 'Premium Edge'; multiplier_paid = 5 }) + $result = @(Merge-ModelData -ReleaseStatus $release -Multipliers $mult) + $result[0].tier | Should -Be 'premium' + } + + It 'Assigns ultra tier for multiplier 15' { + $release = @(@{ name = 'Ultra Model'; release_status = 'GA' }) + $mult = @(@{ name = 'Ultra Model'; multiplier_paid = 15 }) + $result = @(Merge-ModelData -ReleaseStatus $release -Multipliers $mult) + $result[0].tier | Should -Be 'ultra' + } + + It 'Assigns ultra tier for multiplier 30' { + $release = @(@{ name = 'Ultra Max'; release_status = 'GA' }) + $mult = @(@{ name = 'Ultra Max'; multiplier_paid = 30 }) + $result = @(Merge-ModelData -ReleaseStatus $release -Multipliers $mult) + $result[0].tier | Should -Be 'ultra' + } + + It 'Returns a string tier, not an array' { + $release = @(@{ name = 'Tier Check'; release_status = 'GA' }) + $mult = @(@{ name = 'Tier Check'; multiplier_paid = 0.25 }) + $result = @(Merge-ModelData -ReleaseStatus $release -Multipliers $mult) + $result[0].tier | Should -BeOfType [string] + } + } + + Context 'when multiplier_paid is Not applicable' { + It 'Treats Not applicable as multiplier 0 and free tier' { + $release = @(@{ name = 'NA Model'; release_status = 'GA' }) + $mult = @(@{ name = 'NA Model'; multiplier_paid = 'Not applicable' }) + $result = @(Merge-ModelData -ReleaseStatus $release -Multipliers $mult) + $result[0].multiplier | Should -Be 0 + $result[0].tier | Should -Be 'free' + } + } + + Context 'when model has no multiplier entry' { + It 'Defaults to multiplier 1 and standard tier' { + $release = @(@{ name = 'No Mult Model'; release_status = 'GA' }) + $mult = @(@{ name = 'Other Model'; multiplier_paid = 5 }) + $result = @(Merge-ModelData -ReleaseStatus $release -Multipliers $mult) + $result[0].multiplier | Should -Be 1 + $result[0].tier | Should -Be 'standard' + } + } + + Context 'when release status has multiple models' { + It 'Processes all models in order' { + $release = @( + @{ name = 'Model A'; release_status = 'GA' } + @{ name = 'Model B'; release_status = 'Preview' } + @{ name = 'Model C'; release_status = 'GA' } + ) + $mult = @( + @{ name = 'Model A'; multiplier_paid = 0 } + @{ name = 'Model B'; multiplier_paid = 1 } + @{ name = 'Model C'; multiplier_paid = 4 } + ) + $result = @(Merge-ModelData -ReleaseStatus $release -Multipliers $mult) + $result | Should -HaveCount 3 + $result[0].name | Should -Be 'Model A (copilot)' + $result[1].name | Should -Be 'Model B (copilot)' + $result[2].name | Should -Be 'Model C (copilot)' + $result[0].tier | Should -Be 'free' + $result[1].tier | Should -Be 'standard' + $result[2].tier | Should -Be 'premium' + } + } + + Context 'when multiplier_paid is null' { + It 'Defaults to multiplier 1' { + $release = @(@{ name = 'Null Mult'; release_status = 'GA' }) + $mult = @(@{ name = 'Null Mult'; multiplier_paid = $null }) + $result = @(Merge-ModelData -ReleaseStatus $release -Multipliers $mult) + $result[0].multiplier | Should -Be 1 + $result[0].tier | Should -Be 'standard' + } + } +} + +#endregion Merge-ModelData Tests + +#region Compare-Catalogs Tests + +Describe 'Compare-Catalogs' -Tag 'Unit' { + Context 'when catalogs are identical' { + It 'Returns empty added, removed, and changed arrays' { + $models = @( + [PSCustomObject]@{ name = 'Model A (copilot)'; multiplier = 1 } + [PSCustomObject]@{ name = 'Model B (copilot)'; multiplier = 3 } + ) + $result = Compare-Catalogs -Current $models -Discovered $models + $result.added | Should -HaveCount 0 + $result.removed | Should -HaveCount 0 + $result.changed | Should -HaveCount 0 + } + } + + Context 'when new models are added' { + It 'Identifies added models' { + $current = @( + [PSCustomObject]@{ name = 'Model A (copilot)'; multiplier = 1 } + ) + $discovered = @( + [PSCustomObject]@{ name = 'Model A (copilot)'; multiplier = 1 } + [PSCustomObject]@{ name = 'Model B (copilot)'; multiplier = 3 } + ) + $result = Compare-Catalogs -Current $current -Discovered $discovered + $result.added | Should -HaveCount 1 + $result.added[0].name | Should -Be 'Model B (copilot)' + } + } + + Context 'when models are removed' { + It 'Identifies removed models' { + $current = @( + [PSCustomObject]@{ name = 'Model A (copilot)'; multiplier = 1 } + [PSCustomObject]@{ name = 'Model B (copilot)'; multiplier = 3 } + ) + $discovered = @( + [PSCustomObject]@{ name = 'Model A (copilot)'; multiplier = 1 } + ) + $result = Compare-Catalogs -Current $current -Discovered $discovered + $result.removed | Should -HaveCount 1 + $result.removed[0].name | Should -Be 'Model B (copilot)' + } + } + + Context 'when multipliers change' { + It 'Identifies changed multipliers' { + $current = @( + [PSCustomObject]@{ name = 'Model A (copilot)'; multiplier = 1 } + ) + $discovered = @( + [PSCustomObject]@{ name = 'Model A (copilot)'; multiplier = 3 } + ) + $result = Compare-Catalogs -Current $current -Discovered $discovered + $result.changed | Should -HaveCount 1 + $result.changed[0].name | Should -Be 'Model A (copilot)' + $result.changed[0].oldMultiplier | Should -Be 1 + $result.changed[0].newMultiplier | Should -Be 3 + } + } + + Context 'when all types of changes occur simultaneously' { + It 'Reports additions, removals, and changes together' { + $current = @( + [PSCustomObject]@{ name = 'Stable (copilot)'; multiplier = 1 } + [PSCustomObject]@{ name = 'Removed (copilot)'; multiplier = 5 } + [PSCustomObject]@{ name = 'Changed (copilot)'; multiplier = 1 } + ) + $discovered = @( + [PSCustomObject]@{ name = 'Stable (copilot)'; multiplier = 1 } + [PSCustomObject]@{ name = 'Changed (copilot)'; multiplier = 3 } + [PSCustomObject]@{ name = 'Added (copilot)'; multiplier = 0 } + ) + $result = Compare-Catalogs -Current $current -Discovered $discovered + $result.added | Should -HaveCount 1 + $result.removed | Should -HaveCount 1 + $result.changed | Should -HaveCount 1 + $result.added[0].name | Should -Be 'Added (copilot)' + $result.removed[0].name | Should -Be 'Removed (copilot)' + $result.changed[0].name | Should -Be 'Changed (copilot)' + } + } + + Context 'when current catalog is empty' { + It 'Reports all discovered models as added' { + $current = @( + [PSCustomObject]@{ name = 'placeholder'; multiplier = 0 } + ) + # Use a single-element to avoid empty array issues; test with actual additions + $discovered = @( + [PSCustomObject]@{ name = 'New A (copilot)'; multiplier = 1 } + [PSCustomObject]@{ name = 'New B (copilot)'; multiplier = 3 } + ) + $result = Compare-Catalogs -Current $current -Discovered $discovered + $result.added | Should -HaveCount 2 + $result.removed | Should -HaveCount 1 + } + } + + Context 'when multiplier does not change' { + It 'Does not report unchanged models' { + $current = @( + [PSCustomObject]@{ name = 'Model A (copilot)'; multiplier = 1 } + [PSCustomObject]@{ name = 'Model B (copilot)'; multiplier = 3 } + ) + $discovered = @( + [PSCustomObject]@{ name = 'Model A (copilot)'; multiplier = 1 } + [PSCustomObject]@{ name = 'Model B (copilot)'; multiplier = 3 } + ) + $result = Compare-Catalogs -Current $current -Discovered $discovered + $result.changed | Should -HaveCount 0 + } + } +} + +#endregion Compare-Catalogs Tests + +#region Invoke-ModelCatalogUpdate Tests + +Describe 'Invoke-ModelCatalogUpdate' -Tag 'Unit' { + BeforeAll { + $script:CatalogDir = Join-Path ([System.IO.Path]::GetTempPath()) "CatalogUpdateTests_$([guid]::NewGuid().ToString('N'))" + New-Item -ItemType Directory -Path $script:CatalogDir -Force | Out-Null + } + + AfterAll { + if (Test-Path $script:CatalogDir) { + Remove-Item -Path $script:CatalogDir -Recurse -Force -ErrorAction SilentlyContinue + } + } + + Context 'when no existing catalog exists' { + BeforeAll { + $script:NewCatalogPath = Join-Path $script:CatalogDir 'new-catalog.json' + if (Test-Path $script:NewCatalogPath) { Remove-Item $script:NewCatalogPath -Force } + + $release = @( + @{ name = 'Model A'; release_status = 'GA' } + @{ name = 'Model B'; release_status = 'Preview' } + ) + $mult = @( + @{ name = 'Model A'; multiplier_paid = 1 } + @{ name = 'Model B'; multiplier_paid = 0.25 } + ) + + $script:NewResult = Invoke-ModelCatalogUpdate -ReleaseStatus $release -Multipliers $mult -CatalogPath $script:NewCatalogPath + } + + It 'Returns created status' { + $script:NewResult.status | Should -Be 'created' + } + + It 'Returns null diff' { + $script:NewResult.diff | Should -BeNullOrEmpty + } + + It 'Returns all discovered models in finalModels' { + $script:NewResult.finalModels | Should -HaveCount 2 + } + + It 'Writes catalog file to disk' { + Test-Path $script:NewCatalogPath | Should -BeTrue + } + + It 'Written catalog contains correct model count' { + $written = Get-Content $script:NewCatalogPath -Raw | ConvertFrom-Json + $written.models | Should -HaveCount 2 + } + + It 'Written catalog has lastUpdated field' { + $written = Get-Content $script:NewCatalogPath -Raw | ConvertFrom-Json + $written.lastUpdated | Should -Not -BeNullOrEmpty + } + } + + Context 'when catalog exists with no changes' { + BeforeAll { + $script:UnchangedPath = Join-Path $script:CatalogDir 'unchanged-catalog.json' + $existingCatalog = @{ + lastUpdated = '2026-01-01' + source = 'https://example.com' + models = @( + @{ name = 'Model A (copilot)'; tier = 'standard'; multiplier = 1; status = 'ga' } + ) + } + $existingCatalog | ConvertTo-Json -Depth 5 | Set-Content -Path $script:UnchangedPath -Encoding utf8 + + $release = @(@{ name = 'Model A'; release_status = 'GA' }) + $mult = @(@{ name = 'Model A'; multiplier_paid = 1 }) + + $script:UnchangedResult = Invoke-ModelCatalogUpdate -ReleaseStatus $release -Multipliers $mult -CatalogPath $script:UnchangedPath + } + + It 'Returns unchanged status' { + $script:UnchangedResult.status | Should -Be 'unchanged' + } + + It 'Updates lastUpdated timestamp in file' { + $written = Get-Content $script:UnchangedPath -Raw | ConvertFrom-Json + $written.lastUpdated | Should -Be (Get-Date -Format 'yyyy-MM-dd') + } + } + + Context 'when models are added' { + BeforeAll { + $script:AddedPath = Join-Path $script:CatalogDir 'added-catalog.json' + $existingCatalog = @{ + lastUpdated = '2026-01-01' + source = 'https://example.com' + models = @( + @{ name = 'Model A (copilot)'; tier = 'standard'; multiplier = 1; status = 'ga' } + ) + } + $existingCatalog | ConvertTo-Json -Depth 5 | Set-Content -Path $script:AddedPath -Encoding utf8 + + $release = @( + @{ name = 'Model A'; release_status = 'GA' } + @{ name = 'Model B'; release_status = 'Preview' } + ) + $mult = @( + @{ name = 'Model A'; multiplier_paid = 1 } + @{ name = 'Model B'; multiplier_paid = 3 } + ) + + $script:AddedResult = Invoke-ModelCatalogUpdate -ReleaseStatus $release -Multipliers $mult -CatalogPath $script:AddedPath + } + + It 'Returns updated status' { + $script:AddedResult.status | Should -Be 'updated' + } + + It 'Includes new model in finalModels' { + $names = @($script:AddedResult.finalModels | ForEach-Object { if ($_ -is [hashtable]) { $_['name'] } else { $_.name } }) + $names | Should -Contain 'Model B (copilot)' + } + + It 'Reports addition in diff' { + $script:AddedResult.diff.added | Should -HaveCount 1 + } + } + + Context 'when models are removed' { + BeforeAll { + $script:RemovedPath = Join-Path $script:CatalogDir 'removed-catalog.json' + $existingCatalog = @{ + lastUpdated = '2026-01-01' + source = 'https://example.com' + models = @( + @{ name = 'Model A (copilot)'; tier = 'standard'; multiplier = 1; status = 'ga' } + @{ name = 'Model B (copilot)'; tier = 'premium'; multiplier = 3; status = 'ga' } + ) + } + $existingCatalog | ConvertTo-Json -Depth 5 | Set-Content -Path $script:RemovedPath -Encoding utf8 + + $release = @(@{ name = 'Model A'; release_status = 'GA' }) + $mult = @(@{ name = 'Model A'; multiplier_paid = 1 }) + + $script:RemovedResult = Invoke-ModelCatalogUpdate -ReleaseStatus $release -Multipliers $mult -CatalogPath $script:RemovedPath + } + + It 'Returns updated status' { + $script:RemovedResult.status | Should -Be 'updated' + } + + It 'Returns a hashtable with status, diff, and finalModels keys' { + $script:RemovedResult.Keys | Should -Contain 'status' + $script:RemovedResult.Keys | Should -Contain 'diff' + $script:RemovedResult.Keys | Should -Contain 'finalModels' + } + + It 'Marks removed model as retiring PSCustomObject in finalModels' { + $retiring = $script:RemovedResult.finalModels | Where-Object { + if ($_ -is [hashtable]) { $_['name'] -eq 'Model B (copilot)' } + else { $_.name -eq 'Model B (copilot)' } + } + $retiring | Should -BeOfType [PSCustomObject] + $retiring.status | Should -Be 'retiring' + } + + It 'Preserves tier and multiplier on retiring model' { + $retiring = $script:RemovedResult.finalModels | Where-Object { + if ($_ -is [hashtable]) { $_['name'] -eq 'Model B (copilot)' } + else { $_.name -eq 'Model B (copilot)' } + } + $retiring.tier | Should -Be 'premium' + $retiring.multiplier | Should -Be 3 + } + + It 'Sets retiredDate as future date on removed model' { + $retiring = $script:RemovedResult.finalModels | Where-Object { + if ($_ -is [hashtable]) { $_['name'] -eq 'Model B (copilot)' } + else { $_.name -eq 'Model B (copilot)' } + } + $retiring.retiredDate | Should -Not -BeNullOrEmpty + [datetime]::Parse($retiring.retiredDate) | Should -BeGreaterThan (Get-Date) + } + + It 'Keeps non-removed models unchanged' { + $kept = $script:RemovedResult.finalModels | Where-Object { + if ($_ -is [hashtable]) { $_['name'] -eq 'Model A (copilot)' } + else { $_.name -eq 'Model A (copilot)' } + } + $kept | Should -Not -BeNullOrEmpty + } + } + + Context 'when multipliers change' { + BeforeAll { + $script:ChangedPath = Join-Path $script:CatalogDir 'changed-catalog.json' + $existingCatalog = @{ + lastUpdated = '2026-01-01' + source = 'https://example.com' + models = @( + @{ name = 'Model A (copilot)'; tier = 'standard'; multiplier = 1; status = 'ga' } + ) + } + $existingCatalog | ConvertTo-Json -Depth 5 | Set-Content -Path $script:ChangedPath -Encoding utf8 + + $release = @(@{ name = 'Model A'; release_status = 'GA' }) + $mult = @(@{ name = 'Model A'; multiplier_paid = 5 }) + + $script:ChangedResult = Invoke-ModelCatalogUpdate -ReleaseStatus $release -Multipliers $mult -CatalogPath $script:ChangedPath + } + + It 'Returns updated status' { + $script:ChangedResult.status | Should -Be 'updated' + } + + It 'Reports change in diff' { + $script:ChangedResult.diff.changed | Should -HaveCount 1 + $script:ChangedResult.diff.changed[0].oldMultiplier | Should -Be 1 + $script:ChangedResult.diff.changed[0].newMultiplier | Should -Be 5 + } + } + + Context 'when DryRun is specified' { + BeforeAll { + $script:DryRunPath = Join-Path $script:CatalogDir 'dryrun-catalog.json' + $existingCatalog = @{ + lastUpdated = '2026-01-01' + source = 'https://example.com' + models = @( + @{ name = 'Model A (copilot)'; tier = 'standard'; multiplier = 1; status = 'ga' } + ) + } + $existingCatalog | ConvertTo-Json -Depth 5 | Set-Content -Path $script:DryRunPath -Encoding utf8 + + $release = @( + @{ name = 'Model A'; release_status = 'GA' } + @{ name = 'Model B'; release_status = 'GA' } + ) + $mult = @( + @{ name = 'Model A'; multiplier_paid = 1 } + @{ name = 'Model B'; multiplier_paid = 3 } + ) + + $script:DryRunResult = Invoke-ModelCatalogUpdate -ReleaseStatus $release -Multipliers $mult -CatalogPath $script:DryRunPath -DryRun + } + + It 'Returns dryrun status' { + $script:DryRunResult.status | Should -Be 'dryrun' + } + + It 'Does not modify the catalog file' { + $written = Get-Content $script:DryRunPath -Raw | ConvertFrom-Json + $written.lastUpdated | Should -Be '2026-01-01' + $written.models | Should -HaveCount 1 + } + + It 'Still computes finalModels' { + $script:DryRunResult.finalModels.Count | Should -BeGreaterThan 1 + } + } + + Context 'when DryRun with no changes' { + BeforeAll { + $script:DryRunNoChangePath = Join-Path $script:CatalogDir 'dryrun-nochange.json' + $existingCatalog = @{ + lastUpdated = '2026-01-01' + source = 'https://example.com' + models = @( + @{ name = 'Model A (copilot)'; tier = 'standard'; multiplier = 1; status = 'ga' } + ) + } + $existingCatalog | ConvertTo-Json -Depth 5 | Set-Content -Path $script:DryRunNoChangePath -Encoding utf8 + + $release = @(@{ name = 'Model A'; release_status = 'GA' }) + $mult = @(@{ name = 'Model A'; multiplier_paid = 1 }) + + $script:DryRunNoChangeResult = Invoke-ModelCatalogUpdate -ReleaseStatus $release -Multipliers $mult -CatalogPath $script:DryRunNoChangePath -DryRun + } + + It 'Returns unchanged status' { + $script:DryRunNoChangeResult.status | Should -Be 'unchanged' + } + + It 'Does not update lastUpdated in file' { + $written = Get-Content $script:DryRunNoChangePath -Raw | ConvertFrom-Json + $written.lastUpdated | Should -Be '2026-01-01' + } + } + + Context 'when catalog path directory does not exist' { + BeforeAll { + $script:DeepPath = Join-Path $script:CatalogDir 'deep/nested/dir/catalog.json' + if (Test-Path (Split-Path $script:DeepPath -Parent)) { + Remove-Item (Split-Path $script:DeepPath -Parent) -Recurse -Force + } + + $release = @(@{ name = 'Model A'; release_status = 'GA' }) + $mult = @(@{ name = 'Model A'; multiplier_paid = 1 }) + + $script:DeepResult = Invoke-ModelCatalogUpdate -ReleaseStatus $release -Multipliers $mult -CatalogPath $script:DeepPath + } + + It 'Creates directory and writes catalog' { + Test-Path $script:DeepPath | Should -BeTrue + } + + It 'Returns created status' { + $script:DeepResult.status | Should -Be 'created' + } + } +} + +#endregion Invoke-ModelCatalogUpdate Tests