Skip to content

Commit 22cdb59

Browse files
bcorfmanbcorfman
authored
Merge pull request #536 from bcorfman/validation
Phase 1 validation harness
2 parents 0d0b81e + b6834ae commit 22cdb59

7 files changed

Lines changed: 271 additions & 9 deletions

File tree

‎.plans/hosted-deployment-validation-harness-implementation-plan-2026-07-27.md‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Hosted Deployment Validation Harness Implementation Plan
22

3-
Status: proposed
3+
Status: Phase 1 complete
44

55
Date: 2026-07-27
66

@@ -117,16 +117,16 @@ isolation mutations require an explicit flag such as
117117

118118
## Phase 1 — Read-only deployment probes
119119

120-
- [ ] Add configuration parsing and safety validation tests.
121-
- [ ] Implement `GET /api/v1/health` checks with status, JSON shape, and
120+
- [x] Add configuration parsing and safety validation tests.
121+
- [x] Implement `GET /api/v1/health` checks with status, JSON shape, and
122122
bounded response timing.
123-
- [ ] Implement `GET /api/v1/version` checks for expected channel and optional
123+
- [x] Implement `GET /api/v1/version` checks for expected channel and optional
124124
commit.
125-
- [ ] Record sanitized status, headers, timing, and mismatch reasons.
126-
- [ ] Redact response bodies to approved fields only.
127-
- [ ] Add a CLI command such as:
125+
- [x] Record sanitized status, headers, timing, and mismatch reasons.
126+
- [x] Redact response bodies to approved fields only.
127+
- [x] Add a CLI command such as:
128128
`npm run repair:ci -- hosted-probe --config <path>`.
129-
- [ ] Classify DNS, TLS, timeout, 5xx, wrong-channel, and wrong-commit cases
129+
- [x] Classify DNS, TLS, timeout, 5xx, wrong-channel, and wrong-commit cases
130130
distinctly as deployment/infrastructure evidence.
131131

132132
Exit criteria:

‎scripts/repair-harness/cli.ts‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ import { runBoundedRepair } from './repair';
99
import { appendEvent, readState, resolveResumeDirectory, writeState } from './state';
1010
import type { EvidenceEnvelope } from './types';
1111
import { aggregateRepairOutcomes, formatRepairMetrics, readRepairOutcomes, writeRepairMetrics } from './metrics';
12+
import { loadHostedConfig } from './hosted/config';
13+
import { runHostedProbe } from './hosted/run';
1214

1315
const repositoryRoot = path.resolve(new URL('../..', import.meta.url).pathname);
1416
const args = process.argv.slice(2);
@@ -22,7 +24,18 @@ const value = (name: string): string | undefined => {
2224
const has = (name: string): boolean => args.includes(name);
2325

2426
async function main(): Promise<void> {
25-
if (args[0] === 'metrics') {
27+
if (args[0] === 'hosted-probe') {
28+
try {
29+
const configPath = value('--config');
30+
if (!configPath) throw new Error('Expected --config <path> for hosted-probe.');
31+
const result = await runHostedProbe({ repo: value('--repo') ?? repositoryRoot, config: loadHostedConfig(path.resolve(configPath)), runId: value('--run-id') });
32+
console.log(`Hosted probe ${result.status} in ${result.runDirectory}`);
33+
if (result.status !== 'passed') process.exitCode = 1;
34+
} catch (error) {
35+
console.error(error instanceof Error ? error.message : String(error));
36+
process.exitCode = 1;
37+
}
38+
} else if (args[0] === 'metrics') {
2639
try {
2740
const repo = value('--repo') ?? repositoryRoot;
2841
const runsRoot = value('--runs-root') ?? path.join(repo, '.repair-harness', 'runs');

‎scripts/repair-harness/hosted/config.ts‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
import { readFileSync } from 'node:fs';
2+
3+
export type HostedAccountProvider = 'external' | 'manual' | 'fixture';
4+
5+
export interface HostedConfig {
6+
devFrontendUrl: string;
7+
devApiUrl: string;
8+
stableFrontendUrl: string;
9+
stableApiUrl: string;
10+
expectedDevChannel: string;
11+
expectedStableChannel: string;
12+
expectedDevCommit?: string;
13+
expectedStableCommit?: string;
14+
testAccountProvider: HostedAccountProvider;
15+
allowMutations: boolean;
16+
allowOAuth: boolean;
17+
timeoutMs: number;
18+
allowedApiHosts: string[];
19+
}
20+
21+
export interface HostedConfigInput {
22+
HOSTED_DEV_FRONTEND_URL?: string;
23+
HOSTED_DEV_API_URL?: string;
24+
HOSTED_STABLE_FRONTEND_URL?: string;
25+
HOSTED_STABLE_API_URL?: string;
26+
HOSTED_EXPECTED_DEV_CHANNEL?: string;
27+
HOSTED_EXPECTED_STABLE_CHANNEL?: string;
28+
HOSTED_EXPECTED_DEV_COMMIT?: string;
29+
HOSTED_EXPECTED_STABLE_COMMIT?: string;
30+
HOSTED_TEST_ACCOUNT_PROVIDER?: string;
31+
HOSTED_ALLOW_MUTATIONS?: string | boolean;
32+
HOSTED_ALLOW_OAUTH?: string | boolean;
33+
HOSTED_TIMEOUT_MS?: string | number;
34+
HOSTED_ALLOWED_API_HOSTS?: string | string[];
35+
}
36+
37+
export function parseHostedConfig(input: HostedConfigInput): HostedConfig {
38+
const config: HostedConfig = {
39+
devFrontendUrl: required(input.HOSTED_DEV_FRONTEND_URL, 'HOSTED_DEV_FRONTEND_URL'),
40+
devApiUrl: required(input.HOSTED_DEV_API_URL, 'HOSTED_DEV_API_URL'),
41+
stableFrontendUrl: required(input.HOSTED_STABLE_FRONTEND_URL, 'HOSTED_STABLE_FRONTEND_URL'),
42+
stableApiUrl: required(input.HOSTED_STABLE_API_URL, 'HOSTED_STABLE_API_URL'),
43+
expectedDevChannel: input.HOSTED_EXPECTED_DEV_CHANNEL?.trim() || 'dev',
44+
expectedStableChannel: input.HOSTED_EXPECTED_STABLE_CHANNEL?.trim() || 'stable',
45+
expectedDevCommit: optional(input.HOSTED_EXPECTED_DEV_COMMIT),
46+
expectedStableCommit: optional(input.HOSTED_EXPECTED_STABLE_COMMIT),
47+
testAccountProvider: parseProvider(input.HOSTED_TEST_ACCOUNT_PROVIDER),
48+
allowMutations: parseBoolean(input.HOSTED_ALLOW_MUTATIONS, false),
49+
allowOAuth: parseBoolean(input.HOSTED_ALLOW_OAUTH, false),
50+
timeoutMs: parseTimeout(input.HOSTED_TIMEOUT_MS),
51+
allowedApiHosts: parseHosts(input.HOSTED_ALLOWED_API_HOSTS),
52+
};
53+
validateHostedConfig(config);
54+
return config;
55+
}
56+
57+
export function loadHostedConfig(filePath: string): HostedConfig {
58+
const raw = JSON.parse(readFileSync(filePath, 'utf8')) as HostedConfigInput;
59+
return parseHostedConfig(raw);
60+
}
61+
62+
export function validateHostedConfig(config: HostedConfig): void {
63+
const urls = [config.devFrontendUrl, config.devApiUrl, config.stableFrontendUrl, config.stableApiUrl];
64+
for (const value of urls) {
65+
const url = parseUrl(value);
66+
if (url.protocol !== 'https:') throw new Error(`Hosted URL must use HTTPS: ${value}`);
67+
if (url.username || url.password) throw new Error(`Hosted URL must not contain credentials: ${value}`);
68+
}
69+
const devApi = parseUrl(config.devApiUrl);
70+
const stableApi = parseUrl(config.stableApiUrl);
71+
if (devApi.origin === stableApi.origin) throw new Error('Development and stable API origins must be distinct.');
72+
if (!config.allowedApiHosts.includes(devApi.host) || !config.allowedApiHosts.includes(stableApi.host)) {
73+
throw new Error('Development and stable API hosts must be present in the configured API allowlist (HOSTED_ALLOWED_API_HOSTS).');
74+
}
75+
if (!Number.isInteger(config.timeoutMs) || config.timeoutMs < 100 || config.timeoutMs > 120_000) {
76+
throw new Error('HOSTED_TIMEOUT_MS must be an integer between 100 and 120000.');
77+
}
78+
}
79+
80+
function required(value: string | undefined, name: string): string {
81+
if (!value?.trim()) throw new Error(`${name} is required.`);
82+
return value.trim();
83+
}
84+
85+
function optional(value: string | undefined): string | undefined { return value?.trim() || undefined; }
86+
87+
function parseUrl(value: string): URL {
88+
try { return new URL(value); } catch { throw new Error(`Invalid hosted URL: ${value}`); }
89+
}
90+
91+
function parseProvider(value: string | undefined): HostedAccountProvider {
92+
if (value !== 'external' && value !== 'manual' && value !== 'fixture') throw new Error('HOSTED_TEST_ACCOUNT_PROVIDER must be external, manual, or fixture.');
93+
return value;
94+
}
95+
96+
function parseBoolean(value: string | boolean | undefined, fallback: boolean): boolean {
97+
if (value === undefined) return fallback;
98+
if (value === true || value === 'true') return true;
99+
if (value === false || value === 'false') return false;
100+
throw new Error('Hosted boolean options must be true or false.');
101+
}
102+
103+
function parseTimeout(value: string | number | undefined): number { return value === undefined ? 15_000 : Number(value); }
104+
105+
function parseHosts(value: string | string[] | undefined): string[] {
106+
const values = Array.isArray(value) ? value : value?.split(',') ?? [];
107+
return values.map((host) => host.trim().toLowerCase()).filter(Boolean);
108+
}

‎scripts/repair-harness/hosted/evidence.ts‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
import { redactSecrets } from '../artifacts';
2+
3+
const APPROVED_HEADERS = new Set(['cache-control', 'content-type', 'strict-transport-security', 'access-control-allow-origin', 'access-control-allow-credentials']);
4+
5+
export interface HostedResponseProjection {
6+
status: number;
7+
ok: boolean;
8+
headers: Record<string, string>;
9+
body: Record<string, string>;
10+
}
11+
12+
export function projectHostedResponse(status: number, headers: Headers, body: unknown, fields: string[]): HostedResponseProjection {
13+
const source = body && typeof body === 'object' ? body as Record<string, unknown> : {};
14+
const projected = Object.fromEntries(fields.filter((field) => typeof source[field] === 'string').map((field) => [field, String(source[field])]));
15+
return {
16+
status,
17+
ok: status >= 200 && status < 300,
18+
headers: Object.fromEntries([...headers.entries()].filter(([name]) => APPROVED_HEADERS.has(name.toLowerCase())).map(([name, value]) => [name.toLowerCase(), redactSecrets(value)])),
19+
body: projected,
20+
};
21+
}
22+
23+
export function redactHostedReason(reason: string): string { return redactSecrets(reason).slice(0, 500); }

‎scripts/repair-harness/hosted/probes.ts‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
import type { HostedConfig } from './config';
2+
import { projectHostedResponse, redactHostedReason, type HostedResponseProjection } from './evidence';
3+
4+
export type HostedFailureClass = 'dns' | 'tls' | 'timeout' | 'http-5xx' | 'wrong-channel' | 'wrong-commit' | 'http';
5+
export interface HostedProbeResult { endpoint: 'health' | 'version'; url: string; durationMs: number; response?: HostedResponseProjection; failureClass?: HostedFailureClass; reason?: string; }
6+
7+
type FetchLike = (input: string, init?: RequestInit) => Promise<Response>;
8+
9+
export async function probeDeployment(config: HostedConfig, fetchImpl: FetchLike = fetch): Promise<HostedProbeResult[]> {
10+
const results = [
11+
await probeEndpoint('health', config.devApiUrl, fetchImpl, ['status'], config.timeoutMs),
12+
await probeVersion(config, fetchImpl),
13+
];
14+
const health = results[0];
15+
if (!health.failureClass && health.response?.body.status !== 'ok') {
16+
results[0] = { ...health, failureClass: 'http', reason: 'Health response did not contain status=ok.' };
17+
}
18+
return results;
19+
}
20+
21+
async function probeEndpoint(endpoint: 'health' | 'version', baseUrl: string, fetchImpl: FetchLike, fields: string[], timeoutMs = 15_000): Promise<HostedProbeResult> {
22+
const url = `${baseUrl.replace(/\/$/, '')}/api/v1/${endpoint}`;
23+
const started = Date.now();
24+
const controller = new AbortController();
25+
const timer = setTimeout(() => controller.abort(), timeoutMs);
26+
let timeoutTimer: ReturnType<typeof setTimeout> | undefined;
27+
try {
28+
const response = await Promise.race([
29+
fetchImpl(url, { method: 'GET', signal: controller.signal, headers: { accept: 'application/json' } }),
30+
new Promise<Response>((_, reject) => { timeoutTimer = setTimeout(() => reject(new Error('Hosted probe timed out.')), timeoutMs); }),
31+
]);
32+
const body = await response.json().catch(() => ({}));
33+
const projection = projectHostedResponse(response.status, response.headers, body, fields);
34+
return { endpoint, url, durationMs: Date.now() - started, response: projection, ...(response.status >= 500 ? { failureClass: 'http-5xx' as const, reason: `HTTP ${response.status}` } : response.ok ? {} : { failureClass: 'http' as const, reason: `HTTP ${response.status}` }) };
35+
} catch (error) {
36+
return { endpoint, url, durationMs: Date.now() - started, failureClass: classifyNetworkError(error), reason: redactHostedReason(error instanceof Error ? error.message : String(error)) };
37+
} finally { clearTimeout(timer); if (timeoutTimer) clearTimeout(timeoutTimer); }
38+
}
39+
40+
async function probeVersion(config: HostedConfig, fetchImpl: FetchLike): Promise<HostedProbeResult> {
41+
const result = await probeEndpoint('version', config.devApiUrl, fetchImpl, ['channel', 'commit'], config.timeoutMs);
42+
const body = result.response?.body;
43+
if (!result.failureClass && body?.channel !== config.expectedDevChannel) return { ...result, failureClass: 'wrong-channel', reason: `Expected channel ${config.expectedDevChannel}; received ${body?.channel ?? '[missing]'}.` };
44+
if (!result.failureClass && config.expectedDevCommit && body?.commit !== config.expectedDevCommit) return { ...result, failureClass: 'wrong-commit', reason: 'Configured development commit does not match deployed commit.' };
45+
return result;
46+
}
47+
48+
export function classifyNetworkError(error: unknown): 'dns' | 'tls' | 'timeout' {
49+
const code = typeof error === 'object' && error && 'code' in error ? String(error.code) : '';
50+
const message = error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase();
51+
if (code === 'ENOTFOUND' || code === 'EAI_AGAIN' || message.includes('dns')) return 'dns';
52+
if (code.startsWith('ERR_TLS') || code.includes('CERT') || message.includes('certificate') || message.includes('tls')) return 'tls';
53+
return 'timeout';
54+
}

‎scripts/repair-harness/hosted/run.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import { mkdirSync, writeFileSync } from 'node:fs';
2+
import path from 'node:path';
3+
import type { HostedConfig } from './config';
4+
import { probeDeployment, type HostedProbeResult } from './probes';
5+
6+
export interface HostedProbeRun { runId: string; runDirectory: string; results: HostedProbeResult[]; status: 'passed' | 'failed'; }
7+
8+
export async function runHostedProbe(options: { repo: string; config: HostedConfig; runId?: string; fetchImpl?: (input: string, init?: RequestInit) => Promise<Response> }): Promise<HostedProbeRun> {
9+
const runId = options.runId ?? `hosted-${Date.now()}`;
10+
const runDirectory = path.resolve(options.repo, '.repair-harness', 'runs', runId);
11+
mkdirSync(path.join(runDirectory, 'hosted-browser'), { recursive: true });
12+
const results = await probeDeployment(options.config, options.fetchImpl);
13+
const failed = results.filter((result) => result.failureClass);
14+
writeFileSync(path.join(runDirectory, 'hosted-config.json'), `${JSON.stringify({ ...options.config, expectedDevCommit: options.config.expectedDevCommit ? '[configured]' : undefined, expectedStableCommit: options.config.expectedStableCommit ? '[configured]' : undefined }, null, 2)}\n`);
15+
writeFileSync(path.join(runDirectory, 'hosted-evidence.json'), `${JSON.stringify({ version: 1, kind: 'hosted-deployment-probe', runId, status: failed.length ? 'failed' : 'passed', results }, null, 2)}\n`);
16+
writeFileSync(path.join(runDirectory, 'hosted-events.jsonl'), results.map((result) => `${JSON.stringify({ event: 'hosted-probe', endpoint: result.endpoint, status: result.failureClass ? 'failed' : 'passed', failureClass: result.failureClass, durationMs: result.durationMs, reason: result.reason })}\n`).join(''));
17+
writeFileSync(path.join(runDirectory, 'hosted-summary.md'), `# Hosted deployment probe\n\nStatus: ${failed.length ? 'failed' : 'passed'}\n\n${results.map((result) => `- ${result.endpoint}: ${result.failureClass ?? 'passed'} (${result.durationMs}ms)`).join('\n')}\n`);
18+
return { runId, runDirectory, results, status: failed.length ? 'failed' : 'passed' };
19+
}

‎tests/scripts/repair-harness-hosted-phase1.test.ts‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
import { describe, expect, it } from 'vitest';
2+
import { mkdtempSync, readFileSync } from 'node:fs';
3+
import os from 'node:os';
4+
import path from 'node:path';
5+
import { parseHostedConfig } from '../../scripts/repair-harness/hosted/config';
6+
import { projectHostedResponse } from '../../scripts/repair-harness/hosted/evidence';
7+
import { classifyNetworkError, probeDeployment } from '../../scripts/repair-harness/hosted/probes';
8+
import { runHostedProbe } from '../../scripts/repair-harness/hosted/run';
9+
10+
const input = { HOSTED_DEV_FRONTEND_URL: 'https://pages.example/dev/', HOSTED_DEV_API_URL: 'https://dev-api.example', HOSTED_STABLE_FRONTEND_URL: 'https://pages.example/stable/', HOSTED_STABLE_API_URL: 'https://stable-api.example', HOSTED_TEST_ACCOUNT_PROVIDER: 'manual', HOSTED_ALLOWED_API_HOSTS: 'dev-api.example,stable-api.example' };
11+
12+
describe('hosted deployment validation phase 1', () => {
13+
it('requires HTTPS, distinct allowlisted APIs, and safe defaults', () => {
14+
expect(parseHostedConfig(input)).toMatchObject({ expectedDevChannel: 'dev', expectedStableChannel: 'stable', allowMutations: false, allowOAuth: false });
15+
expect(() => parseHostedConfig({ ...input, HOSTED_DEV_API_URL: 'http://dev-api.example' })).toThrow('HTTPS');
16+
expect(() => parseHostedConfig({ ...input, HOSTED_ALLOWED_API_HOSTS: 'dev-api.example' })).toThrow('allowlist');
17+
});
18+
19+
it('projects only approved health/version fields and headers', () => {
20+
const result = projectHostedResponse(200, new Headers({ 'content-type': 'application/json', authorization: 'Bearer secret', 'x-private': 'password=hidden' }), { status: 'ok', channel: 'dev', commit: 'abc', password: 'hidden' }, ['status', 'channel', 'commit']);
21+
expect(result.body).toEqual({ status: 'ok', channel: 'dev', commit: 'abc' });
22+
expect(result.headers).toEqual({ 'content-type': 'application/json' });
23+
expect(JSON.stringify(result)).not.toContain('hidden');
24+
});
25+
26+
it('distinguishes healthy, wrong channel, 5xx, DNS, TLS, and timeout results', async () => {
27+
const response = (body: unknown, status = 200) => new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } });
28+
const healthy = await probeDeployment(parseHostedConfig(input), async (url) => response(url.endsWith('/health') ? { status: 'ok' } : { channel: 'dev', commit: 'abc' }));
29+
expect(healthy.every((item) => !item.failureClass)).toBe(true);
30+
const wrong = await probeDeployment(parseHostedConfig({ ...input, HOSTED_EXPECTED_DEV_CHANNEL: 'preview' }), async (url) => response(url.endsWith('/health') ? { status: 'ok' } : { channel: 'dev', commit: 'abc' }));
31+
expect(wrong[1].failureClass).toBe('wrong-channel');
32+
expect((await probeDeployment(parseHostedConfig(input), async () => response({ error: 'nope' }, 503)))[0].failureClass).toBe('http-5xx');
33+
expect(classifyNetworkError(Object.assign(new Error('lookup failed'), { code: 'ENOTFOUND' }))).toBe('dns');
34+
expect(classifyNetworkError(Object.assign(new Error('certificate expired'), { code: 'CERT_HAS_EXPIRED' }))).toBe('tls');
35+
expect(classifyNetworkError(new Error('aborted'))).toBe('timeout');
36+
});
37+
38+
it('writes a bounded hosted artifact envelope without response secrets', async () => {
39+
const repo = mkdtempSync(path.join(os.tmpdir(), 'phaserforge-hosted-'));
40+
const result = await runHostedProbe({ repo, config: parseHostedConfig(input), runId: 'test-run', fetchImpl: async (url) => new Response(JSON.stringify(url.endsWith('/health') ? { status: 'ok' } : { channel: 'dev', commit: 'abc' }), { headers: { 'content-type': 'application/json' } }) });
41+
expect(result.runDirectory).toContain('test-run');
42+
expect(readFileSync(path.join(result.runDirectory, 'hosted-config.json'), 'utf8')).not.toContain('abc');
43+
expect(readFileSync(path.join(result.runDirectory, 'hosted-evidence.json'), 'utf8')).toContain('hosted-deployment-probe');
44+
});
45+
});

0 commit comments

Comments
 (0)