Merge pull request #532 from bcorfman/phase1repairharness

Implemented phase 1 repair harness; cleaned up testBridge

Author: bcorfman

.plans/ci-repair-harness-implementation-plan-2026-07-26.md

@@ -221,16 +221,16 @@ Exit criteria:
 Goal: produce a compact, redacted, reproducible failure envelope without a
 model call.
 
-- [ ] Refactor the read-only reusable portions of
+- [x] Refactor the read-only reusable portions of
   `scripts/inspect-gh-actions-failures.ts` into shared helpers while preserving
   the existing `ci:checks` and `ci:checks:json` behavior and tests.
-- [ ] Add `github.ts` to fetch one resolved Actions run, failed job logs, and
+- [x] Add `github.ts` to fetch one resolved Actions run, failed job logs, and
   only that run's Playwright artifacts using authenticated `gh` commands.
-- [ ] Implement failure-class parsers for Vitest, TypeScript/build, and
+- [x] Implement failure-class parsers for Vitest, TypeScript/build, and
   Playwright output; classify runner/network/action failures as infrastructure.
-- [ ] Implement artifact metadata extraction and redaction tests. Do not unpack
+- [x] Implement artifact metadata extraction and redaction tests. Do not unpack
   or prompt raw trace/video contents by default.
-- [ ] Add `repair:ci:collect` to write the normalized evidence envelope and
+- [x] Add `repair:ci:collect` to write the normalized evidence envelope and
   human-readable collection summary.
 
 Exit criteria:

package.json

@@ -66,6 +66,7 @@
         "ci:checks": "node --import tsx scripts/inspect-gh-actions-failures.ts",
         "ci:checks:json": "node --import tsx scripts/inspect-gh-actions-failures.ts --json",
         "repair:ci": "node --import tsx scripts/repair-harness/cli.ts",
+        "repair:ci:collect": "node --import tsx scripts/repair-harness/cli.ts collect",
         "test:all": "npm run test:unit && npm run test:e2e",
         "storybook": "storybook dev -p 6006",
         "build-storybook": "storybook build",

scripts/inspect-gh-actions-failures.ts

@@ -1,60 +1,9 @@
-import { spawnSync } from 'node:child_process';
+import { extractFailureSnippet, extractRunIdFromUrl, isFailingCheck, parseAvailableFields } from './repair-harness/ghHelpers';
+import { fetchChecks, resolvePr, runGh } from './repair-harness/github';
 
-type JsonRecord = Record<string, unknown>;
-
-const FAILURE_CONCLUSIONS = new Set(['failure', 'cancelled', 'timed_out', 'action_required']);
-const FAILURE_STATES = new Set(['failure', 'error', 'cancelled', 'timed_out', 'action_required']);
-const FAILURE_BUCKETS = new Set(['fail']);
-const FAILURE_MARKERS = ['error', 'fail', 'failed', 'traceback', 'exception', 'assert', 'panic', 'fatal', 'timeout'];
-
-export function parseAvailableFields(text: string): string[] {
-  const match = text.match(/Available fields:\s*([\s\S]+)/i);
-  if (!match) return [];
-  return match[1]
-    .split('\n')
-    .map((line) => line.trim())
-    .filter((line) => /^[a-zA-Z][\w-]*$/.test(line));
-}
-
-export function isFailingCheck(check: JsonRecord): boolean {
-  const conclusion = String(check.conclusion ?? '').toLowerCase();
-  if (FAILURE_CONCLUSIONS.has(conclusion)) return true;
-  const state = String((check.state ?? check.status ?? '')).toLowerCase();
-  if (FAILURE_STATES.has(state)) return true;
-  const bucket = String(check.bucket ?? '').toLowerCase();
-  return FAILURE_BUCKETS.has(bucket);
-}
-
-export function extractRunIdFromUrl(url: string): string | null {
-  const match = url.match(/\/actions\/runs\/(\d+)/);
-  return match?.[1] ?? null;
-}
-
-export function extractFailureSnippet(logText: string, maxLines = 120, context = 20): string {
-  const lines = logText.split(/\r?\n/);
-  const failureIndex = lines.findIndex((line) => {
-    const normalized = line.toLowerCase();
-    return FAILURE_MARKERS.some((marker) => normalized.includes(marker));
-  });
-  if (failureIndex < 0) {
-    return lines.slice(-maxLines).join('\n').trim();
-  }
-  const start = Math.max(0, failureIndex - context);
-  const end = Math.min(lines.length, start + maxLines);
-  return lines.slice(start, end).join('\n').trim();
-}
+export { extractFailureSnippet, extractRunIdFromUrl, isFailingCheck, parseAvailableFields } from './repair-harness/ghHelpers';
 
-function runGh(args: string[], options: { cwd?: string; allowFailure?: boolean } = {}) {
-  const result = spawnSync('gh', args, {
-    cwd: options.cwd,
-    encoding: 'utf8',
-  });
-  if (result.status !== 0 && !options.allowFailure) {
-    const message = [result.stderr, result.stdout].filter(Boolean).join('\n').trim();
-    throw new Error(message || `gh ${args.join(' ')} failed`);
-  }
-  return result;
-}
+type JsonRecord = Record<string, unknown>;
 
 function parseArgs(argv: string[]) {
   const parsed: { pr?: string; json: boolean; repo: string } = {
@@ -71,29 +20,6 @@ function parseArgs(argv: string[]) {
   return parsed;
 }
 
-function resolvePr(pr: string | undefined, repo: string): string {
-  if (pr) return pr;
-  const result = runGh(['pr', 'view', '--json', 'number,url'], { cwd: repo });
-  const data = JSON.parse(result.stdout) as { number?: number; url?: string };
-  if (!data.number) throw new Error('Unable to resolve current branch PR.');
-  return String(data.number);
-}
-
-function fetchChecks(pr: string, repo: string): JsonRecord[] {
-  const primaryFields = ['name', 'state', 'conclusion', 'detailsUrl', 'startedAt', 'completedAt'];
-  let result = runGh(['pr', 'checks', pr, '--json', primaryFields.join(',')], { cwd: repo, allowFailure: true });
-  if (result.status !== 0) {
-    const availableFields = parseAvailableFields(`${result.stderr}\n${result.stdout}`);
-    const fallbackFields = ['name', 'state', 'bucket', 'link', 'workflow', 'startedAt', 'completedAt'];
-    const selectedFields = fallbackFields.filter((field) => availableFields.includes(field));
-    if (selectedFields.length === 0) {
-      throw new Error([result.stderr, result.stdout].filter(Boolean).join('\n').trim() || 'gh pr checks failed');
-    }
-    result = runGh(['pr', 'checks', pr, '--json', selectedFields.join(',')], { cwd: repo });
-  }
-  return JSON.parse(result.stdout) as JsonRecord[];
-}
-
 function inspectRun(runId: string, repo: string) {
   const metadataResult = runGh(
     ['run', 'view', runId, '--json', 'name,workflowName,conclusion,status,url,event,headBranch,headSha'],

scripts/repair-harness/artifacts.ts

@@ -0,0 +1,21 @@
+import type { ArtifactMetadata } from './types';
+
+const REDACTION_PATTERNS: Array<[RegExp, string]> = [
+  [/(authorization\s*:\s*bearer\s+)[^\s,;]+/gi, '$1[REDACTED]'],
+  [/(\b(?:token|password|passwd|secret|cookie)\s*[:=]\s*)[^\s,;]+/gi, '$1[REDACTED]'],
+  [/(gh[pousr]_[A-Za-z0-9_]+|github_pat_[A-Za-z0-9_]+)/g, '[REDACTED]'],
+  [/(https?:\/\/[^\s/@]+):[^\s/@]+@/gi, '$1:[REDACTED]@'],
+];
+
+export function redactSecrets(value: string): string {
+  return REDACTION_PATTERNS.reduce((result, [pattern, replacement]) => result.replace(pattern, replacement), value);
+}
+
+export function extractArtifactMetadata(paths: string[]): ArtifactMetadata {
+  const normalized = paths.filter((item) => !item.includes('..')).map((item) => item.replaceAll('\\', '/'));
+  return {
+    tracePaths: normalized.filter((item) => /(?:^|\/)trace\.(?:zip|json)$/i.test(item)),
+    screenshotPaths: normalized.filter((item) => /\.(?:png|jpe?g)$/i.test(item)),
+    reportPath: normalized.find((item) => /(?:^|\/)playwright-report\/.*\.html$/i.test(item)),
+  };
+}

scripts/repair-harness/cli.ts

@@ -1,8 +1,24 @@
 import path from 'node:path';
 
 import { formatWorkflowCatalog, getWorkflowCatalog, validateWorkflowCatalog } from './workflowCatalog';
+import { collectEvidence } from './collect';
 
 const repositoryRoot = path.resolve(new URL('../..', import.meta.url).pathname);
+const args = process.argv.slice(2);
+
+if (args[0] === 'collect') {
+  const value = (name: string): string | undefined => {
+    const index = args.indexOf(name);
+    return index >= 0 ? args[index + 1] : undefined;
+  };
+  try {
+    const result = collectEvidence({ repo: value('--repo') ?? repositoryRoot, pr: value('--pr'), run: value('--run'), job: value('--job') });
+    console.log(`Collected ${result.envelope.failure.class} evidence in ${result.runDirectory}`);
+  } catch (error) {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exitCode = 1;
+  }
+} else {
 const validation = validateWorkflowCatalog(repositoryRoot);
 
 if (!validation.valid) {
@@ -11,3 +27,4 @@ if (!validation.valid) {
 } else {
   console.log(formatWorkflowCatalog(getWorkflowCatalog()));
 }
+}

scripts/repair-harness/collect.ts

@@ -0,0 +1,70 @@
+import { mkdirSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+
+import { getWorkflowCatalog } from './workflowCatalog';
+import { extractArtifactMetadata, redactSecrets } from './artifacts';
+import { resolvePr, resolveRun, resolveRunFromPr } from './github';
+import { extractFailure } from './triage';
+import type { EvidenceEnvelope } from './types';
+
+export interface CollectionOptions {
+  repo: string;
+  pr?: string;
+  run?: string;
+  job?: string;
+  outputRoot?: string;
+}
+
+function chooseScope(workflow: string, job: string): string {
+  const key = `${workflow} ${job}`.toLowerCase();
+  if (key.includes('e2e pr') || key.includes('pr chromium')) return 'pr-e2e-chromium';
+  if (key.includes('unit') && key.includes('node')) return 'unit-node';
+  if (key.includes('unit') && key.includes('jsdom')) return 'unit-jsdom';
+  if (key.includes('storybook')) return 'storybook';
+  if (key.includes('build')) return 'build';
+  return 'unknown';
+}
+
+function commandFor(scope: string, job: string): string {
+  const entry = getWorkflowCatalog().find((item) => item.scope === scope);
+  const shard = job.match(/shard\s+(\d+)\s*\/\s*(\d+)/i);
+  return (entry?.reproductionCommand ?? '').replace('{shard}', shard?.[1] ?? '1').replace('{shards}', shard?.[2] ?? '2');
+}
+
+export function collectEvidence(options: CollectionOptions): { envelope: EvidenceEnvelope; runDirectory: string } {
+  const pr = options.pr ? resolvePr(options.pr, options.repo) : undefined;
+  const resolved = options.run ? { runId: options.run, check: undefined } : resolveRunFromPr(pr!, options.repo);
+  const run = resolveRun(resolved.runId, options.repo, options.job ?? (resolved.check ? String(resolved.check.name ?? '') : undefined));
+  const workflow = String(run.metadata.workflowName ?? run.metadata.name ?? 'Unknown workflow');
+  const job = String(run.job.name ?? options.job ?? 'Unknown job');
+  const scope = chooseScope(workflow, job);
+  const rawLog = redactSecrets(run.log);
+  const failure = extractFailure(rawLog);
+  const knownPaths = rawLog.match(/(?:playwright-report|test-results)[/\\][^\s)]+/g) ?? [];
+  const artifacts = extractArtifactMetadata([...knownPaths, ...run.artifacts.map((artifact) => `artifacts/${String(artifact.name ?? '')}`)]);
+  const envelope: EvidenceEnvelope = {
+    workflow: redactSecrets(workflow),
+    job: redactSecrets(job),
+    runId: resolved.runId,
+    commit: String(run.metadata.headSha ?? ''),
+    scope,
+    reproduction: { command: commandFor(scope, job) || 'unsupported' },
+    failure,
+    artifacts,
+    redactionsApplied: ['authorization', 'token', 'password', 'secret', 'cookie', 'credentialed URLs'],
+  };
+  const runDirectory = path.resolve(options.repo, options.outputRoot ?? '.repair-harness/runs', `${Date.now()}-${resolved.runId}`);
+  mkdirSync(runDirectory, { recursive: true });
+  writeFileSync(path.join(runDirectory, 'evidence.json'), `${JSON.stringify(envelope, null, 2)}\n`);
+  writeFileSync(path.join(runDirectory, 'summary.md'), [
+    `# CI failure collection`, '',
+    `- Workflow: ${envelope.workflow}`,
+    `- Job: ${envelope.job}`,
+    `- Run: ${envelope.runId}`,
+    `- Scope: ${envelope.scope}`,
+    `- Failure class: ${envelope.failure.class}`,
+    `- Reproduction: \`${envelope.reproduction.command}\``,
+    '', envelope.failure.stackExcerpt,
+  ].join('\n') + '\n');
+  return { envelope, runDirectory };
+}

scripts/repair-harness/ghHelpers.ts

@@ -0,0 +1,24 @@
+export function parseAvailableFields(text: string): string[] {
+  const match = text.match(/Available fields:\s*([\s\S]+)/i);
+  if (!match) return [];
+  return match[1].split('\n').map((line) => line.trim()).filter((line) => /^[a-zA-Z][\w-]*$/.test(line));
+}
+
+export function isFailingCheck(check: Record<string, unknown>): boolean {
+  const failures = new Set(['failure', 'error', 'cancelled', 'timed_out', 'action_required', 'fail']);
+  return failures.has(String(check.conclusion ?? '').toLowerCase())
+    || failures.has(String(check.state ?? check.status ?? '').toLowerCase())
+    || failures.has(String(check.bucket ?? '').toLowerCase());
+}
+
+export function extractRunIdFromUrl(url: string): string | null {
+  return url.match(/\/actions\/runs\/(\d+)/)?.[1] ?? null;
+}
+
+export function extractFailureSnippet(logText: string, maxLines = 120, context = 20): string {
+  const lines = logText.split(/\r?\n/);
+  const failureIndex = lines.findIndex((line) => /error|fail|failed|traceback|exception|assert|panic|fatal|timeout/i.test(line));
+  if (failureIndex < 0) return lines.slice(-maxLines).join('\n').trim();
+  const start = Math.max(0, failureIndex - context);
+  return lines.slice(start, Math.min(lines.length, start + maxLines)).join('\n').trim();
+}

scripts/repair-harness/github.ts

@@ -0,0 +1,80 @@
+import { spawnSync } from 'node:child_process';
+
+import { extractFailureSnippet, extractRunIdFromUrl, isFailingCheck, parseAvailableFields } from './ghHelpers';
+
+export type JsonRecord = Record<string, unknown>;
+
+export interface GhRunResult {
+  status: number | null;
+  stdout: string;
+  stderr: string;
+}
+
+export function runGh(args: string[], options: { cwd?: string; allowFailure?: boolean } = {}): GhRunResult {
+  const result = spawnSync('gh', args, { cwd: options.cwd, encoding: 'utf8' });
+  const normalized = { status: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '' };
+  if (normalized.status !== 0 && !options.allowFailure) {
+    throw new Error([normalized.stderr, normalized.stdout].filter(Boolean).join('\n').trim() || `gh ${args.join(' ')} failed`);
+  }
+  return normalized;
+}
+
+export function resolvePr(pr: string | undefined, repo: string): string {
+  if (pr) return pr;
+  const result = runGh(['pr', 'view', '--json', 'number'], { cwd: repo });
+  const data = JSON.parse(result.stdout) as { number?: number };
+  if (!data.number) throw new Error('Unable to resolve current branch PR.');
+  return String(data.number);
+}
+
+export function fetchChecks(pr: string, repo: string): JsonRecord[] {
+  const primaryFields = ['name', 'state', 'conclusion', 'detailsUrl', 'startedAt', 'completedAt'];
+  let result = runGh(['pr', 'checks', pr, '--json', primaryFields.join(',')], { cwd: repo, allowFailure: true });
+  if (result.status !== 0) {
+    const availableFields = parseAvailableFields(`${result.stderr}\n${result.stdout}`);
+    const fallbackFields = ['name', 'state', 'bucket', 'link', 'workflow', 'startedAt', 'completedAt'];
+    const selectedFields = fallbackFields.filter((field) => availableFields.includes(field));
+    if (selectedFields.length === 0) throw new Error([result.stderr, result.stdout].filter(Boolean).join('\n').trim() || 'gh pr checks failed');
+    result = runGh(['pr', 'checks', pr, '--json', selectedFields.join(',')], { cwd: repo });
+  }
+  return JSON.parse(result.stdout) as JsonRecord[];
+}
+
+export interface ResolvedRun {
+  metadata: JsonRecord;
+  job: JsonRecord;
+  log: string;
+  artifacts: JsonRecord[];
+}
+
+function parseJson(value: string): JsonRecord {
+  return JSON.parse(value) as JsonRecord;
+}
+
+export function resolveRun(runId: string, repo: string, jobName?: string): ResolvedRun {
+  const metadataResult = runGh(['run', 'view', runId, '--json', 'name,workflowName,conclusion,status,url,event,headBranch,headSha,jobs'], { cwd: repo });
+  const metadata = parseJson(metadataResult.stdout);
+  const jobs = Array.isArray(metadata.jobs) ? metadata.jobs as JsonRecord[] : [];
+  const candidates = jobs.filter((job) => !jobName || String(job.name ?? '').toLowerCase() === jobName.toLowerCase() || String(job.name ?? '').toLowerCase().includes(jobName.toLowerCase()));
+  const job = candidates.find((item) => isFailingCheck(item)) ?? candidates[0] ?? { name: jobName ?? String(metadata.name ?? 'Actions run') };
+  const jobId = job.databaseId ?? job.id;
+  const logArgs = jobId ? ['run', 'view', runId, '--job', String(jobId), '--log'] : ['run', 'view', runId, '--log'];
+  const logResult = runGh(logArgs, { cwd: repo, allowFailure: true });
+  const artifactResult = runGh(['api', `repos/{owner}/{repo}/actions/runs/${runId}/artifacts`, '--paginate'], { cwd: repo, allowFailure: true });
+  let artifacts: JsonRecord[] = [];
+  if (artifactResult.status === 0 && artifactResult.stdout.trim()) {
+    try { artifacts = (parseJson(artifactResult.stdout).artifacts as JsonRecord[]) ?? []; } catch { artifacts = []; }
+  }
+  return { metadata, job, log: logResult.stdout || logResult.stderr || '', artifacts };
+}
+
+export function resolveRunFromPr(pr: string, repo: string): { runId: string; check: JsonRecord } {
+  const check = fetchChecks(pr, repo).find(isFailingCheck);
+  if (!check) throw new Error(`PR #${pr}: no failing GitHub Actions checks detected.`);
+  const url = String(check.detailsUrl ?? check.link ?? '');
+  const runId = extractRunIdFromUrl(url);
+  if (!runId) throw new Error(`Unsupported external check: ${String(check.name ?? 'Unnamed check')}`);
+  return { runId, check };
+}
+
+export { extractFailureSnippet, extractRunIdFromUrl, isFailingCheck, parseAvailableFields };