From d50b3af574e61bfca6f5dc34c7cd307eb04b4066 Mon Sep 17 00:00:00 2001 From: cbro Date: Tue, 5 May 2026 16:46:04 -0400 Subject: [PATCH 01/15] feat(cli): add docs deployment ledger with dual-write support Integrate the new docs-ledger register/finish flow into the CLI's docs publish pipeline. Controlled by FERN_DOCS_DEPLOY_MODE env var: - legacy (default): existing startDocsRegister/finishDocsRegister only - dual: legacy + ledger (non-fatal if ledger fails) - ledger: ledger only (fast, incremental via CAS) The ledger path maps the resolved DocsDefinition to a DocsPublishInput (nav tree + content-addressed page blobs), calls register to get presigned URLs for missing blobs, uploads them, then calls finish. Uses raw fetch for the ledger endpoints because the oRPC contract currently types the input as DeploymentDescriptor while the server accepts DocsPublishInput (server-side structural decomposition). --- .../unreleased/docs-ledger-dual-write.yml | 6 + .../src/__test__/buildLedgerInput.test.ts | 149 +++++++++++++++ .../src/__test__/docsDeployMode.test.ts | 49 +++++ .../src/docsDeployMode.ts | 28 +++ .../src/publishDocs.ts | 69 +++++-- .../src/publishDocsLedger.ts | 180 ++++++++++++++++++ 6 files changed, 465 insertions(+), 16 deletions(-) create mode 100644 packages/cli/cli/changes/unreleased/docs-ledger-dual-write.yml create mode 100644 packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/buildLedgerInput.test.ts create mode 100644 packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/docsDeployMode.test.ts create mode 100644 packages/cli/generation/remote-generation/remote-workspace-runner/src/docsDeployMode.ts create mode 100644 packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts diff --git a/packages/cli/cli/changes/unreleased/docs-ledger-dual-write.yml b/packages/cli/cli/changes/unreleased/docs-ledger-dual-write.yml new file mode 100644 index 000000000000..2eccb0036945 --- /dev/null +++ b/packages/cli/cli/changes/unreleased/docs-ledger-dual-write.yml @@ -0,0 +1,6 @@ +- summary: | + Add docs deployment ledger support with dual-write capability. + Set FERN_DOCS_DEPLOY_MODE=dual to enable writing to both the legacy + and new ledger backends. The new ledger path uses content-addressed + storage for incremental deploys. + type: feat diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/buildLedgerInput.test.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/buildLedgerInput.test.ts new file mode 100644 index 000000000000..442618597356 --- /dev/null +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/buildLedgerInput.test.ts @@ -0,0 +1,149 @@ +import { createHash } from "crypto"; +import { describe, expect, it } from "vitest"; +import { buildLedgerInput } from "../publishDocsLedger.js"; + +function sha256(data: string): string { + return createHash("sha256").update(Buffer.from(data, "utf-8")).digest("hex"); +} + +const MINIMAL_ROOT = { + type: "root" as const, + version: "v1" as const, + id: "root-1", + child: { + type: "unversioned" as const, + id: "uv-1", + child: { type: "sidebarRoot" as const, id: "sr-1", children: [] } + } +}; + +function makeDocsDefinition({ + pages = {}, + root = MINIMAL_ROOT +}: { + pages?: Record; + root?: unknown; +} = {}) { + // Minimal DocsDefinition shape — only the fields buildLedgerInput reads. + return { + pages, + config: { root } + } as Parameters[0]["docsDefinition"]; +} + +describe("buildLedgerInput", () => { + it("hashes page content and creates blob refs", () => { + const markdown = "# Hello World"; + const { input, blobs } = buildLedgerInput({ + docsDefinition: makeDocsDefinition({ pages: { "page-1": { markdown } } }), + organization: "acme", + domain: "docs.acme.com", + basepath: undefined, + previewId: undefined + }); + + const expectedHash = sha256(markdown); + expect(input.pages["page-1"]).toEqual({ + hash: expectedHash, + contentType: "text/markdown", + contentLength: Buffer.byteLength(markdown, "utf-8") + }); + expect(blobs.has(expectedHash)).toBe(true); + }); + + it("skips null/undefined pages", () => { + const pages = { "page-1": null } as unknown as Record; + const { input } = buildLedgerInput({ + docsDefinition: makeDocsDefinition({ pages }), + organization: "acme", + domain: "docs.acme.com", + basepath: undefined, + previewId: undefined + }); + + expect(Object.keys(input.pages)).toHaveLength(0); + }); + + it("serializes config as a JSON blob ref", () => { + const { input, blobs } = buildLedgerInput({ + docsDefinition: makeDocsDefinition(), + organization: "acme", + domain: "docs.acme.com", + basepath: undefined, + previewId: undefined + }); + + expect(input.config).toBeDefined(); + expect(input.config!.contentType).toBe("application/json"); + // The blob should exist and round-trip back to the config. + const configBuf = blobs.get(input.config!.hash); + expect(configBuf).toBeDefined(); + expect(JSON.parse(configBuf!.toString("utf-8"))).toEqual({ root: MINIMAL_ROOT }); + }); + + it("passes through org, domain, basepath, previewId", () => { + const { input } = buildLedgerInput({ + docsDefinition: makeDocsDefinition(), + organization: "acme", + domain: "docs.acme.com", + basepath: "/v2", + previewId: "pr-42" + }); + + expect(input.orgId).toBe("acme"); + expect(input.domain).toBe("docs.acme.com"); + expect(input.basepath).toBe("/v2"); + expect(input.previewId).toBe("pr-42"); + }); + + it("defaults basepath to empty string and previewId to null", () => { + const { input } = buildLedgerInput({ + docsDefinition: makeDocsDefinition(), + organization: "acme", + domain: "docs.acme.com", + basepath: undefined, + previewId: undefined + }); + + expect(input.basepath).toBe(""); + expect(input.previewId).toBeNull(); + }); + + it("uses config.root for the root field", () => { + const { input } = buildLedgerInput({ + docsDefinition: makeDocsDefinition({ root: MINIMAL_ROOT }), + organization: "acme", + domain: "docs.acme.com", + basepath: undefined, + previewId: undefined + }); + + expect(input.root).toEqual(MINIMAL_ROOT); + }); + + it("deduplicates pages with identical content", () => { + const markdown = "same content"; + const { blobs } = buildLedgerInput({ + docsDefinition: makeDocsDefinition({ + pages: { + "page-a": { markdown }, + "page-b": { markdown } + } + }), + organization: "acme", + domain: "docs.acme.com", + basepath: undefined, + previewId: undefined + }); + + // Two pages with the same content should produce one blob (content-addressed). + const pageHashes = new Set( + Object.values(blobs) + .filter((buf) => buf.toString("utf-8") === markdown) + .map((_, i) => i) + ); + // The blobs map is keyed by hash, so duplicate content = one entry. + const markdownHash = sha256(markdown); + expect(blobs.get(markdownHash)?.toString("utf-8")).toBe(markdown); + }); +}); diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/docsDeployMode.test.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/docsDeployMode.test.ts new file mode 100644 index 000000000000..69c36940d9ed --- /dev/null +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/docsDeployMode.test.ts @@ -0,0 +1,49 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { getDocsDeployMode } from "../docsDeployMode.js"; + +describe("getDocsDeployMode", () => { + const originalEnv = process.env.FERN_DOCS_DEPLOY_MODE; + + afterEach(() => { + if (originalEnv === undefined) { + delete process.env.FERN_DOCS_DEPLOY_MODE; + } else { + process.env.FERN_DOCS_DEPLOY_MODE = originalEnv; + } + }); + + it("defaults to legacy when env var is unset", () => { + delete process.env.FERN_DOCS_DEPLOY_MODE; + expect(getDocsDeployMode()).toBe("legacy"); + }); + + it("defaults to legacy when env var is empty", () => { + process.env.FERN_DOCS_DEPLOY_MODE = ""; + expect(getDocsDeployMode()).toBe("legacy"); + }); + + it("returns dual", () => { + process.env.FERN_DOCS_DEPLOY_MODE = "dual"; + expect(getDocsDeployMode()).toBe("dual"); + }); + + it("returns ledger", () => { + process.env.FERN_DOCS_DEPLOY_MODE = "ledger"; + expect(getDocsDeployMode()).toBe("ledger"); + }); + + it("is case-insensitive", () => { + process.env.FERN_DOCS_DEPLOY_MODE = "DUAL"; + expect(getDocsDeployMode()).toBe("dual"); + }); + + it("trims whitespace", () => { + process.env.FERN_DOCS_DEPLOY_MODE = " ledger "; + expect(getDocsDeployMode()).toBe("ledger"); + }); + + it("falls back to legacy for unrecognized values", () => { + process.env.FERN_DOCS_DEPLOY_MODE = "banana"; + expect(getDocsDeployMode()).toBe("legacy"); + }); +}); diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/docsDeployMode.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/docsDeployMode.ts new file mode 100644 index 000000000000..83def9159dd2 --- /dev/null +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/docsDeployMode.ts @@ -0,0 +1,28 @@ +/** + * Controls which docs deployment backend the CLI writes to. + * + * Set via FERN_DOCS_DEPLOY_MODE environment variable: + * - "legacy" (default) — existing startDocsRegister / finishDocsRegister flow only. + * - "dual" — legacy flow + new docs-ledger register/finish (dual write). + * - "ledger" — docs-ledger only (fast, incremental). + */ +export type DocsDeployMode = "legacy" | "dual" | "ledger"; + +const VALID_MODES = new Set(["legacy", "dual", "ledger"]); + +export function getDocsDeployMode(): DocsDeployMode { + const raw = process.env.FERN_DOCS_DEPLOY_MODE?.toLowerCase().trim(); + if (raw == null || raw === "") { + return "legacy"; + } + if (isValidMode(raw)) { + return raw; + } + // Fall back to legacy for unrecognized values, but warn at debug level + // (the caller can surface this via context.logger.debug). + return "legacy"; +} + +function isValidMode(value: string): value is DocsDeployMode { + return VALID_MODES.has(value as DocsDeployMode); +} diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts index 7deed77a97e2..1a90973ccd0b 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts @@ -45,8 +45,10 @@ import { readFile } from "fs/promises"; import { chunk } from "lodash-es"; import * as mime from "mime-types"; import terminalLink from "terminal-link"; +import { getDocsDeployMode } from "./docsDeployMode.js"; import { getDynamicGeneratorConfig } from "./getDynamicGeneratorConfig.js"; import { measureImageSizes } from "./measureImageSizes.js"; +import { publishDocsViaLedger } from "./publishDocsLedger.js"; import { asyncPool } from "./utils/asyncPool.js"; const MEASURE_IMAGE_BATCH_SIZE = 10; @@ -201,6 +203,11 @@ export async function publishDocs({ if (deployerAuthor?.email != null) { headers["X-Deployer-Author-Email"] = deployerAuthor.email; } + const deployMode = getDocsDeployMode(); + if (deployMode !== "legacy") { + context.logger.info(`Docs deploy mode: ${deployMode}`); + } + const fdr = createFdrService({ token: token.value, ...(Object.keys(headers).length > 0 && { headers }) @@ -621,30 +628,60 @@ export async function publishDocs({ `Memory after resolve: RSS=${(resolveMemory.rss / 1024 / 1024).toFixed(2)}MB, Heap=${(resolveMemory.heapUsed / 1024 / 1024).toFixed(2)}MB` ); - if (docsRegistrationId == null) { + if (docsRegistrationId == null && deployMode !== "ledger") { doUnlock(); return context.failAndThrow("Failed to publish docs.", "Docs registration ID is missing.", { code: CliError.Code.InternalError }); } - context.logger.info("Publishing docs to FDR..."); - const publishStart = performance.now(); - try { - await fdr.docs.v2.write.finishDocsRegister({ - docsRegistrationId, - docsDefinition, - excludeApis, - ...(isBasepathAware && !preview && { basepathAware: true }) - }); - } catch (error) { - return context.failAndThrow("Failed to publish docs to " + domain, error, { - code: CliError.Code.NetworkError - }); + // ── Legacy publish path ────────────────────────────────────── + if (deployMode !== "ledger") { + context.logger.info("Publishing docs to FDR..."); + const publishStart = performance.now(); + try { + await fdr.docs.v2.write.finishDocsRegister({ + docsRegistrationId: docsRegistrationId!, + docsDefinition, + excludeApis, + ...(isBasepathAware && !preview && { basepathAware: true }) + }); + } catch (error) { + return context.failAndThrow("Failed to publish docs to " + domain, error, { + code: CliError.Code.NetworkError + }); + } + const publishTime = performance.now() - publishStart; + context.logger.debug(`Docs published to FDR in ${publishTime.toFixed(0)}ms`); } - const publishTime = performance.now() - publishStart; - context.logger.debug(`Docs published to FDR in ${publishTime.toFixed(0)}ms`); + // ── Ledger publish path (dual-write or ledger-only) ────────── + if (deployMode === "dual" || deployMode === "ledger") { + try { + const ledgerResult = await publishDocsViaLedger({ + docsDefinition, + organization, + domain, + basepath: basePath, + previewId, + token: token.value, + fdrOrigin, + headers, + context + }); + context.logger.info( + `[ledger] Deployment ${ledgerResult.reusedDeployment ? "reused" : "created"}: ${ledgerResult.deploymentId}` + ); + } catch (error) { + if (deployMode === "ledger") { + return context.failAndThrow("Failed to publish docs via ledger to " + domain, error, { + code: CliError.Code.NetworkError + }); + } + // In dual-write mode, ledger failure is non-fatal — legacy already succeeded. + context.logger.warn(`[ledger] Dual-write failed (non-fatal): ${String(error)}`); + } + } // Register translated page content for each configured locale. // In preview mode, register translations against the preview URL (not the production domain) diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts new file mode 100644 index 000000000000..c0b04ad5261e --- /dev/null +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts @@ -0,0 +1,180 @@ +import type { DocsV1Write } from "@fern-api/fdr-sdk"; +import { createDocsLedgerClient, type DocsPublishInput } from "@fern-api/fdr-sdk/orpc-client"; +import type { TaskContext } from "@fern-api/task-context"; +import { createHash } from "crypto"; + +type DocsDefinition = DocsV1Write.DocsDefinition; + +function sha256(data: Buffer): string { + return createHash("sha256").update(data).digest("hex"); +} + +interface BlobRef { + hash: string; + contentType: string; + contentLength: number; +} + +/** + * Serializes a value to a JSON buffer and returns a BlobRef + the raw bytes, + * keyed by content hash for later upload. + */ +function jsonBlobRef(value: unknown): { ref: BlobRef; hash: string; buf: Buffer } { + const buf = Buffer.from(JSON.stringify(value), "utf-8"); + const hash = sha256(buf); + return { + ref: { hash, contentType: "application/json", contentLength: buf.length }, + hash, + buf + }; +} + +/** + * Build a DocsPublishInput from a resolved DocsDefinition and collect + * all content blobs that may need uploading. + */ +export function buildLedgerInput({ + docsDefinition, + organization, + domain, + basepath, + previewId +}: { + docsDefinition: DocsDefinition; + organization: string; + domain: string; + basepath: string | undefined; + previewId: string | undefined; +}): { input: DocsPublishInput; blobs: Map } { + const blobs = new Map(); + + // Pages: hash each page's markdown content. + const pages: DocsPublishInput["pages"] = {}; + for (const [pageId, page] of Object.entries(docsDefinition.pages)) { + if (page == null) { + continue; + } + const buf = Buffer.from(page.markdown, "utf-8"); + const hash = sha256(buf); + pages[pageId] = { hash, contentType: "text/markdown", contentLength: buf.length }; + blobs.set(hash, buf); + } + + // Config: serialize the entire config as a JSON blob. + const configBlob = jsonBlobRef(docsDefinition.config); + blobs.set(configBlob.hash, configBlob.buf); + + const input: DocsPublishInput = { + orgId: organization, + domain, + basepath: basepath ?? "", + previewId: previewId ?? null, + root: docsDefinition.config.root ?? docsDefinition.config.navigation, + pages, + config: configBlob.ref, + apiManifest: null, + theme: null, + files: null, + redirects: null + }; + + return { input, blobs }; +} + +export interface LedgerPublishResult { + deploymentId: string; + siteId: string; + deploymentHash: string; + reusedDeployment: boolean; +} + +/** + * Publish docs via the new docs-ledger register → upload → finish flow. + * + * This is a self-contained function that can run alongside (dual-write) + * or instead of (ledger-only) the legacy finishDocsRegister path. + */ +export async function publishDocsViaLedger({ + docsDefinition, + organization, + domain, + basepath, + previewId, + token, + fdrOrigin, + headers, + context +}: { + docsDefinition: DocsDefinition; + organization: string; + domain: string; + basepath: string | undefined; + previewId: string | undefined; + token: string; + fdrOrigin: string; + headers: Record; + context: TaskContext; +}): Promise { + const { input, blobs } = buildLedgerInput({ + docsDefinition, + organization, + domain, + basepath, + previewId + }); + + const client = createDocsLedgerClient({ baseUrl: fdrOrigin, token, headers }); + + // Step 1: Register — server computes deployment hash, returns presigned + // S3 URLs for any blobs it doesn't already have in CAS. + context.logger.debug("[ledger] Registering deployment..."); + const registerStart = performance.now(); + const registerResult = await client.register(input); + const registerTime = performance.now() - registerStart; + context.logger.debug( + `[ledger] Registered in ${registerTime.toFixed(0)}ms — hash=${registerResult.deploymentHash}, missing=${registerResult.missingContent.length} blobs` + ); + + // Step 2: Upload any blobs the server doesn't have yet. + if (registerResult.missingContent.length > 0) { + context.logger.debug(`[ledger] Uploading ${registerResult.missingContent.length} missing blobs...`); + const uploadStart = performance.now(); + + await Promise.all( + registerResult.missingContent.map(async ({ hash, uploadUrl }) => { + const blob = blobs.get(hash); + if (blob == null) { + context.logger.warn(`[ledger] Server requested blob ${hash} but we don't have it — skipping`); + return; + } + const response = await fetch(uploadUrl, { + method: "PUT", + headers: { "Content-Type": "application/octet-stream" }, + body: blob.buffer.slice(blob.byteOffset, blob.byteOffset + blob.byteLength) as ArrayBuffer + }); + if (!response.ok) { + const text = await response.text(); + throw new Error(`[ledger] S3 upload failed for ${hash}: ${response.status} ${text}`); + } + }) + ); + + const uploadTime = performance.now() - uploadStart; + context.logger.debug( + `[ledger] Uploaded ${registerResult.missingContent.length} blobs in ${uploadTime.toFixed(0)}ms` + ); + } else { + context.logger.debug("[ledger] All content already in CAS — no uploads needed"); + } + + // Step 3: Finish — server persists the deployment. + context.logger.debug("[ledger] Finishing deployment..."); + const finishStart = performance.now(); + const finishResult = await client.finish(input); + const finishTime = performance.now() - finishStart; + context.logger.debug( + `[ledger] Finished in ${finishTime.toFixed(0)}ms — deploymentId=${finishResult.deploymentId}, reused=${finishResult.reusedDeployment}` + ); + + return finishResult; +} From e790b169c84d62af7a3b3db9a21cbf5ca6d6163e Mon Sep 17 00:00:00 2001 From: cbro Date: Tue, 5 May 2026 18:02:55 -0400 Subject: [PATCH 02/15] feat(cli): improve ledger blob uploads with concurrency, retry, and S3 status handling - Use asyncPool (sliding window, concurrency=10) instead of unbounded Promise.all - Add retry with exponential backoff (1s, 2s, 4s) for transient failures (429, 5xx) - Handle S3 412 Precondition Failed as success (object already exists in CAS) - Fail fast on 403 (expired presigned URL) and 400 (content integrity mismatch) - Track and log upload stats: uploaded vs already-in-store counts - Remove stale theme field, add locale to DocsPublishInput --- .../src/__test__/buildLedgerInput.test.ts | 3 +- .../src/publishDocsLedger.ts | 100 ++++++++++++++---- 2 files changed, 80 insertions(+), 23 deletions(-) diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/buildLedgerInput.test.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/buildLedgerInput.test.ts index 442618597356..b07e85b4df60 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/buildLedgerInput.test.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/buildLedgerInput.test.ts @@ -96,7 +96,7 @@ describe("buildLedgerInput", () => { expect(input.previewId).toBe("pr-42"); }); - it("defaults basepath to empty string and previewId to null", () => { + it("defaults basepath to empty string, previewId to null, and locale to en", () => { const { input } = buildLedgerInput({ docsDefinition: makeDocsDefinition(), organization: "acme", @@ -107,6 +107,7 @@ describe("buildLedgerInput", () => { expect(input.basepath).toBe(""); expect(input.previewId).toBeNull(); + expect(input.locale).toBe("en"); }); it("uses config.root for the root field", () => { diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts index c0b04ad5261e..422eafd00e88 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts @@ -3,6 +3,12 @@ import { createDocsLedgerClient, type DocsPublishInput } from "@fern-api/fdr-sdk import type { TaskContext } from "@fern-api/task-context"; import { createHash } from "crypto"; +import { asyncPool } from "./utils/asyncPool.js"; + +const UPLOAD_CONCURRENCY = 10; +const UPLOAD_MAX_RETRIES = 3; +const UPLOAD_INITIAL_DELAY_MS = 1_000; + type DocsDefinition = DocsV1Write.DocsDefinition; function sha256(data: Buffer): string { @@ -73,9 +79,9 @@ export function buildLedgerInput({ pages, config: configBlob.ref, apiManifest: null, - theme: null, files: null, - redirects: null + redirects: null, + locale: "en" }; return { input, blobs }; @@ -140,28 +146,20 @@ export async function publishDocsViaLedger({ context.logger.debug(`[ledger] Uploading ${registerResult.missingContent.length} missing blobs...`); const uploadStart = performance.now(); - await Promise.all( - registerResult.missingContent.map(async ({ hash, uploadUrl }) => { - const blob = blobs.get(hash); - if (blob == null) { - context.logger.warn(`[ledger] Server requested blob ${hash} but we don't have it — skipping`); - return; - } - const response = await fetch(uploadUrl, { - method: "PUT", - headers: { "Content-Type": "application/octet-stream" }, - body: blob.buffer.slice(blob.byteOffset, blob.byteOffset + blob.byteLength) as ArrayBuffer - }); - if (!response.ok) { - const text = await response.text(); - throw new Error(`[ledger] S3 upload failed for ${hash}: ${response.status} ${text}`); - } - }) - ); - + const results = await asyncPool(UPLOAD_CONCURRENCY, registerResult.missingContent, async ({ hash, uploadUrl }) => { + const blob = blobs.get(hash); + if (blob == null) { + context.logger.warn(`[ledger] Server requested blob ${hash} but we don't have it — skipping`); + return "skipped" as const; + } + return uploadBlobWithRetry(blob, uploadUrl, hash, context); + }); + + const uploaded = results.filter((r) => r === "uploaded").length; + const alreadyExisted = results.filter((r) => r === "already_exists").length; const uploadTime = performance.now() - uploadStart; context.logger.debug( - `[ledger] Uploaded ${registerResult.missingContent.length} blobs in ${uploadTime.toFixed(0)}ms` + `[ledger] Upload complete in ${uploadTime.toFixed(0)}ms — ${uploaded} uploaded, ${alreadyExisted} already in store` ); } else { context.logger.debug("[ledger] All content already in CAS — no uploads needed"); @@ -178,3 +176,61 @@ export async function publishDocsViaLedger({ return finishResult; } + +type UploadResult = "uploaded" | "already_exists"; + +/** + * Upload a single blob to S3 with retries on transient failures (429, 5xx). + * Exponential backoff: 1s, 2s, 4s. + * + * Returns "already_exists" on 412 Precondition Failed — the presigned URL may + * include an If-None-Match condition, so 412 means the object already exists in S3. + * + * Fails fast on 403 (expired/invalid presigned URL) and 400 (content integrity + * mismatch) since these are not recoverable by retrying. + */ +async function uploadBlobWithRetry(blob: Buffer, uploadUrl: string, hash: string, context: TaskContext): Promise { + const body = blob.buffer.slice(blob.byteOffset, blob.byteOffset + blob.byteLength) as ArrayBuffer; + + for (let attempt = 0; attempt <= UPLOAD_MAX_RETRIES; attempt++) { + const response = await fetch(uploadUrl, { + method: "PUT", + headers: { "Content-Type": "application/octet-stream" }, + body + }); + + if (response.ok) { + return "uploaded"; + } + + if (response.status === 412) { + context.logger.debug(`[ledger] Blob ${hash} already exists in store — skipping`); + return "already_exists"; + } + + if (response.status === 403) { + const text = await response.text(); + throw new Error(`[ledger] Presigned URL rejected for ${hash} (expired or signature mismatch): ${text}`); + } + + if (response.status === 400) { + const text = await response.text(); + throw new Error(`[ledger] Content integrity check failed for ${hash}: ${text}`); + } + + const isRetryable = response.status === 429 || response.status >= 500; + if (isRetryable && attempt < UPLOAD_MAX_RETRIES) { + const delay = UPLOAD_INITIAL_DELAY_MS * 2 ** attempt; + context.logger.debug( + `[ledger] Upload ${hash} got ${response.status}, retrying in ${delay}ms (attempt ${attempt + 1}/${UPLOAD_MAX_RETRIES})` + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + continue; + } + + const text = await response.text(); + throw new Error(`[ledger] S3 upload failed for ${hash}: ${response.status} ${text}`); + } + + throw new Error(`[ledger] Upload exhausted retries for ${hash}`); +} From 039db4dda52c61e68a5fe0d931216384ac8f7875 Mon Sep 17 00:00:00 2001 From: cbro Date: Fri, 8 May 2026 13:19:31 -0400 Subject: [PATCH 03/15] feat(cli): wire API definitions into docs ledger manifest Collect registered API definitions during docs publishing and serialize them as a JSON blob in the ledger's apiManifest field. Previously apiManifest was always null; now it contains the full set of API definitions keyed by FDR definition ID when any are present. Updates buildLedgerInput to accept an apiDefinitions map, adds corresponding test coverage for empty and non-empty manifests, and threads the collector through publishDocs into publishDocsLedger. --- .../src/__test__/buildLedgerInput.test.ts | 82 ++++++++++++++++--- .../src/publishDocs.ts | 7 +- .../src/publishDocsLedger.ts | 49 ++++++++--- 3 files changed, 114 insertions(+), 24 deletions(-) diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/buildLedgerInput.test.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/buildLedgerInput.test.ts index b07e85b4df60..ba2a6550a99f 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/buildLedgerInput.test.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/buildLedgerInput.test.ts @@ -1,3 +1,4 @@ +import type { APIV1Write } from "@fern-api/fdr-sdk"; import { createHash } from "crypto"; import { describe, expect, it } from "vitest"; import { buildLedgerInput } from "../publishDocsLedger.js"; @@ -39,7 +40,8 @@ describe("buildLedgerInput", () => { organization: "acme", domain: "docs.acme.com", basepath: undefined, - previewId: undefined + previewId: undefined, + apiDefinitions: new Map() }); const expectedHash = sha256(markdown); @@ -58,7 +60,8 @@ describe("buildLedgerInput", () => { organization: "acme", domain: "docs.acme.com", basepath: undefined, - previewId: undefined + previewId: undefined, + apiDefinitions: new Map() }); expect(Object.keys(input.pages)).toHaveLength(0); @@ -70,15 +73,16 @@ describe("buildLedgerInput", () => { organization: "acme", domain: "docs.acme.com", basepath: undefined, - previewId: undefined + previewId: undefined, + apiDefinitions: new Map() }); expect(input.config).toBeDefined(); - expect(input.config!.contentType).toBe("application/json"); + expect(input.config?.contentType).toBe("application/json"); // The blob should exist and round-trip back to the config. - const configBuf = blobs.get(input.config!.hash); + const configBuf = blobs.get(input.config?.hash ?? ""); expect(configBuf).toBeDefined(); - expect(JSON.parse(configBuf!.toString("utf-8"))).toEqual({ root: MINIMAL_ROOT }); + expect(JSON.parse(configBuf?.toString("utf-8") ?? "")).toEqual({ root: MINIMAL_ROOT }); }); it("passes through org, domain, basepath, previewId", () => { @@ -87,7 +91,8 @@ describe("buildLedgerInput", () => { organization: "acme", domain: "docs.acme.com", basepath: "/v2", - previewId: "pr-42" + previewId: "pr-42", + apiDefinitions: new Map() }); expect(input.orgId).toBe("acme"); @@ -102,7 +107,8 @@ describe("buildLedgerInput", () => { organization: "acme", domain: "docs.acme.com", basepath: undefined, - previewId: undefined + previewId: undefined, + apiDefinitions: new Map() }); expect(input.basepath).toBe(""); @@ -116,7 +122,8 @@ describe("buildLedgerInput", () => { organization: "acme", domain: "docs.acme.com", basepath: undefined, - previewId: undefined + previewId: undefined, + apiDefinitions: new Map() }); expect(input.root).toEqual(MINIMAL_ROOT); @@ -134,7 +141,8 @@ describe("buildLedgerInput", () => { organization: "acme", domain: "docs.acme.com", basepath: undefined, - previewId: undefined + previewId: undefined, + apiDefinitions: new Map() }); // Two pages with the same content should produce one blob (content-addressed). @@ -147,4 +155,58 @@ describe("buildLedgerInput", () => { const markdownHash = sha256(markdown); expect(blobs.get(markdownHash)?.toString("utf-8")).toBe(markdown); }); + + it("sets apiManifest to null when apiDefinitions is empty", () => { + const { input } = buildLedgerInput({ + docsDefinition: makeDocsDefinition(), + organization: "acme", + domain: "docs.acme.com", + basepath: undefined, + previewId: undefined, + apiDefinitions: new Map() + }); + + expect(input.apiManifest).toBeNull(); + }); + + it("serializes apiManifest as a JSON blob ref when apiDefinitions is non-empty", () => { + const minimalApiDefinition: APIV1Write.ApiDefinition = { + types: {}, + subpackages: {}, + rootPackage: { + endpoints: [], + types: [], + subpackages: [], + websockets: [], + webhooks: [] + }, + auth: undefined, + snippetsConfiguration: {}, + globalHeaders: [] + }; + + const apiDefinitions = new Map(); + apiDefinitions.set("api-def-1", minimalApiDefinition); + + const { input, blobs } = buildLedgerInput({ + docsDefinition: makeDocsDefinition(), + organization: "acme", + domain: "docs.acme.com", + basepath: undefined, + previewId: undefined, + apiDefinitions + }); + + expect(input.apiManifest).not.toBeNull(); + expect(input.apiManifest?.contentType).toBe("application/json"); + expect(input.apiManifest?.contentLength).toBeGreaterThan(0); + + // The blob should exist in the blob map. + const manifestBuf = blobs.get(input.apiManifest?.hash ?? ""); + expect(manifestBuf).toBeDefined(); + + // Round-trip: the blob content should deserialize to match the input map. + const parsed = JSON.parse(manifestBuf?.toString("utf-8") ?? ""); + expect(parsed).toEqual({ "api-def-1": minimalApiDefinition }); + }); }); diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts index 1a90973ccd0b..2f160d443028 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts @@ -264,6 +264,9 @@ export async function publishDocs({ taskContext: context }); + // Collect API definitions (keyed by FDR definition ID) for the ledger manifest. + const apiDefinitionCollector = new Map(); + const resolver = new DocsDefinitionResolver({ domain, docsWorkspace: effectiveWorkspace, @@ -582,6 +585,7 @@ export async function publishDocs({ } context.logger.debug(`Registered API Definition ${apiName}: ${response.apiDefinitionId}`); + apiDefinitionCollector.set(response.apiDefinitionId, apiDefinition); if (response.dynamicIRs && dynamicIRsByLanguage) { if (skipUpload) { @@ -667,7 +671,8 @@ export async function publishDocs({ token: token.value, fdrOrigin, headers, - context + context, + apiDefinitions: apiDefinitionCollector }); context.logger.info( `[ledger] Deployment ${ledgerResult.reusedDeployment ? "reused" : "created"}: ${ledgerResult.deploymentId}` diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts index 422eafd00e88..84a5354aef49 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts @@ -1,4 +1,4 @@ -import type { DocsV1Write } from "@fern-api/fdr-sdk"; +import type { APIV1Write, DocsV1Write } from "@fern-api/fdr-sdk"; import { createDocsLedgerClient, type DocsPublishInput } from "@fern-api/fdr-sdk/orpc-client"; import type { TaskContext } from "@fern-api/task-context"; import { createHash } from "crypto"; @@ -44,13 +44,15 @@ export function buildLedgerInput({ organization, domain, basepath, - previewId + previewId, + apiDefinitions }: { docsDefinition: DocsDefinition; organization: string; domain: string; basepath: string | undefined; previewId: string | undefined; + apiDefinitions: Map; }): { input: DocsPublishInput; blobs: Map } { const blobs = new Map(); @@ -70,6 +72,15 @@ export function buildLedgerInput({ const configBlob = jsonBlobRef(docsDefinition.config); blobs.set(configBlob.hash, configBlob.buf); + // API manifest: serialize all API definitions as a single JSON blob. + let apiManifestRef: BlobRef | null = null; + if (apiDefinitions.size > 0) { + const manifestObj = Object.fromEntries(apiDefinitions); + const manifestBlob = jsonBlobRef(manifestObj); + blobs.set(manifestBlob.hash, manifestBlob.buf); + apiManifestRef = manifestBlob.ref; + } + const input: DocsPublishInput = { orgId: organization, domain, @@ -78,7 +89,7 @@ export function buildLedgerInput({ root: docsDefinition.config.root ?? docsDefinition.config.navigation, pages, config: configBlob.ref, - apiManifest: null, + apiManifest: apiManifestRef, files: null, redirects: null, locale: "en" @@ -109,7 +120,8 @@ export async function publishDocsViaLedger({ token, fdrOrigin, headers, - context + context, + apiDefinitions }: { docsDefinition: DocsDefinition; organization: string; @@ -120,13 +132,15 @@ export async function publishDocsViaLedger({ fdrOrigin: string; headers: Record; context: TaskContext; + apiDefinitions: Map; }): Promise { const { input, blobs } = buildLedgerInput({ docsDefinition, organization, domain, basepath, - previewId + previewId, + apiDefinitions }); const client = createDocsLedgerClient({ baseUrl: fdrOrigin, token, headers }); @@ -146,14 +160,18 @@ export async function publishDocsViaLedger({ context.logger.debug(`[ledger] Uploading ${registerResult.missingContent.length} missing blobs...`); const uploadStart = performance.now(); - const results = await asyncPool(UPLOAD_CONCURRENCY, registerResult.missingContent, async ({ hash, uploadUrl }) => { - const blob = blobs.get(hash); - if (blob == null) { - context.logger.warn(`[ledger] Server requested blob ${hash} but we don't have it — skipping`); - return "skipped" as const; + const results = await asyncPool( + UPLOAD_CONCURRENCY, + registerResult.missingContent, + async ({ hash, uploadUrl }) => { + const blob = blobs.get(hash); + if (blob == null) { + context.logger.warn(`[ledger] Server requested blob ${hash} but we don't have it — skipping`); + return "skipped" as const; + } + return uploadBlobWithRetry(blob, uploadUrl, hash, context); } - return uploadBlobWithRetry(blob, uploadUrl, hash, context); - }); + ); const uploaded = results.filter((r) => r === "uploaded").length; const alreadyExisted = results.filter((r) => r === "already_exists").length; @@ -189,7 +207,12 @@ type UploadResult = "uploaded" | "already_exists"; * Fails fast on 403 (expired/invalid presigned URL) and 400 (content integrity * mismatch) since these are not recoverable by retrying. */ -async function uploadBlobWithRetry(blob: Buffer, uploadUrl: string, hash: string, context: TaskContext): Promise { +async function uploadBlobWithRetry( + blob: Buffer, + uploadUrl: string, + hash: string, + context: TaskContext +): Promise { const body = blob.buffer.slice(blob.byteOffset, blob.byteOffset + blob.byteLength) as ArrayBuffer; for (let attempt = 0; attempt <= UPLOAD_MAX_RETRIES; attempt++) { From 4eb12a02128de109ffbffd8d2b4cf6975b240269 Mon Sep 17 00:00:00 2001 From: cbro Date: Sat, 16 May 2026 00:35:41 -0400 Subject: [PATCH 04/15] fix(cli): docs ledger lazy file loading, stable tokens, ADR wiring (#15932) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(cli): ignore python .venv directories * feat(cli): populate fileManifest for docs ledger publish * feat(cli): adapt to current docs-ledger and SDK contracts * fix(cli): send undefined config (DocsConfig → LedgerConfig mapping TBD) * feat(cli): map DocsConfig to LedgerConfig for docs ledger publish * test(cli): cover docs-ledger snippet payload wiring Populate the remaining snippets metadata path used during API-definition registration and add direct unit coverage for the language-specific payload builder. This keeps docs-ledger dynamic IR checks and uploads aligned with the actual snippet package/version inputs instead of empty placeholders, including the AUTO-version fallback behavior. * fix(cli): stable apiManifest serialization in buildLedgerInput Sort apiDefinitionCollector entries by key before serializing the docs-ledger apiManifest blob. Map iteration order today depends on Promise.all completion order (random run-to-run), which leaks into the manifest blob hash and therefore the docs-ledger deployment hash. Pairs with the server-side (orgId, apiName, contentHash) dedup added in fern-platform: with both fixes, byte-identical publishes produce byte-identical apiManifests and a deterministic deployment hash. * fix(cli,docs): ledger mode skips V2; stable file tokens; ADR 0009/0011/0012 wiring * fix(cli): lazy file blob loading in ledger publish to reduce memory usage Instead of holding all file Buffers in memory for the entire publish duration, store only hash→filePath mappings. Re-read files on demand during the upload step, and only for blobs the server reports as missing. --- .gitignore | 4 + .../unreleased/ledger-adr-0009-0011-0012.yml | 5 + .../src/__test__/buildLedgerInput.test.ts | 307 +++++++++++++++++- .../src/__test__/normalizeRepoUrl.test.ts | 26 ++ .../__test__/resolveVersionFallback.test.ts | 66 +++- .../src/mapDocsConfigToLedgerConfig.ts | 291 +++++++++++++++++ .../src/normalizeRepoUrl.ts | 21 ++ .../src/publishDocs.ts | 215 ++++++++++-- .../src/publishDocsLedger.ts | 182 +++++++++-- .../src/publishDocsLedgerPreview.ts | 128 ++++++++ .../src/runRemoteGenerationForGenerator.ts | 58 +++- 11 files changed, 1225 insertions(+), 78 deletions(-) create mode 100644 packages/cli/cli/changes/unreleased/ledger-adr-0009-0011-0012.yml create mode 100644 packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/normalizeRepoUrl.test.ts create mode 100644 packages/cli/generation/remote-generation/remote-workspace-runner/src/mapDocsConfigToLedgerConfig.ts create mode 100644 packages/cli/generation/remote-generation/remote-workspace-runner/src/normalizeRepoUrl.ts create mode 100644 packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts diff --git a/.gitignore b/.gitignore index af0c6674c88f..88e28feb16c1 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,10 @@ generators/csharp/playground/**/obj/ next-env.d.ts .vercel +# python virtual envs (local dev tooling) +.venv/ +**/.venv/ + # misc .DS_Store *.swp diff --git a/packages/cli/cli/changes/unreleased/ledger-adr-0009-0011-0012.yml b/packages/cli/cli/changes/unreleased/ledger-adr-0009-0011-0012.yml new file mode 100644 index 000000000000..de7429410131 --- /dev/null +++ b/packages/cli/cli/changes/unreleased/ledger-adr-0009-0011-0012.yml @@ -0,0 +1,5 @@ +- summary: | + Wire CLI to docs-ledger publish contracts for ADRs 0009, 0011, and 0012: + structured git provenance (repoUrl, branch, commitSha), multi-domain + customDomains forwarding, and dedicated preview endpoint (/preview/init). + type: feat diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/buildLedgerInput.test.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/buildLedgerInput.test.ts index ba2a6550a99f..aee5d480edbd 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/buildLedgerInput.test.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/buildLedgerInput.test.ts @@ -1,4 +1,5 @@ import type { APIV1Write } from "@fern-api/fdr-sdk"; +import type { FileManifestEntry } from "@fern-api/fdr-sdk/orpc-client"; import { createHash } from "crypto"; import { describe, expect, it } from "vitest"; import { buildLedgerInput } from "../publishDocsLedger.js"; @@ -67,8 +68,8 @@ describe("buildLedgerInput", () => { expect(Object.keys(input.pages)).toHaveLength(0); }); - it("serializes config as a JSON blob ref", () => { - const { input, blobs } = buildLedgerInput({ + it("maps a minimal DocsConfig to a LedgerConfig shape (mostly empty fields)", () => { + const { input } = buildLedgerInput({ docsDefinition: makeDocsDefinition(), organization: "acme", domain: "docs.acme.com", @@ -77,12 +78,107 @@ describe("buildLedgerInput", () => { apiDefinitions: new Map() }); + // With only `root` set on the source DocsConfig every LedgerConfig + // field is `undefined` — but `input.config` itself is the populated + // ledger object (not `undefined`), unlike the prior workaround. expect(input.config).toBeDefined(); - expect(input.config?.contentType).toBe("application/json"); - // The blob should exist and round-trip back to the config. - const configBuf = blobs.get(input.config?.hash ?? ""); - expect(configBuf).toBeDefined(); - expect(JSON.parse(configBuf?.toString("utf-8") ?? "")).toEqual({ root: MINIMAL_ROOT }); + expect(input.config?.title).toBeUndefined(); + expect(input.config?.colorsV3).toBeUndefined(); + expect(input.config?.metadata).toBeUndefined(); + expect(input.config?.redirects).toBeUndefined(); + }); + + it("translates a DocsConfig logo FileId into a LedgerConfig ImageRef using fileManifest dimensions", () => { + // Source DocsConfig: colorsV3.dark.logo points to a FileId that + // matches a fileManifest entry's fullPath (current FDR behaviour). + const docsDefinition = { + pages: {}, + config: { + root: MINIMAL_ROOT, + colorsV3: { + type: "dark" as const, + accentPrimary: { r: 1, g: 2, b: 3 }, + logo: "assets/logo.png" + } + } + } as unknown as Parameters[0]["docsDefinition"]; + + const fileManifest: Record = { + "assets/logo.png": { + hash: "deadbeef", + contentType: "image/png", + contentLength: 1234, + filename: "logo.png", + width: 320, + height: 160 + } + }; + + const { input } = buildLedgerInput({ + docsDefinition, + organization: "acme", + domain: "docs.acme.com", + basepath: undefined, + previewId: undefined, + apiDefinitions: new Map(), + fileManifest, + // Identity map: fileId === sanitizedPath in the current FDR flow. + fileIdToPath: new Map([["assets/logo.png", "assets/logo.png"]]) + }); + + expect(input.config?.colorsV3).toEqual({ + type: "dark", + accentPrimary: { r: 1, g: 2, b: 3 }, + logo: { path: "assets/logo.png", width: 320, height: 160 } + }); + }); + + it("drops a logo ImageRef when the file is not measured (no width/height in manifest)", () => { + const docsDefinition = { + pages: {}, + config: { + root: MINIMAL_ROOT, + colorsV3: { + type: "light" as const, + accentPrimary: { r: 4, g: 5, b: 6 }, + logo: "assets/unmeasured.svg" + } + } + } as unknown as Parameters[0]["docsDefinition"]; + + const fileManifest: Record = { + "assets/unmeasured.svg": { + hash: "cafef00d", + contentType: "image/svg+xml", + contentLength: 42, + filename: "unmeasured.svg" + // width/height intentionally omitted + } + }; + + const { input } = buildLedgerInput({ + docsDefinition, + organization: "acme", + domain: "docs.acme.com", + basepath: undefined, + previewId: undefined, + apiDefinitions: new Map(), + fileManifest, + fileIdToPath: new Map([["assets/unmeasured.svg", "assets/unmeasured.svg"]]) + }); + + // Logo absent rather than emitted with placeholder dimensions. + expect(input.config?.colorsV3).toEqual({ + type: "light", + accentPrimary: { r: 4, g: 5, b: 6 }, + logo: undefined, + backgroundImage: undefined, + background: undefined, + border: undefined, + sidebarBackground: undefined, + headerBackground: undefined, + cardBackground: undefined + }); }); it("passes through org, domain, basepath, previewId", () => { @@ -169,6 +265,58 @@ describe("buildLedgerInput", () => { expect(input.apiManifest).toBeNull(); }); + it("apiManifest blob hash is stable across Map insertion order (determinism guard)", () => { + // Reproduces the docs-ledger deterministic-hash bug: `apiDefinitions` + // is built by `Promise.all` of /api/register calls in CLI, so its Map + // insertion order is whichever round-trip completed first. Without + // `stableStringify`, byte-identical content would hash differently + // across publishes and the docs-ledger "no-op republish" fast-path + // would never fire. The two manifests below have identical entries + // inserted in opposite orders and MUST produce the same blob hash. + const minimalApiDefinition: APIV1Write.ApiDefinition = { + types: {}, + subpackages: {}, + rootPackage: { + endpoints: [], + types: [], + subpackages: [], + websockets: [], + webhooks: [] + }, + auth: undefined, + snippetsConfiguration: {}, + globalHeaders: [] + }; + + const forward = new Map(); + forward.set("api-def-a", minimalApiDefinition); + forward.set("api-def-b", minimalApiDefinition); + + const reverse = new Map(); + reverse.set("api-def-b", minimalApiDefinition); + reverse.set("api-def-a", minimalApiDefinition); + + const { input: inForward } = buildLedgerInput({ + docsDefinition: makeDocsDefinition(), + organization: "acme", + domain: "docs.acme.com", + basepath: undefined, + previewId: undefined, + apiDefinitions: forward + }); + const { input: inReverse } = buildLedgerInput({ + docsDefinition: makeDocsDefinition(), + organization: "acme", + domain: "docs.acme.com", + basepath: undefined, + previewId: undefined, + apiDefinitions: reverse + }); + + expect(inForward.apiManifest?.hash).toBe(inReverse.apiManifest?.hash); + expect(inForward.apiManifest?.contentLength).toBe(inReverse.apiManifest?.contentLength); + }); + it("serializes apiManifest as a JSON blob ref when apiDefinitions is non-empty", () => { const minimalApiDefinition: APIV1Write.ApiDefinition = { types: {}, @@ -209,4 +357,149 @@ describe("buildLedgerInput", () => { const parsed = JSON.parse(manifestBuf?.toString("utf-8") ?? ""); expect(parsed).toEqual({ "api-def-1": minimalApiDefinition }); }); + + it("forwards fileManifest unchanged (file blobs are loaded lazily, not included in blob map)", () => { + // Image entry — exercises width/height fields. + const imageBytes = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); // PNG header + const imageHash = createHash("sha256").update(new Uint8Array(imageBytes)).digest("hex"); + const imageEntry: FileManifestEntry = { + hash: imageHash, + contentType: "image/png", + contentLength: imageBytes.byteLength, + filename: "logo.png", + width: 200, + height: 100 + }; + + // Non-image entry — no width/height. + const docBytes = Buffer.from("hello world", "utf-8"); + const docHash = createHash("sha256").update(new Uint8Array(docBytes)).digest("hex"); + const docEntry: FileManifestEntry = { + hash: docHash, + contentType: "text/plain", + contentLength: docBytes.byteLength, + filename: "notes.txt" + }; + + const fileManifest: Record = { + "assets/logo.png": imageEntry, + "docs/notes.txt": docEntry + }; + + const { input, blobs } = buildLedgerInput({ + docsDefinition: makeDocsDefinition(), + organization: "acme", + domain: "docs.acme.com", + basepath: undefined, + previewId: undefined, + apiDefinitions: new Map(), + fileManifest + }); + + // fileManifest round-trips unchanged. + expect(input.fileManifest).toEqual(fileManifest); + + // File blobs are NOT in the blob map — they are loaded lazily during + // the upload step via filePaths (hash → absolute path). + expect(blobs.has(imageHash)).toBe(false); + expect(blobs.has(docHash)).toBe(false); + }); + + it("legacy behaviour: omitting fileManifest still works", () => { + const { input, blobs } = buildLedgerInput({ + docsDefinition: makeDocsDefinition({ pages: { "page-1": { markdown: "# hi" } } }), + organization: "acme", + domain: "docs.acme.com", + basepath: undefined, + previewId: undefined, + apiDefinitions: new Map() + }); + + expect(input.fileManifest).toBeUndefined(); + // Blob map still contains the page + config blobs. + expect(blobs.size).toBeGreaterThan(0); + }); + + // ── ADR 0011: git provenance ────────────────────────────────────── + + it("forwards git into DocsPublishInput when provided", () => { + const git = { + repoUrl: "https://github.com/acme/docs", + branch: "main", + commitSha: "abc123" + }; + const { input } = buildLedgerInput({ + docsDefinition: makeDocsDefinition(), + organization: "acme", + domain: "docs.acme.com", + basepath: undefined, + previewId: undefined, + git, + apiDefinitions: new Map() + }); + + expect(input.git).toEqual(git); + }); + + it("omits git from DocsPublishInput when not provided", () => { + const { input } = buildLedgerInput({ + docsDefinition: makeDocsDefinition(), + organization: "acme", + domain: "docs.acme.com", + basepath: undefined, + previewId: undefined, + apiDefinitions: new Map() + }); + + expect(input.git).toBeUndefined(); + }); + + it("forwards git without commitSha when commitSha is omitted", () => { + const git = { + repoUrl: "https://gitlab.com/acme/docs", + branch: "feature/x" + }; + const { input } = buildLedgerInput({ + docsDefinition: makeDocsDefinition(), + organization: "acme", + domain: "docs.acme.com", + basepath: undefined, + previewId: undefined, + git, + apiDefinitions: new Map() + }); + + expect(input.git).toEqual(git); + expect(input.git?.commitSha).toBeUndefined(); + }); + + // ── ADR 0009: customDomains ─────────────────────────────────────── + + it("forwards customDomains into DocsPublishInput", () => { + const customDomains = ["docs.acme.com", "alt.acme.com/v2"]; + const { input } = buildLedgerInput({ + docsDefinition: makeDocsDefinition(), + organization: "acme", + domain: "acme.docs.buildwithfern.com", + basepath: undefined, + previewId: undefined, + customDomains, + apiDefinitions: new Map() + }); + + expect(input.customDomains).toEqual(customDomains); + }); + + it("defaults customDomains to [] when omitted", () => { + const { input } = buildLedgerInput({ + docsDefinition: makeDocsDefinition(), + organization: "acme", + domain: "docs.acme.com", + basepath: undefined, + previewId: undefined, + apiDefinitions: new Map() + }); + + expect(input.customDomains).toEqual([]); + }); }); diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/normalizeRepoUrl.test.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/normalizeRepoUrl.test.ts new file mode 100644 index 000000000000..2e1525064417 --- /dev/null +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/normalizeRepoUrl.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import { normalizeRepoUrlToHttps } from "../normalizeRepoUrl.js"; + +describe("normalizeRepoUrlToHttps", () => { + it("converts GitHub slug to HTTPS URL", () => { + expect(normalizeRepoUrlToHttps("acme/docs", "github")).toBe("https://github.com/acme/docs"); + }); + + it("converts GitLab slug to HTTPS URL", () => { + expect(normalizeRepoUrlToHttps("acme/docs", "gitlab")).toBe("https://gitlab.com/acme/docs"); + }); + + it("converts Bitbucket slug to HTTPS URL", () => { + expect(normalizeRepoUrlToHttps("acme/docs", "bitbucket")).toBe("https://bitbucket.org/acme/docs"); + }); + + it("passes through an HTTPS URL unchanged", () => { + const url = "https://github.enterprise.com/acme/docs"; + expect(normalizeRepoUrlToHttps(url, "github")).toBe(url); + }); + + it("passes through an HTTP URL unchanged", () => { + const url = "http://gitlab.internal/acme/docs"; + expect(normalizeRepoUrlToHttps(url, "gitlab")).toBe(url); + }); +}); diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/resolveVersionFallback.test.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/resolveVersionFallback.test.ts index 469f9dfeb7ec..6f7954a62e57 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/resolveVersionFallback.test.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/resolveVersionFallback.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { resolveVersionFallback } from "../runRemoteGenerationForGenerator.js"; +import { buildSnippetsConfigForSdk, resolveVersionFallback } from "../runRemoteGenerationForGenerator.js"; describe("resolveVersionFallback", () => { it("returns the version when an explicit version is provided", () => { @@ -31,3 +31,67 @@ describe("resolveVersionFallback", () => { expect(resolveVersionFallback("v2.0.0")).toBe("v2.0.0"); }); }); + +describe("buildSnippetsConfigForSdk", () => { + it("builds a typed snippets payload for typescript", () => { + expect( + buildSnippetsConfigForSdk({ + language: "typescript", + packageName: "@acme/sdk", + version: "1.2.3" + }) + ).toEqual({ + typescriptSdk: { + package: "@acme/sdk", + version: "1.2.3" + } + }); + }); + + it("maps java package names to coordinates", () => { + expect( + buildSnippetsConfigForSdk({ + language: "java", + packageName: "com.acme:sdk", + version: "2.0.0" + }) + ).toEqual({ + javaSdk: { + coordinate: "com.acme:sdk", + version: "2.0.0" + } + }); + }); + + it("drops AUTO versions from snippet payloads", () => { + expect( + buildSnippetsConfigForSdk({ + language: "go", + packageName: "github.com/acme/sdk", + version: "AUTO" + }) + ).toEqual({ + goSdk: { + githubRepo: "github.com/acme/sdk", + version: undefined + } + }); + }); + + it("returns an empty object when language or package name is missing", () => { + expect( + buildSnippetsConfigForSdk({ + language: undefined, + packageName: "@acme/sdk", + version: "1.2.3" + }) + ).toEqual({}); + expect( + buildSnippetsConfigForSdk({ + language: "typescript", + packageName: undefined, + version: "1.2.3" + }) + ).toEqual({}); + }); +}); diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/mapDocsConfigToLedgerConfig.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/mapDocsConfigToLedgerConfig.ts new file mode 100644 index 000000000000..e427b4ffb49a --- /dev/null +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/mapDocsConfigToLedgerConfig.ts @@ -0,0 +1,291 @@ +import type { DocsV1Write } from "@fern-api/fdr-sdk"; +import type { FileManifestEntry, ImageRef, LedgerConfig, PathOrUrl } from "@fern-api/fdr-sdk/orpc-client"; + +type DocsConfig = DocsV1Write.DocsConfig; + +/** + * Resolve a FileId reference produced by the classic docs publish flow + * (FDR's startDocsRegister) to the `fullPath` string that the ledger flow + * uses to identify file artifacts. + * + * In the current FDR behaviour (`extractFileId` in publishDocs.ts), the + * "fileId" returned by FDR for a freshly-uploaded file is the same + * sanitized path used as the ledger fileManifest key, so the lookup is + * effectively the identity function. We still consult `fileIdToPath` + * first to remain forward-compatible with the legacy server response + * shape ({ uploadUrl, fileId }) where fileId is a separate UUID. + * + * Returns undefined when the FileId can't be mapped — callers MUST + * treat that as "field absent" rather than fabricating a path string. + */ +function resolveFileIdToPath( + fileId: string | undefined, + fileIdToPath: Map | undefined +): string | undefined { + if (fileId == null) { + return undefined; + } + const mapped = fileIdToPath?.get(fileId); + if (mapped != null) { + return mapped; + } + // Fallback: in the current FDR flow the fileId IS the sanitized path, + // so we can pass it through unchanged. + return fileId; +} + +function toImageRef( + fileId: string | undefined, + fileManifest: Record | undefined, + fileIdToPath: Map | undefined +): ImageRef | undefined { + const path = resolveFileIdToPath(fileId, fileIdToPath); + if (path == null) { + return undefined; + } + const entry = fileManifest?.[path]; + if (entry?.width == null || entry?.height == null) { + // Dimensions are required for ImageRef (next/image layout reservation). + // If the file wasn't measured (e.g. SVG or missing manifest entry) we + // drop the logo from the ledger config rather than send a half-valid + // ref that would fail server-side validation. + return undefined; + } + return { path, width: entry.width, height: entry.height }; +} + +function toPathOrUrl( + fileIdOrUrl: { type: "fileId"; value: string } | { type: "url"; value: string } | undefined, + fileIdToPath: Map | undefined +): PathOrUrl | undefined { + if (fileIdOrUrl == null) { + return undefined; + } + if (fileIdOrUrl.type === "url") { + return { type: "url", value: fileIdOrUrl.value }; + } + const path = resolveFileIdToPath(fileIdOrUrl.value, fileIdToPath); + if (path == null) { + return undefined; + } + return { type: "path", value: path }; +} + +type ThemeWithImages = Extract, { type: "dark" } | { type: "light" }>; + +function mapTheme( + theme: ThemeWithImages | NonNullable>["dark"], + fileManifest: Record | undefined, + fileIdToPath: Map | undefined +): { + logo?: ImageRef; + backgroundImage?: string; + accentPrimary: { r: number; g: number; b: number; a?: number }; + background?: { type: "solid"; r: number; g: number; b: number; a?: number } | { type: "gradient" }; + border?: { r: number; g: number; b: number; a?: number }; + sidebarBackground?: { r: number; g: number; b: number; a?: number }; + headerBackground?: { r: number; g: number; b: number; a?: number }; + cardBackground?: { r: number; g: number; b: number; a?: number }; +} { + // DocsConfig's per-theme `background` is a plain RGBA; LedgerConfig + // wraps it in a `solid` discriminated-union variant. We do not produce + // the `gradient` variant because DocsConfig has no source for it. + const background = theme.background != null ? { type: "solid" as const, ...theme.background } : undefined; + const backgroundImagePath = resolveFileIdToPath(theme.backgroundImage, fileIdToPath); + return { + logo: toImageRef(theme.logo, fileManifest, fileIdToPath), + backgroundImage: backgroundImagePath, + accentPrimary: theme.accentPrimary, + background, + border: theme.border, + sidebarBackground: theme.sidebarBackground, + headerBackground: theme.headerBackground, + cardBackground: theme.cardBackground + }; +} + +function mapColorsV3( + colors: DocsConfig["colorsV3"], + fileManifest: Record | undefined, + fileIdToPath: Map | undefined +): LedgerConfig["colorsV3"] { + if (colors == null) { + return undefined; + } + if (colors.type === "dark" || colors.type === "light") { + return { + type: colors.type, + ...mapTheme(colors, fileManifest, fileIdToPath) + }; + } + return { + type: "darkAndLight", + dark: mapTheme(colors.dark, fileManifest, fileIdToPath), + light: mapTheme(colors.light, fileManifest, fileIdToPath) + }; +} + +function mapMetadata( + metadata: DocsConfig["metadata"], + fileIdToPath: Map | undefined +): LedgerConfig["metadata"] { + if (metadata == null) { + return undefined; + } + // Strip image fields and re-add them under PathOrUrl shape. + const { + "og:image": ogImage, + "og:logo": ogLogo, + "twitter:image": twitterImage, + "og:background-image": ogBackgroundImage, + "og:dynamic:background-image": ogDynamicBackgroundImage, + ...rest + } = metadata; + const mapped: Record = { ...rest }; + const ogImageMapped = toPathOrUrl(ogImage, fileIdToPath); + if (ogImageMapped != null) { + mapped["og:image"] = ogImageMapped; + } + const ogLogoMapped = toPathOrUrl(ogLogo, fileIdToPath); + if (ogLogoMapped != null) { + mapped["og:logo"] = ogLogoMapped; + } + const twitterImageMapped = toPathOrUrl(twitterImage, fileIdToPath); + if (twitterImageMapped != null) { + mapped["twitter:image"] = twitterImageMapped; + } + const ogBackgroundImageMapped = toPathOrUrl(ogBackgroundImage, fileIdToPath); + if (ogBackgroundImageMapped != null) { + mapped["og:background-image"] = ogBackgroundImageMapped; + } + const ogDynamicBackgroundImageMapped = toPathOrUrl(ogDynamicBackgroundImage, fileIdToPath); + if (ogDynamicBackgroundImageMapped != null) { + mapped["og:dynamic:background-image"] = ogDynamicBackgroundImageMapped; + } + return mapped as LedgerConfig["metadata"]; +} + +function mapJsFiles(js: DocsConfig["js"], fileIdToPath: Map | undefined): LedgerConfig["js"] { + if (js == null) { + return undefined; + } + type LedgerJsFile = NonNullable["files"] extends Array | undefined ? T : never; + const files: LedgerJsFile[] = []; + for (const file of js.files) { + const path = resolveFileIdToPath(file.fileId, fileIdToPath); + if (path == null) { + continue; + } + files.push({ path, strategy: file.strategy }); + } + return { + remote: js.remote?.map((r) => ({ url: r.url, strategy: r.strategy })), + files, + inline: js.inline + }; +} + +function mapTypography( + typography: DocsConfig["typographyV2"], + fileIdToPath: Map | undefined +): LedgerConfig["typographyV2"] { + if (typography == null) { + return undefined; + } + const mapFont = ["bodyFont"]>(font: F) => { + if (font == null) { + return undefined; + } + return { + type: "custom" as const, + name: font.name, + variants: font.variants.map((variant) => ({ + fontFile: resolveFileIdToPath(variant.fontFile, fileIdToPath) ?? variant.fontFile, + weight: variant.weight, + style: variant.style + })), + display: font.display, + fallback: font.fallback, + fontVariationSettings: font.fontVariationSettings + }; + }; + return { + headingsFont: mapFont(typography.headingsFont), + bodyFont: mapFont(typography.bodyFont), + codeFont: mapFont(typography.codeFont) + }; +} + +function mapAgents(agents: DocsConfig["agents"]): LedgerConfig["agents"] { + if (agents == null) { + return undefined; + } + // llmsTxt / llmsFullTxt are intentionally dropped — the ledger contract + // serves these well-known files by convention via file artifact lookup + // (see LedgerConfigSchema doc comment in docs-ledger/contract.ts). + return { + pageDirective: agents.pageDirective, + pageDescriptionSource: agents.pageDescriptionSource, + siteDescription: agents.siteDescription + }; +} + +/** + * Map a classic DocsConfig (FileId-based) into the ledger-native LedgerConfig + * (path-based) shape. + * + * The two schemas have diverged: + * - DocsConfig.colorsV3.{dark,light}.logo: FileId → LedgerConfig: ImageRef { path, width, height } + * - DocsConfig.colorsV3.{dark,light}.backgroundImage: FileId → LedgerConfig: string (path) + * - DocsConfig.metadata image fields: FileIdOrUrl → LedgerConfig: PathOrUrl + * - DocsConfig.js.files[].fileId → LedgerConfig.js.files[].path + * - DocsConfig.typographyV2.*.variants[].fontFile: FileId → LedgerConfig: string (path) + * + * Dimensions for ImageRef come from `fileManifest` (populated by the + * publishDocs uploadFiles callback using `measureImageSizes`). If an image + * has no measured dimensions, the corresponding logo field is dropped rather + * than emitted with placeholder values. + * + * Fields that exist only in DocsConfig (favicon, agents.llmsTxt, integrations, + * languages, navigation, root, logoV2, colors, colorsV2, typography (v1), + * hideNavLinks, globalTheme, backgroundImage at the top level, logo at the + * top level) are intentionally omitted: LedgerConfig either exposes them by + * convention (favicon, llms*) or has dropped them (legacy v1/v2 variants). + */ +export function mapDocsConfigToLedgerConfig({ + docsConfig, + fileManifest, + fileIdToPath +}: { + docsConfig: DocsConfig; + fileManifest: Record | undefined; + fileIdToPath: Map | undefined; +}): LedgerConfig { + return { + title: docsConfig.title, + defaultLanguage: docsConfig.defaultLanguage, + translations: docsConfig.translations, + announcement: docsConfig.announcement, + navbarLinks: docsConfig.navbarLinks, + footerLinks: docsConfig.footerLinks, + logoHeight: docsConfig.logoHeight, + logoHref: docsConfig.logoHref, + logoRightText: docsConfig.logoRightText, + agents: mapAgents(docsConfig.agents), + metadata: mapMetadata(docsConfig.metadata, fileIdToPath), + redirects: docsConfig.redirects, + colorsV3: mapColorsV3(docsConfig.colorsV3, fileManifest, fileIdToPath), + layout: docsConfig.layout, + theme: docsConfig.theme, + settings: docsConfig.settings, + typographyV2: mapTypography(docsConfig.typographyV2, fileIdToPath), + analyticsConfig: docsConfig.analyticsConfig, + css: docsConfig.css, + js: mapJsFiles(docsConfig.js, fileIdToPath), + aiChatConfig: docsConfig.aiChatConfig, + pageActions: docsConfig.pageActions, + editThisPageLaunch: docsConfig.editThisPageLaunch, + header: docsConfig.header, + footer: docsConfig.footer + }; +} diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/normalizeRepoUrl.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/normalizeRepoUrl.ts new file mode 100644 index 000000000000..f4d25449beb6 --- /dev/null +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/normalizeRepoUrl.ts @@ -0,0 +1,21 @@ +/** + * Converts a CI-source repo slug (e.g. "owner/repo") into an HTTPS URL + * suitable for the docs-ledger `git.repoUrl` field. + * + * If the value is already a full URL it is returned as-is. + */ +export function normalizeRepoUrlToHttps(repo: string, provider: "github" | "gitlab" | "bitbucket"): string { + // Already a URL — passthrough. + if (repo.startsWith("https://") || repo.startsWith("http://")) { + return repo; + } + + switch (provider) { + case "github": + return `https://github.com/${repo}`; + case "gitlab": + return `https://gitlab.com/${repo}`; + case "bitbucket": + return `https://bitbucket.org/${repo}`; + } +} diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts index 2f160d443028..62798f9e09f8 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts @@ -17,6 +17,7 @@ import { wrapWithHttps } from "@fern-api/docs-resolver"; import { APIV1Write, FdrAPI as CjsFdrSdk, DocsV1Write, DocsV2Write, FdrClient } from "@fern-api/fdr-sdk"; +import type { DocsPublishGitInput, FileManifestEntry } from "@fern-api/fdr-sdk/orpc-client"; type DynamicIr = APIV1Write.DynamicIr; type DynamicIRUpload = APIV1Write.DynamicIRUpload; @@ -44,11 +45,14 @@ import { createHash } from "crypto"; import { readFile } from "fs/promises"; import { chunk } from "lodash-es"; import * as mime from "mime-types"; +import { basename } from "path"; import terminalLink from "terminal-link"; import { getDocsDeployMode } from "./docsDeployMode.js"; import { getDynamicGeneratorConfig } from "./getDynamicGeneratorConfig.js"; import { measureImageSizes } from "./measureImageSizes.js"; +import { normalizeRepoUrlToHttps } from "./normalizeRepoUrl.js"; import { publishDocsViaLedger } from "./publishDocsLedger.js"; +import { publishDocsViaLedgerPreview } from "./publishDocsLedgerPreview.js"; import { asyncPool } from "./utils/asyncPool.js"; const MEASURE_IMAGE_BATCH_SIZE = 10; @@ -116,6 +120,17 @@ export async function calculateFileHash(absoluteFilePath: AbsoluteFilePath | str return createHash("sha256").update(new Uint8Array(fileBuffer)).digest("hex"); } +/** + * Read a file once and return its bytes + sha256 hash. Avoids the double-read + * we'd otherwise do for files that need both hashing (legacy register) and + * inclusion in the ledger CAS blob map (publishDocsViaLedger). + */ +async function readAndHashFile(absoluteFilePath: AbsoluteFilePath | string): Promise<{ buffer: Buffer; hash: string }> { + const buffer = await readFile(absoluteFilePath); + const hash = createHash("sha256").update(new Uint8Array(buffer)).digest("hex"); + return { buffer, hash }; +} + export function sanitizeRelativePathForS3(relativeFilePath: RelativeFilePath): RelativeFilePath { // Replace ../ segments with _dot_dot_/ to prevent HTTP client normalization issues // that cause S3 signature mismatches when paths contain parent directory references @@ -267,6 +282,27 @@ export async function publishDocs({ // Collect API definitions (keyed by FDR definition ID) for the ledger manifest. const apiDefinitionCollector = new Map(); + // Collect per-file manifest entries for the ledger publish. + // The manifest is keyed by `sanitizedPath` (fern-host-relative file path), + // which matches the `fullPath` used by the FDR register handler when + // routing fileManifest entries to file artifacts. + // + // File content is NOT kept in memory. Instead, ledgerFilePaths maps + // each content hash to the file's absolute path so that uploadMissingBlobs + // can re-read only the files the server actually needs. + const ledgerFileManifest: Record = {}; + const ledgerFilePaths = new Map(); + // FileId → fullPath lookup used by mapDocsConfigToLedgerConfig to + // translate DocsConfig's FileId-based references (e.g. colorsV3.dark.logo) + // into LedgerConfig path strings. + // + // Populated by the uploadFiles callback below: + // - ledger mode: identity map (fullPath → fullPath) — the FileId we + // emit IS the sanitizedPath, so the lookup just round-trips it. + // - dual/legacy: keyed by the UUID FileId returned by FDR's + // startDocsRegister/startDocsPreviewRegister response. + const ledgerFileIdToPath = new Map(); + const resolver = new DocsDefinitionResolver({ domain, docsWorkspace: effectiveWorkspace, @@ -317,6 +353,23 @@ export async function publishDocs({ } const sanitizedPath = filePath.sanitizedPath; + const { buffer, hash } = await readAndHashFile(filePath.absoluteFilePath); + + // Populate the ledger file manifest entry for this image. + // mediaType is guaranteed non-false here because the file passed + // the mime.lookup filter upstream; fall back defensively anyway. + const contentType = mime.lookup(filePath.absoluteFilePath) || "application/octet-stream"; + ledgerFileManifest[sanitizedPath] = { + hash, + contentType, + contentLength: buffer.byteLength, + filename: basename(filePath.sanitizedPath), + width: image.width, + height: image.height + // blurDataURL: not populated yet (caching is a separate concern) + }; + ledgerFilePaths.set(hash, filePath.absoluteFilePath); + const obj = { filePath: CjsFdrSdk.docs.v1.write.FilePath( convertToFernHostRelativeFilePath(sanitizedPath) @@ -325,7 +378,7 @@ export async function publishDocs({ height: image.height, blurDataUrl: image.blurDataUrl, alt: undefined, - fileHash: await calculateFileHash(filePath.absoluteFilePath) + fileHash: hash } as DocsV2Write.ImageFilePath; return obj; } @@ -349,17 +402,80 @@ export async function publishDocs({ HASH_CONCURRENCY, nonImageFiles, async (file) => { + const { buffer, hash } = await readAndHashFile(file.absoluteFilePath); + + // Populate the ledger file manifest entry for this non-image file. + // If mime.lookup fails (unknown extension), fall back to + // application/octet-stream — both S3 and the docs CDN accept it, + // and the manifest only needs *some* content-type for routing. + const contentType = mime.lookup(file.absoluteFilePath) || "application/octet-stream"; + ledgerFileManifest[file.sanitizedPath] = { + hash, + contentType, + contentLength: buffer.byteLength, + filename: basename(file.sanitizedPath) + // width/height/blurDataURL omitted: non-image + }; + ledgerFilePaths.set(hash, file.absoluteFilePath); + return { path: CjsFdrSdk.docs.v1.write.FilePath( convertToFernHostRelativeFilePath(file.sanitizedPath) ), - fileHash: await calculateFileHash(file.absoluteFilePath) + fileHash: hash }; } ); const hashNonImageTime = performance.now() - hashNonImageStart; context.logger.debug(`Hashed ${filepaths.length} non-image files in ${hashNonImageTime.toFixed(0)}ms`); + // ── Ledger-only path ───────────────────────────────────── + // In ledger mode we do NOT call fdr.docs.v2.write.startDocsRegister + // / startDocsPreviewRegister. The legacy V2 register mints fresh + // FileId UUIDs per request even for byte-identical inputs, which + // then leak into the substituted markdown (via + // replaceImagePathsAndUrls below) and rotate the deployment hash + // on every publish — defeating the ledger's deployment-level dedup. + // + // Instead, we synthesize UploadedFile entries whose `fileId` is + // the file's sanitized fern-host-relative path. The resolver + // substitutes `file:` into the markdown, which: + // - is byte-identical across publishes of byte-identical + // inputs (sanitizedPath is deterministic), so pages dedup + // at the CAS layer; and + // - resolves directly through the existing path-keyed ledger + // reader endpoints (`fileArtifact`/`fileMetadata`, both + // keyed on `fullPath` ≡ sanitizedPath) — no new server- + // side resolver is needed. + // + // File bytes are uploaded later by the ledger missing-blobs + // step in publishDocsViaLedger / publishDocsViaLedgerPreview; + // they do not need a separate V2 upload round-trip. + if (deployMode === "ledger") { + const uploadedFiles: UploadedFile[] = []; + for (const file of filesWithSanitizedPaths) { + const manifestEntry = ledgerFileManifest[file.sanitizedPath]; + if (manifestEntry == null) { + continue; + } + // mapDocsConfigToLedgerConfig keys this lookup by + // whatever string the DocsConfig stores as a FileId. + // In ledger mode the FileId we emit IS the fullPath, + // so the map is identity (fullPath → fullPath) — kept + // populated for parity with the dual/legacy paths. + ledgerFileIdToPath.set(file.sanitizedPath, file.sanitizedPath); + uploadedFiles.push({ + relativeFilePath: file.relativeFilePath, + absoluteFilePath: file.absoluteFilePath, + fileId: file.sanitizedPath + }); + } + context.logger.debug( + `[ledger] Skipping V2 startDocsRegister; resolved ${uploadedFiles.length} files by sanitizedPath` + ); + return uploadedFiles; + } + if (preview) { let startDocsRegisterResponse; try { @@ -404,11 +520,18 @@ export async function publishDocs({ context.logger.debug(`No files to upload (all ${skippedCount} up to date)`); } } - return convertToFilePathPairs( + const uploadedFiles = convertToFilePathPairs( startDocsRegisterResponse.uploadUrls, docsWorkspace.absoluteFilePath, sanitizedToAbsoluteMap ); + for (const uploaded of uploadedFiles) { + const sanitizedPath = filesMap.get(uploaded.absoluteFilePath)?.sanitizedPath; + if (sanitizedPath != null) { + ledgerFileIdToPath.set(uploaded.fileId, sanitizedPath); + } + } + return uploadedFiles; } else { let startDocsRegisterResponse; try { @@ -460,11 +583,18 @@ export async function publishDocs({ context.logger.info("No files to upload (all up to date)"); } } - return convertToFilePathPairs( + const uploadedFiles = convertToFilePathPairs( startDocsRegisterResponse.uploadUrls, docsWorkspace.absoluteFilePath, sanitizedToAbsoluteMap ); + for (const uploaded of uploadedFiles) { + const sanitizedPath = filesMap.get(uploaded.absoluteFilePath)?.sanitizedPath; + if (sanitizedPath != null) { + ledgerFileIdToPath.set(uploaded.fileId, sanitizedPath); + } + } + return uploadedFiles; } }, registerApi: async ({ @@ -661,22 +791,66 @@ export async function publishDocs({ // ── Ledger publish path (dual-write or ledger-only) ────────── if (deployMode === "dual" || deployMode === "ledger") { + // Build structured git provenance from the CI environment (ADR 0011). + // The X-CI-Source header is still sent for other telemetry sinks; this + // puts the same data into the DocsPublishInput so the ledger persists it. + const ledgerGit: DocsPublishGitInput | undefined = + ciSource?.repo != null && ciSource?.branch != null + ? { + repoUrl: normalizeRepoUrlToHttps(ciSource.repo, ciSource.type), + branch: ciSource.branch, + commitSha: ciSource.commitSha + } + : undefined; + try { - const ledgerResult = await publishDocsViaLedger({ - docsDefinition, - organization, - domain, - basepath: basePath, - previewId, - token: token.value, - fdrOrigin, - headers, - context, - apiDefinitions: apiDefinitionCollector - }); - context.logger.info( - `[ledger] Deployment ${ledgerResult.reusedDeployment ? "reused" : "created"}: ${ledgerResult.deploymentId}` - ); + if (preview) { + // ADR 0012: preview publishes go through the dedicated + // /preview/init endpoint so they land on the server-generated + // preview hostname instead of the production domain. + const previewResult = await publishDocsViaLedgerPreview({ + docsDefinition, + organization, + basePath, + previewId: previewId != null ? sanitizePreviewId(previewId) : previewId, + git: ledgerGit, + token: token.value, + fdrOrigin, + headers, + context, + apiDefinitions: apiDefinitionCollector, + fileManifest: Object.keys(ledgerFileManifest).length > 0 ? ledgerFileManifest : undefined, + filePaths: ledgerFilePaths.size > 0 ? ledgerFilePaths : undefined, + fileIdToPath: ledgerFileIdToPath.size > 0 ? ledgerFileIdToPath : undefined + }); + // In ledger-only mode the preview URL comes from the ledger; + // in dual mode the V2 URL was already set above. + if (deployMode === "ledger") { + urlToOutput = previewResult.previewUrl; + } + context.logger.info(`[ledger] Preview deployment created: ${previewResult.deploymentId}`); + } else { + const ledgerResult = await publishDocsViaLedger({ + docsDefinition, + organization, + domain, + basepath: basePath, + previewId, + customDomains, + git: ledgerGit, + token: token.value, + fdrOrigin, + headers, + context, + apiDefinitions: apiDefinitionCollector, + fileManifest: Object.keys(ledgerFileManifest).length > 0 ? ledgerFileManifest : undefined, + filePaths: ledgerFilePaths.size > 0 ? ledgerFilePaths : undefined, + fileIdToPath: ledgerFileIdToPath.size > 0 ? ledgerFileIdToPath : undefined + }); + context.logger.info( + `[ledger] Deployment ${ledgerResult.reusedDeployment ? "reused" : "created"}: ${ledgerResult.deploymentId}` + ); + } } catch (error) { if (deployMode === "ledger") { return context.failAndThrow("Failed to publish docs via ledger to " + domain, error, { @@ -1227,8 +1401,7 @@ async function checkAndDownloadExistingSdkDynamicIRs({ try { const response = await fdr.api.register.checkSdkDynamicIrExists({ orgId: CjsFdrSdk.OrgId(organization), - apiId: "", - irVersions: [] + snippetConfiguration: snippetConfigWithVersions }); const existingDynamicIrs = response.existingDynamicIrs ?? {}; diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts index 84a5354aef49..2a3ead7451e6 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts @@ -1,8 +1,16 @@ import type { APIV1Write, DocsV1Write } from "@fern-api/fdr-sdk"; -import { createDocsLedgerClient, type DocsPublishInput } from "@fern-api/fdr-sdk/orpc-client"; +import { + createDocsLedgerClient, + type DocsPublishGitInput, + type DocsPublishInput, + type FileManifestEntry +} from "@fern-api/fdr-sdk/orpc-client"; +import type { AbsoluteFilePath } from "@fern-api/fs-utils"; import type { TaskContext } from "@fern-api/task-context"; import { createHash } from "crypto"; +import { readFile } from "fs/promises"; +import { mapDocsConfigToLedgerConfig } from "./mapDocsConfigToLedgerConfig.js"; import { asyncPool } from "./utils/asyncPool.js"; const UPLOAD_CONCURRENCY = 10; @@ -22,11 +30,27 @@ interface BlobRef { } /** - * Serializes a value to a JSON buffer and returns a BlobRef + the raw bytes, - * keyed by content hash for later upload. + * `JSON.stringify` with deterministic key ordering at every level. Arrays + * keep their original ordering (positions are meaningful); object keys are + * sorted lexicographically via the replacer. Two inputs that differ only by + * key insertion order serialize identically — required so the apiManifest + * blob hash is stable across publishes (cf. FDR `stableStringify`). + */ +function stableStringify(value: unknown): string { + return JSON.stringify(value, (_key, val) => { + if (val != null && typeof val === "object" && !Array.isArray(val)) { + return Object.fromEntries(Object.entries(val).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))); + } + return val; + }); +} + +/** + * Serializes a value to a JSON buffer using {@link stableStringify} and + * returns a BlobRef + the raw bytes, keyed by content hash for later upload. */ function jsonBlobRef(value: unknown): { ref: BlobRef; hash: string; buf: Buffer } { - const buf = Buffer.from(JSON.stringify(value), "utf-8"); + const buf = Buffer.from(stableStringify(value), "utf-8"); const hash = sha256(buf); return { ref: { hash, contentType: "application/json", contentLength: buf.length }, @@ -38,6 +62,14 @@ function jsonBlobRef(value: unknown): { ref: BlobRef; hash: string; buf: Buffer /** * Build a DocsPublishInput from a resolved DocsDefinition and collect * all content blobs that may need uploading. + * + * If `fileManifest` is provided, it is forwarded as-is into the resulting + * input. File blobs are NOT included in the returned blob map — they are + * loaded lazily during the upload step to avoid holding all file content in + * memory for the entire publish duration. The manifest's keys MUST be the + * same string the FDR register handler treats as `fullPath` (see + * {@link makeFileArtifact} in docsPublishTransform.ts) — the CLI uses + * sanitizedPath (fern-host-relative) which maps 1:1 to fullPath. */ export function buildLedgerInput({ docsDefinition, @@ -45,14 +77,29 @@ export function buildLedgerInput({ domain, basepath, previewId, - apiDefinitions + customDomains, + git, + apiDefinitions, + fileManifest, + fileIdToPath }: { docsDefinition: DocsDefinition; organization: string; domain: string; basepath: string | undefined; previewId: string | undefined; + customDomains?: string[]; + git?: DocsPublishGitInput; apiDefinitions: Map; + fileManifest?: Record; + /** + * Map from FileId (as returned by FDR's legacy startDocsRegister + * `uploadUrls`) to the `fullPath` string used to key `fileManifest`. + * Used by {@link mapDocsConfigToLedgerConfig} to translate DocsConfig's + * FileId-based references (e.g. `colorsV3.dark.logo`) into LedgerConfig's + * path-based references (e.g. `ImageRef { path, width, height }`). + */ + fileIdToPath?: Map; }): { input: DocsPublishInput; blobs: Map } { const blobs = new Map(); @@ -68,11 +115,32 @@ export function buildLedgerInput({ blobs.set(hash, buf); } - // Config: serialize the entire config as a JSON blob. - const configBlob = jsonBlobRef(docsDefinition.config); - blobs.set(configBlob.hash, configBlob.buf); + // Config is sent inline (not a CAS blob) per the docs-ledger contract. + // DocsConfig (FileId-based) is translated to LedgerConfig (path-based) + // up front so the wire payload is already in the schema FDR validates. + const ledgerConfig = mapDocsConfigToLedgerConfig({ + docsConfig: docsDefinition.config, + fileManifest, + fileIdToPath + }); // API manifest: serialize all API definitions as a single JSON blob. + // + // `apiDefinitions` is populated by the `registerApi` callback inside a + // `Promise.all`, so the Map's insertion order reflects whichever HTTP + // round-trip completed first — non-deterministic across publishes. + // {@link jsonBlobRef} uses {@link stableStringify}, which sorts object + // keys at every level, so the resulting bytes are stable regardless of + // Map iteration order. + // + // Determinism caveat: stable apiManifest bytes are necessary but not + // sufficient for a deterministic deployment hash. Page bodies must also + // be byte-identical, which requires that file references substituted + // into markdown (`file:` tokens emitted by replaceImagePathsAndUrls) + // be stable. In `ledger` deploy mode the CLI emits path tokens + // (`file:`) and short-circuits the V2 register call; in + // `dual`/`legacy` modes the V2 register flow mints fresh UUID FileIds per + // request, so deployment-level dedup will not fire there. let apiManifestRef: BlobRef | null = null; if (apiDefinitions.size > 0) { const manifestObj = Object.fromEntries(apiDefinitions); @@ -85,14 +153,16 @@ export function buildLedgerInput({ orgId: organization, domain, basepath: basepath ?? "", + customDomains: customDomains ?? [], previewId: previewId ?? null, root: docsDefinition.config.root ?? docsDefinition.config.navigation, pages, - config: configBlob.ref, + config: ledgerConfig, apiManifest: apiManifestRef, - files: null, + fileManifest, redirects: null, - locale: "en" + locale: "en", + git }; return { input, blobs }; @@ -117,22 +187,33 @@ export async function publishDocsViaLedger({ domain, basepath, previewId, + customDomains, + git, token, fdrOrigin, headers, context, - apiDefinitions + apiDefinitions, + fileManifest, + filePaths, + fileIdToPath }: { docsDefinition: DocsDefinition; organization: string; domain: string; basepath: string | undefined; previewId: string | undefined; + customDomains?: string[]; + git?: DocsPublishGitInput; token: string; fdrOrigin: string; headers: Record; context: TaskContext; apiDefinitions: Map; + fileManifest?: Record; + /** Hash → absolute file path for lazy on-demand reads during upload. */ + filePaths?: Map; + fileIdToPath?: Map; }): Promise { const { input, blobs } = buildLedgerInput({ docsDefinition, @@ -140,7 +221,11 @@ export async function publishDocsViaLedger({ domain, basepath, previewId, - apiDefinitions + customDomains, + git, + apiDefinitions, + fileManifest, + fileIdToPath }); const client = createDocsLedgerClient({ baseUrl: fdrOrigin, token, headers }); @@ -156,22 +241,58 @@ export async function publishDocsViaLedger({ ); // Step 2: Upload any blobs the server doesn't have yet. - if (registerResult.missingContent.length > 0) { - context.logger.debug(`[ledger] Uploading ${registerResult.missingContent.length} missing blobs...`); + // In-memory blobs (pages, config, apiManifest) are checked first; + // file blobs are read lazily from disk via filePaths. + await uploadMissingBlobs(registerResult.missingContent, blobs, context, filePaths); + + // Step 3: Finish — server persists the deployment. + context.logger.debug("[ledger] Finishing deployment..."); + const finishStart = performance.now(); + const finishResult = await client.finish(input); + const finishTime = performance.now() - finishStart; + context.logger.debug( + `[ledger] Finished in ${finishTime.toFixed(0)}ms — deploymentId=${finishResult.deploymentId}, reused=${finishResult.reusedDeployment}` + ); + + return finishResult; +} + +/** + * Upload blobs the server reported as missing after a register call. + * Shared between the production and preview ledger flows. + * + * Small in-memory blobs (pages, config, apiManifest) are looked up in + * `blobs` first. File blobs are loaded lazily from disk via `filePaths` + * (hash → absolute path) to avoid holding every file's bytes in memory + * for the entire publish duration. + */ +export async function uploadMissingBlobs( + missingContent: ReadonlyArray<{ hash: string; uploadUrl: string }>, + blobs: Map, + context: TaskContext, + filePaths?: Map +): Promise { + if (missingContent.length > 0) { + context.logger.debug(`[ledger] Uploading ${missingContent.length} missing blobs...`); const uploadStart = performance.now(); - const results = await asyncPool( - UPLOAD_CONCURRENCY, - registerResult.missingContent, - async ({ hash, uploadUrl }) => { - const blob = blobs.get(hash); - if (blob == null) { - context.logger.warn(`[ledger] Server requested blob ${hash} but we don't have it — skipping`); - return "skipped" as const; + const results = await asyncPool(UPLOAD_CONCURRENCY, [...missingContent], async ({ hash, uploadUrl }) => { + // Prefer in-memory blobs (pages, config, apiManifest). + let blob = blobs.get(hash); + if (blob == null && filePaths != null) { + // Lazy read: only load file content for blobs the server + // actually needs, and discard after upload. + const filePath = filePaths.get(hash); + if (filePath != null) { + blob = await readFile(filePath); } - return uploadBlobWithRetry(blob, uploadUrl, hash, context); } - ); + if (blob == null) { + context.logger.warn(`[ledger] Server requested blob ${hash} but we don't have it — skipping`); + return "skipped" as const; + } + return uploadBlobWithRetry(blob, uploadUrl, hash, context); + }); const uploaded = results.filter((r) => r === "uploaded").length; const alreadyExisted = results.filter((r) => r === "already_exists").length; @@ -182,17 +303,6 @@ export async function publishDocsViaLedger({ } else { context.logger.debug("[ledger] All content already in CAS — no uploads needed"); } - - // Step 3: Finish — server persists the deployment. - context.logger.debug("[ledger] Finishing deployment..."); - const finishStart = performance.now(); - const finishResult = await client.finish(input); - const finishTime = performance.now() - finishStart; - context.logger.debug( - `[ledger] Finished in ${finishTime.toFixed(0)}ms — deploymentId=${finishResult.deploymentId}, reused=${finishResult.reusedDeployment}` - ); - - return finishResult; } type UploadResult = "uploaded" | "already_exists"; diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts new file mode 100644 index 000000000000..8e53607744e9 --- /dev/null +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts @@ -0,0 +1,128 @@ +import type { APIV1Write, DocsV1Write } from "@fern-api/fdr-sdk"; +import { + createDocsLedgerClient, + type DocsPublishGitInput, + type FileManifestEntry +} from "@fern-api/fdr-sdk/orpc-client"; +import type { AbsoluteFilePath } from "@fern-api/fs-utils"; +import type { TaskContext } from "@fern-api/task-context"; + +import { buildLedgerInput, uploadMissingBlobs } from "./publishDocsLedger.js"; + +type DocsDefinition = DocsV1Write.DocsDefinition; + +export interface LedgerPreviewResult { + previewUrl: string; + deploymentId: string; +} + +/** + * Publish a docs preview via the dedicated ledger preview endpoint + * (POST /preview/init) followed by the standard finish call. + * + * Unlike the production {@link publishDocsViaLedger}, this flow: + * - Sends `LedgerPreviewRegisterInput` (no `domain` / `customDomains`). + * - Receives a server-generated preview URL and domain. + * - Finishes with the server-assigned domain + previewId so the deployment + * lands on the preview branch, not production. + */ +export async function publishDocsViaLedgerPreview({ + docsDefinition, + organization, + basePath, + previewId, + git, + token, + fdrOrigin, + headers, + context, + apiDefinitions, + fileManifest, + filePaths, + fileIdToPath +}: { + docsDefinition: DocsDefinition; + organization: string; + basePath: string | undefined; + previewId: string | undefined; + git?: DocsPublishGitInput; + token: string; + fdrOrigin: string; + headers: Record; + context: TaskContext; + apiDefinitions: Map; + fileManifest?: Record; + /** Hash → absolute file path for lazy on-demand reads during upload. */ + filePaths?: Map; + fileIdToPath?: Map; +}): Promise { + // Build the input and blob map using the shared helper. We pass a + // throwaway `domain` — it's required by buildLedgerInput's type but will + // not be sent to the preview endpoint (LedgerPreviewRegisterInput has no + // domain field). + const { input, blobs } = buildLedgerInput({ + docsDefinition, + organization, + domain: "", + basepath: basePath, + previewId, + customDomains: [], + git, + apiDefinitions, + fileManifest, + fileIdToPath + }); + + const client = createDocsLedgerClient({ baseUrl: fdrOrigin, token, headers }); + + // Step 1: Preview register — server picks the preview host and returns + // presigned upload URLs for missing blobs. + context.logger.debug("[ledger-preview] Registering preview deployment..."); + const registerStart = performance.now(); + const registerResult = await client.previewRegister({ + orgId: organization, + previewId: previewId ?? null, + basePath: basePath ?? "", + root: input.root, + pages: input.pages, + apiManifest: input.apiManifest, + config: input.config, + fileManifest: input.fileManifest, + jsFiles: input.jsFiles, + redirects: input.redirects, + locale: input.locale, + version: input.version, + repo: input.repo, + git + }); + const registerTime = performance.now() - registerStart; + context.logger.debug( + `[ledger-preview] Registered in ${registerTime.toFixed(0)}ms — hash=${registerResult.deploymentHash}, ` + + `preview=${registerResult.previewUrl}, missing=${registerResult.missingContent.length} blobs` + ); + + // Step 2: Upload missing blobs. + await uploadMissingBlobs(registerResult.missingContent, blobs, context, filePaths); + + // Step 3: Finish — use the server-assigned domain and previewId so the + // deployment is keyed to the preview branch. + context.logger.debug("[ledger-preview] Finishing preview deployment..."); + const finishStart = performance.now(); + const finishResult = await client.finish({ + ...input, + domain: registerResult.domain, + basepath: registerResult.basepath, + customDomains: [], + previewId: registerResult.previewId + }); + const finishTime = performance.now() - finishStart; + context.logger.debug( + `[ledger-preview] Finished in ${finishTime.toFixed(0)}ms — deploymentId=${finishResult.deploymentId}, ` + + `reused=${finishResult.reusedDeployment}` + ); + + return { + previewUrl: registerResult.previewUrl, + deploymentId: finishResult.deploymentId + }; +} diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts index 9d2d1a90befe..8f35711c5b8e 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts @@ -191,17 +191,11 @@ export async function runRemoteGenerationForGenerator({ const apiDefinition = convertIrToFdrApi({ ir, - snippetsConfig: { - typescriptSdk: undefined, - pythonSdk: undefined, - javaSdk: undefined, - rubySdk: undefined, - goSdk: undefined, - csharpSdk: undefined, - phpSdk: undefined, - swiftSdk: undefined, - rustSdk: undefined - }, + snippetsConfig: buildSnippetsConfigForSdk({ + language: generatorInvocation.language, + packageName, + version: resolvedVersion + }), context: interactiveTaskContext }); try { @@ -452,6 +446,44 @@ const emptyReadmeConfig: FernIr.ReadmeConfig = { exampleStyle: undefined }; +export function buildSnippetsConfigForSdk({ + language, + packageName, + version +}: { + language: generatorsYml.GenerationLanguage | undefined; + packageName: string | undefined; + version: string | undefined; +}): FdrAPI.api.v1.register.SnippetsConfig { + if (language == null || packageName == null) { + return {}; + } + + const resolvedVersion = resolveVersionFallback(version); + switch (language) { + case "typescript": + return { typescriptSdk: { package: packageName, version: resolvedVersion } }; + case "python": + return { pythonSdk: { package: packageName, version: resolvedVersion } }; + case "java": + return { javaSdk: { coordinate: packageName, version: resolvedVersion } }; + case "ruby": + return { rubySdk: { gem: packageName, version: resolvedVersion } }; + case "go": + return { goSdk: { githubRepo: packageName, version: resolvedVersion } }; + case "csharp": + return { csharpSdk: { package: packageName, version: resolvedVersion } }; + case "php": + return { phpSdk: { package: packageName, version: resolvedVersion } }; + case "swift": + return { swiftSdk: { package: packageName, version: resolvedVersion } }; + case "rust": + return { rustSdk: { package: packageName, version: resolvedVersion } }; + default: + return {}; + } +} + async function uploadDynamicIRForSdkGeneration({ fdr, organization, @@ -480,8 +512,8 @@ async function uploadDynamicIRForSdkGeneration({ try { uploadUrlsResponse = await fdr.api.register.getSdkDynamicIrUploadUrls({ orgId: FdrAPI.OrgId(organization), - apiId: "", - irVersions: [] + version, + snippetConfiguration: { [language]: packageName } }); } catch (error) { // Log warning but don't fail the generation - dynamic IR upload is optional From 54f1169fe55776b8d47caf56e987ef394a92a15d Mon Sep 17 00:00:00 2001 From: emjoseph Date: Mon, 18 May 2026 18:11:12 -0700 Subject: [PATCH 05/15] fix(cli): include jsFiles in ledger publish payload (#15973) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- packages/cli/cli/changes/unreleased/ledger-jsfiles.yml | 5 +++++ .../remote-workspace-runner/src/publishDocsLedger.ts | 10 ++++++++++ 2 files changed, 15 insertions(+) create mode 100644 packages/cli/cli/changes/unreleased/ledger-jsfiles.yml diff --git a/packages/cli/cli/changes/unreleased/ledger-jsfiles.yml b/packages/cli/cli/changes/unreleased/ledger-jsfiles.yml new file mode 100644 index 000000000000..87094d69fe3c --- /dev/null +++ b/packages/cli/cli/changes/unreleased/ledger-jsfiles.yml @@ -0,0 +1,5 @@ +- summary: | + Include jsFiles (custom React components, referenced markdown snippets, and + custom header/footer components) in the ledger publish payload so they are + stored in CAS and available on the ledger read path. + type: fix diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts index 2a3ead7451e6..5fc1e6ac991f 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts @@ -149,6 +149,15 @@ export function buildLedgerInput({ apiManifestRef = manifestBlob.ref; } + // jsFiles: custom React components, referenced markdown snippets, and + // custom header/footer components resolved by DocsDefinitionResolver. + let jsFilesRef: BlobRef | null = null; + if (docsDefinition.jsFiles != null && Object.keys(docsDefinition.jsFiles).length > 0) { + const jsFilesBlob = jsonBlobRef(docsDefinition.jsFiles); + blobs.set(jsFilesBlob.hash, jsFilesBlob.buf); + jsFilesRef = jsFilesBlob.ref; + } + const input: DocsPublishInput = { orgId: organization, domain, @@ -159,6 +168,7 @@ export function buildLedgerInput({ pages, config: ledgerConfig, apiManifest: apiManifestRef, + jsFiles: jsFilesRef, fileManifest, redirects: null, locale: "en", From 5b49668419b5a68f4f388dd5dca695e13986f8f0 Mon Sep 17 00:00:00 2001 From: emjoseph Date: Mon, 18 May 2026 18:14:52 -0700 Subject: [PATCH 06/15] feat(cli): add docs-ledger multi-locale translation publishing (#15970) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(cli): add docs-ledger multi-locale translation publishing Port the V2 translation loop into the ledger pipeline: - buildLedgerInput now accepts an optional locale parameter (default "en") - After base deployment finishes, publishDocsViaLedger loops through resolver.getTranslationPages() and publishes each locale via register + finishTranslation - Translation MDX processing (snippet resolution, image rewrites, @/ imports, comment stripping, nav overlays) is shared with V2 - V2 registerTranslation loop is skipped when deployMode=="ledger" (translations handled by the ledger path instead) Co-Authored-By: Eugene Joseph * fix: organize imports for biome check Co-Authored-By: Eugene Joseph * fix: narrow docsRegistrationId type instead of non-null assertion Co-Authored-By: Eugene Joseph * fix(cli): extract shared translation pipeline, add preview translations, parallel locale publishing Issue #1: Extract buildTranslatedDocsDefinition into shared module - Both V2 (publishDocs.ts) and ledger (publishDocsLedger.ts) now import from buildTranslatedDocsDefinition.ts instead of duplicating ~140 lines of MDX processing logic (snippet resolution, image rewrites, nav overlays) Issue #2: Wire resolver into ledger preview path - publishDocsViaLedgerPreview now accepts optional resolver parameter - When translations are available, preview publishes them via finishTranslation after the base deployment completes Issue #3: Parallel locale publishing - Replace sequential for...of with Promise.all in publishTranslationsViaLedger, matching V2's parallel approach Issue #4: Document dual-mode double-publish - Add comment explaining that dual mode intentionally publishes translations via both V2 and ledger for migration parity Co-Authored-By: Eugene Joseph * fix(cli): reuse client in preview translations, add failure summary log 1. Reuse existing 'client' in preview translation path instead of creating a redundant client2 instance 2. Aggregate per-locale failures into a summary log at the end of translation publishing (both preview and non-preview paths): e.g. '2/5 locale(s) failed: es, ja' Co-Authored-By: Eugene Joseph * refactor: build all locales upfront with fail-fast, single register→upload→finish flow Restructure translation publishing so all locale DocsDefinitions are built before any network calls. If any locale fails to build, the entire publish aborts immediately. Production path (publishDocsLedger.ts): - Phase 1: Build base + all translation inputs in parallel (fail-fast) - Phase 2: Merge all blobs, single register → upload → finish for base - Phase 3: Attach translations (blobs already uploaded) Preview path (publishDocsLedgerPreview.ts): - Phase 1: Build base input + all translated DocsDefinitions upfront - Phase 2: previewRegister → build translation ledger inputs (need server-assigned domain) → merge blobs → upload → finish - Phase 3: Attach translations Removes per-locale try/catch error handling — errors now propagate to abort the entire publish rather than warn-and-continue. Co-Authored-By: Eugene Joseph * feat(cli): pass translations inline to finish call, remove Phase 3 Replace per-locale finishTranslation loop with a single finish call that includes a translations[] array. Both production and preview paths now follow a two-phase pipeline: Phase 1: Build all locales upfront (fail-fast) Phase 2: Single register → upload → finish (base + translations) The server persists locale-specific segments for each translation entry after the base deployment is created — no separate round-trips. Co-Authored-By: Eugene Joseph --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../unreleased/ledger-localization.yml | 8 + .../src/buildTranslatedDocsDefinition.ts | 174 +++++++++++++++ .../src/publishDocs.ts | 204 +++--------------- .../src/publishDocsLedger.ts | 183 ++++++++++++++-- .../src/publishDocsLedgerPreview.ts | 145 +++++++++++-- 5 files changed, 511 insertions(+), 203 deletions(-) create mode 100644 packages/cli/cli/changes/unreleased/ledger-localization.yml create mode 100644 packages/cli/generation/remote-generation/remote-workspace-runner/src/buildTranslatedDocsDefinition.ts diff --git a/packages/cli/cli/changes/unreleased/ledger-localization.yml b/packages/cli/cli/changes/unreleased/ledger-localization.yml new file mode 100644 index 000000000000..140dea06fbfe --- /dev/null +++ b/packages/cli/cli/changes/unreleased/ledger-localization.yml @@ -0,0 +1,8 @@ +- summary: | + Add multi-locale support to docs-ledger publishing. All locales are + built upfront and published in a single register → upload → finish + flow. Translations are passed inline to the finish call, eliminating + per-locale finishTranslation round-trips. Extracts the shared MDX + translation pipeline into a reusable module used by both V2 and + ledger paths. Preview mode also supports translations. + type: feat diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/buildTranslatedDocsDefinition.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/buildTranslatedDocsDefinition.ts new file mode 100644 index 000000000000..950e53211647 --- /dev/null +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/buildTranslatedDocsDefinition.ts @@ -0,0 +1,174 @@ +import { + applyTranslatedFrontmatterToNavTree, + applyTranslatedNavigationOverlays, + type DocsDefinitionResolver, + getTranslatedAnnouncement, + replaceImagePathsAndUrls, + replaceReferencedCode, + replaceReferencedMarkdown, + stripMdxComments, + transformAtPrefixImports +} from "@fern-api/docs-resolver"; +import type { DocsV1Write } from "@fern-api/fdr-sdk"; +import { AbsoluteFilePath, doesPathExist, RelativeFilePath, relative, resolve } from "@fern-api/fs-utils"; +import type { TaskContext } from "@fern-api/task-context"; +import { readFile } from "fs/promises"; + +type DocsDefinition = DocsV1Write.DocsDefinition; + +/** + * Build a translated DocsDefinition by overlaying translated pages and + * navigation on top of the base definition. Applies the full MDX processing + * pipeline shared by both the V2 and ledger translation paths: + * + * 1. Locale-aware snippet resolution (prefers translated snippets) + * 2. `` reference resolution + * 3. `@/` import transforms + * 4. MDX comment stripping + * 5. Image path rewrites (using base page location) + * 6. `editThisPageUrl` rewriting to point to the translated file + * 7. Nav tree overlay (translated sidebar titles from frontmatter + docs.yml) + * 8. Translated announcement and navbar links + */ +export async function buildTranslatedDocsDefinition({ + docsDefinition, + locale, + localePages, + translationNavigationOverlays, + resolver, + context +}: { + docsDefinition: DocsDefinition; + locale: string; + localePages: Record; + translationNavigationOverlays: + | Record + | undefined; + resolver: DocsDefinitionResolver; + context: TaskContext; +}): Promise { + const collectedFileIds = resolver.getCollectedFileIds(); + const docsWorkspacePath = resolver.getDocsWorkspacePath(); + + const resolveLocalePath = async (filepath: AbsoluteFilePath): Promise => { + const relPath = relative(docsWorkspacePath, filepath); + const translatedPath = resolve(docsWorkspacePath, RelativeFilePath.of(`translations/${locale}/${relPath}`)); + return (await doesPathExist(translatedPath)) ? translatedPath : filepath; + }; + + const localeAwareMarkdownLoader = async (filepath: AbsoluteFilePath): Promise => { + const pathToRead = await resolveLocalePath(filepath); + const raw = await readFile(pathToRead, "utf-8"); + const fmMatch = raw.match(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/); + return fmMatch != null ? raw.slice(fmMatch[0].length) : raw; + }; + + const localeAwareFileLoader = async (filepath: AbsoluteFilePath): Promise => { + const pathToRead = await resolveLocalePath(filepath); + return readFile(pathToRead, "utf-8"); + }; + + const translatedPageEntries = await Promise.all( + Object.entries(localePages).map(async ([path, rawMarkdown]) => { + try { + const basePage = docsDefinition.pages[path as DocsV1Write.PageId]; + const absolutePathToMarkdownFile = resolve(docsWorkspacePath, RelativeFilePath.of(path)); + + const { markdown: markdownResolved } = await replaceReferencedMarkdown({ + markdown: rawMarkdown, + absolutePathToFernFolder: docsWorkspacePath, + absolutePathToMarkdownFile, + context, + markdownLoader: localeAwareMarkdownLoader + }); + + const codeResolved = await replaceReferencedCode({ + markdown: markdownResolved, + absolutePathToFernFolder: docsWorkspacePath, + absolutePathToMarkdownFile, + context, + fileLoader: localeAwareFileLoader + }); + + const importsResolved = transformAtPrefixImports({ + markdown: codeResolved, + absolutePathToFernFolder: docsWorkspacePath, + absolutePathToMarkdownFile + }); + + let processedMarkdown = stripMdxComments(importsResolved); + + processedMarkdown = replaceImagePathsAndUrls( + processedMarkdown, + collectedFileIds, + {}, + { + absolutePathToMarkdownFile, + absolutePathToFernFolder: docsWorkspacePath + }, + context + ); + + let editThisPageUrl = basePage?.editThisPageUrl; + if (editThisPageUrl != null) { + const fernPathPattern = `/fern/${path}`; + const translatedPathStr = `/fern/translations/${locale}/${path}`; + editThisPageUrl = editThisPageUrl.replace( + fernPathPattern, + translatedPathStr + ) as typeof editThisPageUrl; + } + + return [ + path, + { + markdown: processedMarkdown, + rawMarkdown: processedMarkdown, + editThisPageUrl, + editThisPageLaunch: basePage?.editThisPageLaunch + } + ]; + } catch (pageError) { + context.logger.warn( + `Failed to process translated page "${path}" for locale "${locale}": ${String(pageError)}. Falling back to base page.` + ); + return undefined; + } + }) + ); + + const translatedPages = { + ...docsDefinition.pages, + ...Object.fromEntries( + translatedPageEntries.filter((entry): entry is NonNullable => entry != null) + ) + }; + + let updatedRoot = applyTranslatedFrontmatterToNavTree( + docsDefinition.config.root, + localePages as Record, + context + ); + + const localeNavOverlay = translationNavigationOverlays?.[locale]; + let translatedAnnouncement = docsDefinition.config.announcement; + let translatedNavbarLinks = docsDefinition.config.navbarLinks; + if (localeNavOverlay != null) { + updatedRoot = applyTranslatedNavigationOverlays(updatedRoot, localeNavOverlay); + translatedAnnouncement = getTranslatedAnnouncement(localeNavOverlay) ?? translatedAnnouncement; + if (localeNavOverlay.navbarLinks != null) { + translatedNavbarLinks = localeNavOverlay.navbarLinks; + } + } + + return { + ...docsDefinition, + pages: translatedPages, + config: { + ...docsDefinition.config, + root: updatedRoot, + announcement: translatedAnnouncement, + navbarLinks: translatedNavbarLinks + } + }; +} diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts index 62798f9e09f8..3e576a3106ac 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts @@ -3,19 +3,7 @@ import { SourceResolverImpl } from "@fern-api/cli-source-resolver"; import { docsYml, generatorsYml } from "@fern-api/configuration"; import { createFdrService } from "@fern-api/core"; import { MediaType, replaceEnvVariables } from "@fern-api/core-utils"; -import { - applyTranslatedFrontmatterToNavTree, - applyTranslatedNavigationOverlays, - DocsDefinitionResolver, - getTranslatedAnnouncement, - replaceImagePathsAndUrls, - replaceReferencedCode, - replaceReferencedMarkdown, - stripMdxComments, - transformAtPrefixImports, - UploadedFile, - wrapWithHttps -} from "@fern-api/docs-resolver"; +import { DocsDefinitionResolver, UploadedFile, wrapWithHttps } from "@fern-api/docs-resolver"; import { APIV1Write, FdrAPI as CjsFdrSdk, DocsV1Write, DocsV2Write, FdrClient } from "@fern-api/fdr-sdk"; import type { DocsPublishGitInput, FileManifestEntry } from "@fern-api/fdr-sdk/orpc-client"; @@ -25,14 +13,7 @@ type SnippetsConfig = APIV1Write.SnippetsConfig; type DocsDefinition = DocsV1Write.DocsDefinition; import { stitchGlobalTheme } from "@fern-api/docs-resolver"; -import { - AbsoluteFilePath, - convertToFernHostRelativeFilePath, - doesPathExist, - RelativeFilePath, - relative, - resolve -} from "@fern-api/fs-utils"; +import { AbsoluteFilePath, convertToFernHostRelativeFilePath, RelativeFilePath, resolve } from "@fern-api/fs-utils"; import { convertIrToDynamicSnippetsIr, generateIntermediateRepresentation } from "@fern-api/ir-generator"; import { getOriginalName } from "@fern-api/ir-utils"; import { detectAirGappedMode, OSSWorkspace } from "@fern-api/lazy-fern-workspace"; @@ -47,6 +28,7 @@ import { chunk } from "lodash-es"; import * as mime from "mime-types"; import { basename } from "path"; import terminalLink from "terminal-link"; +import { buildTranslatedDocsDefinition } from "./buildTranslatedDocsDefinition.js"; import { getDocsDeployMode } from "./docsDeployMode.js"; import { getDynamicGeneratorConfig } from "./getDynamicGeneratorConfig.js"; import { measureImageSizes } from "./measureImageSizes.js"; @@ -770,12 +752,12 @@ export async function publishDocs({ } // ── Legacy publish path ────────────────────────────────────── - if (deployMode !== "ledger") { + if (deployMode !== "ledger" && docsRegistrationId != null) { context.logger.info("Publishing docs to FDR..."); const publishStart = performance.now(); try { await fdr.docs.v2.write.finishDocsRegister({ - docsRegistrationId: docsRegistrationId!, + docsRegistrationId, docsDefinition, excludeApis, ...(isBasepathAware && !preview && { basepathAware: true }) @@ -821,7 +803,8 @@ export async function publishDocs({ apiDefinitions: apiDefinitionCollector, fileManifest: Object.keys(ledgerFileManifest).length > 0 ? ledgerFileManifest : undefined, filePaths: ledgerFilePaths.size > 0 ? ledgerFilePaths : undefined, - fileIdToPath: ledgerFileIdToPath.size > 0 ? ledgerFileIdToPath : undefined + fileIdToPath: ledgerFileIdToPath.size > 0 ? ledgerFileIdToPath : undefined, + resolver }); // In ledger-only mode the preview URL comes from the ledger; // in dual mode the V2 URL was already set above. @@ -845,7 +828,8 @@ export async function publishDocs({ apiDefinitions: apiDefinitionCollector, fileManifest: Object.keys(ledgerFileManifest).length > 0 ? ledgerFileManifest : undefined, filePaths: ledgerFilePaths.size > 0 ? ledgerFilePaths : undefined, - fileIdToPath: ledgerFileIdToPath.size > 0 ? ledgerFileIdToPath : undefined + fileIdToPath: ledgerFileIdToPath.size > 0 ? ledgerFileIdToPath : undefined, + resolver }); context.logger.info( `[ledger] Deployment ${ledgerResult.reusedDeployment ? "reused" : "created"}: ${ledgerResult.deploymentId}` @@ -862,171 +846,33 @@ export async function publishDocs({ } } - // Register translated page content for each configured locale. + // Register translated page content for each configured locale via the V2 endpoint. + // In ledger-only mode, translations are handled by publishDocsViaLedger (above), + // so this block only runs for legacy and dual-write modes. + // In dual mode, translations are intentionally published through BOTH the V2 + // endpoint (here) and the ledger finishTranslation endpoint (inside + // publishDocsViaLedger above). This ensures migration parity — both stores + // receive identical translated content until the ledger path is promoted to + // sole owner. // In preview mode, register translations against the preview URL (not the production domain) // so that translated docs are visible in preview without overwriting production translations. const translationPages = resolver.getTranslationPages(); const translationNavigationOverlays = resolver.getTranslationNavigationOverlays(); const translationDomain = preview ? urlToOutput : domain; - if (translationPages != null && Object.keys(translationPages).length > 0) { + if (deployMode !== "ledger" && translationPages != null && Object.keys(translationPages).length > 0) { context.logger.info(`Registering translations for ${Object.keys(translationPages).length} locale(s)...`); await Promise.all( Object.entries(translationPages).map(async ([locale, localePages]) => { try { - // Build a translated DocsDefinition by taking the base definition, - // overriding translated pages, and updating the nav tree to reflect - // any sidebar-title / slug frontmatter in the translated pages. - // - // For each translated page, we apply the same transformations as default locale pages: - // 1. Resolve and snippet references - // 2. Transform @/ prefix imports to relative paths - // 3. Strip MDX comments to prevent leakage - // 4. Replace relative image paths with file IDs (using base page path for resolution) - // 5. Preserve editThisPageUrl/editThisPageLaunch from the base page - const collectedFileIds = resolver.getCollectedFileIds(); - const docsWorkspacePath = resolver.getDocsWorkspacePath(); - - // Create a locale-aware file loader that prefers translated snippets - // (e.g., translations/zh/snippets/foo.mdx) over base snippets. - const resolveLocalePath = async (filepath: AbsoluteFilePath): Promise => { - const relPath = relative(docsWorkspacePath, filepath); - const translatedPath = resolve( - docsWorkspacePath, - RelativeFilePath.of(`translations/${locale}/${relPath}`) - ); - return (await doesPathExist(translatedPath)) ? translatedPath : filepath; - }; - - const localeAwareMarkdownLoader = async (filepath: AbsoluteFilePath): Promise => { - const pathToRead = await resolveLocalePath(filepath); - const raw = await readFile(pathToRead, "utf-8"); - // Strip frontmatter (---\n...\n---) from snippet files - const fmMatch = raw.match(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/); - return fmMatch != null ? raw.slice(fmMatch[0].length) : raw; - }; - - const localeAwareFileLoader = async (filepath: AbsoluteFilePath): Promise => { - const pathToRead = await resolveLocalePath(filepath); - return readFile(pathToRead, "utf-8"); - }; - - const translatedPageEntries = await Promise.all( - Object.entries(localePages).map(async ([path, rawMarkdown]) => { - try { - const basePage = docsDefinition.pages[path as DocsV1Write.PageId]; - const absolutePathToMarkdownFile = resolve( - docsWorkspacePath, - RelativeFilePath.of(path) - ); - - // Resolve snippets (must happen before image processing). - // Uses locale-aware loader to prefer translated snippets when available. - const { markdown: markdownResolved } = await replaceReferencedMarkdown({ - markdown: rawMarkdown, - absolutePathToFernFolder: docsWorkspacePath, - absolutePathToMarkdownFile, - context, - markdownLoader: localeAwareMarkdownLoader - }); - - // Resolve references (also locale-aware) - const codeResolved = await replaceReferencedCode({ - markdown: markdownResolved, - absolutePathToFernFolder: docsWorkspacePath, - absolutePathToMarkdownFile, - context, - fileLoader: localeAwareFileLoader - }); - - // Transform @/ prefix imports to relative paths - const importsResolved = transformAtPrefixImports({ - markdown: codeResolved, - absolutePathToFernFolder: docsWorkspacePath, - absolutePathToMarkdownFile - }); - - // Strip MDX comments - let processedMarkdown = stripMdxComments(importsResolved); - - // Replace image paths using the base page's location for resolution - // (translated pages reference the same images as the default locale) - processedMarkdown = replaceImagePathsAndUrls( - processedMarkdown, - collectedFileIds, - {}, // markdownFilesToPathName not needed for translations - { - absolutePathToMarkdownFile, - absolutePathToFernFolder: docsWorkspacePath - }, - context - ); - - // Rewrite editThisPageUrl to point to the translated file - let editThisPageUrl = basePage?.editThisPageUrl; - if (editThisPageUrl != null) { - const fernPathPattern = `/fern/${path}`; - const translatedPath = `/fern/translations/${locale}/${path}`; - editThisPageUrl = editThisPageUrl.replace( - fernPathPattern, - translatedPath - ) as typeof editThisPageUrl; - } - return [ - path, - { - markdown: processedMarkdown, - rawMarkdown: processedMarkdown, - editThisPageUrl, - editThisPageLaunch: basePage?.editThisPageLaunch - } - ]; - } catch (pageError) { - context.logger.warn( - `Failed to process translated page "${path}" for locale "${locale}": ${String(pageError)}. Falling back to base page.` - ); - return undefined; - } - }) - ); - - const translatedPages = { - ...docsDefinition.pages, - ...Object.fromEntries( - translatedPageEntries.filter( - (entry): entry is NonNullable => entry != null - ) - ) - }; - let updatedRoot = applyTranslatedFrontmatterToNavTree( - docsDefinition.config.root, - // localePages is Record (path -> raw markdown) - localePages as Record, + const translatedDefinition = await buildTranslatedDocsDefinition({ + docsDefinition, + locale, + localePages, + translationNavigationOverlays, + resolver, context - ); - - // Apply navigation overlay (translated display-names, titles, etc.) - const localeNavOverlay = translationNavigationOverlays?.[locale]; - let translatedAnnouncement = docsDefinition.config.announcement; - let translatedNavbarLinks = docsDefinition.config.navbarLinks; - if (localeNavOverlay != null) { - updatedRoot = applyTranslatedNavigationOverlays(updatedRoot, localeNavOverlay); - translatedAnnouncement = - getTranslatedAnnouncement(localeNavOverlay) ?? translatedAnnouncement; - if (localeNavOverlay.navbarLinks != null) { - translatedNavbarLinks = localeNavOverlay.navbarLinks; - } - } + }); - const translatedDefinition: DocsDefinition = { - ...docsDefinition, - pages: translatedPages, - config: { - ...docsDefinition.config, - root: updatedRoot, - announcement: translatedAnnouncement, - navbarLinks: translatedNavbarLinks - } - }; const pageCount = Object.keys(localePages).length; context.logger.debug( `Sending translation for locale "${locale}" (${pageCount} page${pageCount === 1 ? "" : "s"})` diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts index 5fc1e6ac991f..9cba91350881 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts @@ -1,15 +1,18 @@ +import { type DocsDefinitionResolver } from "@fern-api/docs-resolver"; import type { APIV1Write, DocsV1Write } from "@fern-api/fdr-sdk"; import { createDocsLedgerClient, type DocsPublishGitInput, type DocsPublishInput, - type FileManifestEntry + type FileManifestEntry, + type TranslationEntry } from "@fern-api/fdr-sdk/orpc-client"; -import type { AbsoluteFilePath } from "@fern-api/fs-utils"; +import { AbsoluteFilePath } from "@fern-api/fs-utils"; import type { TaskContext } from "@fern-api/task-context"; import { createHash } from "crypto"; import { readFile } from "fs/promises"; +import { buildTranslatedDocsDefinition } from "./buildTranslatedDocsDefinition.js"; import { mapDocsConfigToLedgerConfig } from "./mapDocsConfigToLedgerConfig.js"; import { asyncPool } from "./utils/asyncPool.js"; @@ -81,7 +84,8 @@ export function buildLedgerInput({ git, apiDefinitions, fileManifest, - fileIdToPath + fileIdToPath, + locale = "en" }: { docsDefinition: DocsDefinition; organization: string; @@ -100,6 +104,8 @@ export function buildLedgerInput({ * path-based references (e.g. `ImageRef { path, width, height }`). */ fileIdToPath?: Map; + /** Locale to stamp on segments. Defaults to "en". */ + locale?: string; }): { input: DocsPublishInput; blobs: Map } { const blobs = new Map(); @@ -171,7 +177,7 @@ export function buildLedgerInput({ jsFiles: jsFilesRef, fileManifest, redirects: null, - locale: "en", + locale, git }; @@ -188,6 +194,12 @@ export interface LedgerPublishResult { /** * Publish docs via the new docs-ledger register → upload → finish flow. * + * All locales (base + translations) are built upfront before any network + * calls. If any locale fails to build, the entire publish aborts. After + * building, a single register → upload → finish pipeline runs for the + * base locale, and then finishTranslation is called for each translation + * locale (blobs are already uploaded from the combined pool). + * * This is a self-contained function that can run alongside (dual-write) * or instead of (ledger-only) the legacy finishDocsRegister path. */ @@ -206,7 +218,8 @@ export async function publishDocsViaLedger({ apiDefinitions, fileManifest, filePaths, - fileIdToPath + fileIdToPath, + resolver }: { docsDefinition: DocsDefinition; organization: string; @@ -224,7 +237,13 @@ export async function publishDocsViaLedger({ /** Hash → absolute file path for lazy on-demand reads during upload. */ filePaths?: Map; fileIdToPath?: Map; + /** Resolver instance for accessing translation pages/overlays. Optional. */ + resolver?: DocsDefinitionResolver; }): Promise { + // ── Phase 1: Build all locales upfront ────────────────────────────── + // If any locale fails to build, the entire publish aborts before any + // network calls are made. + const { input, blobs } = buildLedgerInput({ docsDefinition, organization, @@ -238,10 +257,38 @@ export async function publishDocsViaLedger({ fileIdToPath }); + const translationInputs = buildAllTranslationInputs({ + docsDefinition, + organization, + domain, + basepath, + git, + apiDefinitions, + fileManifest, + fileIdToPath, + resolver, + context + }); + + // Wait for all translation builds to complete (fail-fast). + const builtTranslations = await translationInputs; + + // Merge all translation blobs into the base blob pool so the single + // upload phase covers everything. + for (const t of builtTranslations) { + for (const [hash, buf] of t.blobs) { + blobs.set(hash, buf); + } + } + + // ── Phase 2: Single register → upload → finish ───────────────────── + // Translations are passed inline to the finish call so the server + // persists base + all locale segments in a single request. + const client = createDocsLedgerClient({ baseUrl: fdrOrigin, token, headers }); - // Step 1: Register — server computes deployment hash, returns presigned - // S3 URLs for any blobs it doesn't already have in CAS. + // Register — server computes deployment hash, returns presigned S3 + // URLs for any blobs it doesn't already have in CAS. context.logger.debug("[ledger] Registering deployment..."); const registerStart = performance.now(); const registerResult = await client.register(input); @@ -250,20 +297,49 @@ export async function publishDocsViaLedger({ `[ledger] Registered in ${registerTime.toFixed(0)}ms — hash=${registerResult.deploymentHash}, missing=${registerResult.missingContent.length} blobs` ); - // Step 2: Upload any blobs the server doesn't have yet. - // In-memory blobs (pages, config, apiManifest) are checked first; - // file blobs are read lazily from disk via filePaths. + // Upload all blobs the server doesn't have yet (base + translations + // combined). In-memory blobs are checked first; file blobs are read + // lazily from disk via filePaths. await uploadMissingBlobs(registerResult.missingContent, blobs, context, filePaths); - // Step 3: Finish — server persists the deployment. + // Build the translations array for the finish call. Each entry + // carries only the content fields + locale — orgId/domain/basepath + // are inherited from the base input on the server side. + const translations: TranslationEntry[] = builtTranslations.map((t) => ({ + locale: t.locale, + root: t.input.root, + pages: t.input.pages, + apiManifest: t.input.apiManifest, + config: t.input.config, + fileManifest: t.input.fileManifest, + jsFiles: t.input.jsFiles, + redirects: t.input.redirects, + version: t.input.version, + repo: t.input.repo, + git: t.input.git + })); + + // Finish — server persists the base deployment + all translations + // in a single call. + const finishInput: DocsPublishInput = { + ...input, + translations: translations.length > 0 ? translations : undefined + }; context.logger.debug("[ledger] Finishing deployment..."); const finishStart = performance.now(); - const finishResult = await client.finish(input); + const finishResult = await client.finish(finishInput); const finishTime = performance.now() - finishStart; context.logger.debug( `[ledger] Finished in ${finishTime.toFixed(0)}ms — deploymentId=${finishResult.deploymentId}, reused=${finishResult.reusedDeployment}` ); + // Log translation results if any were processed. + if (finishResult.translationsProcessed != null) { + for (const tp of finishResult.translationsProcessed) { + context.logger.info(`[ledger] Locale "${tp.locale}": ${tp.segmentsAdded} segment(s) added`); + } + } + return finishResult; } @@ -377,3 +453,86 @@ async function uploadBlobWithRetry( throw new Error(`[ledger] Upload exhausted retries for ${hash}`); } + +// ── Translation build helpers ────────────────────────────────────────── + +interface BuiltTranslation { + locale: string; + localePages: Record; + translatedDefinition: DocsDefinition; + input: DocsPublishInput; + blobs: Map; +} + +/** + * Build ledger inputs for every translation locale the resolver discovered. + * + * All locales are built in parallel. If ANY locale fails, the returned + * promise rejects — callers should let the error propagate to abort the + * entire publish. + */ +async function buildAllTranslationInputs({ + docsDefinition, + organization, + domain, + basepath, + git, + apiDefinitions, + fileManifest, + fileIdToPath, + resolver, + context +}: { + docsDefinition: DocsDefinition; + organization: string; + domain: string; + basepath: string | undefined; + git?: DocsPublishGitInput; + apiDefinitions: Map; + fileManifest?: Record; + fileIdToPath?: Map; + resolver?: DocsDefinitionResolver; + context: TaskContext; +}): Promise { + if (resolver == null) { + return []; + } + + const translationPages = resolver.getTranslationPages(); + const translationNavigationOverlays = resolver.getTranslationNavigationOverlays(); + + if (translationPages == null || Object.keys(translationPages).length === 0) { + return []; + } + + const localeEntries = Object.entries(translationPages); + context.logger.info(`[ledger] Building ${localeEntries.length} translation locale(s)...`); + + return Promise.all( + localeEntries.map(async ([locale, localePages]): Promise => { + const translatedDefinition = await buildTranslatedDocsDefinition({ + docsDefinition, + locale, + localePages, + translationNavigationOverlays, + resolver, + context + }); + + const { input, blobs } = buildLedgerInput({ + docsDefinition: translatedDefinition, + organization, + domain, + basepath, + previewId: undefined, + git, + apiDefinitions, + fileManifest, + fileIdToPath, + locale + }); + + return { locale, localePages, translatedDefinition, input, blobs }; + }) + ); +} diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts index 8e53607744e9..fa24b8d6be06 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts @@ -1,12 +1,15 @@ +import type { DocsDefinitionResolver } from "@fern-api/docs-resolver"; import type { APIV1Write, DocsV1Write } from "@fern-api/fdr-sdk"; import { createDocsLedgerClient, type DocsPublishGitInput, - type FileManifestEntry + type FileManifestEntry, + type TranslationEntry } from "@fern-api/fdr-sdk/orpc-client"; import type { AbsoluteFilePath } from "@fern-api/fs-utils"; import type { TaskContext } from "@fern-api/task-context"; +import { buildTranslatedDocsDefinition } from "./buildTranslatedDocsDefinition.js"; import { buildLedgerInput, uploadMissingBlobs } from "./publishDocsLedger.js"; type DocsDefinition = DocsV1Write.DocsDefinition; @@ -20,6 +23,9 @@ export interface LedgerPreviewResult { * Publish a docs preview via the dedicated ledger preview endpoint * (POST /preview/init) followed by the standard finish call. * + * All translation locales are built upfront before any network calls. + * If any locale fails to build, the entire preview publish aborts. + * * Unlike the production {@link publishDocsViaLedger}, this flow: * - Sends `LedgerPreviewRegisterInput` (no `domain` / `customDomains`). * - Receives a server-generated preview URL and domain. @@ -39,7 +45,8 @@ export async function publishDocsViaLedgerPreview({ apiDefinitions, fileManifest, filePaths, - fileIdToPath + fileIdToPath, + resolver }: { docsDefinition: DocsDefinition; organization: string; @@ -55,11 +62,15 @@ export async function publishDocsViaLedgerPreview({ /** Hash → absolute file path for lazy on-demand reads during upload. */ filePaths?: Map; fileIdToPath?: Map; + /** Resolver instance for accessing translation pages/overlays. Optional. */ + resolver?: DocsDefinitionResolver; }): Promise { - // Build the input and blob map using the shared helper. We pass a - // throwaway `domain` — it's required by buildLedgerInput's type but will - // not be sent to the preview endpoint (LedgerPreviewRegisterInput has no - // domain field). + // ── Phase 1: Build all locales upfront ────────────────────────────── + // Translated DocsDefinitions are built before any network calls so a + // single locale failure aborts the entire preview publish. The cheap + // sync buildLedgerInput for translations is deferred until after + // previewRegister because we need the server-assigned domain. + const { input, blobs } = buildLedgerInput({ docsDefinition, organization, @@ -73,10 +84,16 @@ export async function publishDocsViaLedgerPreview({ fileIdToPath }); + const builtTranslationDefs = await buildAllTranslationDefinitions({ + docsDefinition, + resolver, + context + }); + + // ── Phase 2: Single register → upload → finish ───────────────────── + const client = createDocsLedgerClient({ baseUrl: fdrOrigin, token, headers }); - // Step 1: Preview register — server picks the preview host and returns - // presigned upload URLs for missing blobs. context.logger.debug("[ledger-preview] Registering preview deployment..."); const registerStart = performance.now(); const registerResult = await client.previewRegister({ @@ -101,11 +118,50 @@ export async function publishDocsViaLedgerPreview({ `preview=${registerResult.previewUrl}, missing=${registerResult.missingContent.length} blobs` ); - // Step 2: Upload missing blobs. + // Build translation ledger inputs now that we have the server domain. + // This is a cheap sync operation (serialization only). + const translationInputs = builtTranslationDefs.map((t) => { + const { input: translationInput, blobs: translationBlobs } = buildLedgerInput({ + docsDefinition: t.translatedDefinition, + organization, + domain: registerResult.domain, + basepath: registerResult.basepath, + previewId: registerResult.previewId, + git, + apiDefinitions, + fileManifest, + fileIdToPath, + locale: t.locale + }); + return { ...t, input: translationInput, blobs: translationBlobs }; + }); + + // Merge all translation blobs into the base pool for the upload phase. + for (const t of translationInputs) { + for (const [hash, buf] of t.blobs) { + blobs.set(hash, buf); + } + } + await uploadMissingBlobs(registerResult.missingContent, blobs, context, filePaths); - // Step 3: Finish — use the server-assigned domain and previewId so the - // deployment is keyed to the preview branch. + // Build the translations array for the finish call. + const translations: TranslationEntry[] = translationInputs.map((t) => ({ + locale: t.locale, + root: t.input.root, + pages: t.input.pages, + apiManifest: t.input.apiManifest, + config: t.input.config, + fileManifest: t.input.fileManifest, + jsFiles: t.input.jsFiles, + redirects: t.input.redirects, + version: t.input.version, + repo: t.input.repo, + git: t.input.git + })); + + // Finish — server persists the preview deployment + all translations + // in a single call. context.logger.debug("[ledger-preview] Finishing preview deployment..."); const finishStart = performance.now(); const finishResult = await client.finish({ @@ -113,7 +169,8 @@ export async function publishDocsViaLedgerPreview({ domain: registerResult.domain, basepath: registerResult.basepath, customDomains: [], - previewId: registerResult.previewId + previewId: registerResult.previewId, + translations: translations.length > 0 ? translations : undefined }); const finishTime = performance.now() - finishStart; context.logger.debug( @@ -121,8 +178,72 @@ export async function publishDocsViaLedgerPreview({ `reused=${finishResult.reusedDeployment}` ); + // Log translation results if any were processed. + if (finishResult.translationsProcessed != null) { + for (const tp of finishResult.translationsProcessed) { + context.logger.info(`[ledger-preview] Locale "${tp.locale}": ${tp.segmentsAdded} segment(s) added`); + } + } + return { previewUrl: registerResult.previewUrl, deploymentId: finishResult.deploymentId }; } + +// ── Translation build helpers ────────────────────────────────────────── + +interface BuiltTranslationDef { + locale: string; + localePages: Record; + translatedDefinition: DocsDefinition; +} + +/** + * Build translated DocsDefinitions for every locale the resolver discovered. + * + * All locales are built in parallel. If ANY locale fails, the returned + * promise rejects — callers should let the error propagate to abort the + * entire publish. + * + * This only builds the DocsDefinitions (the expensive async part). The + * cheap sync `buildLedgerInput` is deferred by the caller because the + * preview flow needs the server-assigned domain first. + */ +async function buildAllTranslationDefinitions({ + docsDefinition, + resolver, + context +}: { + docsDefinition: DocsDefinition; + resolver?: DocsDefinitionResolver; + context: TaskContext; +}): Promise { + if (resolver == null) { + return []; + } + + const translationPages = resolver.getTranslationPages(); + const translationNavigationOverlays = resolver.getTranslationNavigationOverlays(); + + if (translationPages == null || Object.keys(translationPages).length === 0) { + return []; + } + + const localeEntries = Object.entries(translationPages); + context.logger.info(`[ledger-preview] Building ${localeEntries.length} translation locale(s)...`); + + return Promise.all( + localeEntries.map(async ([locale, localePages]): Promise => { + const translatedDefinition = await buildTranslatedDocsDefinition({ + docsDefinition, + locale, + localePages, + translationNavigationOverlays, + resolver, + context + }); + return { locale, localePages, translatedDefinition }; + }) + ); +} From 9436bbd0b57e9ad4019fcfec7fcc13d6e6f7abcb Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 20 May 2026 23:52:27 +0000 Subject: [PATCH 07/15] fix: include translations in docs-ledger register call Send the translations array in the register input (not just finish) so the server can process translation content refs and issue presigned S3 upload URLs for locale-specific blobs. Without this, translation pages with unique hashes (different from base) got no upload URL and would cause phantom S3 references after finish. Both register and finish now receive the same DocsPublishInput shape (base + translations), eliminating the asymmetry. Co-Authored-By: cbro --- .../fix-ledger-translation-register.yml | 6 +++ .../src/publishDocsLedger.ts | 50 ++++++++++--------- 2 files changed, 33 insertions(+), 23 deletions(-) create mode 100644 packages/cli/cli/changes/unreleased/fix-ledger-translation-register.yml diff --git a/packages/cli/cli/changes/unreleased/fix-ledger-translation-register.yml b/packages/cli/cli/changes/unreleased/fix-ledger-translation-register.yml new file mode 100644 index 000000000000..e01f50fb4029 --- /dev/null +++ b/packages/cli/cli/changes/unreleased/fix-ledger-translation-register.yml @@ -0,0 +1,6 @@ +- summary: | + Include translations in docs-ledger register call so the server issues + presigned upload URLs for translation-unique blobs. Previously only base + locale content was sent to register, causing locale-specific pages with + different hashes to get no upload URL. + type: fix diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts index 9cba91350881..68a2f3448d8c 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts @@ -282,29 +282,15 @@ export async function publishDocsViaLedger({ } // ── Phase 2: Single register → upload → finish ───────────────────── - // Translations are passed inline to the finish call so the server - // persists base + all locale segments in a single request. + // Translations are passed inline to both register and finish so the + // server can issue presigned URLs for translation-unique blobs during + // register, and persist base + all locale segments during finish. const client = createDocsLedgerClient({ baseUrl: fdrOrigin, token, headers }); - // Register — server computes deployment hash, returns presigned S3 - // URLs for any blobs it doesn't already have in CAS. - context.logger.debug("[ledger] Registering deployment..."); - const registerStart = performance.now(); - const registerResult = await client.register(input); - const registerTime = performance.now() - registerStart; - context.logger.debug( - `[ledger] Registered in ${registerTime.toFixed(0)}ms — hash=${registerResult.deploymentHash}, missing=${registerResult.missingContent.length} blobs` - ); - - // Upload all blobs the server doesn't have yet (base + translations - // combined). In-memory blobs are checked first; file blobs are read - // lazily from disk via filePaths. - await uploadMissingBlobs(registerResult.missingContent, blobs, context, filePaths); - - // Build the translations array for the finish call. Each entry - // carries only the content fields + locale — orgId/domain/basepath - // are inherited from the base input on the server side. + // Build the translations array before register so the server sees + // translation content refs and can issue presigned upload URLs for + // locale-specific blobs that don't share a hash with the base. const translations: TranslationEntry[] = builtTranslations.map((t) => ({ locale: t.locale, root: t.input.root, @@ -319,12 +305,30 @@ export async function publishDocsViaLedger({ git: t.input.git })); - // Finish — server persists the base deployment + all translations - // in a single call. - const finishInput: DocsPublishInput = { + const registerInput: DocsPublishInput = { ...input, translations: translations.length > 0 ? translations : undefined }; + + // Register — server computes deployment hash, returns presigned S3 + // URLs for any blobs it doesn't already have in CAS (base + + // translations combined). + context.logger.debug("[ledger] Registering deployment..."); + const registerStart = performance.now(); + const registerResult = await client.register(registerInput); + const registerTime = performance.now() - registerStart; + context.logger.debug( + `[ledger] Registered in ${registerTime.toFixed(0)}ms — hash=${registerResult.deploymentHash}, missing=${registerResult.missingContent.length} blobs` + ); + + // Upload all blobs the server doesn't have yet (base + translations + // combined). In-memory blobs are checked first; file blobs are read + // lazily from disk via filePaths. + await uploadMissingBlobs(registerResult.missingContent, blobs, context, filePaths); + + // Finish — server persists the base deployment + all translations + // in a single call. Uses the same input shape as register. + const finishInput: DocsPublishInput = registerInput; context.logger.debug("[ledger] Finishing deployment..."); const finishStart = performance.now(); const finishResult = await client.finish(finishInput); From 7b5425f18ffa98b8f0c8cc5454c60524f73d4ba0 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 21 May 2026 00:18:22 +0000 Subject: [PATCH 08/15] fix: unify docs-ledger publish client to use locales[] wire format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor the CLI docs-ledger client to match the server's unified locales[] wire format: - buildLedgerInput returns { localeEntry, blobs } instead of { input, blobs } - Deployment-level fields (orgId, domain, basepath, etc.) removed from buildLedgerInput — assembled separately in publishDocsViaLedger - buildAllTranslationInputs returns localeEntry per translation - Both production and preview flows send unified locales[] array - Base locale is locales[0], translations follow - Tests updated to match new return shape Co-Authored-By: cbro --- .../fix-ledger-translation-register.yml | 8 +- .../src/__test__/buildLedgerInput.test.ts | 196 ++++-------------- .../src/publishDocsLedger.ts | 107 +++------- .../src/publishDocsLedgerPreview.ts | 61 ++---- 4 files changed, 92 insertions(+), 280 deletions(-) diff --git a/packages/cli/cli/changes/unreleased/fix-ledger-translation-register.yml b/packages/cli/cli/changes/unreleased/fix-ledger-translation-register.yml index e01f50fb4029..fabf4df4af06 100644 --- a/packages/cli/cli/changes/unreleased/fix-ledger-translation-register.yml +++ b/packages/cli/cli/changes/unreleased/fix-ledger-translation-register.yml @@ -1,6 +1,6 @@ - summary: | - Include translations in docs-ledger register call so the server issues - presigned upload URLs for translation-unique blobs. Previously only base - locale content was sent to register, causing locale-specific pages with - different hashes to get no upload URL. + Refactor docs-ledger publish client to use unified locales[] wire format. + Base locale and translations are now sent as a single locales array where + all entries go through the same register/finish pipeline, eliminating the + separate translation bolt-on codepath. type: fix diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/buildLedgerInput.test.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/buildLedgerInput.test.ts index aee5d480edbd..7ae8c041656d 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/buildLedgerInput.test.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/buildLedgerInput.test.ts @@ -36,17 +36,13 @@ function makeDocsDefinition({ describe("buildLedgerInput", () => { it("hashes page content and creates blob refs", () => { const markdown = "# Hello World"; - const { input, blobs } = buildLedgerInput({ + const { localeEntry, blobs } = buildLedgerInput({ docsDefinition: makeDocsDefinition({ pages: { "page-1": { markdown } } }), - organization: "acme", - domain: "docs.acme.com", - basepath: undefined, - previewId: undefined, apiDefinitions: new Map() }); const expectedHash = sha256(markdown); - expect(input.pages["page-1"]).toEqual({ + expect(localeEntry.pages["page-1"]).toEqual({ hash: expectedHash, contentType: "text/markdown", contentLength: Buffer.byteLength(markdown, "utf-8") @@ -56,36 +52,28 @@ describe("buildLedgerInput", () => { it("skips null/undefined pages", () => { const pages = { "page-1": null } as unknown as Record; - const { input } = buildLedgerInput({ + const { localeEntry } = buildLedgerInput({ docsDefinition: makeDocsDefinition({ pages }), - organization: "acme", - domain: "docs.acme.com", - basepath: undefined, - previewId: undefined, apiDefinitions: new Map() }); - expect(Object.keys(input.pages)).toHaveLength(0); + expect(Object.keys(localeEntry.pages)).toHaveLength(0); }); it("maps a minimal DocsConfig to a LedgerConfig shape (mostly empty fields)", () => { - const { input } = buildLedgerInput({ + const { localeEntry } = buildLedgerInput({ docsDefinition: makeDocsDefinition(), - organization: "acme", - domain: "docs.acme.com", - basepath: undefined, - previewId: undefined, apiDefinitions: new Map() }); // With only `root` set on the source DocsConfig every LedgerConfig - // field is `undefined` — but `input.config` itself is the populated + // field is `undefined` — but `localeEntry.config` itself is the populated // ledger object (not `undefined`), unlike the prior workaround. - expect(input.config).toBeDefined(); - expect(input.config?.title).toBeUndefined(); - expect(input.config?.colorsV3).toBeUndefined(); - expect(input.config?.metadata).toBeUndefined(); - expect(input.config?.redirects).toBeUndefined(); + expect(localeEntry.config).toBeDefined(); + expect(localeEntry.config?.title).toBeUndefined(); + expect(localeEntry.config?.colorsV3).toBeUndefined(); + expect(localeEntry.config?.metadata).toBeUndefined(); + expect(localeEntry.config?.redirects).toBeUndefined(); }); it("translates a DocsConfig logo FileId into a LedgerConfig ImageRef using fileManifest dimensions", () => { @@ -114,19 +102,15 @@ describe("buildLedgerInput", () => { } }; - const { input } = buildLedgerInput({ + const { localeEntry } = buildLedgerInput({ docsDefinition, - organization: "acme", - domain: "docs.acme.com", - basepath: undefined, - previewId: undefined, apiDefinitions: new Map(), fileManifest, // Identity map: fileId === sanitizedPath in the current FDR flow. fileIdToPath: new Map([["assets/logo.png", "assets/logo.png"]]) }); - expect(input.config?.colorsV3).toEqual({ + expect(localeEntry.config?.colorsV3).toEqual({ type: "dark", accentPrimary: { r: 1, g: 2, b: 3 }, logo: { path: "assets/logo.png", width: 320, height: 160 } @@ -156,19 +140,15 @@ describe("buildLedgerInput", () => { } }; - const { input } = buildLedgerInput({ + const { localeEntry } = buildLedgerInput({ docsDefinition, - organization: "acme", - domain: "docs.acme.com", - basepath: undefined, - previewId: undefined, apiDefinitions: new Map(), fileManifest, fileIdToPath: new Map([["assets/unmeasured.svg", "assets/unmeasured.svg"]]) }); // Logo absent rather than emitted with placeholder dimensions. - expect(input.config?.colorsV3).toEqual({ + expect(localeEntry.config?.colorsV3).toEqual({ type: "light", accentPrimary: { r: 4, g: 5, b: 6 }, logo: undefined, @@ -181,48 +161,22 @@ describe("buildLedgerInput", () => { }); }); - it("passes through org, domain, basepath, previewId", () => { - const { input } = buildLedgerInput({ + it("defaults locale to en", () => { + const { localeEntry } = buildLedgerInput({ docsDefinition: makeDocsDefinition(), - organization: "acme", - domain: "docs.acme.com", - basepath: "/v2", - previewId: "pr-42", apiDefinitions: new Map() }); - expect(input.orgId).toBe("acme"); - expect(input.domain).toBe("docs.acme.com"); - expect(input.basepath).toBe("/v2"); - expect(input.previewId).toBe("pr-42"); - }); - - it("defaults basepath to empty string, previewId to null, and locale to en", () => { - const { input } = buildLedgerInput({ - docsDefinition: makeDocsDefinition(), - organization: "acme", - domain: "docs.acme.com", - basepath: undefined, - previewId: undefined, - apiDefinitions: new Map() - }); - - expect(input.basepath).toBe(""); - expect(input.previewId).toBeNull(); - expect(input.locale).toBe("en"); + expect(localeEntry.locale).toBe("en"); }); it("uses config.root for the root field", () => { - const { input } = buildLedgerInput({ + const { localeEntry } = buildLedgerInput({ docsDefinition: makeDocsDefinition({ root: MINIMAL_ROOT }), - organization: "acme", - domain: "docs.acme.com", - basepath: undefined, - previewId: undefined, apiDefinitions: new Map() }); - expect(input.root).toEqual(MINIMAL_ROOT); + expect(localeEntry.root).toEqual(MINIMAL_ROOT); }); it("deduplicates pages with identical content", () => { @@ -234,10 +188,6 @@ describe("buildLedgerInput", () => { "page-b": { markdown } } }), - organization: "acme", - domain: "docs.acme.com", - basepath: undefined, - previewId: undefined, apiDefinitions: new Map() }); @@ -253,16 +203,12 @@ describe("buildLedgerInput", () => { }); it("sets apiManifest to null when apiDefinitions is empty", () => { - const { input } = buildLedgerInput({ + const { localeEntry } = buildLedgerInput({ docsDefinition: makeDocsDefinition(), - organization: "acme", - domain: "docs.acme.com", - basepath: undefined, - previewId: undefined, apiDefinitions: new Map() }); - expect(input.apiManifest).toBeNull(); + expect(localeEntry.apiManifest).toBeNull(); }); it("apiManifest blob hash is stable across Map insertion order (determinism guard)", () => { @@ -296,20 +242,12 @@ describe("buildLedgerInput", () => { reverse.set("api-def-b", minimalApiDefinition); reverse.set("api-def-a", minimalApiDefinition); - const { input: inForward } = buildLedgerInput({ + const { localeEntry: inForward } = buildLedgerInput({ docsDefinition: makeDocsDefinition(), - organization: "acme", - domain: "docs.acme.com", - basepath: undefined, - previewId: undefined, apiDefinitions: forward }); - const { input: inReverse } = buildLedgerInput({ + const { localeEntry: inReverse } = buildLedgerInput({ docsDefinition: makeDocsDefinition(), - organization: "acme", - domain: "docs.acme.com", - basepath: undefined, - previewId: undefined, apiDefinitions: reverse }); @@ -336,21 +274,17 @@ describe("buildLedgerInput", () => { const apiDefinitions = new Map(); apiDefinitions.set("api-def-1", minimalApiDefinition); - const { input, blobs } = buildLedgerInput({ + const { localeEntry, blobs } = buildLedgerInput({ docsDefinition: makeDocsDefinition(), - organization: "acme", - domain: "docs.acme.com", - basepath: undefined, - previewId: undefined, apiDefinitions }); - expect(input.apiManifest).not.toBeNull(); - expect(input.apiManifest?.contentType).toBe("application/json"); - expect(input.apiManifest?.contentLength).toBeGreaterThan(0); + expect(localeEntry.apiManifest).not.toBeNull(); + expect(localeEntry.apiManifest?.contentType).toBe("application/json"); + expect(localeEntry.apiManifest?.contentLength).toBeGreaterThan(0); // The blob should exist in the blob map. - const manifestBuf = blobs.get(input.apiManifest?.hash ?? ""); + const manifestBuf = blobs.get(localeEntry.apiManifest?.hash ?? ""); expect(manifestBuf).toBeDefined(); // Round-trip: the blob content should deserialize to match the input map. @@ -386,18 +320,14 @@ describe("buildLedgerInput", () => { "docs/notes.txt": docEntry }; - const { input, blobs } = buildLedgerInput({ + const { localeEntry, blobs } = buildLedgerInput({ docsDefinition: makeDocsDefinition(), - organization: "acme", - domain: "docs.acme.com", - basepath: undefined, - previewId: undefined, apiDefinitions: new Map(), fileManifest }); // fileManifest round-trips unchanged. - expect(input.fileManifest).toEqual(fileManifest); + expect(localeEntry.fileManifest).toEqual(fileManifest); // File blobs are NOT in the blob map — they are loaded lazily during // the upload step via filePaths (hash → absolute path). @@ -406,16 +336,12 @@ describe("buildLedgerInput", () => { }); it("legacy behaviour: omitting fileManifest still works", () => { - const { input, blobs } = buildLedgerInput({ + const { localeEntry, blobs } = buildLedgerInput({ docsDefinition: makeDocsDefinition({ pages: { "page-1": { markdown: "# hi" } } }), - organization: "acme", - domain: "docs.acme.com", - basepath: undefined, - previewId: undefined, apiDefinitions: new Map() }); - expect(input.fileManifest).toBeUndefined(); + expect(localeEntry.fileManifest).toBeUndefined(); // Blob map still contains the page + config blobs. expect(blobs.size).toBeGreaterThan(0); }); @@ -428,30 +354,22 @@ describe("buildLedgerInput", () => { branch: "main", commitSha: "abc123" }; - const { input } = buildLedgerInput({ + const { localeEntry } = buildLedgerInput({ docsDefinition: makeDocsDefinition(), - organization: "acme", - domain: "docs.acme.com", - basepath: undefined, - previewId: undefined, git, apiDefinitions: new Map() }); - expect(input.git).toEqual(git); + expect(localeEntry.git).toEqual(git); }); it("omits git from DocsPublishInput when not provided", () => { - const { input } = buildLedgerInput({ + const { localeEntry } = buildLedgerInput({ docsDefinition: makeDocsDefinition(), - organization: "acme", - domain: "docs.acme.com", - basepath: undefined, - previewId: undefined, apiDefinitions: new Map() }); - expect(input.git).toBeUndefined(); + expect(localeEntry.git).toBeUndefined(); }); it("forwards git without commitSha when commitSha is omitted", () => { @@ -459,47 +377,13 @@ describe("buildLedgerInput", () => { repoUrl: "https://gitlab.com/acme/docs", branch: "feature/x" }; - const { input } = buildLedgerInput({ + const { localeEntry } = buildLedgerInput({ docsDefinition: makeDocsDefinition(), - organization: "acme", - domain: "docs.acme.com", - basepath: undefined, - previewId: undefined, git, apiDefinitions: new Map() }); - expect(input.git).toEqual(git); - expect(input.git?.commitSha).toBeUndefined(); - }); - - // ── ADR 0009: customDomains ─────────────────────────────────────── - - it("forwards customDomains into DocsPublishInput", () => { - const customDomains = ["docs.acme.com", "alt.acme.com/v2"]; - const { input } = buildLedgerInput({ - docsDefinition: makeDocsDefinition(), - organization: "acme", - domain: "acme.docs.buildwithfern.com", - basepath: undefined, - previewId: undefined, - customDomains, - apiDefinitions: new Map() - }); - - expect(input.customDomains).toEqual(customDomains); - }); - - it("defaults customDomains to [] when omitted", () => { - const { input } = buildLedgerInput({ - docsDefinition: makeDocsDefinition(), - organization: "acme", - domain: "docs.acme.com", - basepath: undefined, - previewId: undefined, - apiDefinitions: new Map() - }); - - expect(input.customDomains).toEqual([]); + expect(localeEntry.git).toEqual(git); + expect(localeEntry.git?.commitSha).toBeUndefined(); }); }); diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts index 68a2f3448d8c..62492912de09 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts @@ -5,7 +5,7 @@ import { type DocsPublishGitInput, type DocsPublishInput, type FileManifestEntry, - type TranslationEntry + type LocaleEntry } from "@fern-api/fdr-sdk/orpc-client"; import { AbsoluteFilePath } from "@fern-api/fs-utils"; import type { TaskContext } from "@fern-api/task-context"; @@ -76,11 +76,6 @@ function jsonBlobRef(value: unknown): { ref: BlobRef; hash: string; buf: Buffer */ export function buildLedgerInput({ docsDefinition, - organization, - domain, - basepath, - previewId, - customDomains, git, apiDefinitions, fileManifest, @@ -88,11 +83,6 @@ export function buildLedgerInput({ locale = "en" }: { docsDefinition: DocsDefinition; - organization: string; - domain: string; - basepath: string | undefined; - previewId: string | undefined; - customDomains?: string[]; git?: DocsPublishGitInput; apiDefinitions: Map; fileManifest?: Record; @@ -106,11 +96,11 @@ export function buildLedgerInput({ fileIdToPath?: Map; /** Locale to stamp on segments. Defaults to "en". */ locale?: string; -}): { input: DocsPublishInput; blobs: Map } { +}): { localeEntry: LocaleEntry; blobs: Map } { const blobs = new Map(); // Pages: hash each page's markdown content. - const pages: DocsPublishInput["pages"] = {}; + const pages: LocaleEntry["pages"] = {}; for (const [pageId, page] of Object.entries(docsDefinition.pages)) { if (page == null) { continue; @@ -164,12 +154,7 @@ export function buildLedgerInput({ jsFilesRef = jsFilesBlob.ref; } - const input: DocsPublishInput = { - orgId: organization, - domain, - basepath: basepath ?? "", - customDomains: customDomains ?? [], - previewId: previewId ?? null, + const localeEntry: LocaleEntry = { root: docsDefinition.config.root ?? docsDefinition.config.navigation, pages, config: ledgerConfig, @@ -181,7 +166,7 @@ export function buildLedgerInput({ git }; - return { input, blobs }; + return { localeEntry, blobs }; } export interface LedgerPublishResult { @@ -244,24 +229,16 @@ export async function publishDocsViaLedger({ // If any locale fails to build, the entire publish aborts before any // network calls are made. - const { input, blobs } = buildLedgerInput({ + const { localeEntry: baseLocale, blobs } = buildLedgerInput({ docsDefinition, - organization, - domain, - basepath, - previewId, - customDomains, git, apiDefinitions, fileManifest, fileIdToPath }); - const translationInputs = buildAllTranslationInputs({ + const builtTranslations = await buildAllTranslationInputs({ docsDefinition, - organization, - domain, - basepath, git, apiDefinitions, fileManifest, @@ -270,9 +247,6 @@ export async function publishDocsViaLedger({ context }); - // Wait for all translation builds to complete (fail-fast). - const builtTranslations = await translationInputs; - // Merge all translation blobs into the base blob pool so the single // upload phase covers everything. for (const t of builtTranslations) { @@ -282,56 +256,41 @@ export async function publishDocsViaLedger({ } // ── Phase 2: Single register → upload → finish ───────────────────── - // Translations are passed inline to both register and finish so the - // server can issue presigned URLs for translation-unique blobs during - // register, and persist base + all locale segments during finish. + // Build a unified locales[] array where the base locale is the first + // entry and translations follow. The server processes all locales + // through the same pipeline. const client = createDocsLedgerClient({ baseUrl: fdrOrigin, token, headers }); - // Build the translations array before register so the server sees - // translation content refs and can issue presigned upload URLs for - // locale-specific blobs that don't share a hash with the base. - const translations: TranslationEntry[] = builtTranslations.map((t) => ({ - locale: t.locale, - root: t.input.root, - pages: t.input.pages, - apiManifest: t.input.apiManifest, - config: t.input.config, - fileManifest: t.input.fileManifest, - jsFiles: t.input.jsFiles, - redirects: t.input.redirects, - version: t.input.version, - repo: t.input.repo, - git: t.input.git - })); - - const registerInput: DocsPublishInput = { - ...input, - translations: translations.length > 0 ? translations : undefined + const locales: LocaleEntry[] = [baseLocale, ...builtTranslations.map((t) => t.localeEntry)]; + + const publishInput: DocsPublishInput = { + orgId: organization, + domain, + basepath: basepath ?? "", + customDomains: customDomains ?? [], + previewId: previewId ?? null, + locales }; // Register — server computes deployment hash, returns presigned S3 - // URLs for any blobs it doesn't already have in CAS (base + - // translations combined). + // URLs for any blobs it doesn't already have in CAS (all locales). context.logger.debug("[ledger] Registering deployment..."); const registerStart = performance.now(); - const registerResult = await client.register(registerInput); + const registerResult = await client.register(publishInput); const registerTime = performance.now() - registerStart; context.logger.debug( `[ledger] Registered in ${registerTime.toFixed(0)}ms — hash=${registerResult.deploymentHash}, missing=${registerResult.missingContent.length} blobs` ); - // Upload all blobs the server doesn't have yet (base + translations - // combined). In-memory blobs are checked first; file blobs are read - // lazily from disk via filePaths. + // Upload all blobs the server doesn't have yet (all locales combined). await uploadMissingBlobs(registerResult.missingContent, blobs, context, filePaths); - // Finish — server persists the base deployment + all translations - // in a single call. Uses the same input shape as register. - const finishInput: DocsPublishInput = registerInput; + // Finish — server persists the deployment + all locale segments in + // a single atomic transaction. Same input shape as register. context.logger.debug("[ledger] Finishing deployment..."); const finishStart = performance.now(); - const finishResult = await client.finish(finishInput); + const finishResult = await client.finish(publishInput); const finishTime = performance.now() - finishStart; context.logger.debug( `[ledger] Finished in ${finishTime.toFixed(0)}ms — deploymentId=${finishResult.deploymentId}, reused=${finishResult.reusedDeployment}` @@ -464,7 +423,7 @@ interface BuiltTranslation { locale: string; localePages: Record; translatedDefinition: DocsDefinition; - input: DocsPublishInput; + localeEntry: LocaleEntry; blobs: Map; } @@ -477,9 +436,6 @@ interface BuiltTranslation { */ async function buildAllTranslationInputs({ docsDefinition, - organization, - domain, - basepath, git, apiDefinitions, fileManifest, @@ -488,9 +444,6 @@ async function buildAllTranslationInputs({ context }: { docsDefinition: DocsDefinition; - organization: string; - domain: string; - basepath: string | undefined; git?: DocsPublishGitInput; apiDefinitions: Map; fileManifest?: Record; @@ -523,12 +476,8 @@ async function buildAllTranslationInputs({ context }); - const { input, blobs } = buildLedgerInput({ + const { localeEntry, blobs } = buildLedgerInput({ docsDefinition: translatedDefinition, - organization, - domain, - basepath, - previewId: undefined, git, apiDefinitions, fileManifest, @@ -536,7 +485,7 @@ async function buildAllTranslationInputs({ locale }); - return { locale, localePages, translatedDefinition, input, blobs }; + return { locale, localePages, translatedDefinition, localeEntry, blobs }; }) ); } diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts index fa24b8d6be06..dac02e5fbf27 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts @@ -4,7 +4,7 @@ import { createDocsLedgerClient, type DocsPublishGitInput, type FileManifestEntry, - type TranslationEntry + type LocaleEntry } from "@fern-api/fdr-sdk/orpc-client"; import type { AbsoluteFilePath } from "@fern-api/fs-utils"; import type { TaskContext } from "@fern-api/task-context"; @@ -71,13 +71,8 @@ export async function publishDocsViaLedgerPreview({ // sync buildLedgerInput for translations is deferred until after // previewRegister because we need the server-assigned domain. - const { input, blobs } = buildLedgerInput({ + const { localeEntry: baseLocale, blobs } = buildLedgerInput({ docsDefinition, - organization, - domain: "", - basepath: basePath, - previewId, - customDomains: [], git, apiDefinitions, fileManifest, @@ -100,16 +95,16 @@ export async function publishDocsViaLedgerPreview({ orgId: organization, previewId: previewId ?? null, basePath: basePath ?? "", - root: input.root, - pages: input.pages, - apiManifest: input.apiManifest, - config: input.config, - fileManifest: input.fileManifest, - jsFiles: input.jsFiles, - redirects: input.redirects, - locale: input.locale, - version: input.version, - repo: input.repo, + root: baseLocale.root, + pages: baseLocale.pages, + apiManifest: baseLocale.apiManifest, + config: baseLocale.config, + fileManifest: baseLocale.fileManifest, + jsFiles: baseLocale.jsFiles, + redirects: baseLocale.redirects, + locale: baseLocale.locale, + version: baseLocale.version, + repo: baseLocale.repo, git }); const registerTime = performance.now() - registerStart; @@ -121,19 +116,15 @@ export async function publishDocsViaLedgerPreview({ // Build translation ledger inputs now that we have the server domain. // This is a cheap sync operation (serialization only). const translationInputs = builtTranslationDefs.map((t) => { - const { input: translationInput, blobs: translationBlobs } = buildLedgerInput({ + const { localeEntry, blobs: translationBlobs } = buildLedgerInput({ docsDefinition: t.translatedDefinition, - organization, - domain: registerResult.domain, - basepath: registerResult.basepath, - previewId: registerResult.previewId, git, apiDefinitions, fileManifest, fileIdToPath, locale: t.locale }); - return { ...t, input: translationInput, blobs: translationBlobs }; + return { ...t, localeEntry, blobs: translationBlobs }; }); // Merge all translation blobs into the base pool for the upload phase. @@ -145,32 +136,20 @@ export async function publishDocsViaLedgerPreview({ await uploadMissingBlobs(registerResult.missingContent, blobs, context, filePaths); - // Build the translations array for the finish call. - const translations: TranslationEntry[] = translationInputs.map((t) => ({ - locale: t.locale, - root: t.input.root, - pages: t.input.pages, - apiManifest: t.input.apiManifest, - config: t.input.config, - fileManifest: t.input.fileManifest, - jsFiles: t.input.jsFiles, - redirects: t.input.redirects, - version: t.input.version, - repo: t.input.repo, - git: t.input.git - })); - - // Finish — server persists the preview deployment + all translations + // Build the unified locales array for the finish call. + const locales: LocaleEntry[] = [baseLocale, ...translationInputs.map((t) => t.localeEntry)]; + + // Finish — server persists the preview deployment + all locale segments // in a single call. context.logger.debug("[ledger-preview] Finishing preview deployment..."); const finishStart = performance.now(); const finishResult = await client.finish({ - ...input, + orgId: organization, domain: registerResult.domain, basepath: registerResult.basepath, customDomains: [], previewId: registerResult.previewId, - translations: translations.length > 0 ? translations : undefined + locales }); const finishTime = performance.now() - finishStart; context.logger.debug( From ba48792901a25b00e09800e236b6a4853952a443 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 21 May 2026 00:51:04 +0000 Subject: [PATCH 09/15] fix: send defaultLocale in docs-ledger publish input Co-Authored-By: cbro --- .../remote-workspace-runner/src/publishDocsLedger.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts index 62492912de09..c5f59b4bb8e3 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts @@ -270,6 +270,7 @@ export async function publishDocsViaLedger({ basepath: basepath ?? "", customDomains: customDomains ?? [], previewId: previewId ?? null, + defaultLocale: baseLocale.locale, locales }; From 02cd8a3f8f085d0539f2c50ae55bd34f0ca20212 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 21 May 2026 03:20:34 +0000 Subject: [PATCH 10/15] feat(fdr): add gzip compression to docs-ledger publish requests Co-Authored-By: cbro --- .../feat-ledger-gzip-compression.yml | 4 ++ .../src/compressedLedgerFetch.ts | 54 ++++++++++++++++ .../src/publishDocsLedger.ts | 27 ++++++-- .../src/publishDocsLedgerPreview.ts | 62 ++++++++++++------- 4 files changed, 118 insertions(+), 29 deletions(-) create mode 100644 packages/cli/cli/changes/unreleased/feat-ledger-gzip-compression.yml create mode 100644 packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedLedgerFetch.ts diff --git a/packages/cli/cli/changes/unreleased/feat-ledger-gzip-compression.yml b/packages/cli/cli/changes/unreleased/feat-ledger-gzip-compression.yml new file mode 100644 index 000000000000..b1ff7b98f9a4 --- /dev/null +++ b/packages/cli/cli/changes/unreleased/feat-ledger-gzip-compression.yml @@ -0,0 +1,4 @@ +- summary: | + Gzip-compress docs-ledger register and finish request bodies to reduce + transfer time for large multi-locale deployments. + type: feat diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedLedgerFetch.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedLedgerFetch.ts new file mode 100644 index 000000000000..59c49f8e4a9d --- /dev/null +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedLedgerFetch.ts @@ -0,0 +1,54 @@ +import type { TaskContext } from "@fern-api/task-context"; +import { promisify } from "util"; +import { gzip } from "zlib"; + +const gzipAsync = promisify(gzip); + +/** + * POST a JSON body to a docs-ledger endpoint with gzip compression. + * + * The FDR server registers `@fastify/compress` which transparently + * decompresses incoming `Content-Encoding: gzip` request bodies. Compressing + * the publish payloads on the wire can significantly reduce transfer time for + * large multi-locale deployments where the JSON body contains many page refs, + * nav trees, and config objects. + */ +export async function compressedLedgerPost({ + url, + body, + token, + headers, + context +}: { + url: string; + body: unknown; + token: string; + headers?: Record; + context: TaskContext; +}): Promise { + const jsonBytes = Buffer.from(JSON.stringify(body), "utf-8"); + const compressed = await gzipAsync(jsonBytes); + + const ratio = ((1 - compressed.length / jsonBytes.byteLength) * 100).toFixed(1); + context.logger.debug( + `[ledger] Compressed request from ${jsonBytes.byteLength} to ${compressed.length} bytes (${ratio}% reduction)` + ); + + const response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Encoding": "gzip", + Authorization: `Bearer ${token}`, + ...headers + }, + body: compressed + }); + + if (!response.ok) { + const text = await response.text(); + throw new Error(`[ledger] POST ${url} failed: ${response.status} ${text}`); + } + + return (await response.json()) as T; +} diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts index c5f59b4bb8e3..4a3d61ad4ec7 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts @@ -1,17 +1,20 @@ import { type DocsDefinitionResolver } from "@fern-api/docs-resolver"; import type { APIV1Write, DocsV1Write } from "@fern-api/fdr-sdk"; import { - createDocsLedgerClient, type DocsPublishGitInput, type DocsPublishInput, type FileManifestEntry, - type LocaleEntry + type FinishResponse, + type LocaleEntry, + type RegisterResponse } from "@fern-api/fdr-sdk/orpc-client"; import { AbsoluteFilePath } from "@fern-api/fs-utils"; import type { TaskContext } from "@fern-api/task-context"; import { createHash } from "crypto"; import { readFile } from "fs/promises"; +import { compressedLedgerPost } from "./compressedLedgerFetch.js"; + import { buildTranslatedDocsDefinition } from "./buildTranslatedDocsDefinition.js"; import { mapDocsConfigToLedgerConfig } from "./mapDocsConfigToLedgerConfig.js"; import { asyncPool } from "./utils/asyncPool.js"; @@ -260,7 +263,7 @@ export async function publishDocsViaLedger({ // entry and translations follow. The server processes all locales // through the same pipeline. - const client = createDocsLedgerClient({ baseUrl: fdrOrigin, token, headers }); + const ledgerBaseUrl = `${fdrOrigin.replace(/\/+$/, "")}/docs-ledger`; const locales: LocaleEntry[] = [baseLocale, ...builtTranslations.map((t) => t.localeEntry)]; @@ -276,9 +279,17 @@ export async function publishDocsViaLedger({ // Register — server computes deployment hash, returns presigned S3 // URLs for any blobs it doesn't already have in CAS (all locales). + // Request body is gzip-compressed to reduce transfer time for large + // multi-locale deployments. context.logger.debug("[ledger] Registering deployment..."); const registerStart = performance.now(); - const registerResult = await client.register(publishInput); + const registerResult = await compressedLedgerPost({ + url: `${ledgerBaseUrl}/register`, + body: publishInput, + token, + headers, + context + }); const registerTime = performance.now() - registerStart; context.logger.debug( `[ledger] Registered in ${registerTime.toFixed(0)}ms — hash=${registerResult.deploymentHash}, missing=${registerResult.missingContent.length} blobs` @@ -291,7 +302,13 @@ export async function publishDocsViaLedger({ // a single atomic transaction. Same input shape as register. context.logger.debug("[ledger] Finishing deployment..."); const finishStart = performance.now(); - const finishResult = await client.finish(publishInput); + const finishResult = await compressedLedgerPost({ + url: `${ledgerBaseUrl}/register/finish`, + body: publishInput, + token, + headers, + context + }); const finishTime = performance.now() - finishStart; context.logger.debug( `[ledger] Finished in ${finishTime.toFixed(0)}ms — deploymentId=${finishResult.deploymentId}, reused=${finishResult.reusedDeployment}` diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts index dac02e5fbf27..e1534d59dee8 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts @@ -1,15 +1,17 @@ import type { DocsDefinitionResolver } from "@fern-api/docs-resolver"; import type { APIV1Write, DocsV1Write } from "@fern-api/fdr-sdk"; import { - createDocsLedgerClient, type DocsPublishGitInput, type FileManifestEntry, + type FinishResponse, + type LedgerPreviewRegisterResponse, type LocaleEntry } from "@fern-api/fdr-sdk/orpc-client"; import type { AbsoluteFilePath } from "@fern-api/fs-utils"; import type { TaskContext } from "@fern-api/task-context"; import { buildTranslatedDocsDefinition } from "./buildTranslatedDocsDefinition.js"; +import { compressedLedgerPost } from "./compressedLedgerFetch.js"; import { buildLedgerInput, uploadMissingBlobs } from "./publishDocsLedger.js"; type DocsDefinition = DocsV1Write.DocsDefinition; @@ -87,25 +89,31 @@ export async function publishDocsViaLedgerPreview({ // ── Phase 2: Single register → upload → finish ───────────────────── - const client = createDocsLedgerClient({ baseUrl: fdrOrigin, token, headers }); + const ledgerBaseUrl = `${fdrOrigin.replace(/\/+$/, "")}/docs-ledger`; context.logger.debug("[ledger-preview] Registering preview deployment..."); const registerStart = performance.now(); - const registerResult = await client.previewRegister({ - orgId: organization, - previewId: previewId ?? null, - basePath: basePath ?? "", - root: baseLocale.root, - pages: baseLocale.pages, - apiManifest: baseLocale.apiManifest, - config: baseLocale.config, - fileManifest: baseLocale.fileManifest, - jsFiles: baseLocale.jsFiles, - redirects: baseLocale.redirects, - locale: baseLocale.locale, - version: baseLocale.version, - repo: baseLocale.repo, - git + const registerResult = await compressedLedgerPost({ + url: `${ledgerBaseUrl}/preview/init`, + body: { + orgId: organization, + previewId: previewId ?? null, + basePath: basePath ?? "", + root: baseLocale.root, + pages: baseLocale.pages, + apiManifest: baseLocale.apiManifest, + config: baseLocale.config, + fileManifest: baseLocale.fileManifest, + jsFiles: baseLocale.jsFiles, + redirects: baseLocale.redirects, + locale: baseLocale.locale, + version: baseLocale.version, + repo: baseLocale.repo, + git + }, + token, + headers, + context }); const registerTime = performance.now() - registerStart; context.logger.debug( @@ -143,13 +151,19 @@ export async function publishDocsViaLedgerPreview({ // in a single call. context.logger.debug("[ledger-preview] Finishing preview deployment..."); const finishStart = performance.now(); - const finishResult = await client.finish({ - orgId: organization, - domain: registerResult.domain, - basepath: registerResult.basepath, - customDomains: [], - previewId: registerResult.previewId, - locales + const finishResult = await compressedLedgerPost({ + url: `${ledgerBaseUrl}/register/finish`, + body: { + orgId: organization, + domain: registerResult.domain, + basepath: registerResult.basepath, + customDomains: [], + previewId: registerResult.previewId, + locales + }, + token, + headers, + context }); const finishTime = performance.now() - finishStart; context.logger.debug( From 0910dfe72a9cf2902887b98a9db2a4d037e3db98 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 21 May 2026 03:23:07 +0000 Subject: [PATCH 11/15] fix: sort imports to satisfy biome lint Co-Authored-By: cbro --- .../remote-workspace-runner/src/publishDocsLedger.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts index 4a3d61ad4ec7..c33a3912f8f9 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts @@ -13,9 +13,8 @@ import type { TaskContext } from "@fern-api/task-context"; import { createHash } from "crypto"; import { readFile } from "fs/promises"; -import { compressedLedgerPost } from "./compressedLedgerFetch.js"; - import { buildTranslatedDocsDefinition } from "./buildTranslatedDocsDefinition.js"; +import { compressedLedgerPost } from "./compressedLedgerFetch.js"; import { mapDocsConfigToLedgerConfig } from "./mapDocsConfigToLedgerConfig.js"; import { asyncPool } from "./utils/asyncPool.js"; From b4d212029a0bf8288b65b711e2501848680596ad Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 21 May 2026 03:40:09 +0000 Subject: [PATCH 12/15] refactor: use oRPC client with custom gzip fetch instead of raw fetch calls Co-Authored-By: cbro --- .../src/compressedLedgerFetch.ts | 98 +++++++++++-------- .../src/publishDocsLedger.ts | 27 ++--- .../src/publishDocsLedgerPreview.ts | 63 +++++------- 3 files changed, 86 insertions(+), 102 deletions(-) diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedLedgerFetch.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedLedgerFetch.ts index 59c49f8e4a9d..17cfc089f4c1 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedLedgerFetch.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedLedgerFetch.ts @@ -5,50 +5,62 @@ import { gzip } from "zlib"; const gzipAsync = promisify(gzip); /** - * POST a JSON body to a docs-ledger endpoint with gzip compression. + * Returns a `fetch` function that gzip-compresses JSON request bodies before + * sending. Designed to be passed as the `fetch` option to + * {@link createDocsLedgerClient} so all ledger requests are transparently + * compressed while retaining full oRPC SDK typing. * * The FDR server registers `@fastify/compress` which transparently - * decompresses incoming `Content-Encoding: gzip` request bodies. Compressing - * the publish payloads on the wire can significantly reduce transfer time for - * large multi-locale deployments where the JSON body contains many page refs, - * nav trees, and config objects. + * decompresses incoming `Content-Encoding: gzip` request bodies. */ -export async function compressedLedgerPost({ - url, - body, - token, - headers, - context -}: { - url: string; - body: unknown; - token: string; - headers?: Record; - context: TaskContext; -}): Promise { - const jsonBytes = Buffer.from(JSON.stringify(body), "utf-8"); - const compressed = await gzipAsync(jsonBytes); - - const ratio = ((1 - compressed.length / jsonBytes.byteLength) * 100).toFixed(1); - context.logger.debug( - `[ledger] Compressed request from ${jsonBytes.byteLength} to ${compressed.length} bytes (${ratio}% reduction)` - ); - - const response = await fetch(url, { - method: "POST", - headers: { - "Content-Type": "application/json", - "Content-Encoding": "gzip", - Authorization: `Bearer ${token}`, - ...headers - }, - body: compressed - }); - - if (!response.ok) { - const text = await response.text(); - throw new Error(`[ledger] POST ${url} failed: ${response.status} ${text}`); - } - - return (await response.json()) as T; +export function createGzipFetch(context: TaskContext): typeof globalThis.fetch { + return async (input: RequestInfo | URL, init?: RequestInit): Promise => { + // The oRPC `LinkFetchClient` calls fetch with a fully-formed Request + // object as the first argument. The body is baked into the Request. + if (input instanceof Request && init?.body == null) { + const body = await input.clone().text(); + if (body.length > 0) { + const jsonBytes = Buffer.from(body, "utf-8"); + const compressed = await gzipAsync(jsonBytes); + + const ratio = ((1 - compressed.length / jsonBytes.byteLength) * 100).toFixed(1); + context.logger.debug( + `[ledger] Compressed request from ${jsonBytes.byteLength} to ${compressed.length} bytes (${ratio}% reduction)` + ); + + const headers = new Headers(input.headers); + headers.set("Content-Encoding", "gzip"); + + const compressedRequest = new Request(input.url, { + method: input.method, + headers, + body: compressed, + signal: input.signal + }); + return globalThis.fetch(compressedRequest, init); + } + } + + // Fallback: plain string body in init (non-oRPC callers). + if (init?.body != null && typeof init.body === "string") { + const jsonBytes = Buffer.from(init.body, "utf-8"); + const compressed = await gzipAsync(jsonBytes); + + const ratio = ((1 - compressed.length / jsonBytes.byteLength) * 100).toFixed(1); + context.logger.debug( + `[ledger] Compressed request from ${jsonBytes.byteLength} to ${compressed.length} bytes (${ratio}% reduction)` + ); + + const headers = new Headers(init.headers); + headers.set("Content-Encoding", "gzip"); + + return globalThis.fetch(input, { + ...init, + headers, + body: compressed + }); + } + + return globalThis.fetch(input, init); + }; } diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts index c33a3912f8f9..d2df3f258a35 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts @@ -1,12 +1,11 @@ import { type DocsDefinitionResolver } from "@fern-api/docs-resolver"; import type { APIV1Write, DocsV1Write } from "@fern-api/fdr-sdk"; import { + createDocsLedgerClient, type DocsPublishGitInput, type DocsPublishInput, type FileManifestEntry, - type FinishResponse, - type LocaleEntry, - type RegisterResponse + type LocaleEntry } from "@fern-api/fdr-sdk/orpc-client"; import { AbsoluteFilePath } from "@fern-api/fs-utils"; import type { TaskContext } from "@fern-api/task-context"; @@ -14,7 +13,7 @@ import { createHash } from "crypto"; import { readFile } from "fs/promises"; import { buildTranslatedDocsDefinition } from "./buildTranslatedDocsDefinition.js"; -import { compressedLedgerPost } from "./compressedLedgerFetch.js"; +import { createGzipFetch } from "./compressedLedgerFetch.js"; import { mapDocsConfigToLedgerConfig } from "./mapDocsConfigToLedgerConfig.js"; import { asyncPool } from "./utils/asyncPool.js"; @@ -262,7 +261,7 @@ export async function publishDocsViaLedger({ // entry and translations follow. The server processes all locales // through the same pipeline. - const ledgerBaseUrl = `${fdrOrigin.replace(/\/+$/, "")}/docs-ledger`; + const client = createDocsLedgerClient({ baseUrl: fdrOrigin, token, headers, fetch: createGzipFetch(context) }); const locales: LocaleEntry[] = [baseLocale, ...builtTranslations.map((t) => t.localeEntry)]; @@ -278,17 +277,9 @@ export async function publishDocsViaLedger({ // Register — server computes deployment hash, returns presigned S3 // URLs for any blobs it doesn't already have in CAS (all locales). - // Request body is gzip-compressed to reduce transfer time for large - // multi-locale deployments. context.logger.debug("[ledger] Registering deployment..."); const registerStart = performance.now(); - const registerResult = await compressedLedgerPost({ - url: `${ledgerBaseUrl}/register`, - body: publishInput, - token, - headers, - context - }); + const registerResult = await client.register(publishInput); const registerTime = performance.now() - registerStart; context.logger.debug( `[ledger] Registered in ${registerTime.toFixed(0)}ms — hash=${registerResult.deploymentHash}, missing=${registerResult.missingContent.length} blobs` @@ -301,13 +292,7 @@ export async function publishDocsViaLedger({ // a single atomic transaction. Same input shape as register. context.logger.debug("[ledger] Finishing deployment..."); const finishStart = performance.now(); - const finishResult = await compressedLedgerPost({ - url: `${ledgerBaseUrl}/register/finish`, - body: publishInput, - token, - headers, - context - }); + const finishResult = await client.finish(publishInput); const finishTime = performance.now() - finishStart; context.logger.debug( `[ledger] Finished in ${finishTime.toFixed(0)}ms — deploymentId=${finishResult.deploymentId}, reused=${finishResult.reusedDeployment}` diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts index e1534d59dee8..c9e3f4c8dfce 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts @@ -1,17 +1,16 @@ import type { DocsDefinitionResolver } from "@fern-api/docs-resolver"; import type { APIV1Write, DocsV1Write } from "@fern-api/fdr-sdk"; import { + createDocsLedgerClient, type DocsPublishGitInput, type FileManifestEntry, - type FinishResponse, - type LedgerPreviewRegisterResponse, type LocaleEntry } from "@fern-api/fdr-sdk/orpc-client"; import type { AbsoluteFilePath } from "@fern-api/fs-utils"; import type { TaskContext } from "@fern-api/task-context"; import { buildTranslatedDocsDefinition } from "./buildTranslatedDocsDefinition.js"; -import { compressedLedgerPost } from "./compressedLedgerFetch.js"; +import { createGzipFetch } from "./compressedLedgerFetch.js"; import { buildLedgerInput, uploadMissingBlobs } from "./publishDocsLedger.js"; type DocsDefinition = DocsV1Write.DocsDefinition; @@ -89,31 +88,25 @@ export async function publishDocsViaLedgerPreview({ // ── Phase 2: Single register → upload → finish ───────────────────── - const ledgerBaseUrl = `${fdrOrigin.replace(/\/+$/, "")}/docs-ledger`; + const client = createDocsLedgerClient({ baseUrl: fdrOrigin, token, headers, fetch: createGzipFetch(context) }); context.logger.debug("[ledger-preview] Registering preview deployment..."); const registerStart = performance.now(); - const registerResult = await compressedLedgerPost({ - url: `${ledgerBaseUrl}/preview/init`, - body: { - orgId: organization, - previewId: previewId ?? null, - basePath: basePath ?? "", - root: baseLocale.root, - pages: baseLocale.pages, - apiManifest: baseLocale.apiManifest, - config: baseLocale.config, - fileManifest: baseLocale.fileManifest, - jsFiles: baseLocale.jsFiles, - redirects: baseLocale.redirects, - locale: baseLocale.locale, - version: baseLocale.version, - repo: baseLocale.repo, - git - }, - token, - headers, - context + const registerResult = await client.previewRegister({ + orgId: organization, + previewId: previewId ?? null, + basePath: basePath ?? "", + root: baseLocale.root, + pages: baseLocale.pages, + apiManifest: baseLocale.apiManifest, + config: baseLocale.config, + fileManifest: baseLocale.fileManifest, + jsFiles: baseLocale.jsFiles, + redirects: baseLocale.redirects, + locale: baseLocale.locale, + version: baseLocale.version, + repo: baseLocale.repo, + git }); const registerTime = performance.now() - registerStart; context.logger.debug( @@ -151,19 +144,13 @@ export async function publishDocsViaLedgerPreview({ // in a single call. context.logger.debug("[ledger-preview] Finishing preview deployment..."); const finishStart = performance.now(); - const finishResult = await compressedLedgerPost({ - url: `${ledgerBaseUrl}/register/finish`, - body: { - orgId: organization, - domain: registerResult.domain, - basepath: registerResult.basepath, - customDomains: [], - previewId: registerResult.previewId, - locales - }, - token, - headers, - context + const finishResult = await client.finish({ + orgId: organization, + domain: registerResult.domain, + basepath: registerResult.basepath, + customDomains: [], + previewId: registerResult.previewId, + locales }); const finishTime = performance.now() - finishStart; context.logger.debug( From 69d04149c237be3066a5fea27448e27d4928b907 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 21 May 2026 04:01:54 +0000 Subject: [PATCH 13/15] refactor: use streaming CompressionStream for gzip fetch Co-Authored-By: cbro --- .../src/compressedLedgerFetch.ts | 67 +++++-------------- .../src/publishDocsLedger.ts | 2 +- .../src/publishDocsLedgerPreview.ts | 2 +- 3 files changed, 18 insertions(+), 53 deletions(-) diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedLedgerFetch.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedLedgerFetch.ts index 17cfc089f4c1..cfd9f26c57ee 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedLedgerFetch.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedLedgerFetch.ts @@ -1,64 +1,29 @@ -import type { TaskContext } from "@fern-api/task-context"; -import { promisify } from "util"; -import { gzip } from "zlib"; - -const gzipAsync = promisify(gzip); - /** - * Returns a `fetch` function that gzip-compresses JSON request bodies before - * sending. Designed to be passed as the `fetch` option to - * {@link createDocsLedgerClient} so all ledger requests are transparently - * compressed while retaining full oRPC SDK typing. + * Returns a `fetch` function that gzip-compresses request bodies using the + * Web Streams `CompressionStream` API. Designed to be passed as the `fetch` + * option to {@link createDocsLedgerClient} so all ledger requests are + * transparently compressed while retaining full oRPC SDK typing. * * The FDR server registers `@fastify/compress` which transparently * decompresses incoming `Content-Encoding: gzip` request bodies. */ -export function createGzipFetch(context: TaskContext): typeof globalThis.fetch { +export function createGzipFetch(): typeof globalThis.fetch { return async (input: RequestInfo | URL, init?: RequestInit): Promise => { - // The oRPC `LinkFetchClient` calls fetch with a fully-formed Request - // object as the first argument. The body is baked into the Request. - if (input instanceof Request && init?.body == null) { - const body = await input.clone().text(); - if (body.length > 0) { - const jsonBytes = Buffer.from(body, "utf-8"); - const compressed = await gzipAsync(jsonBytes); - - const ratio = ((1 - compressed.length / jsonBytes.byteLength) * 100).toFixed(1); - context.logger.debug( - `[ledger] Compressed request from ${jsonBytes.byteLength} to ${compressed.length} bytes (${ratio}% reduction)` - ); - - const headers = new Headers(input.headers); - headers.set("Content-Encoding", "gzip"); - - const compressedRequest = new Request(input.url, { - method: input.method, - headers, - body: compressed, - signal: input.signal - }); - return globalThis.fetch(compressedRequest, init); - } - } - - // Fallback: plain string body in init (non-oRPC callers). - if (init?.body != null && typeof init.body === "string") { - const jsonBytes = Buffer.from(init.body, "utf-8"); - const compressed = await gzipAsync(jsonBytes); - - const ratio = ((1 - compressed.length / jsonBytes.byteLength) * 100).toFixed(1); - context.logger.debug( - `[ledger] Compressed request from ${jsonBytes.byteLength} to ${compressed.length} bytes (${ratio}% reduction)` - ); - - const headers = new Headers(init.headers); + if (input instanceof Request && input.body != null) { + const headers = new Headers(input.headers); headers.set("Content-Encoding", "gzip"); + headers.delete("Content-Length"); + + const compressedBody = input.body.pipeThrough(new CompressionStream("gzip")); - return globalThis.fetch(input, { - ...init, + const compressedRequest = new Request(input.url, { + method: input.method, headers, - body: compressed + body: compressedBody, + // @ts-expect-error duplex required for streaming request bodies in Node.js + duplex: "half" }); + return globalThis.fetch(compressedRequest, init); } return globalThis.fetch(input, init); diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts index d2df3f258a35..7b73808e255f 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts @@ -261,7 +261,7 @@ export async function publishDocsViaLedger({ // entry and translations follow. The server processes all locales // through the same pipeline. - const client = createDocsLedgerClient({ baseUrl: fdrOrigin, token, headers, fetch: createGzipFetch(context) }); + const client = createDocsLedgerClient({ baseUrl: fdrOrigin, token, headers, fetch: createGzipFetch() }); const locales: LocaleEntry[] = [baseLocale, ...builtTranslations.map((t) => t.localeEntry)]; diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts index c9e3f4c8dfce..aede5fa2dbda 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts @@ -88,7 +88,7 @@ export async function publishDocsViaLedgerPreview({ // ── Phase 2: Single register → upload → finish ───────────────────── - const client = createDocsLedgerClient({ baseUrl: fdrOrigin, token, headers, fetch: createGzipFetch(context) }); + const client = createDocsLedgerClient({ baseUrl: fdrOrigin, token, headers, fetch: createGzipFetch() }); context.logger.debug("[ledger-preview] Registering preview deployment..."); const registerStart = performance.now(); From a1189439bab506356b79f80374714551cf3ddc2e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 21 May 2026 04:45:46 +0000 Subject: [PATCH 14/15] feat(cli): add gzip compression to v2 docs publish requests Co-Authored-By: cbro --- .../unreleased/feat-v2-gzip-compression.yml | 5 ++ .../src/compressedFetch.ts | 48 +++++++++++++++++++ .../src/publishDocs.ts | 36 +++++++++----- 3 files changed, 77 insertions(+), 12 deletions(-) create mode 100644 packages/cli/cli/changes/unreleased/feat-v2-gzip-compression.yml create mode 100644 packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedFetch.ts diff --git a/packages/cli/cli/changes/unreleased/feat-v2-gzip-compression.yml b/packages/cli/cli/changes/unreleased/feat-v2-gzip-compression.yml new file mode 100644 index 000000000000..4d59dd2bd177 --- /dev/null +++ b/packages/cli/cli/changes/unreleased/feat-v2-gzip-compression.yml @@ -0,0 +1,5 @@ +- summary: | + Gzip-compress v2 docs publish request bodies (register, finish, API + registration, translation registration) to reduce transfer time for + large deployments. + type: feat diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedFetch.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedFetch.ts new file mode 100644 index 000000000000..7a37e585c182 --- /dev/null +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedFetch.ts @@ -0,0 +1,48 @@ +/** + * Returns a `fetch` function that gzip-compresses request bodies using the + * Web Streams `CompressionStream` API. + * + * Designed to be captured by oRPC's `LinkFetchClient` at construction time + * so all subsequent requests through the client are transparently compressed. + * + * The FDR server registers `@fastify/compress` which transparently + * decompresses incoming `Content-Encoding: gzip` request bodies. + */ +export function createGzipFetch(): typeof globalThis.fetch { + return async (input: RequestInfo | URL, init?: RequestInit): Promise => { + if (input instanceof Request && input.body != null) { + const headers = new Headers(input.headers); + headers.set("Content-Encoding", "gzip"); + headers.delete("Content-Length"); + + const compressedBody = input.body.pipeThrough(new CompressionStream("gzip")); + + const compressedRequest = new Request(input.url, { + method: input.method, + headers, + body: compressedBody, + // @ts-expect-error duplex required for streaming request bodies in Node.js + duplex: "half" + }); + return globalThis.fetch(compressedRequest, init); + } + + return globalThis.fetch(input, init); + }; +} + +/** + * Gzip-compresses a JSON body and returns a `RequestInit` suitable for + * `fetch()`, with the correct `Content-Encoding` and `Content-Type` headers. + */ +export async function gzipJsonBody(body: unknown): Promise<{ body: ReadableStream; headers: Record }> { + const json = JSON.stringify(body); + const stream = new Blob([json]).stream().pipeThrough(new CompressionStream("gzip")); + return { + body: stream, + headers: { + "Content-Encoding": "gzip", + "Content-Type": "application/json" + } + }; +} diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts index 909a0b4ee3f7..6a8dc201ecf2 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts @@ -29,6 +29,7 @@ import * as mime from "mime-types"; import { basename } from "path"; import terminalLink from "terminal-link"; import { buildTranslatedDocsDefinition } from "./buildTranslatedDocsDefinition.js"; +import { createGzipFetch, gzipJsonBody } from "./compressedFetch.js"; import { getDocsDeployMode } from "./docsDeployMode.js"; import { getDynamicGeneratorConfig } from "./getDynamicGeneratorConfig.js"; import { measureImageSizes } from "./measureImageSizes.js"; @@ -209,10 +210,17 @@ export async function publishDocs({ context.logger.info(`Docs deploy mode: ${deployMode}`); } + // Capture a gzip-compressing fetch into the oRPC client so all FDR + // requests with a body are transparently compressed. LinkFetchClient + // snapshots globalThis.fetch at construction time, so we swap it in + // before building the client and restore immediately after. + const savedFetch = globalThis.fetch; + globalThis.fetch = createGzipFetch(); const fdr = createFdrService({ token: token.value, ...(Object.keys(headers).length > 0 && { headers }) }); + globalThis.fetch = savedFetch; const authConfig = { type: "public" as const }; if (excludeApis) { @@ -883,24 +891,28 @@ export async function publishDocs({ ); // Use a raw fetch instead of the oRPC client to send `docsDefinition` // (the live server expects that field; the published fdr-sdk still uses `content`). + const translationPayload = { + domain: translationDomain, + // Send customDomains in production so FDR fans the translation + // S3 write out across every URL the docs are published to. + // Skipped in preview because preview deploys to a single + // ephemeral URL with no custom-domain mirrors. + customDomains: preview ? [] : customDomains, + orgId: organization, + locale, + docsDefinition: translatedDefinition + }; + const compressed = await gzipJsonBody(translationPayload); const translationResponse = await fetch(`${fdrOrigin}/v2/registry/docs/translations/register`, { method: "POST", headers: { - "Content-Type": "application/json", + ...compressed.headers, Authorization: `Bearer ${token.value}`, ...headers // Include telemetry headers (X-CLI-Version, X-CI-Source, etc.) }, - body: JSON.stringify({ - domain: translationDomain, - // Send customDomains in production so FDR fans the translation - // S3 write out across every URL the docs are published to. - // Skipped in preview because preview deploys to a single - // ephemeral URL with no custom-domain mirrors. - customDomains: preview ? [] : customDomains, - orgId: organization, - locale, - docsDefinition: translatedDefinition - }) + body: compressed.body, + // @ts-expect-error duplex required for streaming request bodies in Node.js + duplex: "half" }); if (!translationResponse.ok) { const body = await translationResponse.text(); From 6605a3aa3dcbb4b42ff625bdcfabb9b34ee4269d Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 21 May 2026 05:23:19 +0000 Subject: [PATCH 15/15] refactor: consolidate compressedLedgerFetch into compressedFetch Co-Authored-By: cbro --- .../src/compressedLedgerFetch.ts | 31 ------------------- .../src/publishDocsLedger.ts | 2 +- .../src/publishDocsLedgerPreview.ts | 2 +- 3 files changed, 2 insertions(+), 33 deletions(-) delete mode 100644 packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedLedgerFetch.ts diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedLedgerFetch.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedLedgerFetch.ts deleted file mode 100644 index cfd9f26c57ee..000000000000 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedLedgerFetch.ts +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Returns a `fetch` function that gzip-compresses request bodies using the - * Web Streams `CompressionStream` API. Designed to be passed as the `fetch` - * option to {@link createDocsLedgerClient} so all ledger requests are - * transparently compressed while retaining full oRPC SDK typing. - * - * The FDR server registers `@fastify/compress` which transparently - * decompresses incoming `Content-Encoding: gzip` request bodies. - */ -export function createGzipFetch(): typeof globalThis.fetch { - return async (input: RequestInfo | URL, init?: RequestInit): Promise => { - if (input instanceof Request && input.body != null) { - const headers = new Headers(input.headers); - headers.set("Content-Encoding", "gzip"); - headers.delete("Content-Length"); - - const compressedBody = input.body.pipeThrough(new CompressionStream("gzip")); - - const compressedRequest = new Request(input.url, { - method: input.method, - headers, - body: compressedBody, - // @ts-expect-error duplex required for streaming request bodies in Node.js - duplex: "half" - }); - return globalThis.fetch(compressedRequest, init); - } - - return globalThis.fetch(input, init); - }; -} diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts index 7b73808e255f..cd54fc265942 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts @@ -13,7 +13,7 @@ import { createHash } from "crypto"; import { readFile } from "fs/promises"; import { buildTranslatedDocsDefinition } from "./buildTranslatedDocsDefinition.js"; -import { createGzipFetch } from "./compressedLedgerFetch.js"; +import { createGzipFetch } from "./compressedFetch.js"; import { mapDocsConfigToLedgerConfig } from "./mapDocsConfigToLedgerConfig.js"; import { asyncPool } from "./utils/asyncPool.js"; diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts index aede5fa2dbda..cdcabbc5f5b5 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts @@ -10,7 +10,7 @@ import type { AbsoluteFilePath } from "@fern-api/fs-utils"; import type { TaskContext } from "@fern-api/task-context"; import { buildTranslatedDocsDefinition } from "./buildTranslatedDocsDefinition.js"; -import { createGzipFetch } from "./compressedLedgerFetch.js"; +import { createGzipFetch } from "./compressedFetch.js"; import { buildLedgerInput, uploadMissingBlobs } from "./publishDocsLedger.js"; type DocsDefinition = DocsV1Write.DocsDefinition;