diff --git a/packages/cli/cli/changes/unreleased/feat-ledger-gzip-compression.yml b/packages/cli/cli/changes/unreleased/feat-ledger-gzip-compression.yml new file mode 100644 index 000000000000..b1ff7b98f9a4 --- /dev/null +++ b/packages/cli/cli/changes/unreleased/feat-ledger-gzip-compression.yml @@ -0,0 +1,4 @@ +- summary: | + Gzip-compress docs-ledger register and finish request bodies to reduce + transfer time for large multi-locale deployments. + type: feat diff --git a/packages/cli/cli/changes/unreleased/feat-v2-gzip-compression.yml b/packages/cli/cli/changes/unreleased/feat-v2-gzip-compression.yml new file mode 100644 index 000000000000..4d59dd2bd177 --- /dev/null +++ b/packages/cli/cli/changes/unreleased/feat-v2-gzip-compression.yml @@ -0,0 +1,5 @@ +- summary: | + Gzip-compress v2 docs publish request bodies (register, finish, API + registration, translation registration) to reduce transfer time for + large deployments. + type: feat diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedFetch.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedFetch.ts new file mode 100644 index 000000000000..7a37e585c182 --- /dev/null +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/compressedFetch.ts @@ -0,0 +1,48 @@ +/** + * Returns a `fetch` function that gzip-compresses request bodies using the + * Web Streams `CompressionStream` API. + * + * Designed to be captured by oRPC's `LinkFetchClient` at construction time + * so all subsequent requests through the client are transparently compressed. + * + * The FDR server registers `@fastify/compress` which transparently + * decompresses incoming `Content-Encoding: gzip` request bodies. + */ +export function createGzipFetch(): typeof globalThis.fetch { + return async (input: RequestInfo | URL, init?: RequestInit): Promise => { + if (input instanceof Request && input.body != null) { + const headers = new Headers(input.headers); + headers.set("Content-Encoding", "gzip"); + headers.delete("Content-Length"); + + const compressedBody = input.body.pipeThrough(new CompressionStream("gzip")); + + const compressedRequest = new Request(input.url, { + method: input.method, + headers, + body: compressedBody, + // @ts-expect-error duplex required for streaming request bodies in Node.js + duplex: "half" + }); + return globalThis.fetch(compressedRequest, init); + } + + return globalThis.fetch(input, init); + }; +} + +/** + * Gzip-compresses a JSON body and returns a `RequestInit` suitable for + * `fetch()`, with the correct `Content-Encoding` and `Content-Type` headers. + */ +export async function gzipJsonBody(body: unknown): Promise<{ body: ReadableStream; headers: Record }> { + const json = JSON.stringify(body); + const stream = new Blob([json]).stream().pipeThrough(new CompressionStream("gzip")); + return { + body: stream, + headers: { + "Content-Encoding": "gzip", + "Content-Type": "application/json" + } + }; +} diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts index 909a0b4ee3f7..6a8dc201ecf2 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocs.ts @@ -29,6 +29,7 @@ import * as mime from "mime-types"; import { basename } from "path"; import terminalLink from "terminal-link"; import { buildTranslatedDocsDefinition } from "./buildTranslatedDocsDefinition.js"; +import { createGzipFetch, gzipJsonBody } from "./compressedFetch.js"; import { getDocsDeployMode } from "./docsDeployMode.js"; import { getDynamicGeneratorConfig } from "./getDynamicGeneratorConfig.js"; import { measureImageSizes } from "./measureImageSizes.js"; @@ -209,10 +210,17 @@ export async function publishDocs({ context.logger.info(`Docs deploy mode: ${deployMode}`); } + // Capture a gzip-compressing fetch into the oRPC client so all FDR + // requests with a body are transparently compressed. LinkFetchClient + // snapshots globalThis.fetch at construction time, so we swap it in + // before building the client and restore immediately after. + const savedFetch = globalThis.fetch; + globalThis.fetch = createGzipFetch(); const fdr = createFdrService({ token: token.value, ...(Object.keys(headers).length > 0 && { headers }) }); + globalThis.fetch = savedFetch; const authConfig = { type: "public" as const }; if (excludeApis) { @@ -883,24 +891,28 @@ export async function publishDocs({ ); // Use a raw fetch instead of the oRPC client to send `docsDefinition` // (the live server expects that field; the published fdr-sdk still uses `content`). + const translationPayload = { + domain: translationDomain, + // Send customDomains in production so FDR fans the translation + // S3 write out across every URL the docs are published to. + // Skipped in preview because preview deploys to a single + // ephemeral URL with no custom-domain mirrors. + customDomains: preview ? [] : customDomains, + orgId: organization, + locale, + docsDefinition: translatedDefinition + }; + const compressed = await gzipJsonBody(translationPayload); const translationResponse = await fetch(`${fdrOrigin}/v2/registry/docs/translations/register`, { method: "POST", headers: { - "Content-Type": "application/json", + ...compressed.headers, Authorization: `Bearer ${token.value}`, ...headers // Include telemetry headers (X-CLI-Version, X-CI-Source, etc.) }, - body: JSON.stringify({ - domain: translationDomain, - // Send customDomains in production so FDR fans the translation - // S3 write out across every URL the docs are published to. - // Skipped in preview because preview deploys to a single - // ephemeral URL with no custom-domain mirrors. - customDomains: preview ? [] : customDomains, - orgId: organization, - locale, - docsDefinition: translatedDefinition - }) + body: compressed.body, + // @ts-expect-error duplex required for streaming request bodies in Node.js + duplex: "half" }); if (!translationResponse.ok) { const body = await translationResponse.text(); diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts index c5f59b4bb8e3..cd54fc265942 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedger.ts @@ -13,6 +13,7 @@ import { createHash } from "crypto"; import { readFile } from "fs/promises"; import { buildTranslatedDocsDefinition } from "./buildTranslatedDocsDefinition.js"; +import { createGzipFetch } from "./compressedFetch.js"; import { mapDocsConfigToLedgerConfig } from "./mapDocsConfigToLedgerConfig.js"; import { asyncPool } from "./utils/asyncPool.js"; @@ -260,7 +261,7 @@ export async function publishDocsViaLedger({ // entry and translations follow. The server processes all locales // through the same pipeline. - const client = createDocsLedgerClient({ baseUrl: fdrOrigin, token, headers }); + const client = createDocsLedgerClient({ baseUrl: fdrOrigin, token, headers, fetch: createGzipFetch() }); const locales: LocaleEntry[] = [baseLocale, ...builtTranslations.map((t) => t.localeEntry)]; diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts index dac02e5fbf27..cdcabbc5f5b5 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/publishDocsLedgerPreview.ts @@ -10,6 +10,7 @@ import type { AbsoluteFilePath } from "@fern-api/fs-utils"; import type { TaskContext } from "@fern-api/task-context"; import { buildTranslatedDocsDefinition } from "./buildTranslatedDocsDefinition.js"; +import { createGzipFetch } from "./compressedFetch.js"; import { buildLedgerInput, uploadMissingBlobs } from "./publishDocsLedger.js"; type DocsDefinition = DocsV1Write.DocsDefinition; @@ -87,7 +88,7 @@ export async function publishDocsViaLedgerPreview({ // ── Phase 2: Single register → upload → finish ───────────────────── - const client = createDocsLedgerClient({ baseUrl: fdrOrigin, token, headers }); + const client = createDocsLedgerClient({ baseUrl: fdrOrigin, token, headers, fetch: createGzipFetch() }); context.logger.debug("[ledger-preview] Registering preview deployment..."); const registerStart = performance.now();