Try MP4 instead of GIF

stever · stever · commit 74984e7fa4a8 · 2026-06-15T21:57:02.000+01:00
diff --git a/apps/gif-service/Dockerfile b/apps/gif-service/Dockerfile
@@ -2,6 +2,11 @@
 # ROMs from apps/web/public (see emulator.ts paths relative to apps/gif-service).
 FROM node:22-slim
 
+# ffmpeg is required for MP4 output.
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends ffmpeg \
+    && rm -rf /var/lib/apt/lists/*
+
 WORKDIR /app/apps/gif-service
 
 # Install this service's own dependencies (not workspace-hoisted).
diff --git a/apps/gif-service/src/gif-generator.ts b/apps/gif-service/src/gif-generator.ts
@@ -1,7 +1,9 @@
 import GIFEncoder from 'gif-encoder-2';
+import { spawn } from 'child_process';
+import { tmpdir } from 'os';
 import { Emulator } from './emulator.js';
 import { FrameDecoder } from './frame-decoder.js';
-import { readFile } from 'fs/promises';
+import { readFile, unlink } from 'fs/promises';
 import { join, dirname } from 'path';
 import { fileURLToPath } from 'url';
 
@@ -65,19 +67,11 @@ export class GIFGenerator {
         for (let i = 0; i < gapFrames; i++) this.emulator.runFrame();
     }
 
-    async generateFromTAP(tapData: Buffer, machineType: number = 48): Promise<Buffer> {
-        const width = this.decoder.getWidth();
-        const height = this.decoder.getHeight();
-
-        // Encode every Nth emulated frame. The core runs at 50fps; encoding at
-        // 25fps halves GIF size and encode time with little visible loss.
-        const frameStep = 2;
-        const encoder = new GIFEncoder(width, height, 'neuquant');
-        encoder.setDelay(20 * frameStep); // match the decimated frame rate
-        encoder.setRepeat(0); // loop forever
-        encoder.setQuality(10);
-        encoder.start();
-
+    /**
+     * Boot the machine, load the tape, run the program, and return the kept raw
+     * frame buffers (50fps, trailing static trimmed). Shared by both encoders.
+     */
+    private async captureFrames(tapData: Buffer, machineType: number): Promise<Uint8Array[]> {
         this.emulator.setMachineType(machineType);
         this.emulator.setTapeTraps(true); // instant block loading; no pulse playback needed
         this.emulator.loadTAPFile(tapData);
@@ -104,9 +98,8 @@ export class GIFGenerator {
         // Let the trap loader inject every block and the program start.
         for (let i = 0; i < 15; i++) this.emulator.runFrame();
 
-        // Phase 4: capture the running program. Stop once the screen has been
-        // static for `staleFrameThreshold` consecutive frames (the program has
-        // finished or paused), then trim the trailing static frames.
+        // Capture the running program. Stop once the screen has been static for
+        // `staleFrameThreshold` consecutive frames, then trim trailing static.
         const maxFrames = Math.floor(this.options.maxDurationMs / 20);
         const staleStop = this.options.staleFrameThreshold;
         const tailFrames = 25; // ~0.5s of static tail kept for readability
@@ -134,21 +127,80 @@ export class GIFGenerator {
         }
 
         if (lastChangeIndex < 0) {
-            // Nothing ever changed; keep a short clip so the GIF is not empty.
+            // Nothing ever changed; keep a short clip so the output is not empty.
             lastChangeIndex = Math.min(frames.length, tailFrames) - 1;
         }
         const keep = Math.min(frames.length, lastChangeIndex + 1 + tailFrames);
-        let encoded = 0;
-        for (let i = 0; i < keep; i += frameStep) {
+        console.log(`Captured ${frames.length} frames, keeping ${keep}`);
+        return frames.slice(0, keep);
+    }
+
+    /** Render the program to an animated GIF (25fps to keep size sane). */
+    async generateFromTAP(tapData: Buffer, machineType: number = 48): Promise<Buffer> {
+        const frames = await this.captureFrames(tapData, machineType);
+
+        // The core runs at 50fps; encode every 2nd frame (25fps) to halve GIF
+        // size and encode time with little visible loss.
+        const frameStep = 2;
+        const encoder = new GIFEncoder(this.decoder.getWidth(), this.decoder.getHeight(), 'neuquant');
+        encoder.setDelay(20 * frameStep);
+        encoder.setRepeat(0); // loop forever
+        encoder.setQuality(10);
+        encoder.start();
+
+        for (let i = 0; i < frames.length; i += frameStep) {
             encoder.addFrame(this.decoder.decode(frames[i]));
-            encoded++;
         }
-        console.log(`Encoding ${encoded} frames (${frames.length} captured, ${keep} kept)`);
-
         encoder.finish();
         return Buffer.from(encoder.out.getData());
     }
 
+    /** Render the program to an H.264 MP4 at the full 50fps. */
+    async generateMp4FromTAP(tapData: Buffer, machineType: number = 48): Promise<Buffer> {
+        const frames = await this.captureFrames(tapData, machineType);
+        return this.encodeMp4(frames, 50);
+    }
+
+    /** Pipe decoded RGBA frames through ffmpeg to a temporary MP4 and return it. */
+    private async encodeMp4(frames: Uint8Array[], fps: number): Promise<Buffer> {
+        const width = this.decoder.getWidth();
+        const height = this.decoder.getHeight();
+        const outPath = join(tmpdir(), `zxplay-${process.pid}-${Date.now()}.mp4`);
+
+        const args = [
+            '-f', 'rawvideo', '-pix_fmt', 'rgba', '-s', `${width}x${height}`, '-r', String(fps),
+            '-i', 'pipe:0',
+            '-an', // no audio
+            '-c:v', 'libx264', '-pix_fmt', 'yuv420p', '-preset', 'veryfast', '-crf', '20',
+            '-movflags', '+faststart',
+            '-y', outPath,
+        ];
+        const ff = spawn('ffmpeg', args, { stdio: ['pipe', 'ignore', 'pipe'] });
+        let stderr = '';
+        ff.stderr.on('data', (d) => { stderr += d.toString(); });
+
+        const finished = new Promise<void>((resolve, reject) => {
+            ff.on('error', reject);
+            ff.on('close', (code) =>
+                code === 0 ? resolve() : reject(new Error(`ffmpeg exited ${code}: ${stderr.slice(-500)}`)),
+            );
+        });
+
+        for (const raw of frames) {
+            const rgba = this.decoder.decode(raw);
+            if (!ff.stdin.write(rgba)) {
+                await new Promise<void>((resolve) => ff.stdin.once('drain', () => resolve()));
+            }
+        }
+        ff.stdin.end();
+        await finished;
+
+        const buffer = await readFile(outPath);
+        await unlink(outPath).catch(() => undefined);
+        console.log(`Encoded MP4: ${frames.length} frames, ${buffer.length} bytes`);
+        return buffer;
+    }
+
     private async parseSZXSnapshot(data: Buffer): Promise<any> {
         const file = new DataView(data.buffer, data.byteOffset, data.byteLength);
 
diff --git a/apps/gif-service/src/routes/basic.ts b/apps/gif-service/src/routes/basic.ts
@@ -1,41 +1,51 @@
-import { Router } from 'express';
+import { Router, Request, Response } from 'express';
 import zmakebas, { ZmakebasMessage } from 'zmakebas';
 import { GIFGenerator } from '../gif-generator.js';
 
 const router = Router();
 
+type Format = 'gif' | 'mp4';
+
+function readCode(req: Request): string | null {
+    const code: unknown = typeof req.body === 'string' ? req.body : req.body?.code;
+    return typeof code === 'string' && code.trim() ? code : null;
+}
+
+async function compile(code: string): Promise<{ tap: Uint8Array | null; detail: string }> {
+    try {
+        return { tap: await zmakebas(code), detail: '' };
+    } catch (messages) {
+        const detail = Array.isArray(messages)
+            ? (messages as ZmakebasMessage[])
+                .filter((m) => m.type === 'err')
+                .map((m) => m.text)
+                .join('\n')
+            : String(messages);
+        return { tap: null, detail: detail || 'unknown error' };
+    }
+}
+
 /**
- * POST /api/basic-to-gif
- *
- * Body: JSON `{ "code": "10 PRINT ..." }` or raw `text/plain` BASIC source.
- * Compiles the source to a .tap with zmakebas, runs it, and returns an
- * animated GIF. On a compile error responds 400 with the compiler messages so
- * the caller can show the user what to fix.
+ * Compile BASIC and render it. GIF defaults to 2x (size-constrained), MP4 to 4x
+ * (H.264 stays small at high resolution). On a compile error responds 400 with
+ * the compiler messages so the caller can show the user what to fix.
  */
-router.post('/basic-to-gif', async (req, res) => {
+async function handle(format: Format, req: Request, res: Response): Promise<void> {
     try {
-        const code: unknown = typeof req.body === 'string' ? req.body : req.body?.code;
-        if (typeof code !== 'string' || !code.trim()) {
+        const code = readCode(req);
+        if (!code) {
             res.status(400).json({ error: 'No BASIC source provided' });
             return;
         }
 
         const maxSeconds = parseInt(req.query.maxSeconds as string) || 30;
         const staleThreshold = parseInt(req.query.staleThreshold as string) || 150;
         const machineType = parseInt(req.query.machineType as string) || 48;
-        const scale = parseInt(req.query.scale as string) || 2;
-
-        let tap: Uint8Array;
-        try {
-            tap = await zmakebas(code);
-        } catch (messages) {
-            const detail = Array.isArray(messages)
-                ? (messages as ZmakebasMessage[])
-                    .filter((m) => m.type === 'err')
-                    .map((m) => m.text)
-                    .join('\n')
-                : String(messages);
-            res.status(400).json({ error: 'BASIC compilation failed', detail: detail || 'unknown error' });
+        const scale = parseInt(req.query.scale as string) || (format === 'mp4' ? 4 : 2);
+
+        const { tap: compiledTap, detail } = await compile(code);
+        if (!compiledTap) {
+            res.status(400).json({ error: 'BASIC compilation failed', detail });
             return;
         }
 
@@ -46,16 +56,24 @@ router.post('/basic-to-gif', async (req, res) => {
         });
         await generator.initialize();
 
-        console.log(`Generating GIF from ${code.length} bytes of BASIC...`);
-        const gifBuffer = await generator.generateFromTAP(Buffer.from(tap), machineType);
-        console.log(`GIF generated: ${gifBuffer.length} bytes`);
-
-        res.setHeader('Content-Type', 'image/gif');
-        res.send(gifBuffer);
+        console.log(`Generating ${format.toUpperCase()} from ${code.length} bytes of BASIC...`);
+        const tap = Buffer.from(compiledTap);
+        if (format === 'mp4') {
+            const mp4 = await generator.generateMp4FromTAP(tap, machineType);
+            res.setHeader('Content-Type', 'video/mp4');
+            res.send(mp4);
+        } else {
+            const gif = await generator.generateFromTAP(tap, machineType);
+            res.setHeader('Content-Type', 'image/gif');
+            res.send(gif);
+        }
     } catch (error: any) {
-        console.error('Error generating GIF from BASIC:', error);
+        console.error(`Error generating ${format} from BASIC:`, error);
         res.status(500).json({ error: error.message });
     }
-});
+}
+
+router.post('/basic-to-gif', (req, res) => handle('gif', req, res));
+router.post('/basic-to-mp4', (req, res) => handle('mp4', req, res));
 
 export default router;
diff --git a/apps/mastodon-bot/.env-dist b/apps/mastodon-bot/.env-dist
@@ -10,8 +10,8 @@ STATE_FILE=/data/state.json
 POLL_INTERVAL_MS=15000
 # Capture and size limits.
 MAX_SECONDS=30
-MAX_GIF_BYTES=8000000
-# Optional text posted alongside the GIF (leave empty for image-only replies).
+MAX_MEDIA_BYTES=8000000
+# Optional text posted alongside the video (leave empty for media-only replies).
 REPLY_CAPTION=
 # Set true to log what would run without posting anything.
 DRY_RUN=false
diff --git a/apps/mastodon-bot/src/cli.ts b/apps/mastodon-bot/src/cli.ts
@@ -1,7 +1,7 @@
 import { readFile, writeFile } from 'fs/promises';
-import { basicToGif } from './gif.js';
+import { basicToMedia } from './media.js';
 
-// Local end-to-end test of the BASIC -> GIF chain without any Mastodon account.
+// Local end-to-end test of the BASIC -> MP4 chain without any Mastodon account.
 // Usage:
 //   GIF_SERVICE_URL=http://localhost:5001 npm run cli -- program.bas
 //   echo '10 PRINT "HI"' | GIF_SERVICE_URL=http://localhost:5001 npm run cli
@@ -22,15 +22,15 @@ async function main(): Promise<void> {
         process.exit(1);
     }
 
-    const result = await basicToGif(code);
+    const result = await basicToMedia(code);
     if (!result.ok) {
         console.error('Error:', result.error);
         process.exit(1);
     }
 
-    const out = process.env.OUT ?? 'program.gif';
-    await writeFile(out, result.gif);
-    console.log(`Wrote ${out} (${result.gif.length} bytes)`);
+    const out = process.env.OUT ?? result.filename;
+    await writeFile(out, result.data);
+    console.log(`Wrote ${out} (${result.data.length} bytes)`);
 }
 
 main().catch((err) => {
diff --git a/apps/mastodon-bot/src/config.ts b/apps/mastodon-bot/src/config.ts
@@ -9,7 +9,7 @@ export const config = {
     stateFile: process.env.STATE_FILE ?? './state.json',
     pollIntervalMs: parseInt(process.env.POLL_INTERVAL_MS ?? '15000', 10),
     maxSeconds: parseInt(process.env.MAX_SECONDS ?? '30', 10),
-    maxGifBytes: parseInt(process.env.MAX_GIF_BYTES ?? '8000000', 10),
+    maxMediaBytes: parseInt(process.env.MAX_MEDIA_BYTES ?? '8000000', 10),
     replyCaption: process.env.REPLY_CAPTION ?? '',
     dryRun: process.env.DRY_RUN === 'true',
 };
diff --git a/apps/mastodon-bot/src/index.ts b/apps/mastodon-bot/src/index.ts
@@ -1,6 +1,6 @@
 import { config, assertRuntimeConfig } from './config.js';
 import { htmlToBasic } from './basic.js';
-import { basicToGif } from './gif.js';
+import { basicToMedia } from './media.js';
 import { loadState, saveState } from './state.js';
 import {
     MastodonAccount,
@@ -10,7 +10,7 @@ import {
     fetchMentions,
     postReply,
     sleep,
-    uploadGif,
+    uploadMedia,
     verifyCredentials,
 } from './mastodon.js';
 
@@ -43,7 +43,7 @@ async function handleMention(self: MastodonAccount, n: MastodonNotification): Pr
         return;
     }
 
-    const result = await basicToGif(code);
+    const result = await basicToMedia(code);
 
     if (!result.ok) {
         await postReply({
@@ -54,23 +54,23 @@ async function handleMention(self: MastodonAccount, n: MastodonNotification): Pr
         return;
     }
 
-    if (result.gif.length > config.maxGifBytes) {
+    if (result.data.length > config.maxMediaBytes) {
         await postReply({
             inReplyToId: status.id,
-            statusText: 'That program produced a GIF too large to post here.',
+            statusText: 'That program produced a file too large to post here.',
             visibility,
         });
         return;
     }
 
-    const mediaId = await uploadGif(result.gif, code);
+    const mediaId = await uploadMedia(result.data, result.contentType, result.filename, code);
     await postReply({
         inReplyToId: status.id,
         statusText: config.replyCaption,
         mediaIds: [mediaId],
         visibility,
     });
-    console.log(`Replied to ${n.id} with ${result.gif.length} byte GIF`);
+    console.log(`Replied to ${n.id} with ${result.data.length} byte ${result.contentType}`);
 }
 
 async function pollOnce(self: MastodonAccount): Promise<void> {
diff --git a/apps/mastodon-bot/src/mastodon.ts b/apps/mastodon-bot/src/mastodon.ts
@@ -64,10 +64,15 @@ export async function fetchMentions(sinceId: string | null): Promise<MastodonNot
     return notifications.reverse();
 }
 
-/** Upload a GIF, waiting for server-side processing, and return its media id. */
-export async function uploadGif(gif: Buffer, description: string): Promise<string> {
+/** Upload media, waiting for server-side processing, and return its media id. */
+export async function uploadMedia(
+    data: Buffer,
+    contentType: string,
+    filename: string,
+    description: string,
+): Promise<string> {
     const form = new FormData();
-    form.append('file', new Blob([gif], { type: 'image/gif' }), 'program.gif');
+    form.append('file', new Blob([data], { type: contentType }), filename);
     if (description) {
         form.append('description', description.slice(0, 1400));
     }
@@ -76,9 +81,10 @@ export async function uploadGif(gif: Buffer, description: string): Promise<strin
         headers: authHeaders(),
         body: form,
     });
-    // v2 media may return with url=null while the server processes it.
+    // v2 media may return with url=null while the server processes it. Video
+    // transcoding can take longer than images, so allow up to ~60s.
     let media = created;
-    for (let i = 0; i < 30 && !media.url; i++) {
+    for (let i = 0; i < 60 && !media.url; i++) {
         await sleep(1000);
         media = await api<MediaAttachment>(`/api/v1/media/${created.id}`, { headers: authHeaders() });
     }
diff --git a/apps/mastodon-bot/src/media.ts b/apps/mastodon-bot/src/media.ts
