From ba032713e6332526794781f741daed8f5efcbec1 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Fri, 12 Dec 2025 16:38:31 -0600 Subject: [PATCH 01/38] docs to useService --- packages/server/src/service.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/server/src/service.ts b/packages/server/src/service.ts index dbb350c1..720eaafb 100644 --- a/packages/server/src/service.ts +++ b/packages/server/src/service.ts @@ -21,6 +21,15 @@ type ServiceOptions = { processOptions?: ProcessOptions; }; +/** + * Start a process and return an Operation that represents the running service. + * + * The Operation returned by useService returns when the process has started and, + * if a wellnessCheck is provided, once the wellnessCheck passes. When run in an + * effection scope, the operation remains active in that scope. When the operation + * goes out of scope, effection will automatically shut down the + * process and clean up and shut down the process. + */ export function useService( _name: string, cmd: string, From 418b9ca7bdd3a11c2b8a44276637c4bac20be8b6 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Fri, 12 Dec 2025 16:44:27 -0600 Subject: [PATCH 02/38] spike out dag to start services --- packages/server/src/index.ts | 1 + packages/server/src/services.ts | 175 ++++++++++++++++++++++++++++++++ 2 files changed, 176 insertions(+) create mode 100644 packages/server/src/services.ts diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index d1df1916..5b9d59f2 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -1,2 +1,3 @@ export * from "./logging.ts"; export * from "./service.ts"; +export * from "./services.ts"; diff --git a/packages/server/src/services.ts b/packages/server/src/services.ts new file mode 100644 index 00000000..db34be9b --- /dev/null +++ b/packages/server/src/services.ts @@ -0,0 +1,175 @@ +import { + type Operation, + resource, + spawn, + suspend, + withResolvers, +} from "effection"; + +export type ServiceDefinition = { + // The operation that starts the service and returns when the service is ready. + operation: Operation; + deps?: string[]; + options?: { + // Keep an options object for future expansion or hooks; currently unused when operation is present + }; + // lifecycle hooks + beforeStart?: () => Operation; + afterStart?: () => Operation; + beforeStop?: () => Operation; + afterStop?: () => Operation; +}; + +export type ServicesMap = Record; + +function computeLevels(services: ServicesMap): string[][] { + const indeg: Record = {}; + const graph: Record> = {}; + for (const name of Object.keys(services)) { + indeg[name] = 0; + graph[name] = new Set(); + } + for (const [name, def] of Object.entries(services)) { + for (const dep of def.deps ?? []) { + if (!(dep in services)) { + throw new Error( + `Service '${name}' depends on unknown service '${dep}'` + ); + } + graph[dep].add(name); + indeg[name] = (indeg[name] || 0) + 1; + } + } + + const levels: string[][] = []; + let q: string[] = []; + for (const [k, v] of Object.entries(indeg)) { + if (v === 0) q.push(k); + } + + let processed = 0; + while (q.length) { + const currentLayer = q.slice(); + levels.push(currentLayer); + processed += currentLayer.length; + const next: string[] = []; + for (const n of currentLayer) { + for (const m of graph[n]) { + indeg[m] -= 1; + if (indeg[m] === 0) next.push(m); + } + } + q = next; + } + if (processed !== Object.keys(services).length) { + throw new Error(`Cycle detected in services`); + } + return levels; +} + +function* waitForAllReady( + names: string[], + readyResolvers: Map +): Operation { + for (const n of names) { + const r = readyResolvers.get(n); + if (r) { + yield* r.operation; + } + } +} + +/** + * useServiceGraph + * + * Start a set of services with dependencies (a DAG). Each service must provide an + * Operation that starts the service and returns once the service is ready. + * + * Example usage: + * + * yield* useServiceGraph({ + * A: { operation: useService('A', 'node --import tsx ./test/services/service-a.ts') }, + * B: { operation: useService('B', 'node --import tsx ./test/services/service-b.ts'), deps: ['A'] } + * }); + * + * Services within the same topological layer are started concurrently. Lifecycle + * hooks can be used to perform actions before or after each service starts or stops. + */ +export function useServiceGraph(services: ServicesMap): Operation { + return resource(function* (provide) { + const layers = computeLevels(services); + + // Map of readiness resolvers returned by `withResolvers` + const readyResolvers = new Map< + string, + { + operation: Operation; + resolve: () => void; + reject: (err: Error) => void; + } + >(); + for (const name of Object.keys(services)) { + const r = withResolvers(); + readyResolvers.set(name, { + operation: r.operation, + resolve: r.resolve, + reject: r.reject, + }); + } + + // Keep track of start order so we can run beforeStop hooks in reverse + const startOrder: string[] = []; + + for (const layer of layers) { + // spawn all services in this layer in parallel + for (const name of layer) { + const def = services[name]; + // wait for deps to be ready (yield the underlying Promise) + if (def.deps) { + for (const dep of def.deps) { + const r = readyResolvers.get(dep); + if (!r) + throw new Error( + `missing readiness resolver for dependency '${dep}'` + ); + yield* r.operation; + } + } + + // spawn a child operation that runs the service and keeps it alive with suspend() + yield* spawn(function* () { + try { + if (def.beforeStart) yield* def.beforeStart(); + + // The caller-supplied operation starts the service and returns when ready. + yield* def.operation; + startOrder.push(name); + const res = readyResolvers.get(name); + if (res) res.resolve(); + if (def.afterStart) yield* def.afterStart(); + + yield* suspend(); + } finally { + // run afterStop hooks in child finalizer so they are executed after the + // process has cleaned up + if (def.afterStop) yield* def.afterStop(); + } + }); + } + // after spawning the whole layer, wait until every service in the layer is ready + yield* waitForAllReady(layer, readyResolvers); + } + + try { + yield* provide(); + } finally { + // Run beforeStop hooks in reverse start order + for (const name of startOrder.slice().reverse()) { + const def = services[name]; + if (def.beforeStop) { + yield* def.beforeStop(); + } + } + } + }); +} From 785a24e22d172da208c978c55e69548fcb07902c Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Sat, 27 Dec 2025 01:14:27 -0600 Subject: [PATCH 03/38] tests and service filter --- packages/server/README.md | 149 ++++++++++ packages/server/example/README.md | 27 ++ packages/server/example/basic-graph.ts | 55 ++++ packages/server/example/concurrency-layers.ts | 68 +++++ packages/server/example/debug-run.ts | 10 + packages/server/example/lifecycle-hooks.ts | 81 ++++++ .../server/example/operation/basic-graph.ts | 28 ++ .../example/operation/concurrency-layers.ts | 32 +++ .../example/operation/lifecycle-hooks.ts | 49 ++++ packages/server/example/services/a.ts | 4 + packages/server/example/services/b.ts | 4 + packages/server/example/services/fast.ts | 4 + .../server/example/services/http-server.ts | 45 +++ packages/server/example/services/slow.ts | 4 + packages/server/package.json | 4 +- packages/server/src/cli.ts | 44 +++ packages/server/src/services.ts | 100 ++++++- packages/server/test/examples-smoke.test.ts | 111 ++++++++ packages/server/test/service.test.ts | 2 +- packages/server/test/services.test.ts | 262 ++++++++++++++++++ packages/server/test/services/service-a.ts | 4 + packages/server/test/services/service-b.ts | 4 + packages/server/test/services/service-fast.ts | 4 + .../test/{ => services}/service-main.ts | 0 packages/server/test/services/service-slow.ts | 4 + packages/server/tsconfig.json | 2 +- 26 files changed, 1088 insertions(+), 13 deletions(-) create mode 100644 packages/server/example/README.md create mode 100644 packages/server/example/basic-graph.ts create mode 100644 packages/server/example/concurrency-layers.ts create mode 100644 packages/server/example/debug-run.ts create mode 100644 packages/server/example/lifecycle-hooks.ts create mode 100644 packages/server/example/operation/basic-graph.ts create mode 100644 packages/server/example/operation/concurrency-layers.ts create mode 100644 packages/server/example/operation/lifecycle-hooks.ts create mode 100644 packages/server/example/services/a.ts create mode 100644 packages/server/example/services/b.ts create mode 100644 packages/server/example/services/fast.ts create mode 100644 packages/server/example/services/http-server.ts create mode 100644 packages/server/example/services/slow.ts create mode 100644 packages/server/src/cli.ts create mode 100644 packages/server/test/examples-smoke.test.ts create mode 100644 packages/server/test/services.test.ts create mode 100644 packages/server/test/services/service-a.ts create mode 100644 packages/server/test/services/service-b.ts create mode 100644 packages/server/test/services/service-fast.ts rename packages/server/test/{ => services}/service-main.ts (100%) create mode 100644 packages/server/test/services/service-slow.ts diff --git a/packages/server/README.md b/packages/server/README.md index 2ec953b7..122f1770 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -6,3 +6,152 @@ https://github.com/thefrontside/simulacrum > [!WARNING] > The server is undergoing a refactor, and this may not be required for your use case. The refactor includes allow for more simply running single simulators so this package will be primarily useful as a control plane for cases where there are many simulators under test and in use. For the previous iterations, see the `v0` branch which contain the previous functionality. + +## Operation-based service orchestration + +`@simulacrum/server` provides operations to start and manage services with lifecycle hooks. The recommended pattern is to create `Operation` instances for each service (typically via `useService`) and pass them to `useServiceGraph` which starts the services respecting a dependency DAG and provides lifecycle hooks for startup and shutdown. + +Key points: + +- `useServiceGraph(services: ServicesMap): Operation` — starts a DAG of services. Each service in the map is declared as a `ServiceDefinition`. +- `ServiceDefinition.operation` (required) — an `Operation` which indicates the service has started. This operation may be long-lived (e.g. `useService`) or may return once the service is ready while a background child keeps the service running. See the example below. +- `deps` — an optional list of service names this service depends on; services without dependencies in the same layer are started concurrently. +- Lifecycle hooks: `beforeStart`, `afterStart`, `beforeStop`, `afterStop` — each is an `Operation` that runs at the appropriate time. + +Example: + +```ts +import { main, spawn, sleep } from "effection"; +import { useServiceGraph, useService } from "@simulacrum/server"; + +main(function* () { + yield* spawn(function* () { + // In many situations, pass `useService` directly: it returns once the + // process is spawned and, if a wellnessCheck is provided, once the + // wellnessCheck passes. The service is automatically shut down by + // effection when the operation goes out of scope. + yield* useServiceGraph({ + A: { + operation: useService( + "A", + "node --import tsx ./test/services/service-a.ts" + ), + }, + B: { + operation: useService( + "B", + "node --import tsx ./test/services/service-b.ts" + ), + deps: ["A"], + }, + }); + }); +}); +``` + +Notes: + +- `useServiceGraph` returns an `Operation` that holds while services run and only cancels on parent scope termination. +- If you want to start services sequentially or add more advanced concurrency control, compose operations yourself and use `spawn` to control how operations run. + +### Lifecycle hooks + +Each `ServiceDefinition` supports lifecycle hook operations. These hooks run in the parent scope and are useful for performing orchestration tasks, logging, or writing sentinel files for integration tests. Hooks are `Operation` as well. + +```ts +const services = { + A: { + operation: useService( + "A", + "node --import tsx ./test/services/service-a.ts" + ), + afterStart: () => + (function* () { + // runs after the operation returns + console.log("A has started"); + })(), + beforeStop: () => + (function* () { + // runs during shutdown in reverse order + console.log("A is stopping"); + })(), + }, +}; +``` + +Notes: + +- `afterStart` runs after `operation` returns (service is ready) +- `beforeStop` runs during cleanup in reverse-order of startup +- Hooks are optional and can be used together with a passed `operation` or a custom operation + +Try it + +```bash +# Run the server package tests +cd packages/server +npm test +``` + +## Examples + +The `example` folder contains runnable examples demonstrating `useServiceGraph` and `useService`. + +Run the basic dependency example: + +```bash +cd packages/server +npm run example:basic +``` + +Run lifecycle hooks example: + +```bash +cd packages/server +npm run example:lifecycle +``` + +Run concurrency layers example: + +````bash +cd packages/server +npm run example:concurrency + +Run examples directly (each example has its own npm script). You can also run the TypeScript module with `tsx`. + +```bash +cd packages/server +npm run example:basic +npm run example:lifecycle +npm run example:concurrency +# or run a module directly: +node --import tsx ./example/basic-graph.ts +``` + +### Typed exports between services 💡 + +Services may return a value from their `operation`. That value is exposed to dependent services via an `exportsOperation` on the provider. + +```ts +const services = { + provider: { + // provider operation returns { url: string } + operation: useService<{ url: string }>( + "provider", + "node --import tsx ./example/services/provider.ts" + ), + }, + consumer: { + deps: ["provider"], + operation() { + return (function* () { + // access provider exports via the `services` variable + const providerExports = yield* services.provider.exportsOperation; + console.log("provider url:", providerExports.url); + })(); + }, + }, +}; + +// pass `services` to `useServiceGraph` or `simulationCLI` +```` diff --git a/packages/server/example/README.md b/packages/server/example/README.md new file mode 100644 index 00000000..330da1e7 --- /dev/null +++ b/packages/server/example/README.md @@ -0,0 +1,27 @@ +# Server package examples + +This folder contains runnable examples demonstrating `useServiceGraph` and `useService`. + +There are two sets of examples: + +- **use-service** (top-level files like `basic-graph.ts`, `lifecycle-hooks.ts`, `concurrency-layers.ts`) — these spawn separate processes using `useService` (e.g. `node --import tsx ./example/services/*.ts`). Use these to exercise the process-based behavior. + +- **operation** (under `operation/`) — these use the `httpServer()` operation directly and run entirely in-process. They are faster and more deterministic for tests and quick iteration. + +Quick commands: + +Run the basic dependency example (use-service): + +```bash +cd packages/server +node --import tsx ./example/basic-graph.ts +``` + +Run the basic dependency example (operation): + +```bash +cd packages/server +node --import tsx ./example/operation/basic-graph.ts +``` + +These examples make use of the small service implementations in `./example/services`. diff --git a/packages/server/example/basic-graph.ts b/packages/server/example/basic-graph.ts new file mode 100644 index 00000000..bdc3c809 --- /dev/null +++ b/packages/server/example/basic-graph.ts @@ -0,0 +1,55 @@ +#!/usr/bin/env node +import { sleep, each, type Stream } from "effection"; +import { useService } from "../src/service.ts"; +import { useServiceGraph } from "../src/services.ts"; +import { simulationCLI } from "../src/cli.ts"; + +const services = { + A: { + operation: useService("A", "node --import tsx ./example/services/a.ts", { + wellnessCheck: { + frequency: 10, + *operation(stdio: Stream) { + for (let line of yield* each(stdio)) { + if (line.includes("started")) { + console.log("A ready (wellnessCheck)"); + return { ok: true } as any; + } + yield* each.next(); + } + }, + }, + }), + }, + B: { + operation: useService("B", "node --import tsx ./example/services/b.ts", { + wellnessCheck: { + frequency: 10, + *operation(stdio: Stream) { + for (let line of yield* each(stdio)) { + if (line.includes("started")) { + console.log("B ready (wellnessCheck)"); + return { ok: true } as any; + } + yield* each.next(); + } + }, + }, + }), + deps: ["A"], + }, +}; + +export function example(opts: { duration?: number } = {}) { + return (function* () { + yield* useServiceGraph(services as any); + yield* sleep(opts.duration ?? 300); + console.log(`Basic example complete`); + })(); +} + +import { fileURLToPath } from "node:url"; +if (process.argv[1] === fileURLToPath(import.meta.url)) { + // run via CLI when executed directly + simulationCLI(services); +} diff --git a/packages/server/example/concurrency-layers.ts b/packages/server/example/concurrency-layers.ts new file mode 100644 index 00000000..68c7dca9 --- /dev/null +++ b/packages/server/example/concurrency-layers.ts @@ -0,0 +1,68 @@ +#!/usr/bin/env node +import { sleep, each, type Stream } from "effection"; +import { useServiceGraph } from "../src/services.ts"; +import { useService } from "../src/service.ts"; +import { simulationCLI } from "../src/cli.ts"; + +const services = { + fast: { + operation: useService( + "fast", + "node --import tsx ./example/services/fast.ts", + { + wellnessCheck: { + frequency: 10, + *operation(stdio: Stream) { + for (let line of yield* each(stdio)) { + if (line.includes("started")) { + console.log("fast ready"); + return { ok: true } as any; + } + yield* each.next(); + } + }, + }, + } + ), + }, + slow: { + operation: useService( + "slow", + "node --import tsx ./example/services/slow.ts", + { + wellnessCheck: { + frequency: 10, + *operation(stdio: Stream) { + for (let line of yield* each(stdio)) { + if (line.includes("started")) { + console.log("slow ready"); + return { ok: true } as any; + } + yield* each.next(); + } + }, + }, + } + ), + }, + dependent: { + deps: ["fast", "slow"], + operation: (function* () { + console.log("dependent: all deps started; running dependent logic"); + yield* sleep(50); + })(), + }, +}; + +import { fileURLToPath } from "node:url"; +if (process.argv[1] === fileURLToPath(import.meta.url)) { + simulationCLI(services as any); +} + +export function example(opts: { duration?: number } = {}) { + return (function* () { + yield* useServiceGraph(services as any); + yield* sleep(opts.duration ?? 300); + console.log(`Concurrency example complete`); + })(); +} diff --git a/packages/server/example/debug-run.ts b/packages/server/example/debug-run.ts new file mode 100644 index 00000000..57024b0d --- /dev/null +++ b/packages/server/example/debug-run.ts @@ -0,0 +1,10 @@ +import { run, spawn, sleep } from "effection"; + +run(function* () { + yield* spawn(function* () { + console.log("debug - spawn start"); + yield* sleep(100); + console.log("debug - spawn done"); + }); + console.log("debug - after spawn (should only print after spawn completes)"); +}); diff --git a/packages/server/example/lifecycle-hooks.ts b/packages/server/example/lifecycle-hooks.ts new file mode 100644 index 00000000..1636ef29 --- /dev/null +++ b/packages/server/example/lifecycle-hooks.ts @@ -0,0 +1,81 @@ +#!/usr/bin/env node +import { sleep, each, type Stream } from "effection"; +import { useService } from "../src/service.ts"; +import { useServiceGraph } from "../src/services.ts"; +import { simulationCLI } from "../src/cli.ts"; + +const services = { + provider: { + operation: useService( + "provider", + "node --import tsx ./example/services/fast.ts", + { + wellnessCheck: { + frequency: 10, + *operation(stdio: Stream) { + for (let line of yield* each(stdio)) { + if (line.includes("started")) { + return { ok: true } as any; + } + yield* each.next(); + } + }, + }, + } + ), + afterStart() { + return (function* () { + console.log("provider: afterStart"); + })(); + }, + beforeStop() { + return (function* () { + console.log("provider: beforeStop"); + })(); + }, + }, + consumer: { + deps: ["provider"], + operation: useService( + "consumer", + "node --import tsx ./example/services/a.ts", + { + wellnessCheck: { + frequency: 10, + *operation(stdio: Stream) { + for (let line of yield* each(stdio)) { + if (line.includes("started")) { + return { ok: true } as any; + } + yield* each.next(); + } + }, + }, + } + ), + afterStart() { + return (function* () { + console.log("consumer: afterStart"); + })(); + }, + beforeStop() { + return (function* () { + console.log("consumer: beforeStop"); + })(); + }, + }, +}; + +import { fileURLToPath } from "node:url"; +if (process.argv[1] === fileURLToPath(import.meta.url)) { + simulationCLI(services); +} + +export function example(opts: { duration?: number } = {}) { + return (function* () { + console.log(`Starting lifecycle hooks example`); + yield* useServiceGraph(services as any); + yield* sleep(opts.duration ?? 150); + console.log(`Lifecycle example complete`); + })(); +} diff --git a/packages/server/example/operation/basic-graph.ts b/packages/server/example/operation/basic-graph.ts new file mode 100644 index 00000000..b8d2d11a --- /dev/null +++ b/packages/server/example/operation/basic-graph.ts @@ -0,0 +1,28 @@ +#!/usr/bin/env node +import { sleep } from "effection"; +import { useServiceGraph } from "../../src/services.ts"; +import { httpServer } from "../services/http-server.ts"; +import { simulationCLI } from "../../src/cli.ts"; + +export const services = { + A: { + operation: httpServer({ startDelay: 10 }), + }, + B: { + deps: ["A"], + operation: httpServer({ startDelay: 20 }), + }, +}; + +export function example(opts: { duration?: number } = {}) { + return (function* () { + yield* useServiceGraph(services as any); + yield* sleep(opts.duration ?? 300); + console.log(`Basic (operation) example complete`); + })(); +} + +import { fileURLToPath } from "node:url"; +if (process.argv[1] === fileURLToPath(import.meta.url)) { + simulationCLI(services as any); +} diff --git a/packages/server/example/operation/concurrency-layers.ts b/packages/server/example/operation/concurrency-layers.ts new file mode 100644 index 00000000..8ec052f8 --- /dev/null +++ b/packages/server/example/operation/concurrency-layers.ts @@ -0,0 +1,32 @@ +#!/usr/bin/env node +import { sleep } from "effection"; +import { useServiceGraph } from "../../src/services.ts"; +import { httpServer } from "../services/http-server.ts"; +import { simulationCLI } from "../../src/cli.ts"; + +export const services = { + fast: { operation: httpServer({ startDelay: 10 }) }, + slow: { operation: httpServer({ startDelay: 100 }) }, + dependent: { + deps: ["fast", "slow"], + operation: (function* () { + console.log( + "dependent: all deps started; running dependent logic (operation)" + ); + yield* sleep(50); + })(), + }, +}; + +import { fileURLToPath } from "node:url"; +if (process.argv[1] === fileURLToPath(import.meta.url)) { + simulationCLI(services as any); +} + +export function example(opts: { duration?: number } = {}) { + return (function* () { + yield* useServiceGraph(services as any); + yield* sleep(opts.duration ?? 300); + console.log(`Concurrency example (operation) complete`); + })(); +} diff --git a/packages/server/example/operation/lifecycle-hooks.ts b/packages/server/example/operation/lifecycle-hooks.ts new file mode 100644 index 00000000..9543d1d2 --- /dev/null +++ b/packages/server/example/operation/lifecycle-hooks.ts @@ -0,0 +1,49 @@ +#!/usr/bin/env node +import { sleep } from "effection"; +import { useServiceGraph } from "../../src/services.ts"; +import { httpServer } from "../services/http-server.ts"; +import { simulationCLI } from "../../src/cli.ts"; + +export const services = { + provider: { + operation: httpServer({ startDelay: 10 }), + afterStart() { + return (function* () { + console.log("provider: afterStart (operation)"); + })(); + }, + beforeStop() { + return (function* () { + console.log("provider: beforeStop (operation)"); + })(); + }, + }, + consumer: { + deps: ["provider"], + operation: httpServer({ startDelay: 10 }), + afterStart() { + return (function* () { + console.log("consumer: afterStart (operation)"); + })(); + }, + beforeStop() { + return (function* () { + console.log("consumer: beforeStop (operation)"); + })(); + }, + }, +}; + +export function example(opts: { duration?: number } = {}) { + return (function* () { + console.log(`Starting lifecycle hooks example (operation)`); + yield* useServiceGraph(services as any); + yield* sleep(opts.duration ?? 150); + console.log(`Lifecycle example (operation) complete`); + })(); +} + +import { fileURLToPath } from "node:url"; +if (process.argv[1] === fileURLToPath(import.meta.url)) { + simulationCLI(services as any); +} diff --git a/packages/server/example/services/a.ts b/packages/server/example/services/a.ts new file mode 100644 index 00000000..c54314b0 --- /dev/null +++ b/packages/server/example/services/a.ts @@ -0,0 +1,4 @@ +import { main } from "effection"; +import { httpServer } from "./http-server.ts"; + +main(() => httpServer({ startDelay: 10 })); diff --git a/packages/server/example/services/b.ts b/packages/server/example/services/b.ts new file mode 100644 index 00000000..8e54a1e2 --- /dev/null +++ b/packages/server/example/services/b.ts @@ -0,0 +1,4 @@ +import { main } from "effection"; +import { httpServer } from "./http-server.ts"; + +main(() => httpServer({ startDelay: 40 })); diff --git a/packages/server/example/services/fast.ts b/packages/server/example/services/fast.ts new file mode 100644 index 00000000..c54314b0 --- /dev/null +++ b/packages/server/example/services/fast.ts @@ -0,0 +1,4 @@ +import { main } from "effection"; +import { httpServer } from "./http-server.ts"; + +main(() => httpServer({ startDelay: 10 })); diff --git a/packages/server/example/services/http-server.ts b/packages/server/example/services/http-server.ts new file mode 100644 index 00000000..d5cc5a15 --- /dev/null +++ b/packages/server/example/services/http-server.ts @@ -0,0 +1,45 @@ +import http from "node:http"; +import type { Operation } from "effection"; +import { sleep, suspend, resource, withResolvers } from "effection"; + +export type HttpServerOptions = { + port?: number; + startDelay?: number; // ms +}; + +export function httpServer(options: HttpServerOptions = {}): Operation { + return resource(function* () { + if (options.startDelay) { + yield* sleep(options.startDelay); + } + + const port = options.port ?? 0; + const server = http.createServer((req, res) => { + if (req.url === "/status") { + res.writeHead(200, { "content-type": "text/plain" }); + res.end("ok"); + return; + } + res.writeHead(404); + res.end(); + }); + + const ready = withResolvers(); + server.listen(port, () => { + const address = server.address() as any; + const p = typeof address === "object" && address ? address.port : port; + console.log(`http server started on port ${p}`); + ready.resolve(); + }); + server.on("error", (err) => ready.reject(err as Error)); + + // wait for server to be listening + yield* ready.operation; + + try { + yield* suspend(); + } finally { + server.close(); + } + }) as Operation; +} diff --git a/packages/server/example/services/slow.ts b/packages/server/example/services/slow.ts new file mode 100644 index 00000000..05fa6da6 --- /dev/null +++ b/packages/server/example/services/slow.ts @@ -0,0 +1,4 @@ +import { main } from "effection"; +import { httpServer } from "./http-server.ts"; + +main(() => httpServer({ startDelay: 100 })); diff --git a/packages/server/package.json b/packages/server/package.json index ab57cc1f..1dd2b5e9 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -32,7 +32,9 @@ "build": "tsdown", "lint": "echo noop", "tsc": "tsc --noEmit", - "test:service-main": "node --import tsx ./test/service-main.ts" + "example:basic": "node --import tsx ./example/basic-graph.ts", + "example:lifecycle": "node --import tsx ./example/lifecycle-hooks.ts", + "example:concurrency": "node --import tsx ./example/concurrency-layers.ts" }, "dependencies": { "effection": "^3.6.0", diff --git a/packages/server/src/cli.ts b/packages/server/src/cli.ts new file mode 100644 index 00000000..92a6e246 --- /dev/null +++ b/packages/server/src/cli.ts @@ -0,0 +1,44 @@ +import { parseArgs } from "node:util"; +import { suspend, main } from "effection"; +import { useServiceGraph } from "./index.ts"; + +// Internal generator operation used by both the CLI (via main) and programmatically +export function* simulationCLIOp( + services: Parameters[0] +) { + const { values } = parseArgs({ + options: { + services: { type: "string", short: "s" }, + debug: { type: "boolean", short: "d" }, + help: { type: "boolean", short: "h" }, + }, + allowPositionals: true, + allowNegative: true, + allowUnknown: true, + }); + + function* printUsage() { + console.log(`Usage: cli [-s|--services serviceName] +Available services: ${Object.keys(services).join(", ")} +`); + } + + if (values.help) { + return yield* printUsage(); + } + + const subset = values.services + ? (values.services as string) + .split(",") + .map((s) => s.trim()) + .filter(Boolean) + : undefined; + + yield* useServiceGraph(services, subset); + yield* suspend(); +} + +// Public helper: call this from examples or CLI entrypoints — it will invoke effection's main() +export function simulationCLI(services: Parameters[0]) { + return main(() => simulationCLIOp(services)); +} diff --git a/packages/server/src/services.ts b/packages/server/src/services.ts index db34be9b..0187d6f1 100644 --- a/packages/server/src/services.ts +++ b/packages/server/src/services.ts @@ -6,9 +6,16 @@ import { withResolvers, } from "effection"; -export type ServiceDefinition = { +export type ServiceDefinition = { // The operation that starts the service and returns when the service is ready. - operation: Operation; + // The operation may be provided either as an `Operation` (for example the + // `Operation` returned by `useService(...)`) or as a factory that + // returns an `Operation`. The operation may return a value of any type + // which will be exposed to dependent services via an `exportsOperation` + // on the service definition at runtime. + operation: Operation | (() => Operation); + // optional runtime field - the operation that resolves to the exported value + exportsOperation?: Operation; deps?: string[]; options?: { // Keep an options object for future expansion or hooks; currently unused when operation is present @@ -20,7 +27,23 @@ export type ServiceDefinition = { afterStop?: () => Operation; }; -export type ServicesMap = Record; +export type ServicesMap = Record>; + +// helper type to extract the return type from an Operation or an operation factory +type OpReturn = T extends Operation + ? U + : T extends () => Operation + ? U + : never; + +// Augment a service map type to include an optional `exportsOperation` for each +// service keyed by the return type of its `operation`. +export type ServicesWithExports> = + { + [K in keyof S]: S[K] & { + exportsOperation?: Operation>; + }; + }; function computeLevels(services: ServicesMap): string[][] { const indeg: Record = {}; @@ -95,11 +118,38 @@ function* waitForAllReady( * Services within the same topological layer are started concurrently. Lifecycle * hooks can be used to perform actions before or after each service starts or stops. */ -export function useServiceGraph(services: ServicesMap): Operation { +export function useServiceGraph< + S extends Record> +>( + services: ServicesWithExports | ServicesMap, + subset?: string[] | string +): Operation { return resource(function* (provide) { - const layers = computeLevels(services); + // If a subset is provided, compute the closure including dependencies + let effectiveServices: ServicesMap = services; + if (subset) { + const want = new Set( + (typeof subset === "string" ? subset.split(",") : subset).map((s) => + s.trim() + ) + ); + const included = new Set(); + function include(name: string) { + if (included.has(name)) return; + if (!(name in services)) + throw new Error(`Requested service '${name}' not found`); + included.add(name); + for (const dep of services[name].deps ?? []) include(dep); + } + for (const name of want) include(name); + effectiveServices = {} as ServicesMap; + for (const name of included) effectiveServices[name] = services[name]; + } + + const layers = computeLevels(effectiveServices); - // Map of readiness resolvers returned by `withResolvers` + // Map of readiness resolvers returned by `withResolvers` for the + // effective services we plan to start. const readyResolvers = new Map< string, { @@ -108,7 +158,7 @@ export function useServiceGraph(services: ServicesMap): Operation { reject: (err: Error) => void; } >(); - for (const name of Object.keys(services)) { + for (const name of Object.keys(effectiveServices)) { const r = withResolvers(); readyResolvers.set(name, { operation: r.operation, @@ -117,13 +167,33 @@ export function useServiceGraph(services: ServicesMap): Operation { }); } + // Map of export resolvers so services can expose a value to dependents + const exportResolvers = new Map< + string, + { + operation: Operation; + resolve: (v: any) => void; + reject: (err: Error) => void; + } + >(); + for (const name of Object.keys(effectiveServices)) { + const r = withResolvers(); + exportResolvers.set(name, { + operation: r.operation, + resolve: r.resolve, + reject: r.reject, + }); + // attach exportsOperation so dependents can yield it + (effectiveServices as any)[name].exportsOperation = r.operation; + } + // Keep track of start order so we can run beforeStop hooks in reverse const startOrder: string[] = []; for (const layer of layers) { // spawn all services in this layer in parallel for (const name of layer) { - const def = services[name]; + const def = effectiveServices[name]; // wait for deps to be ready (yield the underlying Promise) if (def.deps) { for (const dep of def.deps) { @@ -141,8 +211,18 @@ export function useServiceGraph(services: ServicesMap): Operation { try { if (def.beforeStart) yield* def.beforeStart(); - // The caller-supplied operation starts the service and returns when ready. - yield* def.operation; + // The caller-supplied operation starts the service and may be + // provided as a factory function or as an already-created + // Operation. Resolve it to an Operation here so we can yield it + // and capture any exported value it returns. + const operation: Operation = + typeof def.operation === "function" + ? (def.operation as () => Operation)() + : (def.operation as Operation); + + const exported = yield* operation; + const exportRes = exportResolvers.get(name); + if (exportRes) exportRes.resolve(exported); startOrder.push(name); const res = readyResolvers.get(name); if (res) res.resolve(); diff --git a/packages/server/test/examples-smoke.test.ts b/packages/server/test/examples-smoke.test.ts new file mode 100644 index 00000000..2677d7eb --- /dev/null +++ b/packages/server/test/examples-smoke.test.ts @@ -0,0 +1,111 @@ +import { it } from "node:test"; +import http from "node:http"; +import { run, spawn, sleep } from "effection"; + +import { services as basicServices } from "../example/operation/basic-graph.ts"; +import { services as lifecycleServices } from "../example/operation/lifecycle-hooks.ts"; +import { services as concurrencyServices } from "../example/operation/concurrency-layers.ts"; + +function checkStatus(port: number): Promise { + return new Promise((resolve, reject) => { + const req = http.get( + { hostname: "127.0.0.1", port, path: "/status" }, + (res) => { + resolve(res.statusCode ?? 0); + } + ); + req.on("error", reject); + }); +} + +import { useServiceGraph } from "../src/services.ts"; + +it("basic example imports and runs", async () => { + const services = basicServices; + + // start the graph and await exported ports in a single run operation + const ports: number[] = await run(function* () { + yield* spawn(function* () { + yield* useServiceGraph(services as any); + }); + + const ps: number[] = []; + for (const name of Object.keys(services)) { + const exportsOp = (services as any)[name].exportsOperation; + if (!exportsOp) throw new Error(`no exportsOperation on ${name}`); + ps.push(yield* exportsOp); + } + + // keep the graph alive briefly to allow HTTP checks + yield* sleep(200); + return ps as number[]; + }); + + // check each port + for (const p of ports) { + let ok = false; + for (let i = 0; i < 100; i++) { + try { + const status = await checkStatus(p); + if (status === 200) { + ok = true; + break; + } + } catch (err) { + // ignore and retry + } + await new Promise((r) => setTimeout(r, 10)); + } + if (!ok) throw new Error(`port ${p} did not return 200`); + } +}); + +it("lifecycle example imports and runs", async () => { + const services = lifecycleServices; + + const ports: number[] = await run(function* () { + yield* spawn(function* () { + yield* useServiceGraph(services as any); + }); + + const ps: number[] = []; + for (const name of Object.keys(services)) { + const exportsOp = (services as any)[name].exportsOperation; + if (!exportsOp) throw new Error(`no exportsOperation on ${name}`); + ps.push(yield* exportsOp); + } + + yield* sleep(200); + return ps as number[]; + }); + + for (const p of ports) { + const status = await checkStatus(p); + if (status !== 200) throw new Error(`port ${p} did not return 200`); + } +}); + +it("concurrency example imports and runs", async () => { + const services = concurrencyServices; + + const ports: number[] = await run(function* () { + yield* spawn(function* () { + yield* useServiceGraph(services as any); + }); + + const ps: number[] = []; + for (const name of Object.keys(services)) { + const exportsOp = (services as any)[name].exportsOperation; + if (!exportsOp) throw new Error(`no exportsOperation on ${name}`); + ps.push(yield* exportsOp); + } + + yield* sleep(200); + return ps as number[]; + }); + + for (const p of ports) { + const status = await checkStatus(p); + if (status !== 200) throw new Error(`port ${p} did not return 200`); + } +}); diff --git a/packages/server/test/service.test.ts b/packages/server/test/service.test.ts index 9340e6ef..be05096b 100644 --- a/packages/server/test/service.test.ts +++ b/packages/server/test/service.test.ts @@ -5,7 +5,7 @@ import { each, Err, Ok, run } from "effection"; // these npm scripts don't work, but this is what we are trying to run // const scriptDoesNotWork = "npm run test:service-main"; -const nodeScriptWorks = "node --import tsx ./test/service-main.ts"; +const nodeScriptWorks = "node --import tsx ./test/services/service-main.ts"; it("test service", async () => { let assertionCount = 0; diff --git a/packages/server/test/services.test.ts b/packages/server/test/services.test.ts new file mode 100644 index 00000000..d49a42dc --- /dev/null +++ b/packages/server/test/services.test.ts @@ -0,0 +1,262 @@ +import { it } from "node:test"; +import assert from "node:assert"; +import { run, sleep, spawn, Ok, suspend, type Operation } from "effection"; +import { useServiceGraph } from "../src/services.ts"; +import { useService } from "../src/service.ts"; + +it("starts services in dependency order", async () => { + const startTimes = new Map(); + try { + await run(function* () { + yield* spawn(function* () { + yield* useServiceGraph({ + A: { + operation: useService( + "A", + "node --import tsx ./test/services/service-a.ts", + { + wellnessCheck: { + frequency: 10, + *operation(_stdio) { + yield* sleep(20); + startTimes.set("A", Date.now()); + return Ok(void 0); + }, + }, + } + ), + }, + B: { + operation: useService( + "B", + "node --import tsx ./test/services/service-b.ts", + { + wellnessCheck: { + frequency: 10, + *operation(_stdio) { + yield* sleep(40); + startTimes.set("B", Date.now()); + return Ok(void 0); + }, + }, + } + ), + deps: ["A"], + }, + }); + }); + // The graph is running; sleep a short time to let the services start + yield* sleep(200); + }); + } catch (err) { + console.log("run error:", err instanceof Error ? err.stack : err); + } + + const aStarted = startTimes.get("A"); + const bStarted = startTimes.get("B"); + assert.ok(typeof aStarted === "number", "A started should be recorded"); + assert.ok(typeof bStarted === "number", "B started should be recorded"); + assert(aStarted! <= bStarted!, "A should start before B"); +}); + +it("throws on cycles in dependency graph", async () => { + await assert.rejects(async () => { + await run(function* () { + yield* useServiceGraph({ + A: { + operation: useService( + "A", + "node --import tsx ./test/services/service-a.ts" + ), + deps: ["B"], + }, + B: { + operation: useService( + "B", + "node --import tsx ./test/services/service-b.ts" + ), + deps: ["A"], + }, + }); + }); + }, /Cycle detected in services/); +}); + +it("runs beforeStop hooks in reverse order", async () => { + const stopOrder: string[] = []; + const startedOrder: string[] = []; + await run(function* () { + // spawn and cancel automatically when run returns + yield* spawn(function* () { + yield* useServiceGraph({ + A: { + operation: useService( + "A", + "node --import tsx ./test/services/service-a.ts", + { + wellnessCheck: { + frequency: 10, + *operation(_stdio) { + yield* sleep(20); + startedOrder.push("A"); + return Ok(void 0); + }, + }, + } + ), + beforeStop() { + return (function* () { + stopOrder.push("A"); + })() as unknown as Operation; + }, + }, + B: { + operation: useService( + "B", + "node --import tsx ./test/services/service-b.ts", + { + wellnessCheck: { + frequency: 10, + *operation(_stdio) { + yield* sleep(40); + startedOrder.push("B"); + return Ok(void 0); + }, + }, + } + ), + deps: ["A"], + beforeStop() { + return (function* () { + stopOrder.push("B"); + })() as unknown as Operation; + }, + }, + }); + }); + // let them start + yield* sleep(200); + }); + assert.strictEqual(startedOrder.join(""), "AB"); + assert.strictEqual(stopOrder.join(""), "BA"); +}); + +it("starts independent services in parallel", async () => { + const startTimes = new Map(); + try { + await run(function* () { + yield* spawn(function* () { + yield* useServiceGraph({ + fast: { + operation: useService( + "fast", + "node --import tsx ./test/services/service-fast.ts", + { + wellnessCheck: { + frequency: 10, + *operation(_stdio) { + yield* sleep(20); + startTimes.set("fast", Date.now()); + return Ok(void 0); + }, + }, + } + ), + }, + slow: { + operation: useService( + "slow", + "node --import tsx ./test/services/service-slow.ts", + { + wellnessCheck: { + frequency: 10, + *operation(_stdio) { + yield* sleep(50); + startTimes.set("slow", Date.now()); + return Ok(void 0); + }, + }, + } + ), + }, + }); + }); + yield* sleep(250); + }); + const fastStarted = startTimes.get("fast"); + const slowStarted = startTimes.get("slow"); + assert.ok( + typeof fastStarted === "number", + "fast started should be recorded" + ); + assert.ok( + typeof slowStarted === "number", + "slow started should be recorded" + ); + assert(fastStarted! <= slowStarted!, "fast should start before slow"); + } finally { + // cleanup + } +}); + +it("runs subset of services with dependencies", async () => { + const startTimes = new Map(); + await run(function* () { + yield* spawn(function* () { + const services = { + fast: { + operation: useService( + "fast", + "node --import tsx ./test/services/service-fast.ts", + { + wellnessCheck: { + frequency: 10, + *operation(_stdio) { + yield* sleep(20); + startTimes.set("fast", Date.now()); + return Ok(void 0); + }, + }, + } + ), + }, + slow: { + operation: useService( + "slow", + "node --import tsx ./test/services/service-slow.ts", + { + wellnessCheck: { + frequency: 10, + *operation(_stdio) { + yield* sleep(50); + startTimes.set("slow", Date.now()); + return Ok(void 0); + }, + }, + } + ), + }, + dependent: { + deps: ["fast", "slow"], + operation: (function* () { + startTimes.set("dependent", Date.now()); + yield* suspend(); + })() as unknown as Operation, + }, + } as any; + + // only request dependent; fast and slow should be included as deps + yield* useServiceGraph(services, ["dependent"]); + }); + yield* sleep(300); + }); + + const f = startTimes.get("fast"); + const s = startTimes.get("slow"); + const d = startTimes.get("dependent"); + assert.ok(typeof f === "number", "fast should start"); + assert.ok(typeof s === "number", "slow should start"); + assert.ok(typeof d === "number", "dependent should start"); + assert(f! <= d!, "fast should start before dependent"); + assert(s! <= d!, "slow should start before dependent"); +}); diff --git a/packages/server/test/services/service-a.ts b/packages/server/test/services/service-a.ts new file mode 100644 index 00000000..7d3da53d --- /dev/null +++ b/packages/server/test/services/service-a.ts @@ -0,0 +1,4 @@ +import { main } from "effection"; +import { httpServer } from "../../example/services/http-server.ts"; + +main(() => httpServer({ startDelay: 10 })); diff --git a/packages/server/test/services/service-b.ts b/packages/server/test/services/service-b.ts new file mode 100644 index 00000000..2d285ac5 --- /dev/null +++ b/packages/server/test/services/service-b.ts @@ -0,0 +1,4 @@ +import { main } from "effection"; +import { httpServer } from "../../example/services/http-server.ts"; + +main(() => httpServer({ startDelay: 40 })); diff --git a/packages/server/test/services/service-fast.ts b/packages/server/test/services/service-fast.ts new file mode 100644 index 00000000..7d3da53d --- /dev/null +++ b/packages/server/test/services/service-fast.ts @@ -0,0 +1,4 @@ +import { main } from "effection"; +import { httpServer } from "../../example/services/http-server.ts"; + +main(() => httpServer({ startDelay: 10 })); diff --git a/packages/server/test/service-main.ts b/packages/server/test/services/service-main.ts similarity index 100% rename from packages/server/test/service-main.ts rename to packages/server/test/services/service-main.ts diff --git a/packages/server/test/services/service-slow.ts b/packages/server/test/services/service-slow.ts new file mode 100644 index 00000000..245679aa --- /dev/null +++ b/packages/server/test/services/service-slow.ts @@ -0,0 +1,4 @@ +import { main } from "effection"; +import { httpServer } from "../../example/services/http-server.ts"; + +main(() => httpServer({ startDelay: 200 })); diff --git a/packages/server/tsconfig.json b/packages/server/tsconfig.json index 43e82621..f338bcfd 100644 --- a/packages/server/tsconfig.json +++ b/packages/server/tsconfig.json @@ -4,5 +4,5 @@ "baseUrl": ".", "outDir": "dist" }, - "include": ["src/**/*.ts", "test/**/*.ts"] + "include": ["src/**/*.ts", "test/**/*.ts", "example/**/*.ts"] } From 6645a876fc54afbfb52fc088772911b5e296c73d Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Sat, 3 Jan 2026 01:20:43 -0600 Subject: [PATCH 04/38] adjust useServiceGraph to more directly pass dep returns --- packages/server/README.md | 10 +- packages/server/example/basic-graph.ts | 17 +- packages/server/example/concurrency-layers.ts | 26 +- packages/server/example/lifecycle-hooks.ts | 15 +- .../server/example/operation/basic-graph.ts | 11 +- .../example/operation/concurrency-layers.ts | 11 +- .../server/example/operation/data-sharing.ts | 152 +++++++++ .../example/operation/lifecycle-hooks.ts | 11 +- packages/server/example/services/a.ts | 4 +- packages/server/example/services/b.ts | 4 +- packages/server/example/services/fast.ts | 4 +- .../server/example/services/http-server.ts | 16 +- packages/server/example/services/slow.ts | 4 +- packages/server/package.json | 2 +- packages/server/src/cli.ts | 19 +- packages/server/src/services.ts | 297 +++++++++++++----- packages/server/test/data-sharing.test.ts | 222 +++++++++++++ packages/server/test/examples-smoke.test.ts | 204 +++++++++--- packages/server/test/service.test.ts | 2 +- packages/server/test/services.test.ts | 15 +- packages/server/test/services/service-a.ts | 4 +- packages/server/test/services/service-b.ts | 4 +- packages/server/test/services/service-fast.ts | 4 +- packages/server/test/services/service-slow.ts | 4 +- 24 files changed, 871 insertions(+), 191 deletions(-) create mode 100644 packages/server/example/operation/data-sharing.ts create mode 100644 packages/server/test/data-sharing.test.ts diff --git a/packages/server/README.md b/packages/server/README.md index 122f1770..326ca85a 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -13,9 +13,9 @@ https://github.com/thefrontside/simulacrum Key points: -- `useServiceGraph(services: ServicesMap): Operation` — starts a DAG of services. Each service in the map is declared as a `ServiceDefinition`. +- `useServiceGraph(services: ServicesMap, options?: { sequential?: boolean }): ServiceRunner` — returns a _runner function_ which you call to start the graph: `const run = useServiceGraph(services, options); yield* run(subset?: string[] | string);`. By default services in the same topological layer run concurrently; pass `options.sequential = true` to run services in each layer serially. - `ServiceDefinition.operation` (required) — an `Operation` which indicates the service has started. This operation may be long-lived (e.g. `useService`) or may return once the service is ready while a background child keeps the service running. See the example below. -- `deps` — an optional list of service names this service depends on; services without dependencies in the same layer are started concurrently. +- `deps` — an optional list of service names this service depends on; services without dependencies in the same layer are started concurrently by default, or serially when `options.sequential` is true. - Lifecycle hooks: `beforeStart`, `afterStart`, `beforeStop`, `afterStop` — each is an `Operation` that runs at the appropriate time. Example: @@ -30,7 +30,7 @@ main(function* () { // process is spawned and, if a wellnessCheck is provided, once the // wellnessCheck passes. The service is automatically shut down by // effection when the operation goes out of scope. - yield* useServiceGraph({ + const run = useServiceGraph({ A: { operation: useService( "A", @@ -51,7 +51,7 @@ main(function* () { Notes: -- `useServiceGraph` returns an `Operation` that holds while services run and only cancels on parent scope termination. +- `useServiceGraph` returns a _runner function_; calling the runner (e.g. `yield* run()`) returns an `Operation` that holds while services run and only cancels on parent scope termination. The returned runner has a `.services` property for introspection and can be passed directly to `simulationCLI`. - If you want to start services sequentially or add more advanced concurrency control, compose operations yourself and use `spawn` to control how operations run. ### Lifecycle hooks @@ -153,5 +153,5 @@ const services = { }, }; -// pass `services` to `useServiceGraph` or `simulationCLI` +// create a runner via `useServiceGraph(services)` and pass that runner to `simulationCLI` if desired, e.g. `simulationCLI(useServiceGraph(services))` ```` diff --git a/packages/server/example/basic-graph.ts b/packages/server/example/basic-graph.ts index bdc3c809..ce1d2108 100644 --- a/packages/server/example/basic-graph.ts +++ b/packages/server/example/basic-graph.ts @@ -4,7 +4,7 @@ import { useService } from "../src/service.ts"; import { useServiceGraph } from "../src/services.ts"; import { simulationCLI } from "../src/cli.ts"; -const services = { +const servicesMap = { A: { operation: useService("A", "node --import tsx ./example/services/a.ts", { wellnessCheck: { @@ -13,10 +13,12 @@ const services = { for (let line of yield* each(stdio)) { if (line.includes("started")) { console.log("A ready (wellnessCheck)"); - return { ok: true } as any; + return { ok: true, value: undefined }; } yield* each.next(); } + // default: return success so the result type is well-formed + return { ok: true, value: undefined }; }, }, }), @@ -29,20 +31,25 @@ const services = { for (let line of yield* each(stdio)) { if (line.includes("started")) { console.log("B ready (wellnessCheck)"); - return { ok: true } as any; + return { ok: true, value: undefined }; } yield* each.next(); } + // default: return success so the result type is well-formed + return { ok: true, value: undefined }; }, }, }), - deps: ["A"], + deps: ["A"] as const, }, }; +export const services = useServiceGraph(servicesMap); + export function example(opts: { duration?: number } = {}) { return (function* () { - yield* useServiceGraph(services as any); + const run = services; + yield* run(); yield* sleep(opts.duration ?? 300); console.log(`Basic example complete`); })(); diff --git a/packages/server/example/concurrency-layers.ts b/packages/server/example/concurrency-layers.ts index 68c7dca9..953260e2 100644 --- a/packages/server/example/concurrency-layers.ts +++ b/packages/server/example/concurrency-layers.ts @@ -4,7 +4,7 @@ import { useServiceGraph } from "../src/services.ts"; import { useService } from "../src/service.ts"; import { simulationCLI } from "../src/cli.ts"; -const services = { +const servicesMap = { fast: { operation: useService( "fast", @@ -16,10 +16,12 @@ const services = { for (let line of yield* each(stdio)) { if (line.includes("started")) { console.log("fast ready"); - return { ok: true } as any; + return { ok: true, value: undefined }; } yield* each.next(); } + // default success + return { ok: true, value: undefined }; }, }, } @@ -36,17 +38,19 @@ const services = { for (let line of yield* each(stdio)) { if (line.includes("started")) { console.log("slow ready"); - return { ok: true } as any; + return { ok: true, value: undefined }; } yield* each.next(); } + // default success + return { ok: true, value: undefined }; }, }, } ), }, dependent: { - deps: ["fast", "slow"], + deps: ["fast", "slow"] as const, operation: (function* () { console.log("dependent: all deps started; running dependent logic"); yield* sleep(50); @@ -54,14 +58,22 @@ const services = { }, }; +export const services = useServiceGraph(servicesMap); + import { fileURLToPath } from "node:url"; if (process.argv[1] === fileURLToPath(import.meta.url)) { - simulationCLI(services as any); + simulationCLI(services); } -export function example(opts: { duration?: number } = {}) { +export function example( + opts: { duration?: number; sequential?: boolean } = {} +) { return (function* () { - yield* useServiceGraph(services as any); + if (opts.sequential) { + console.log("Running concurrency example in sequential mode"); + } + const run = services; + yield* run(); yield* sleep(opts.duration ?? 300); console.log(`Concurrency example complete`); })(); diff --git a/packages/server/example/lifecycle-hooks.ts b/packages/server/example/lifecycle-hooks.ts index 1636ef29..9366b5ab 100644 --- a/packages/server/example/lifecycle-hooks.ts +++ b/packages/server/example/lifecycle-hooks.ts @@ -4,7 +4,7 @@ import { useService } from "../src/service.ts"; import { useServiceGraph } from "../src/services.ts"; import { simulationCLI } from "../src/cli.ts"; -const services = { +const servicesMap = { provider: { operation: useService( "provider", @@ -15,10 +15,11 @@ const services = { *operation(stdio: Stream) { for (let line of yield* each(stdio)) { if (line.includes("started")) { - return { ok: true } as any; + return { ok: true, value: undefined }; } yield* each.next(); } + return { ok: true, value: undefined }; }, }, } @@ -35,7 +36,7 @@ const services = { }, }, consumer: { - deps: ["provider"], + deps: ["provider"] as const, operation: useService( "consumer", "node --import tsx ./example/services/a.ts", @@ -45,10 +46,11 @@ const services = { *operation(stdio: Stream) { for (let line of yield* each(stdio)) { if (line.includes("started")) { - return { ok: true } as any; + return { ok: true, value: undefined }; } yield* each.next(); } + return { ok: true, value: undefined }; }, }, } @@ -66,6 +68,8 @@ const services = { }, }; +export const services = useServiceGraph(servicesMap); + import { fileURLToPath } from "node:url"; if (process.argv[1] === fileURLToPath(import.meta.url)) { simulationCLI(services); @@ -74,7 +78,8 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) { export function example(opts: { duration?: number } = {}) { return (function* () { console.log(`Starting lifecycle hooks example`); - yield* useServiceGraph(services as any); + const run = services; + yield* run(); yield* sleep(opts.duration ?? 150); console.log(`Lifecycle example complete`); })(); diff --git a/packages/server/example/operation/basic-graph.ts b/packages/server/example/operation/basic-graph.ts index b8d2d11a..61abbe98 100644 --- a/packages/server/example/operation/basic-graph.ts +++ b/packages/server/example/operation/basic-graph.ts @@ -4,19 +4,22 @@ import { useServiceGraph } from "../../src/services.ts"; import { httpServer } from "../services/http-server.ts"; import { simulationCLI } from "../../src/cli.ts"; -export const services = { +const servicesMap = { A: { operation: httpServer({ startDelay: 10 }), }, B: { - deps: ["A"], + deps: ["A"] as const, operation: httpServer({ startDelay: 20 }), }, }; +export const services = useServiceGraph(servicesMap); + export function example(opts: { duration?: number } = {}) { return (function* () { - yield* useServiceGraph(services as any); + const run = services; + yield* run(); yield* sleep(opts.duration ?? 300); console.log(`Basic (operation) example complete`); })(); @@ -24,5 +27,5 @@ export function example(opts: { duration?: number } = {}) { import { fileURLToPath } from "node:url"; if (process.argv[1] === fileURLToPath(import.meta.url)) { - simulationCLI(services as any); + simulationCLI(services); } diff --git a/packages/server/example/operation/concurrency-layers.ts b/packages/server/example/operation/concurrency-layers.ts index 8ec052f8..5e55c93a 100644 --- a/packages/server/example/operation/concurrency-layers.ts +++ b/packages/server/example/operation/concurrency-layers.ts @@ -4,11 +4,11 @@ import { useServiceGraph } from "../../src/services.ts"; import { httpServer } from "../services/http-server.ts"; import { simulationCLI } from "../../src/cli.ts"; -export const services = { +const servicesMap = { fast: { operation: httpServer({ startDelay: 10 }) }, slow: { operation: httpServer({ startDelay: 100 }) }, dependent: { - deps: ["fast", "slow"], + deps: ["fast", "slow"] as const, operation: (function* () { console.log( "dependent: all deps started; running dependent logic (operation)" @@ -18,14 +18,17 @@ export const services = { }, }; +export const services = useServiceGraph(servicesMap); + import { fileURLToPath } from "node:url"; if (process.argv[1] === fileURLToPath(import.meta.url)) { - simulationCLI(services as any); + simulationCLI(services); } export function example(opts: { duration?: number } = {}) { return (function* () { - yield* useServiceGraph(services as any); + const run = services; + yield* run(); yield* sleep(opts.duration ?? 300); console.log(`Concurrency example (operation) complete`); })(); diff --git a/packages/server/example/operation/data-sharing.ts b/packages/server/example/operation/data-sharing.ts new file mode 100644 index 00000000..6a844c0a --- /dev/null +++ b/packages/server/example/operation/data-sharing.ts @@ -0,0 +1,152 @@ +#!/usr/bin/env node +import { simulationCLI } from "../../src/cli.ts"; +import { useServiceGraph } from "../../src/services.ts"; +import { spawn, suspend, until } from "effection"; +import { createFoundationSimulationServer } from "@simulacrum/foundation-simulator"; +import http from "node:http"; + +export const createServiceASimulation = (seed: number): any => + createFoundationSimulationServer({ + port: 0, + extendRouter(router) { + router.get("/info", (_req, res) => res.json({ seed, handledWith: seed })); + }, + }); + +export const createServiceBSimulation = (used: number): any => + createFoundationSimulationServer({ + port: 0, + extendRouter(router) { + router.get("/info", (_req, res) => res.json({ used })); + }, + }); + +export const createServiceCSimulation = (message: string): any => + createFoundationSimulationServer({ + port: 0, + extendRouter(router) { + router.get("/info", (_req, res) => res.json({ dataMessage: message })); + }, + }); + +export const services = useServiceGraph({ + // short-lived data generator: returns a shared payload and completes + data: { + *operation() { + // generate some data for dependents + const payload = { seed: 42, message: "hello from data" }; + return payload; + }, + }, + + // serviceA depends on data and keeps running (long-running provider) + serviceA: { + deps: ["data"], + *operation({ data }) { + const createSim = createServiceASimulation(data.seed)(); + + const listening: any = yield* until(createSim.listen()); + + // debug log so tests can see the assigned port (left as example output) + // eslint-disable-next-line no-console + console.log( + `[data-sharing] started foundation sim on port ${listening.port}` + ); + + // self-check + try { + const local = yield* until( + new Promise<{ status?: number; body?: string }>((resolve, reject) => { + const req = http.get( + { + hostname: "127.0.0.1", + port: listening.port, + path: "/info", + agent: false, + }, + (res: any) => { + let body = ""; + res.on("data", (c: any) => (body += c)); + res.on("end", () => resolve({ status: res.statusCode, body })); + } + ); + req.on("error", reject); + }) + ); + // eslint-disable-next-line no-console + console.log(`[data-sharing] self-check /info:`, local); + } catch (err) { + // eslint-disable-next-line no-console + console.log(`[data-sharing] self-check error:`, err); + } + + // spawn background keeper which calls ensureClose when finalized + yield* spawn(function* () { + try { + yield* suspend(); + } finally { + // eslint-disable-next-line no-console + console.log( + `[data-sharing] ensuring close for port ${listening.port}` + ); + yield* until(listening.ensureClose()); + // eslint-disable-next-line no-console + console.log( + `[data-sharing] closed foundation sim on port ${listening.port}` + ); + } + }); + + return { handledWith: data.seed, port: listening.port }; + }, + }, + + // serviceB depends on serviceA and data + serviceB: { + deps: ["serviceA", "data"], + *operation({ serviceA, data }) { + const createSim = createServiceBSimulation(serviceA.handledWith)(); + + const listening: any = yield* until(createSim.listen()); + + yield* spawn(function* () { + try { + yield* suspend(); + } finally { + yield* until(listening.ensureClose()); + } + }); + + // include data seed in the export so tests can verify multi-dependency wiring + return { + used: serviceA.handledWith, + dataSeed: data.seed, + port: listening.port, + }; + }, + }, + + // serviceC depends only on data + serviceC: { + deps: ["data"], + *operation({ data }) { + const createSim = createServiceCSimulation(data.message)(); + + const listening: any = yield* until(createSim.listen()); + + yield* spawn(function* () { + try { + yield* suspend(); + } finally { + yield* until(listening.ensureClose()); + } + }); + + return { dataMessage: data.message, port: listening.port }; + }, + }, +}); + +if (import.meta.url === `file://${process.argv[1]}`) { + simulationCLI(services); +} diff --git a/packages/server/example/operation/lifecycle-hooks.ts b/packages/server/example/operation/lifecycle-hooks.ts index 9543d1d2..776b178d 100644 --- a/packages/server/example/operation/lifecycle-hooks.ts +++ b/packages/server/example/operation/lifecycle-hooks.ts @@ -4,7 +4,7 @@ import { useServiceGraph } from "../../src/services.ts"; import { httpServer } from "../services/http-server.ts"; import { simulationCLI } from "../../src/cli.ts"; -export const services = { +const servicesMap = { provider: { operation: httpServer({ startDelay: 10 }), afterStart() { @@ -19,7 +19,7 @@ export const services = { }, }, consumer: { - deps: ["provider"], + deps: ["provider"] as const, operation: httpServer({ startDelay: 10 }), afterStart() { return (function* () { @@ -34,10 +34,13 @@ export const services = { }, }; +export const services = useServiceGraph(servicesMap); + export function example(opts: { duration?: number } = {}) { return (function* () { console.log(`Starting lifecycle hooks example (operation)`); - yield* useServiceGraph(services as any); + const run = services; + yield* run(); yield* sleep(opts.duration ?? 150); console.log(`Lifecycle example (operation) complete`); })(); @@ -45,5 +48,5 @@ export function example(opts: { duration?: number } = {}) { import { fileURLToPath } from "node:url"; if (process.argv[1] === fileURLToPath(import.meta.url)) { - simulationCLI(services as any); + simulationCLI(services); } diff --git a/packages/server/example/services/a.ts b/packages/server/example/services/a.ts index c54314b0..9b9ce494 100644 --- a/packages/server/example/services/a.ts +++ b/packages/server/example/services/a.ts @@ -1,4 +1,6 @@ import { main } from "effection"; import { httpServer } from "./http-server.ts"; -main(() => httpServer({ startDelay: 10 })); +main(function* () { + yield* httpServer({ startDelay: 10 }); +}); diff --git a/packages/server/example/services/b.ts b/packages/server/example/services/b.ts index 8e54a1e2..47421f18 100644 --- a/packages/server/example/services/b.ts +++ b/packages/server/example/services/b.ts @@ -1,4 +1,6 @@ import { main } from "effection"; import { httpServer } from "./http-server.ts"; -main(() => httpServer({ startDelay: 40 })); +main(function* () { + yield* httpServer({ startDelay: 40 }); +}); diff --git a/packages/server/example/services/fast.ts b/packages/server/example/services/fast.ts index c54314b0..9b9ce494 100644 --- a/packages/server/example/services/fast.ts +++ b/packages/server/example/services/fast.ts @@ -1,4 +1,6 @@ import { main } from "effection"; import { httpServer } from "./http-server.ts"; -main(() => httpServer({ startDelay: 10 })); +main(function* () { + yield* httpServer({ startDelay: 10 }); +}); diff --git a/packages/server/example/services/http-server.ts b/packages/server/example/services/http-server.ts index d5cc5a15..7b212af3 100644 --- a/packages/server/example/services/http-server.ts +++ b/packages/server/example/services/http-server.ts @@ -1,14 +1,15 @@ import http from "node:http"; +import type { AddressInfo } from "node:net"; import type { Operation } from "effection"; -import { sleep, suspend, resource, withResolvers } from "effection"; +import { sleep, resource, withResolvers } from "effection"; export type HttpServerOptions = { port?: number; startDelay?: number; // ms }; -export function httpServer(options: HttpServerOptions = {}): Operation { - return resource(function* () { +export function httpServer(options: HttpServerOptions = {}): Operation { + return resource(function* (provide) { if (options.startDelay) { yield* sleep(options.startDelay); } @@ -26,7 +27,7 @@ export function httpServer(options: HttpServerOptions = {}): Operation { const ready = withResolvers(); server.listen(port, () => { - const address = server.address() as any; + const address = server.address() as AddressInfo | string | null; const p = typeof address === "object" && address ? address.port : port; console.log(`http server started on port ${p}`); ready.resolve(); @@ -36,10 +37,13 @@ export function httpServer(options: HttpServerOptions = {}): Operation { // wait for server to be listening yield* ready.operation; + const address = server.address() as AddressInfo | string | null; + const p = typeof address === "object" && address ? address.port : port; + try { - yield* suspend(); + yield* provide(p); } finally { server.close(); } - }) as Operation; + }); } diff --git a/packages/server/example/services/slow.ts b/packages/server/example/services/slow.ts index 05fa6da6..e038eeff 100644 --- a/packages/server/example/services/slow.ts +++ b/packages/server/example/services/slow.ts @@ -1,4 +1,6 @@ import { main } from "effection"; import { httpServer } from "./http-server.ts"; -main(() => httpServer({ startDelay: 100 })); +main(function* () { + yield* httpServer({ startDelay: 100 }); +}); diff --git a/packages/server/package.json b/packages/server/package.json index 7cbc6e11..7c7da4e8 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -27,7 +27,7 @@ ], "scripts": { "clean": "echo skip", - "test": "node --test-timeout=60000 --import tsx --test test/*.test.ts", + "test": "node --import tsx --test --test-timeout=60000 test/*.test.ts", "prepack": "npm run build", "build": "tsdown", "lint": "echo noop", diff --git a/packages/server/src/cli.ts b/packages/server/src/cli.ts index 92a6e246..e9ae2fd4 100644 --- a/packages/server/src/cli.ts +++ b/packages/server/src/cli.ts @@ -1,10 +1,10 @@ import { parseArgs } from "node:util"; import { suspend, main } from "effection"; -import { useServiceGraph } from "./index.ts"; +import type { ServiceRunner } from "./services.ts"; // Internal generator operation used by both the CLI (via main) and programmatically -export function* simulationCLIOp( - services: Parameters[0] +export function* simulationCLIOp>( + runner: ServiceRunner ) { const { values } = parseArgs({ options: { @@ -18,8 +18,11 @@ export function* simulationCLIOp( }); function* printUsage() { + const available = Object.keys( + runner.services as unknown as Record + ).join(", "); console.log(`Usage: cli [-s|--services serviceName] -Available services: ${Object.keys(services).join(", ")} +Available services: ${available} `); } @@ -34,11 +37,13 @@ Available services: ${Object.keys(services).join(", ")} .filter(Boolean) : undefined; - yield* useServiceGraph(services, subset); + yield* runner(subset); yield* suspend(); } // Public helper: call this from examples or CLI entrypoints — it will invoke effection's main() -export function simulationCLI(services: Parameters[0]) { - return main(() => simulationCLIOp(services)); +export function simulationCLI>( + runner: ServiceRunner +) { + return main(() => simulationCLIOp(runner)); } diff --git a/packages/server/src/services.ts b/packages/server/src/services.ts index 0187d6f1..5080929f 100644 --- a/packages/server/src/services.ts +++ b/packages/server/src/services.ts @@ -1,10 +1,4 @@ -import { - type Operation, - resource, - spawn, - suspend, - withResolvers, -} from "effection"; +import { type Operation, spawn, suspend, withResolvers } from "effection"; export type ServiceDefinition = { // The operation that starts the service and returns when the service is ready. @@ -13,7 +7,8 @@ export type ServiceDefinition = { // returns an `Operation`. The operation may return a value of any type // which will be exposed to dependent services via an `exportsOperation` // on the service definition at runtime. - operation: Operation | (() => Operation); + // Accept either an `Operation` or a factory `() => Operation`. + operation: Operation | ((...args: any[]) => Operation); // optional runtime field - the operation that resolves to the exported value exportsOperation?: Operation; deps?: string[]; @@ -27,15 +22,59 @@ export type ServiceDefinition = { afterStop?: () => Operation; }; -export type ServicesMap = Record>; - // helper type to extract the return type from an Operation or an operation factory type OpReturn = T extends Operation ? U : T extends () => Operation ? U + : T extends (...args: any[]) => Operation + ? U : never; +// Build a tuple of dependency return types for a given service key +type DepKeys = S[K] extends { deps: readonly (infer D)[] } + ? D + : []; + +type DepArgs = DepKeys extends readonly any[] + ? { + [I in keyof DepKeys]: DepKeys[I] extends keyof S + ? S[DepKeys[I]] extends { operation: infer OP } + ? OpReturn + : unknown + : unknown; + } + : []; + +// Ensure we have a tuple/array type for rest-parameter compatibility +type ArgsTuple = DepArgs extends readonly any[] + ? DepArgs + : any[]; + +// A strongly-typed service definition for use within a concrete ServicesMap S. +export type ServiceDefinitionFor< + S extends Record, + K extends keyof S, + T = any +> = { + // operation may be a simple Operation or a factory that accepts the + // exported values of the declared `deps` and returns Operation + operation: Operation | ((...args: ArgsTuple) => Operation); + exportsOperation?: Operation; + deps?: readonly (keyof S)[]; + options?: { + // placeholder for future options + }; + beforeStart?: () => Operation; + afterStart?: () => Operation; + beforeStop?: () => Operation; + afterStop?: () => Operation; +}; + +// Generic Services map - callers can use this shape, and useServiceGraph will +// enforce stronger typing via its own generic parameter +export type ServicesMap = Record>; + // Augment a service map type to include an optional `exportsOperation` for each // service keyed by the return type of its `operation`. export type ServicesWithExports> = @@ -54,13 +93,14 @@ function computeLevels(services: ServicesMap): string[][] { } for (const [name, def] of Object.entries(services)) { for (const dep of def.deps ?? []) { - if (!(dep in services)) { + const depKey = String(dep); + if (!(depKey in services)) { throw new Error( - `Service '${name}' depends on unknown service '${dep}'` + `Service '${name}' depends on unknown service '${depKey}'` ); } - graph[dep].add(name); - indeg[name] = (indeg[name] || 0) + 1; + graph[depKey].add(String(name)); + indeg[String(name)] = (indeg[String(name)] || 0) + 1; } } @@ -115,16 +155,44 @@ function* waitForAllReady( * B: { operation: useService('B', 'node --import tsx ./test/services/service-b.ts'), deps: ['A'] } * }); * - * Services within the same topological layer are started concurrently. Lifecycle - * hooks can be used to perform actions before or after each service starts or stops. + * Services within the same topological layer are started concurrently by default. + * Pass an optional `options` object with `{ sequential: true }` to force services + * within the same layer to start sequentially. Lifecycle hooks can be used to + * perform actions before or after each service starts or stops. */ -export function useServiceGraph< - S extends Record> ->( - services: ServicesWithExports | ServicesMap, - subset?: string[] | string -): Operation { - return resource(function* (provide) { +export type ServiceRunner> = { + (subset?: string[] | string): Operation; + services: S; +}; + +export function useServiceGraph>( + services: { [K in keyof S]: ServiceDefinitionFor } & S, + options?: { sequential?: boolean } +): ServiceRunner { + // Create export resolvers and attach `exportsOperation` on the original + // `services` object synchronously so callers (even those that spawn the + // graph) can access exported values immediately. + const exportResolvers = new Map< + string, + { + operation: Operation; + resolve: (v: any) => void; + reject: (err: Error) => void; + } + >(); + for (const name of Object.keys(services)) { + const r = withResolvers(); + exportResolvers.set(name, { + operation: r.operation, + resolve: r.resolve, + reject: (err: Error) => r.reject(err), + }); + (services as any)[name].exportsOperation = r.operation; + } + + const runner = function* (subset?: string[] | string) { + const sequential = options?.sequential ?? false; // when true, start services in each layer serially + // If a subset is provided, compute the closure including dependencies let effectiveServices: ServicesMap = services; if (subset) { @@ -167,81 +235,140 @@ export function useServiceGraph< }); } - // Map of export resolvers so services can expose a value to dependents - const exportResolvers = new Map< - string, - { - operation: Operation; - resolve: (v: any) => void; - reject: (err: Error) => void; - } - >(); - for (const name of Object.keys(effectiveServices)) { - const r = withResolvers(); - exportResolvers.set(name, { - operation: r.operation, - resolve: r.resolve, - reject: r.reject, - }); - // attach exportsOperation so dependents can yield it - (effectiveServices as any)[name].exportsOperation = r.operation; - } - // Keep track of start order so we can run beforeStop hooks in reverse const startOrder: string[] = []; - for (const layer of layers) { - // spawn all services in this layer in parallel - for (const name of layer) { - const def = effectiveServices[name]; - // wait for deps to be ready (yield the underlying Promise) - if (def.deps) { - for (const dep of def.deps) { - const r = readyResolvers.get(dep); - if (!r) - throw new Error( - `missing readiness resolver for dependency '${dep}'` - ); - yield* r.operation; - } - } - - // spawn a child operation that runs the service and keeps it alive with suspend() - yield* spawn(function* () { + // helper to spawn and run a single service name + function startChild(name: string): Operation { + const def = effectiveServices[name]; + return spawn(function* () { + try { try { if (def.beforeStart) yield* def.beforeStart(); + } catch (err) { + const exportRes = exportResolvers.get(name); + if (exportRes) exportRes.reject(err as Error); + const ready = readyResolvers.get(name); + if (ready) ready.resolve(); + return; + } + + // Collect dependency exported values into an object (keyed by dep name) + const depObj: Record = {}; + if (def.deps) { + for (const dep of def.deps) { + const depKey = String(dep); + const exportRes = exportResolvers.get(depKey); + if (!exportRes) { + throw new Error( + `Service '${name}' depends on unknown service '${depKey}'` + ); + } + const val = yield* exportRes.operation; + depObj[depKey] = val; + } + } - // The caller-supplied operation starts the service and may be - // provided as a factory function or as an already-created - // Operation. Resolve it to an Operation here so we can yield it - // and capture any exported value it returns. - const operation: Operation = + // Resolve the caller-supplied operation (factory or operation). + // If it's a factory, call it with the dependency object as a single arg. + let operation: Operation; + try { + operation = typeof def.operation === "function" - ? (def.operation as () => Operation)() + ? ( + def.operation as ( + args: Record + ) => Operation + )(depObj) : (def.operation as Operation); + } catch (err) { + const exportRes = exportResolvers.get(name); + if (exportRes) exportRes.reject(err as Error); + const ready = readyResolvers.get(name); + if (ready) ready.resolve(); + return; + } - const exported = yield* operation; + let exported: any; + try { + exported = yield* operation; const exportRes = exportResolvers.get(name); if (exportRes) exportRes.resolve(exported); - startOrder.push(name); - const res = readyResolvers.get(name); - if (res) res.resolve(); - if (def.afterStart) yield* def.afterStart(); + } catch (err) { + const exportRes = exportResolvers.get(name); + if (exportRes) exportRes.reject(err as Error); + const ready = readyResolvers.get(name); + if (ready) ready.resolve(); + // don't rethrow here; a failing provider should reject its exportsOperation + // so dependents can observe the error without crashing the whole runner + return; + } - yield* suspend(); - } finally { - // run afterStop hooks in child finalizer so they are executed after the - // process has cleaned up - if (def.afterStop) yield* def.afterStop(); + startOrder.push(name); + const res = readyResolvers.get(name); + if (res) res.resolve(); + try { + if (def.afterStart) yield* def.afterStart(); + } catch (err) { + const exportRes = exportResolvers.get(name); + if (exportRes) exportRes.reject(err as Error); + const ready = readyResolvers.get(name); + if (ready) ready.resolve(); + return; } - }); + + yield* suspend(); + } finally { + if (def.afterStop) yield* def.afterStop(); + } + }); + } + + // small helper to await a service's dependencies + function* waitDeps(name: string): Operation { + const def = effectiveServices[name]; + if (def.deps) { + for (const dep of def.deps) { + const depKey = String(dep); + const r = readyResolvers.get(depKey); + if (!r) + throw new Error( + `missing readiness resolver for dependency '${depKey}'` + ); + yield* r.operation; + } + } + } + + for (const layer of layers) { + if (!sequential) { + // spawn all services in this layer in parallel + for (const name of layer) { + // wait for deps to be ready (yield the underlying Promise) + yield* waitDeps(name); + + // start without waiting; we'll wait for the whole layer below + yield* startChild(name); + } + // after spawning the whole layer, wait until every service in the layer is ready + yield* waitForAllReady(layer, readyResolvers); + } else { + // sequential startup within this layer + for (const name of layer) { + // wait for deps to be ready (yield the underlying Promise) + yield* waitDeps(name); + + // start and then wait for readiness before proceeding + yield* startChild(name); + + const res = readyResolvers.get(name); + if (res) yield* res.operation; + } } - // after spawning the whole layer, wait until every service in the layer is ready - yield* waitForAllReady(layer, readyResolvers); } try { - yield* provide(); + yield* suspend(); } finally { // Run beforeStop hooks in reverse start order for (const name of startOrder.slice().reverse()) { @@ -251,5 +378,11 @@ export function useServiceGraph< } } } - }); + } as any as ServiceRunner; + + // attach the source services for introspection (CLI helpers can access) + (runner as any).services = services; + + // return a function (generator) that can be invoked to run the graph + return runner; } diff --git a/packages/server/test/data-sharing.test.ts b/packages/server/test/data-sharing.test.ts new file mode 100644 index 00000000..088eb0b5 --- /dev/null +++ b/packages/server/test/data-sharing.test.ts @@ -0,0 +1,222 @@ +import { it } from "node:test"; +import { run, spawn, sleep, suspend, until, createScope } from "effection"; +import { services as dataServices } from "../example/operation/data-sharing.ts"; + +import { useServiceGraph } from "../src/services.ts"; +import http from "node:http"; + +// log any uncaught errors so we can debug failing teardown +// eslint-disable-next-line no-console +process.on("uncaughtException", (err) => + console.error("uncaughtException in test:", err) +); +// eslint-disable-next-line no-console +process.on("unhandledRejection", (reason) => + console.error("unhandledRejection in test:", reason) +); + +it("data sharing: short-lived provider and dependent services", async () => { + // create scope at test level so we can control shutdown after assertions + const [scope, destroy] = createScope(); + + try { + const results = await run(function* () { + // start the graph in the provided scope + const runner = dataServices as unknown as any; // runner + scope.run(function* () { + yield* runner(); + // keep the graph task alive until the scope is destroyed + yield* suspend(); + }); + + // yield so spawned children get scheduled and exportsOperations are available + yield* sleep(0); + + const svcMap: any = (dataServices as any).services; + const res: any = {}; + res.data = yield* svcMap.data.exportsOperation; + res.a = yield* svcMap.serviceA.exportsOperation; + res.b = yield* svcMap.serviceB.exportsOperation; + res.c = yield* svcMap.serviceC.exportsOperation; + + // ensure the simulator is reachable while still in the run scope + res.simulatorSeed = undefined as number | undefined; + for (let i = 0; i < 100; i++) { + try { + const fetched = yield* until( + new Promise((resolve, reject) => { + const req = http.get( + { + hostname: "127.0.0.1", + port: res.a.port, + path: "/info", + agent: false, + }, + (r: any) => { + let body = ""; + r.on("data", (c: any) => (body += c)); + r.on("end", () => { + try { + const json = JSON.parse(body); + if (typeof json.seed === "number") { + resolve(json.seed as number); + return; + } + reject(new Error("no seed")); + } catch (err) { + reject(err); + } + }); + } + ); + req.on("error", reject); + }) + ); + res.simulatorSeed = fetched; + break; + } catch (err) { + yield* sleep(10); + } + } + + return res; + }); + + if (results.data.seed !== 42) throw new Error("data seed mismatch"); + if (results.a.handledWith !== 42) + throw new Error("serviceA did not get data"); + if (results.b.used !== 42) + throw new Error("serviceB did not get serviceA's export"); + if (results.c.dataMessage !== "hello from data") + throw new Error("serviceC did not get data message"); + + // verify that the foundation simulator created by serviceA is reachable + if (typeof results.a.port !== "number") + throw new Error("serviceA did not expose port"); + + // ensure the simulator returned the expected seed while the scope was active + if (results.simulatorSeed !== 42) + throw new Error("simulator /info did not return expected seed"); + } catch (err) { + console.error("data-sharing test error:", err); + throw err; + } finally { + console.log("data-sharing test cleanup: starting"); + + // best-effort: close any Server handles first so servers stop accepting + // connections and their finalizers can complete during destroy. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const preHandles: any[] = (process as any)._getActiveHandles + ? (process as any)._getActiveHandles() + : []; + for (const h of preHandles) { + try { + const name = h && h.constructor && h.constructor.name; + if (name === "Server" && typeof h.close === "function") { + try { + h.close(); + } catch (e) {} + } + } catch (e) {} + } + + // give servers a moment to close + await new Promise((r) => setTimeout(r, 50)); + console.log("data-sharing test cleanup: closed preHandles"); + + // ensure destroy completes within 10s for debugging + await Promise.race([destroy(), new Promise((r) => setTimeout(r, 10000))]); + console.log("data-sharing test cleanup: destroy completed (or timed out)"); + + // best-effort: close any remaining socket handles so the test process exits + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const handles: any[] = (process as any)._getActiveHandles + ? (process as any)._getActiveHandles() + : []; + // log of handles removed; we attempt to close/destroy remaining handles below + for (const h of handles) { + try { + if (typeof h.destroy === "function") h.destroy(); + else if (typeof h.end === "function") h.end(); + } catch (e) { + // ignore + } + } + // allow handles to close + await new Promise((r) => setTimeout(r, 20)); + console.log("data-sharing test: completed OK"); + } +}); + +it("exportsOperation rejects when provider throws", async () => { + console.log("exportsOperation rejects test: start"); + const services = { + data: { + operation: (function* () { + throw new Error("boom"); + })(), + }, + dependent: { + deps: ["data"], + operation: (function* () { + return (yield* (services as any).data.exportsOperation) as any; + })(), + }, + } as any; + + await run(function* () { + yield* spawn(function* () { + try { + const run = useServiceGraph(services as any); + // start only the 'data' service so dependents don't consume the rejection + yield* run(["data"]); + } catch (err) { + // swallow; the test will observe rejection via exportsOperation + } + }); + + yield* sleep(0); + + const op = services.data.exportsOperation; + // eslint-disable-next-line no-console + console.log( + "exportsOperation typeof", + typeof op, + "isIterator", + !!(op && typeof op.next === "function"), + "op", + op + ); + // timebox waiting for exportsOperation to reject to accommodate scheduler timing + let caught = false; + // eslint-disable-next-line no-console + console.log("exportsOperation waiting for rejection (timeboxed)"); + for (let i = 0; i < 100; i++) { + try { + yield* (services as any).data.exportsOperation; + // if it resolves, that's unexpected; break to the final check + // eslint-disable-next-line no-console + console.log("exportsOperation unexpectedly resolved"); + break; + } catch (err: any) { + // eslint-disable-next-line no-console + console.log( + "exportsOperation attempt rejected with", + err && err.message + ); + if (err && err.message === "boom") { + caught = true; + break; + } + throw err; + } finally { + if (!caught) { + // give scheduler a bit of time to make progress + yield* sleep(1); + } + } + } + + if (!caught) throw new Error("expected exportsOperation to reject"); + }); +}); diff --git a/packages/server/test/examples-smoke.test.ts b/packages/server/test/examples-smoke.test.ts index 2677d7eb..e4676f58 100644 --- a/packages/server/test/examples-smoke.test.ts +++ b/packages/server/test/examples-smoke.test.ts @@ -1,6 +1,7 @@ import { it } from "node:test"; import http from "node:http"; -import { run, spawn, sleep } from "effection"; +import { run, sleep, suspend, createScope, until } from "effection"; +import { timebox } from "@effectionx/timebox"; import { services as basicServices } from "../example/operation/basic-graph.ts"; import { services as lifecycleServices } from "../example/operation/lifecycle-hooks.ts"; @@ -9,7 +10,7 @@ import { services as concurrencyServices } from "../example/operation/concurrenc function checkStatus(port: number): Promise { return new Promise((resolve, reject) => { const req = http.get( - { hostname: "127.0.0.1", port, path: "/status" }, + { hostname: "127.0.0.1", port, path: "/status", agent: false }, (res) => { resolve(res.statusCode ?? 0); } @@ -18,94 +19,203 @@ function checkStatus(port: number): Promise { }); } -import { useServiceGraph } from "../src/services.ts"; +import type { ServicesMap } from "../src/services.ts"; it("basic example imports and runs", async () => { - const services = basicServices; + const runner = basicServices as unknown as any; // runner // start the graph and await exported ports in a single run operation - const ports: number[] = await run(function* () { - yield* spawn(function* () { - yield* useServiceGraph(services as any); + await run(function* () { + const [scope, destroy] = createScope(); + scope.run(function* () { + yield* runner(); + // keep the graph task alive until the scope is destroyed + yield* suspend(); }); + // allow spawned graph to attach `exportsOperation` properties + yield* sleep(0); + + const svcMap: ServicesMap = (basicServices as any).services; const ps: number[] = []; - for (const name of Object.keys(services)) { - const exportsOp = (services as any)[name].exportsOperation; + for (const name of Object.keys(svcMap)) { + const exportsOp = svcMap[name].exportsOperation; if (!exportsOp) throw new Error(`no exportsOperation on ${name}`); - ps.push(yield* exportsOp); + const val = yield* exportsOp; + ps.push(val); } // keep the graph alive briefly to allow HTTP checks yield* sleep(200); - return ps as number[]; - }); - // check each port - for (const p of ports) { - let ok = false; - for (let i = 0; i < 100; i++) { + // check each port while the graph is still running + for (const p of ps) { + let ok = false; + for (let i = 0; i < 100; i++) { + try { + const status = yield* until(checkStatus(p)); + if (status === 200) { + ok = true; + break; + } + } catch (_) {} + yield* sleep(10); + } + if (!ok) { + throw new Error( + `(examples-smoke basic) port ${p} did not return 200 while graph was running` + ); + } + } + + // best-effort: close any Server handles before requesting shutdown + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const preHandles: any[] = (process as any)._getActiveHandles + ? (process as any)._getActiveHandles() + : []; + for (const h of preHandles) { try { - const status = await checkStatus(p); - if (status === 200) { - ok = true; - break; + const name = h && h.constructor && h.constructor.name; + if (name === "Server" && typeof h.close === "function") { + try { + h.close(); + } catch (e) {} } - } catch (err) { - // ignore and retry + } catch (e) {} + } + // give servers a moment to close + yield* sleep(50); + + // request the graph be shut down and wait for up to 1s for cleanup + const tb = yield* timebox(1000, () => until(destroy())); + if (tb.timeout) { + // eslint-disable-next-line no-console + console.warn("cleanup timed out for example graph"); + } + + // best-effort: close any remaining socket handles so tests don't hang + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const handles: any[] = (process as any)._getActiveHandles + ? (process as any)._getActiveHandles() + : []; + for (const h of handles) { + try { + if (typeof h.destroy === "function") h.destroy(); + else if (typeof h.end === "function") h.end(); + } catch (e) { + // ignore } - await new Promise((r) => setTimeout(r, 10)); } - if (!ok) throw new Error(`port ${p} did not return 200`); - } + // allow handles to close + yield* sleep(20); + + return ps as number[]; + }); }); it("lifecycle example imports and runs", async () => { - const services = lifecycleServices; - - const ports: number[] = await run(function* () { - yield* spawn(function* () { - yield* useServiceGraph(services as any); + const runner = lifecycleServices as unknown as any; // runner + + await run(function* () { + const [scope, destroy] = createScope(); + scope.run(function* () { + yield* runner(); + // keep the graph task alive until the scope is destroyed + yield* suspend(); }); + // allow spawned graph to attach `exportsOperation` properties + yield* sleep(0); + + const svcMap: ServicesMap = (lifecycleServices as any).services; const ps: number[] = []; - for (const name of Object.keys(services)) { - const exportsOp = (services as any)[name].exportsOperation; + for (const name of Object.keys(svcMap)) { + const exportsOp = svcMap[name].exportsOperation; if (!exportsOp) throw new Error(`no exportsOperation on ${name}`); ps.push(yield* exportsOp); } yield* sleep(200); + + // check each port while the graph is still running + for (const p of ps) { + let ok = false; + for (let i = 0; i < 100; i++) { + try { + const status = yield* until(checkStatus(p)); + if (status === 200) { + ok = true; + break; + } + } catch (_) {} + yield* sleep(10); + } + if (!ok) { + throw new Error( + `(examples-smoke lifecycle) port ${p} did not return 200 while graph was running` + ); + } + } + + // shut down the graph to avoid hanging the test process + yield* until(destroy()); return ps as number[]; }); - for (const p of ports) { - const status = await checkStatus(p); - if (status !== 200) throw new Error(`port ${p} did not return 200`); - } + // nothing to check here; checks already happened while graph was running }); it("concurrency example imports and runs", async () => { - const services = concurrencyServices; - - const ports: number[] = await run(function* () { - yield* spawn(function* () { - yield* useServiceGraph(services as any); + const runner = concurrencyServices as unknown as any; // runner + + await run(function* () { + const [scope, destroy] = createScope(); + scope.run(function* () { + yield* runner(); + // keep the graph task alive until the scope is destroyed + yield* suspend(); }); + // allow spawned graph to attach `exportsOperation` properties + yield* sleep(0); + + const svcMap: ServicesMap = (concurrencyServices as any).services; const ps: number[] = []; - for (const name of Object.keys(services)) { - const exportsOp = (services as any)[name].exportsOperation; + for (const name of Object.keys(svcMap)) { + const exportsOp = svcMap[name].exportsOperation; if (!exportsOp) throw new Error(`no exportsOperation on ${name}`); ps.push(yield* exportsOp); } yield* sleep(200); + + // check each port while the graph is still running + for (const p of ps) { + if (typeof p !== "number") { + continue; + } + let ok = false; + for (let i = 0; i < 100; i++) { + try { + const status = yield* until(checkStatus(p)); + if (status === 200) { + ok = true; + break; + } + } catch (_) {} + yield* sleep(10); + } + if (!ok) { + throw new Error( + `(examples-smoke concurrency) port ${p} did not return 200 while graph was running` + ); + } + } + + // shut down the graph to avoid hanging the test process + yield* until(destroy()); return ps as number[]; }); - for (const p of ports) { - const status = await checkStatus(p); - if (status !== 200) throw new Error(`port ${p} did not return 200`); - } + // nothing to check here; checks already happened while graph was running }); diff --git a/packages/server/test/service.test.ts b/packages/server/test/service.test.ts index 0dbd02f1..f26a960b 100644 --- a/packages/server/test/service.test.ts +++ b/packages/server/test/service.test.ts @@ -72,7 +72,7 @@ describe("useService with wellness check", () => { await run(function* () { yield* useService("test-service", nodeScriptWorks, { wellnessCheck: { - timeout: 300, + timeout: 700, frequency: 200, *operation(stdio) { for (let line of yield* each(stdio)) { diff --git a/packages/server/test/services.test.ts b/packages/server/test/services.test.ts index d49a42dc..1b11aff9 100644 --- a/packages/server/test/services.test.ts +++ b/packages/server/test/services.test.ts @@ -9,7 +9,7 @@ it("starts services in dependency order", async () => { try { await run(function* () { yield* spawn(function* () { - yield* useServiceGraph({ + const run = useServiceGraph({ A: { operation: useService( "A", @@ -44,6 +44,7 @@ it("starts services in dependency order", async () => { deps: ["A"], }, }); + yield* run(); }); // The graph is running; sleep a short time to let the services start yield* sleep(200); @@ -62,7 +63,7 @@ it("starts services in dependency order", async () => { it("throws on cycles in dependency graph", async () => { await assert.rejects(async () => { await run(function* () { - yield* useServiceGraph({ + const runGraph = useServiceGraph({ A: { operation: useService( "A", @@ -78,6 +79,7 @@ it("throws on cycles in dependency graph", async () => { deps: ["A"], }, }); + yield* runGraph(); }); }, /Cycle detected in services/); }); @@ -88,7 +90,7 @@ it("runs beforeStop hooks in reverse order", async () => { await run(function* () { // spawn and cancel automatically when run returns yield* spawn(function* () { - yield* useServiceGraph({ + const run = useServiceGraph({ A: { operation: useService( "A", @@ -133,6 +135,7 @@ it("runs beforeStop hooks in reverse order", async () => { }, }, }); + yield* run(); }); // let them start yield* sleep(200); @@ -146,7 +149,7 @@ it("starts independent services in parallel", async () => { try { await run(function* () { yield* spawn(function* () { - yield* useServiceGraph({ + const run = useServiceGraph({ fast: { operation: useService( "fast", @@ -180,6 +183,7 @@ it("starts independent services in parallel", async () => { ), }, }); + yield* run(); }); yield* sleep(250); }); @@ -246,7 +250,8 @@ it("runs subset of services with dependencies", async () => { } as any; // only request dependent; fast and slow should be included as deps - yield* useServiceGraph(services, ["dependent"]); + const run = useServiceGraph(services); + yield* run(["dependent"]); }); yield* sleep(300); }); diff --git a/packages/server/test/services/service-a.ts b/packages/server/test/services/service-a.ts index 7d3da53d..bd25e10d 100644 --- a/packages/server/test/services/service-a.ts +++ b/packages/server/test/services/service-a.ts @@ -1,4 +1,6 @@ import { main } from "effection"; import { httpServer } from "../../example/services/http-server.ts"; -main(() => httpServer({ startDelay: 10 })); +main(function* () { + yield* httpServer({ startDelay: 10 }); +}); diff --git a/packages/server/test/services/service-b.ts b/packages/server/test/services/service-b.ts index 2d285ac5..9d9a261e 100644 --- a/packages/server/test/services/service-b.ts +++ b/packages/server/test/services/service-b.ts @@ -1,4 +1,6 @@ import { main } from "effection"; import { httpServer } from "../../example/services/http-server.ts"; -main(() => httpServer({ startDelay: 40 })); +main(function* () { + yield* httpServer({ startDelay: 40 }); +}); diff --git a/packages/server/test/services/service-fast.ts b/packages/server/test/services/service-fast.ts index 7d3da53d..bd25e10d 100644 --- a/packages/server/test/services/service-fast.ts +++ b/packages/server/test/services/service-fast.ts @@ -1,4 +1,6 @@ import { main } from "effection"; import { httpServer } from "../../example/services/http-server.ts"; -main(() => httpServer({ startDelay: 10 })); +main(function* () { + yield* httpServer({ startDelay: 10 }); +}); diff --git a/packages/server/test/services/service-slow.ts b/packages/server/test/services/service-slow.ts index 245679aa..65cec31a 100644 --- a/packages/server/test/services/service-slow.ts +++ b/packages/server/test/services/service-slow.ts @@ -1,4 +1,6 @@ import { main } from "effection"; import { httpServer } from "../../example/services/http-server.ts"; -main(() => httpServer({ startDelay: 200 })); +main(function* () { + yield* httpServer({ startDelay: 200 }); +}); From 0ba7f2d3cedfc1ef2b9ca4906232c8efefcd4a94 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Sat, 3 Jan 2026 22:04:38 -0600 Subject: [PATCH 05/38] useSimulation helper --- .../server/example/operation/data-sharing.ts | 80 ++++++------------- packages/server/src/index.ts | 1 + packages/server/src/simulation.ts | 38 +++++++++ 3 files changed, 63 insertions(+), 56 deletions(-) create mode 100644 packages/server/src/simulation.ts diff --git a/packages/server/example/operation/data-sharing.ts b/packages/server/example/operation/data-sharing.ts index 6a844c0a..a48c6a34 100644 --- a/packages/server/example/operation/data-sharing.ts +++ b/packages/server/example/operation/data-sharing.ts @@ -1,11 +1,14 @@ #!/usr/bin/env node import { simulationCLI } from "../../src/cli.ts"; import { useServiceGraph } from "../../src/services.ts"; -import { spawn, suspend, until } from "effection"; +import { until } from "effection"; import { createFoundationSimulationServer } from "@simulacrum/foundation-simulator"; import http from "node:http"; +import { useSimulation } from "../../src/index.ts"; -export const createServiceASimulation = (seed: number): any => +export const createServiceASimulation = ( + seed: number +): ReturnType => createFoundationSimulationServer({ port: 0, extendRouter(router) { @@ -13,7 +16,9 @@ export const createServiceASimulation = (seed: number): any => }, }); -export const createServiceBSimulation = (used: number): any => +export const createServiceBSimulation = ( + used: number +): ReturnType => createFoundationSimulationServer({ port: 0, extendRouter(router) { @@ -21,7 +26,9 @@ export const createServiceBSimulation = (used: number): any => }, }); -export const createServiceCSimulation = (message: string): any => +export const createServiceCSimulation = ( + message: string +): ReturnType => createFoundationSimulationServer({ port: 0, extendRouter(router) { @@ -43,15 +50,9 @@ export const services = useServiceGraph({ serviceA: { deps: ["data"], *operation({ data }) { - const createSim = createServiceASimulation(data.seed)(); - - const listening: any = yield* until(createSim.listen()); - - // debug log so tests can see the assigned port (left as example output) - // eslint-disable-next-line no-console - console.log( - `[data-sharing] started foundation sim on port ${listening.port}` - ); + const { port: listeningPort } = yield* useSimulation( + createServiceASimulation + )(data.seed); // self-check try { @@ -60,7 +61,7 @@ export const services = useServiceGraph({ const req = http.get( { hostname: "127.0.0.1", - port: listening.port, + port: listeningPort, path: "/info", agent: false, }, @@ -80,24 +81,7 @@ export const services = useServiceGraph({ console.log(`[data-sharing] self-check error:`, err); } - // spawn background keeper which calls ensureClose when finalized - yield* spawn(function* () { - try { - yield* suspend(); - } finally { - // eslint-disable-next-line no-console - console.log( - `[data-sharing] ensuring close for port ${listening.port}` - ); - yield* until(listening.ensureClose()); - // eslint-disable-next-line no-console - console.log( - `[data-sharing] closed foundation sim on port ${listening.port}` - ); - } - }); - - return { handledWith: data.seed, port: listening.port }; + return { handledWith: data.seed, port: listeningPort }; }, }, @@ -105,23 +89,15 @@ export const services = useServiceGraph({ serviceB: { deps: ["serviceA", "data"], *operation({ serviceA, data }) { - const createSim = createServiceBSimulation(serviceA.handledWith)(); - - const listening: any = yield* until(createSim.listen()); - - yield* spawn(function* () { - try { - yield* suspend(); - } finally { - yield* until(listening.ensureClose()); - } - }); + const { port: listeningPort } = yield* useSimulation( + createServiceBSimulation + )(serviceA.handledWith); // include data seed in the export so tests can verify multi-dependency wiring return { used: serviceA.handledWith, dataSeed: data.seed, - port: listening.port, + port: listeningPort, }; }, }, @@ -130,19 +106,11 @@ export const services = useServiceGraph({ serviceC: { deps: ["data"], *operation({ data }) { - const createSim = createServiceCSimulation(data.message)(); - - const listening: any = yield* until(createSim.listen()); - - yield* spawn(function* () { - try { - yield* suspend(); - } finally { - yield* until(listening.ensureClose()); - } - }); + const { port: listeningPort } = yield* useSimulation( + createServiceCSimulation + )(data.message); - return { dataMessage: data.message, port: listening.port }; + return { dataMessage: data.message, port: listeningPort }; }, }, }); diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 5b9d59f2..0a5441f3 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -1,3 +1,4 @@ export * from "./logging.ts"; export * from "./service.ts"; export * from "./services.ts"; +export * from "./simulation.ts"; diff --git a/packages/server/src/simulation.ts b/packages/server/src/simulation.ts new file mode 100644 index 00000000..c7597c7c --- /dev/null +++ b/packages/server/src/simulation.ts @@ -0,0 +1,38 @@ +import { spawn, suspend, until } from "effection"; +import type { Operation } from "effection"; +import type { + FoundationSimulator, + FoundationSimulatorListening, +} from "@simulacrum/foundation-simulator"; + +/** + * Helper to start a foundation simulation server factory and return the listening + * information in a typed way. + */ +export function useSimulation( + createFactory: (...args: A) => () => FoundationSimulator +): (...args: A) => Operation> { + return function* (...args: A) { + const createSim = createFactory(...args)(); + const listening: FoundationSimulatorListening = yield* until( + createSim.listen() + ); + + // small debug log to make it visible in tests + // eslint-disable-next-line no-console + console.log(`simulation started on port ${listening.port}`); + + // ensure server is closed when this operation is finalized + yield* spawn(function* () { + try { + yield* suspend(); + } finally { + yield* until(listening.ensureClose()); + // eslint-disable-next-line no-console + console.log(`simulation closed on port ${listening.port}`); + } + }); + + return listening; + }; +} From e60b0927bb932c959293f200ca78a3efae1d69ea Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Wed, 7 Jan 2026 22:01:48 -0600 Subject: [PATCH 06/38] spawn simulators each as a child_process --- package-lock.json | 79 +- packages/server/bin/run-simulation-child.ts | 70 ++ packages/server/example/basic-graph.ts | 2 +- packages/server/example/concurrency-layers.ts | 2 +- packages/server/example/debug-run.ts | 10 - packages/server/example/lifecycle-hooks.ts | 2 +- .../server/example/operation/basic-graph.ts | 2 +- .../example/operation/concurrency-layers.ts | 2 +- .../server/example/operation/data-sharing.ts | 120 --- .../example/operation/lifecycle-hooks.ts | 2 +- packages/server/package.json | 9 +- packages/server/src/cli.ts | 54 +- packages/server/src/services.ts | 771 +++++++++++++----- packages/server/src/simulation.ts | 113 ++- packages/server/src/watch.ts | 84 ++ packages/server/test/child-simulation.test.ts | 21 + packages/server/test/data-sharing.test.ts | 222 ----- packages/server/test/examples-smoke.test.ts | 131 ++- packages/server/test/fixtures/simple-sim.ts | 10 + packages/server/test/service.test.ts | 2 +- packages/server/test/services.test.ts | 24 +- packages/server/test/simulation.test.ts | 68 ++ packages/server/test/watch.test.ts | 82 ++ 23 files changed, 1237 insertions(+), 645 deletions(-) create mode 100644 packages/server/bin/run-simulation-child.ts delete mode 100644 packages/server/example/debug-run.ts delete mode 100644 packages/server/example/operation/data-sharing.ts create mode 100644 packages/server/src/watch.ts create mode 100644 packages/server/test/child-simulation.test.ts delete mode 100644 packages/server/test/data-sharing.test.ts create mode 100644 packages/server/test/fixtures/simple-sim.ts create mode 100644 packages/server/test/simulation.test.ts create mode 100644 packages/server/test/watch.test.ts diff --git a/package-lock.json b/package-lock.json index 6b9e61fa..a520c417 100644 --- a/package-lock.json +++ b/package-lock.json @@ -391,6 +391,39 @@ "node": ">= 16" } }, + "node_modules/@effectionx/signals": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@effectionx/signals/-/signals-0.4.1.tgz", + "integrity": "sha512-y6oJwpQwqTd2rVPgC2yMQXzQV848MJpRg4zjAL2rIH9znFakPdZN7H3OU4iuY5fnBnS/JaAp43SjHw2hqMnkyA==", + "license": "MIT", + "dependencies": { + "effection": "^3 || ^4.0.0-0", + "immutable": "^5" + }, + "engines": { + "node": ">= 16" + } + }, + "node_modules/@effectionx/signals/node_modules/immutable": { + "version": "5.1.4", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-5.1.4.tgz", + "integrity": "sha512-p6u1bG3YSnINT5RQmx/yRZBpenIl30kVxkTLDyHLIMk0gict704Q9n+thfDI7lTRm9vXdDYutVzXhzcThxTnXA==", + "license": "MIT" + }, + "node_modules/@effectionx/stream-helpers": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@effectionx/stream-helpers/-/stream-helpers-0.4.1.tgz", + "integrity": "sha512-zYnUYbKJcX5pMbMtFLlurj9KO6ZhoG1bedhoG1mU8Fh7Fz06WKIAxQ9dIOYefBP3Kczr8T1+lDcsq62n3YtBug==", + "license": "MIT", + "dependencies": { + "@effectionx/signals": "^0.4.0", + "@effectionx/timebox": "^0.3.0", + "effection": "^3 || ^4.0.0-0" + }, + "engines": { + "node": ">= 16" + } + }, "node_modules/@effectionx/timebox": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/@effectionx/timebox/-/timebox-0.3.1.tgz", @@ -2778,6 +2811,13 @@ "undici-types": "~7.10.0" } }, + "node_modules/@types/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-qHHxQ+P9PysNEGbALT8f8YOSHW0KJu6l2xU8DYY0fu/EmGxXdVnuTLvFUvBgPJMSqXq29SYHveejeAha+4AYgA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/qs": { "version": "6.14.0", "dev": true, @@ -5852,9 +5892,7 @@ }, "node_modules/picomatch": { "version": "4.0.3", - "devOptional": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -7553,10 +7591,43 @@ "dependencies": { "@effectionx/context-api": "^0.2.1", "@effectionx/process": "^0.6.2", + "@effectionx/stream-helpers": "^0.4.1", "@effectionx/timebox": "^0.3.1", - "effection": "^4.0.0" + "chokidar": "^5.0.0", + "effection": "^4.0.0", + "picomatch": "^4.0.3" }, - "devDependencies": {} + "devDependencies": { + "@types/picomatch": "^4.0.2" + } + }, + "packages/server/node_modules/chokidar": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz", + "integrity": "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==", + "license": "MIT", + "dependencies": { + "readdirp": "^5.0.0" + }, + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "packages/server/node_modules/readdirp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.0.0.tgz", + "integrity": "sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ==", + "license": "MIT", + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } }, "packages/ui": { "name": "@simulacrum/ui", diff --git a/packages/server/bin/run-simulation-child.ts b/packages/server/bin/run-simulation-child.ts new file mode 100644 index 00000000..6f92e420 --- /dev/null +++ b/packages/server/bin/run-simulation-child.ts @@ -0,0 +1,70 @@ +#!/usr/bin/env node +import { main, suspend, until } from "effection"; +import { pathToFileURL } from "node:url"; + +main(function* () { + const args = process.argv.slice(2); + if (args.length < 1) { + throw new Error("usage: run-simulation-child.js [jsonArgs]"); + } + + const modulePath = args[0]; + const jsonArgs = args[1] ? JSON.parse(args[1]) : []; + + // Resolve and import module inside the operation + let mod: any; + try { + const url = + modulePath.startsWith("./") || modulePath.startsWith("/") + ? pathToFileURL(modulePath).href + : modulePath; + mod = yield* until(import(url)); + } catch (err) { + throw new Error(`failed to import module: ${String(err)}`); + } + + const exportNames = ["default", "simulation"]; + let factory: Function | undefined = undefined; + for (const name of exportNames) { + if (name in mod && typeof mod[name] === "function") { + factory = mod[name]; + break; + } + } + // fallback: module itself is a function + if (!factory && typeof mod === "function") factory = mod; + + if (!factory) { + throw new Error(`no factory function found in module: ${modulePath}`); + } + + // call factory with provided args + let sim = factory(...(Array.isArray(jsonArgs) ? jsonArgs : [jsonArgs])); + if (typeof sim === "function") { + sim = sim(); + } + + if (!sim || typeof sim.listen !== "function") { + throw new Error("factory did not return a simulator with .listen()"); + } + + let listening: any; + try { + listening = yield* until(sim.listen()); + const out = JSON.stringify({ + ready: true, + port: listening.port, + pid: process.pid, + }); + console.log(out); + yield* suspend(); + } finally { + try { + if (listening && typeof listening.ensureClose === "function") { + yield* until(listening.ensureClose()); + } + } catch (err) { + // ignore + } + } +}); diff --git a/packages/server/example/basic-graph.ts b/packages/server/example/basic-graph.ts index ce1d2108..386db224 100644 --- a/packages/server/example/basic-graph.ts +++ b/packages/server/example/basic-graph.ts @@ -58,5 +58,5 @@ export function example(opts: { duration?: number } = {}) { import { fileURLToPath } from "node:url"; if (process.argv[1] === fileURLToPath(import.meta.url)) { // run via CLI when executed directly - simulationCLI(services); + simulationCLI(services()); } diff --git a/packages/server/example/concurrency-layers.ts b/packages/server/example/concurrency-layers.ts index 953260e2..2dafc1c7 100644 --- a/packages/server/example/concurrency-layers.ts +++ b/packages/server/example/concurrency-layers.ts @@ -62,7 +62,7 @@ export const services = useServiceGraph(servicesMap); import { fileURLToPath } from "node:url"; if (process.argv[1] === fileURLToPath(import.meta.url)) { - simulationCLI(services); + simulationCLI(services()); } export function example( diff --git a/packages/server/example/debug-run.ts b/packages/server/example/debug-run.ts deleted file mode 100644 index 57024b0d..00000000 --- a/packages/server/example/debug-run.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { run, spawn, sleep } from "effection"; - -run(function* () { - yield* spawn(function* () { - console.log("debug - spawn start"); - yield* sleep(100); - console.log("debug - spawn done"); - }); - console.log("debug - after spawn (should only print after spawn completes)"); -}); diff --git a/packages/server/example/lifecycle-hooks.ts b/packages/server/example/lifecycle-hooks.ts index 9366b5ab..6adec34e 100644 --- a/packages/server/example/lifecycle-hooks.ts +++ b/packages/server/example/lifecycle-hooks.ts @@ -72,7 +72,7 @@ export const services = useServiceGraph(servicesMap); import { fileURLToPath } from "node:url"; if (process.argv[1] === fileURLToPath(import.meta.url)) { - simulationCLI(services); + simulationCLI(services()); } export function example(opts: { duration?: number } = {}) { diff --git a/packages/server/example/operation/basic-graph.ts b/packages/server/example/operation/basic-graph.ts index 61abbe98..1754e0b0 100644 --- a/packages/server/example/operation/basic-graph.ts +++ b/packages/server/example/operation/basic-graph.ts @@ -27,5 +27,5 @@ export function example(opts: { duration?: number } = {}) { import { fileURLToPath } from "node:url"; if (process.argv[1] === fileURLToPath(import.meta.url)) { - simulationCLI(services); + simulationCLI(services()); } diff --git a/packages/server/example/operation/concurrency-layers.ts b/packages/server/example/operation/concurrency-layers.ts index 5e55c93a..a8c49f8f 100644 --- a/packages/server/example/operation/concurrency-layers.ts +++ b/packages/server/example/operation/concurrency-layers.ts @@ -22,7 +22,7 @@ export const services = useServiceGraph(servicesMap); import { fileURLToPath } from "node:url"; if (process.argv[1] === fileURLToPath(import.meta.url)) { - simulationCLI(services); + simulationCLI(services()); } export function example(opts: { duration?: number } = {}) { diff --git a/packages/server/example/operation/data-sharing.ts b/packages/server/example/operation/data-sharing.ts deleted file mode 100644 index a48c6a34..00000000 --- a/packages/server/example/operation/data-sharing.ts +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env node -import { simulationCLI } from "../../src/cli.ts"; -import { useServiceGraph } from "../../src/services.ts"; -import { until } from "effection"; -import { createFoundationSimulationServer } from "@simulacrum/foundation-simulator"; -import http from "node:http"; -import { useSimulation } from "../../src/index.ts"; - -export const createServiceASimulation = ( - seed: number -): ReturnType => - createFoundationSimulationServer({ - port: 0, - extendRouter(router) { - router.get("/info", (_req, res) => res.json({ seed, handledWith: seed })); - }, - }); - -export const createServiceBSimulation = ( - used: number -): ReturnType => - createFoundationSimulationServer({ - port: 0, - extendRouter(router) { - router.get("/info", (_req, res) => res.json({ used })); - }, - }); - -export const createServiceCSimulation = ( - message: string -): ReturnType => - createFoundationSimulationServer({ - port: 0, - extendRouter(router) { - router.get("/info", (_req, res) => res.json({ dataMessage: message })); - }, - }); - -export const services = useServiceGraph({ - // short-lived data generator: returns a shared payload and completes - data: { - *operation() { - // generate some data for dependents - const payload = { seed: 42, message: "hello from data" }; - return payload; - }, - }, - - // serviceA depends on data and keeps running (long-running provider) - serviceA: { - deps: ["data"], - *operation({ data }) { - const { port: listeningPort } = yield* useSimulation( - createServiceASimulation - )(data.seed); - - // self-check - try { - const local = yield* until( - new Promise<{ status?: number; body?: string }>((resolve, reject) => { - const req = http.get( - { - hostname: "127.0.0.1", - port: listeningPort, - path: "/info", - agent: false, - }, - (res: any) => { - let body = ""; - res.on("data", (c: any) => (body += c)); - res.on("end", () => resolve({ status: res.statusCode, body })); - } - ); - req.on("error", reject); - }) - ); - // eslint-disable-next-line no-console - console.log(`[data-sharing] self-check /info:`, local); - } catch (err) { - // eslint-disable-next-line no-console - console.log(`[data-sharing] self-check error:`, err); - } - - return { handledWith: data.seed, port: listeningPort }; - }, - }, - - // serviceB depends on serviceA and data - serviceB: { - deps: ["serviceA", "data"], - *operation({ serviceA, data }) { - const { port: listeningPort } = yield* useSimulation( - createServiceBSimulation - )(serviceA.handledWith); - - // include data seed in the export so tests can verify multi-dependency wiring - return { - used: serviceA.handledWith, - dataSeed: data.seed, - port: listeningPort, - }; - }, - }, - - // serviceC depends only on data - serviceC: { - deps: ["data"], - *operation({ data }) { - const { port: listeningPort } = yield* useSimulation( - createServiceCSimulation - )(data.message); - - return { dataMessage: data.message, port: listeningPort }; - }, - }, -}); - -if (import.meta.url === `file://${process.argv[1]}`) { - simulationCLI(services); -} diff --git a/packages/server/example/operation/lifecycle-hooks.ts b/packages/server/example/operation/lifecycle-hooks.ts index 776b178d..a9e0940b 100644 --- a/packages/server/example/operation/lifecycle-hooks.ts +++ b/packages/server/example/operation/lifecycle-hooks.ts @@ -48,5 +48,5 @@ export function example(opts: { duration?: number } = {}) { import { fileURLToPath } from "node:url"; if (process.argv[1] === fileURLToPath(import.meta.url)) { - simulationCLI(services); + simulationCLI(services()); } diff --git a/packages/server/package.json b/packages/server/package.json index 7c7da4e8..0a11666f 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -40,9 +40,14 @@ "effection": "^4.0.0", "@effectionx/context-api": "^0.2.1", "@effectionx/process": "^0.6.2", - "@effectionx/timebox": "^0.3.1" + "@effectionx/stream-helpers": "^0.4.1", + "@effectionx/timebox": "^0.3.1", + "chokidar": "^5.0.0", + "picomatch": "^4.0.3" + }, + "devDependencies": { + "@types/picomatch": "^4.0.2" }, - "devDependencies": {}, "exports": { ".": { "development": "./src/index.ts", diff --git a/packages/server/src/cli.ts b/packages/server/src/cli.ts index e9ae2fd4..6880799f 100644 --- a/packages/server/src/cli.ts +++ b/packages/server/src/cli.ts @@ -1,16 +1,18 @@ import { parseArgs } from "node:util"; -import { suspend, main } from "effection"; -import type { ServiceRunner } from "./services.ts"; +import { suspend, main, type Operation } from "effection"; +import type { ServiceGraphValue } from "./services.ts"; // Internal generator operation used by both the CLI (via main) and programmatically export function* simulationCLIOp>( - runner: ServiceRunner + runnerOp: Operation> ) { const { values } = parseArgs({ options: { services: { type: "string", short: "s" }, debug: { type: "boolean", short: "d" }, help: { type: "boolean", short: "h" }, + watch: { type: "boolean" }, + "watch-debounce": { type: "string" }, }, allowPositionals: true, allowNegative: true, @@ -18,12 +20,11 @@ export function* simulationCLIOp>( }); function* printUsage() { - const available = Object.keys( - runner.services as unknown as Record - ).join(", "); - console.log(`Usage: cli [-s|--services serviceName] -Available services: ${available} -`); + // When we only have the operation form we cannot enumerate services + // without starting the graph — so show a succinct help message. + console.log( + `Usage: cli [-s|--services serviceName] [--watch] [--watch-debounce ms]` + ); } if (values.help) { @@ -36,14 +37,43 @@ Available services: ${available} .map((s) => s.trim()) .filter(Boolean) : undefined; + if (subset) { + // subset-run no longer supported for operation-style graphs; warn + // (consumer should construct a graph for the subset explicitly) + // eslint-disable-next-line no-console + console.warn( + "--services subset not supported with operation-style runner; starting full graph" + ); + } + + const runOptions: { watch?: boolean; watchDebounce?: number } = { + watch: !!values.watch, + }; + if (values["watch-debounce"]) + runOptions.watchDebounce = Number(values["watch-debounce"]); + + // Start the graph and fetch the provided info + const servicesVal = yield* runnerOp; + if (runOptions.watch) { + // eslint-disable-next-line no-console + console.log("starting in watch mode; watched paths:"); + for (const name of Object.keys(servicesVal.services)) { + const def = servicesVal.services[name]; + const watch = def.watch + ? typeof def.watch === "function" + ? (def.watch as () => string[])() + : def.watch + : undefined; + if (watch) console.log(` - ${name}: ${watch.join(", ")}`); + } + } - yield* runner(subset); yield* suspend(); } // Public helper: call this from examples or CLI entrypoints — it will invoke effection's main() export function simulationCLI>( - runner: ServiceRunner + runnerOp: Operation> ) { - return main(() => simulationCLIOp(runner)); + return main(() => simulationCLIOp(runnerOp)); } diff --git a/packages/server/src/services.ts b/packages/server/src/services.ts index 5080929f..fac60991 100644 --- a/packages/server/src/services.ts +++ b/packages/server/src/services.ts @@ -1,16 +1,45 @@ -import { type Operation, spawn, suspend, withResolvers } from "effection"; +import { + type Operation, + resource, + spawn, + suspend, + withResolvers, + createChannel, + each, + sleep, + until, + type Stream, + type Task, +} from "effection"; + +// Types for watcher and channels +export type ServiceUpdate = { service: string; path: string }; + +type WatchesStream = Stream & { + send: (value: string) => Operation; + close: (value?: void) => Operation; +}; + +type Watcher = { + serviceUpdates: Stream; + add: (service: string, paths: string[]) => void; +}; +import * as fs from "node:fs/promises"; +import type { Stats } from "node:fs"; +import { useWatcher } from "./watch.ts"; export type ServiceDefinition = { // The operation that starts the service and returns when the service is ready. // The operation may be provided either as an `Operation` (for example the // `Operation` returned by `useService(...)`) or as a factory that // returns an `Operation`. The operation may return a value of any type - // which will be exposed to dependent services via an `exportsOperation` - // on the service definition at runtime. + // which will be delivered to dependent service factories at runtime. // Accept either an `Operation` or a factory `() => Operation`. operation: Operation | ((...args: any[]) => Operation); - // optional runtime field - the operation that resolves to the exported value - exportsOperation?: Operation; + // folders/files to watch for changes which should cause a restart + watch?: string[]; + // debounce in milliseconds to coalesce rapid changes for this service + watchDebounce?: number; deps?: string[]; options?: { // Keep an options object for future expansion or hooks; currently unused when operation is present @@ -59,8 +88,14 @@ export type ServiceDefinitionFor< > = { // operation may be a simple Operation or a factory that accepts the // exported values of the declared `deps` and returns Operation - operation: Operation | ((...args: ArgsTuple) => Operation); - exportsOperation?: Operation; + operation: + | Operation + | ((...args: ArgsTuple) => Operation) + | ((...args: any[]) => Generator); + // folders/files to watch for changes which should cause a restart + watch?: string[]; + // debounce in milliseconds to coalesce rapid changes for this service + watchDebounce?: number; deps?: readonly (keyof S)[]; options?: { // placeholder for future options @@ -75,14 +110,11 @@ export type ServiceDefinitionFor< // enforce stronger typing via its own generic parameter export type ServicesMap = Record>; -// Augment a service map type to include an optional `exportsOperation` for each +// Previously we exposed a public `exportsOperation` for each service; that has been removed. // service keyed by the return type of its `operation`. -export type ServicesWithExports> = - { - [K in keyof S]: S[K] & { - exportsOperation?: Operation>; - }; - }; +// Note: we no longer attach an `exportsOperation` on the public service map. +// Internal exported values are still resolved and passed to dependent service factories +// but are not exposed as a separate operation on the services object. function computeLevels(services: ServicesMap): string[][] { const indeg: Record = {}; @@ -132,7 +164,14 @@ function computeLevels(services: ServicesMap): string[][] { function* waitForAllReady( names: string[], - readyResolvers: Map + readyResolvers: Map< + string, + { + operation: Operation; + resolve: () => void; + reject: (err: Error) => void; + } + > ): Operation { for (const n of names) { const r = readyResolvers.get(n); @@ -142,6 +181,17 @@ function* waitForAllReady( } } +// A runner returned by `useServiceGraph` — callable to start the graph and +// also exposes properties used by the CLI and tests. +// The object provided by the service graph resource when started +export type ServiceGraphValue> = { + services: S; + serviceUpdates?: Stream | undefined; + watches: WatchesStream; + // map of last known ports for services that expose a `port` in their export value + servicePorts: Map; +}; + /** * useServiceGraph * @@ -160,229 +210,556 @@ function* waitForAllReady( * within the same layer to start sequentially. Lifecycle hooks can be used to * perform actions before or after each service starts or stops. */ -export type ServiceRunner> = { - (subset?: string[] | string): Operation; - services: S; -}; - -export function useServiceGraph>( +export function useServiceGraph< + S extends Record> +>( services: { [K in keyof S]: ServiceDefinitionFor } & S, - options?: { sequential?: boolean } -): ServiceRunner { - // Create export resolvers and attach `exportsOperation` on the original - // `services` object synchronously so callers (even those that spawn the - // graph) can access exported values immediately. + options?: { sequential?: boolean; watch?: boolean; watchDebounce?: number } +): (subset?: string[] | string) => Operation> { + // Create internal export resolvers for provider-returned values so dependent + // services can obtain them during startup. const exportResolvers = new Map< string, { - operation: Operation; - resolve: (v: any) => void; + operation: Operation; + resolve: (v: unknown) => void; reject: (err: Error) => void; } >(); for (const name of Object.keys(services)) { - const r = withResolvers(); + const r = withResolvers(); exportResolvers.set(name, { operation: r.operation, resolve: r.resolve, reject: (err: Error) => r.reject(err), }); - (services as any)[name].exportsOperation = r.operation; + // note: we intentionally do not expose a public `exportsOperation` value on the services map + // previously we exposed an `exportsOperation` on the public service map; + // that behavior has been removed. Internal resolvers will still be used to + // deliver provider-exported values to dependent factories. } - const runner = function* (subset?: string[] | string) { - const sequential = options?.sequential ?? false; // when true, start services in each layer serially - - // If a subset is provided, compute the closure including dependencies - let effectiveServices: ServicesMap = services; - if (subset) { - const want = new Set( - (typeof subset === "string" ? subset.split(",") : subset).map((s) => - s.trim() - ) - ); - const included = new Set(); - function include(name: string) { - if (included.has(name)) return; - if (!(name in services)) - throw new Error(`Requested service '${name}' not found`); - included.add(name); - for (const dep of services[name].deps ?? []) include(dep); + let runnerWatcher: + | { serviceUpdates: Stream } + | undefined; + const providedWatches: WatchesStream = createChannel(); + + // create a simple channel that emits service names when they change. + // We intentionally do not buffer updates; missing the first few updates + // is acceptable and sometimes desirable because they will be used to + // restart services. + + const setup = withResolvers(); + + return (subset?: string[] | string) => + resource(function* (provide) { + const sequential = options?.sequential ?? false; // when true, start services in each layer serially + + // If a subset is provided, compute the closure including dependencies + let effectiveServices: ServicesMap = services; + if (subset) { + const want = new Set( + (typeof subset === "string" ? subset.split(",") : subset).map((s) => + s.trim() + ) + ); + const included = new Set(); + function include(name: string) { + if (included.has(name)) return; + if (!(name in services)) + throw new Error(`Requested service '${name}' not found`); + included.add(name); + for (const dep of services[name].deps ?? []) include(String(dep)); + } + for (const name of want) include(name); + effectiveServices = {} as ServicesMap; + for (const name of included) effectiveServices[name] = services[name]; } - for (const name of want) include(name); - effectiveServices = {} as ServicesMap; - for (const name of included) effectiveServices[name] = services[name]; - } - const layers = computeLevels(effectiveServices); + const layers = computeLevels(effectiveServices); + console.log(`runner: starting layers ${JSON.stringify(layers)}`); + + const watcher = (yield* useWatcher()) as Watcher; + runnerWatcher = watcher; + + // channel to emit file contents for watcher consumers + const watches: WatchesStream = providedWatches; + + // Register any configured watch paths and emit initial file contents + for (const name of Object.keys(effectiveServices)) { + const def = effectiveServices[name]; + if (def.watch) { + watcher.add(name, def.watch); + for (const p of def.watch) { + try { + let stat: Stats | undefined; + try { + stat = (yield* until(fs.stat(p))) as Stats; + } catch (e) { + continue; + } + if (!stat) continue; + if ( + stat && + typeof stat.isDirectory === "function" && + stat.isDirectory() + ) { + const entries: string[] = yield* until(fs.readdir(p)); + + for (const e of entries) { + const full = `${p}/${e}`; + try { + let content = String( + (yield* until(fs.readFile(full, "utf8"))) as string + ); + // if the content is empty, retry a few times in case of a race + if (content === "") { + for (let i = 0; i < 5; i++) { + yield* sleep(10); + try { + content = String( + (yield* until(fs.readFile(full, "utf8"))) as string + ); + if (content !== "") break; + } catch (e) {} + } + } + + yield* watches.send(name); + } catch (e) {} + } + } else { + try { + let content = String( + (yield* until(fs.readFile(p, "utf8"))) as string + ); + if (content === "") { + for (let i = 0; i < 5; i++) { + yield* sleep(10); + try { + content = String( + (yield* until(fs.readFile(p, "utf8"))) as string + ); + if (content !== "") break; + } catch (e) {} + } + } + yield* watches.send(name); + } catch (e) {} + } + } catch (e) { + // ignore + } + } + } + } - // Map of readiness resolvers returned by `withResolvers` for the - // effective services we plan to start. - const readyResolvers = new Map< - string, - { - operation: Operation; - resolve: () => void; - reject: (err: Error) => void; + // signal that we've registered watches and emitted all initial values + // so callers can start their subscriptions with a guarantee that + // initial state has already been produced. + for (const n of Object.keys(effectiveServices)) { + const d = effectiveServices[n]; + console.log( + `setup: service ${n} has beforeStop=${ + typeof d.beforeStop === "function" + }` + ); + } + setup.resolve(); + + // Map to manage per-service debounce state and worker + const state = new Map< + string, + { + lastPath?: string; + lastAt?: number; + worker?: Operation> | undefined; + } + >(); + + // track running tasks so we can halt them for restarts + const runningTasks = new Map(); + // track the last exported ports for services that expose a port + const servicePorts = new Map(); + + // build reverse dependency graph to compute dependents closure + const reverseDeps: Record> = {}; + for (const name of Object.keys(effectiveServices)) + reverseDeps[name] = new Set(); + for (const [name, def] of Object.entries(effectiveServices)) { + for (const dep of def.deps ?? []) { + reverseDeps[String(dep)].add(String(name)); + } } - >(); - for (const name of Object.keys(effectiveServices)) { - const r = withResolvers(); - readyResolvers.set(name, { - operation: r.operation, - resolve: r.resolve, - reject: r.reject, + + // Spawn a listener to collect service update events and debounce per-service + yield* spawn(function* () { + for (const ev of yield* each(watcher.serviceUpdates)) { + const { service, path: p } = ev as ServiceUpdate; + const def = effectiveServices[service]; + const debounceMs = + (def && def.watchDebounce) ?? options?.watchDebounce ?? 20; + const s = state.get(service) ?? {}; + s.lastPath = p; + s.lastAt = Date.now(); + state.set(service, s); + if (!s.worker) { + // start a worker that waits for a quiet period then reads file and emits + + s.worker = spawn(function* () { + while (true) { + const elapsed = Date.now() - (s.lastAt ?? 0); + const wait = Math.max(0, debounceMs - elapsed); + if (wait > 0) yield* sleep(wait); + if (Date.now() - (s.lastAt ?? 0) >= debounceMs) { + try { + // after debounce, send the service name (we don't need the file content) + yield* watches.send(service); + } catch (e) { + // ignore send errors + } + s.worker = undefined; + break; + } + } + }); + } + // required by `each` to allow the loop to continue correctly + yield* each.next(); + } }); - } - // Keep track of start order so we can run beforeStop hooks in reverse - const startOrder: string[] = []; + // Map of readiness resolvers returned by `withResolvers` for the + // effective services we plan to start. + const readyResolvers = new Map< + string, + { + operation: Operation; + resolve: () => void; + reject: (err: Error) => void; + } + >(); + for (const name of Object.keys(effectiveServices)) { + const r = withResolvers(); + readyResolvers.set(name, { + operation: r.operation, + resolve: r.resolve, + reject: r.reject, + }); + } - // helper to spawn and run a single service name - function startChild(name: string): Operation { - const def = effectiveServices[name]; - return spawn(function* () { - try { - try { - if (def.beforeStart) yield* def.beforeStart(); - } catch (err) { - const exportRes = exportResolvers.get(name); - if (exportRes) exportRes.reject(err as Error); - const ready = readyResolvers.get(name); - if (ready) ready.resolve(); - return; + // Keep track of start order so we can run beforeStop hooks in reverse + const startOrder: string[] = []; + + // Restart coordinator: listen for debounced watch events and restart affected services + yield* spawn(function* () { + const pending = new Set(); + let processing = false; + + function addClosure(name: string, set: Set) { + if (set.has(name)) return; + set.add(name); + for (const dep of reverseDeps[name] ?? []) addClosure(dep, set); + } + + for (const ev of yield* each(watches)) { + const name = ev as string; + pending.add(name); + if (processing) { + yield* each.next(); + continue; + } + processing = true; + // small debounce to coalesce multiple rapid events + yield* sleep(20); + const toProcess = Array.from(pending); + pending.clear(); + + // compute closure of affected services + const affected = new Set(); + for (const n of toProcess) addClosure(n, affected); + + if (affected.size === 0) { + processing = false; + yield* each.next(); + continue; } - // Collect dependency exported values into an object (keyed by dep name) - const depObj: Record = {}; - if (def.deps) { - for (const dep of def.deps) { - const depKey = String(dep); - const exportRes = exportResolvers.get(depKey); - if (!exportRes) { - throw new Error( - `Service '${name}' depends on unknown service '${depKey}'` - ); - } - const val = yield* exportRes.operation; - depObj[depKey] = val; - } + // create new export & ready resolvers for affected services + for (const n of affected) { + const r = withResolvers(); + exportResolvers.set(n, { + operation: r.operation, + resolve: r.resolve, + reject: r.reject, + }); + // previously we exposed an `exportsOperation` on the public service map; + // that behavior has been removed. Internal resolvers will still be used to + // deliver provider-exported values to dependent factories. + + const rr = withResolvers(); + readyResolvers.set(n, { + operation: rr.operation, + resolve: rr.resolve, + reject: rr.reject, + }); } - // Resolve the caller-supplied operation (factory or operation). - // If it's a factory, call it with the dependency object as a single arg. - let operation: Operation; - try { - operation = - typeof def.operation === "function" - ? ( - def.operation as ( - args: Record - ) => Operation - )(depObj) - : (def.operation as Operation); - } catch (err) { - const exportRes = exportResolvers.get(name); - if (exportRes) exportRes.reject(err as Error); - const ready = readyResolvers.get(name); - if (ready) ready.resolve(); - return; + console.log( + `runner: restarting services ${Array.from(affected).join(",")}` + ); + + // stop in reverse start order so dependents stop before providers + const stopOrder = startOrder + .filter((s) => affected.has(s)) + .slice() + .reverse(); + for (const n of stopOrder) { + const task = runningTasks.get(n); + if (task) { + try { + console.log(`runner: halting ${n}`); + task.halt(); + } catch (e) {} + runningTasks.delete(n); + } + // wait for port to close if known + const port = servicePorts.get(n); + if (typeof port === "number") { + const net = (yield* until( + import("node:net") + )) as typeof import("node:net"); + const start = Date.now(); + while (Date.now() - start < 2000) { + try { + yield* until( + new Promise((resolve, reject) => { + const s = net.connect({ port, host: "127.0.0.1" }, () => { + s.end(); + reject(new Error("still listening")); + }); + s.on("error", () => { + s.destroy(); + resolve(); + }); + }) + ); + break; + } catch (e) { + // still listening; wait + yield* sleep(20); + } + } + } } - let exported: any; - try { - exported = yield* operation; - const exportRes = exportResolvers.get(name); - if (exportRes) exportRes.resolve(exported); - } catch (err) { - const exportRes = exportResolvers.get(name); - if (exportRes) exportRes.reject(err as Error); - const ready = readyResolvers.get(name); - if (ready) ready.resolve(); - // don't rethrow here; a failing provider should reject its exportsOperation - // so dependents can observe the error without crashing the whole runner - return; + // remove affected from startOrder so they will be re-appended in the right order + for (const n of affected) { + const i = startOrder.indexOf(n); + if (i >= 0) startOrder.splice(i, 1); } - startOrder.push(name); - const res = readyResolvers.get(name); - if (res) res.resolve(); - try { - if (def.afterStart) yield* def.afterStart(); - } catch (err) { - const exportRes = exportResolvers.get(name); - if (exportRes) exportRes.reject(err as Error); - const ready = readyResolvers.get(name); - if (ready) ready.resolve(); - return; + // start affected services in topological order (providers first) + for (const layer of layers) { + const layerAffected = layer.filter((s) => affected.has(s)); + if (layerAffected.length === 0) continue; + for (const n of layerAffected) { + // wait for deps + yield* waitDeps(n); + console.log(`runner: respawning child ${n}`); + const task = yield* startChild(n); + runningTasks.set(n, task); + } + // after spawning the layer, wait for them to be ready + yield* waitForAllReady(layerAffected, readyResolvers); } - yield* suspend(); - } finally { - if (def.afterStop) yield* def.afterStop(); + processing = false; + yield* each.next(); } }); - } - // small helper to await a service's dependencies - function* waitDeps(name: string): Operation { - const def = effectiveServices[name]; - if (def.deps) { - for (const dep of def.deps) { - const depKey = String(dep); - const r = readyResolvers.get(depKey); - if (!r) - throw new Error( - `missing readiness resolver for dependency '${depKey}'` - ); - yield* r.operation; - } - } - } + // helper to spawn and run a single service name + function startChild(name: string): Operation> { + const def = effectiveServices[name]; + return spawn(function* () { + try { + console.log(`startChild: starting ${name}`); + try { + if (def.beforeStart) yield* def.beforeStart(); + } catch (err) { + const exportRes = exportResolvers.get(name); + if (exportRes) exportRes.reject(err as Error); + const ready = readyResolvers.get(name); + if (ready) ready.resolve(); + return; + } - for (const layer of layers) { - if (!sequential) { - // spawn all services in this layer in parallel - for (const name of layer) { - // wait for deps to be ready (yield the underlying Promise) - yield* waitDeps(name); + // Collect dependency exported values into an object (keyed by dep name) + const depObj: Record = {}; + if (def.deps) { + for (const dep of def.deps) { + const depKey = String(dep); + const exportRes = exportResolvers.get(depKey); + if (!exportRes) { + throw new Error( + `Service '${name}' depends on unknown service '${depKey}'` + ); + } + const val = yield* exportRes.operation; + depObj[depKey] = val; + } + } - // start without waiting; we'll wait for the whole layer below - yield* startChild(name); - } - // after spawning the whole layer, wait until every service in the layer is ready - yield* waitForAllReady(layer, readyResolvers); - } else { - // sequential startup within this layer - for (const name of layer) { - // wait for deps to be ready (yield the underlying Promise) - yield* waitDeps(name); - - // start and then wait for readiness before proceeding - yield* startChild(name); - - const res = readyResolvers.get(name); - if (res) yield* res.operation; - } + // Resolve the caller-supplied operation (factory or operation). + // If it's a factory, call it with the dependency object as a single arg. + let operation: Operation; + try { + operation = + typeof def.operation === "function" + ? ( + def.operation as ( + args: Record + ) => Operation + )(depObj) + : (def.operation as Operation); + } catch (err) { + const exportRes = exportResolvers.get(name); + if (exportRes) exportRes.reject(err as Error); + const ready = readyResolvers.get(name); + if (ready) ready.resolve(); + return; + } + + let exported: unknown; + try { + exported = yield* operation; + const exportRes = exportResolvers.get(name); + if (exportRes) exportRes.resolve(exported); + console.log( + `startChild: ${name} started, exported=${typeof exported}` + ); + // record port if exported value contains one + if ( + exported && + typeof exported === "object" && + "port" in exported && + typeof (exported as Record).port === "number" + ) { + const portVal = (exported as Record).port; + if (typeof portVal === "number") { + servicePorts.set(name, portVal); + } + } + } catch (err) { + const exportRes = exportResolvers.get(name); + if (exportRes) exportRes.reject(err as Error); + const ready = readyResolvers.get(name); + if (ready) ready.resolve(); + // don't rethrow here; a failing provider should reject its export resolver + // so dependents can observe the error without crashing the whole runner + return; + } + + startOrder.push(name); + const res = readyResolvers.get(name); + if (res) res.resolve(); + try { + if (def.afterStart) yield* def.afterStart(); + } catch (err) { + const exportRes = exportResolvers.get(name); + if (exportRes) exportRes.reject(err as Error); + const ready = readyResolvers.get(name); + if (ready) ready.resolve(); + return; + } + + yield* suspend(); + } finally { + if (def.afterStop) yield* def.afterStop(); + } + }); } - } - try { - yield* suspend(); - } finally { - // Run beforeStop hooks in reverse start order - for (const name of startOrder.slice().reverse()) { - const def = services[name]; - if (def.beforeStop) { - yield* def.beforeStop(); + // small helper to await a service's dependencies + function* waitDeps(name: string): Operation { + const def = effectiveServices[name]; + if (def.deps) { + for (const dep of def.deps) { + const depKey = String(dep); + const r = readyResolvers.get(depKey); + if (!r) + throw new Error( + `missing readiness resolver for dependency '${depKey}'` + ); + yield* r.operation; + } } } - } - } as any as ServiceRunner; - // attach the source services for introspection (CLI helpers can access) - (runner as any).services = services; + try { + for (const layer of layers) { + if (!sequential) { + // spawn all services in this layer in parallel + for (const name of layer) { + // wait for deps to be ready (yield the underlying Promise) + yield* waitDeps(name); + + // start without waiting; we'll wait for the whole layer below + console.log(`runner: spawning child ${name}`); + const task = yield* startChild(name); + runningTasks.set(name, task); + } + // after spawning the whole layer, wait until every service in the layer is ready + yield* waitForAllReady(layer, readyResolvers); + } else { + // sequential startup within this layer + for (const name of layer) { + // wait for deps to be ready (yield the underlying Promise) + yield* waitDeps(name); + + // start and then wait for readiness before proceeding + const task = yield* startChild(name); + runningTasks.set(name, task); + + const res = readyResolvers.get(name); + if (res) yield* res.operation; + } + } + } - // return a function (generator) that can be invoked to run the graph - return runner; + yield* provide({ + services: services as S, + serviceUpdates: runnerWatcher?.serviceUpdates, + watches: providedWatches, + servicePorts, + }); + } finally { + console.log("shutting down service graph"); + // Run beforeStop hooks in reverse start order + console.log( + `runner: running beforeStop hooks for ${startOrder + .slice() + .reverse() + .join(",")}` + ); + for (const name of startOrder.slice().reverse()) { + const def = services[name]; + console.log( + `runner: beforeStop def for ${name}: hasBeforeStop=${ + typeof def?.beforeStop === "function" + }` + ); + if (def?.beforeStop) { + console.log(`runner: running beforeStop for ${name}`); + try { + yield* def.beforeStop(); + console.log(`runner: beforeStop for ${name} completed`); + } catch (err) { + console.log(`runner: beforeStop for ${name} threw`, err); + } + } + } + } + }); } diff --git a/packages/server/src/simulation.ts b/packages/server/src/simulation.ts index c7597c7c..d0eb1a39 100644 --- a/packages/server/src/simulation.ts +++ b/packages/server/src/simulation.ts @@ -1,38 +1,111 @@ -import { spawn, suspend, until } from "effection"; +import { resource, until, spawn, each, withResolvers, Ok } from "effection"; import type { Operation } from "effection"; +import { exec } from "@effectionx/process"; +import { stderr, stdout } from "./logging.ts"; import type { FoundationSimulator, FoundationSimulatorListening, } from "@simulacrum/foundation-simulator"; /** - * Helper to start a foundation simulation server factory and return the listening - * information in a typed way. + * Helper to start a foundation simulation server factory + * + * This is implemented as an Effection `resource` so cleanup is handled by the + * `provide` finalizer when the operation's scope is closed. */ -export function useSimulation( - createFactory: (...args: A) => () => FoundationSimulator -): (...args: A) => Operation> { - return function* (...args: A) { - const createSim = createFactory(...args)(); +export function useSimulation>( + name: string, + createFactory: () => FoundationSimulator +): Operation> { + return resource(function* (provide) { + const createSim = createFactory(); const listening: FoundationSimulatorListening = yield* until( createSim.listen() ); - // small debug log to make it visible in tests - // eslint-disable-next-line no-console - console.log(`simulation started on port ${listening.port}`); + console.log(`${name} simulation started on port ${listening.port}`); + + try { + yield* provide(listening); + } finally { + yield* until(listening.ensureClose()); + console.log(`${name} simulation closed on port ${listening.port}`); + } + }); +} + +// Spawn a child Node process to run a simulation factory in a fresh module +// environment. This avoids sharing module cache and allows restarts to pick up +// new code. The runtime uses `bin/run-simulation-child.ts`. +export function useChildSimulation>( + name: string, + modulePath: string, + args: unknown[] = [] +): Operation> { + return resource(function* (provide) { + const cmd = [ + "node", + "--import", + "tsx", + "./bin/run-simulation-child.ts", + modulePath, + JSON.stringify(args), + ] + .map((s) => (s.includes(" ") ? `'${s}'` : s)) + .join(" "); + + const process = yield* exec(cmd); + + // read the first stdout JSON line to get the listening info + let listening: FoundationSimulatorListening | undefined = undefined; + let ready = withResolvers( + "wait until the port is returned to signal ready" + ); + + // forward raw stdout for logging in chunk form (no reassembly) + yield* spawn(function* () { + for (let line of yield* each(process.stdout)) { + const buf = Buffer.from(line); + const str = buf.toString(); + stdout(str); + + if (!listening) { + try { + const parsed = JSON.parse(str); + if (parsed && parsed.ready && typeof parsed.port === "number") { + listening = { + port: parsed.port, + } as FoundationSimulatorListening; + ready.resolve(Ok(listening)); + } + } catch (_) { + // ignore lines that are not JSON + } + } + + yield* each.next(); + } + }); - // ensure server is closed when this operation is finalized yield* spawn(function* () { - try { - yield* suspend(); - } finally { - yield* until(listening.ensureClose()); - // eslint-disable-next-line no-console - console.log(`simulation closed on port ${listening.port}`); + for (let line of yield* each(process.stderr)) { + const str = Buffer.from(line).toString(); + stderr(str); + yield* each.next(); } }); - return listening; - }; + // wait to get the listening info from stdout + yield* ready.operation; + // we know listening is defined here + listening = listening!; + + console.log(`${name} process simulation started on port ${listening.port}`); + + try { + yield* provide(listening); + } finally { + console.log(`${name} simulation closed on port ${listening.port}`); + } + }); } diff --git a/packages/server/src/watch.ts b/packages/server/src/watch.ts new file mode 100644 index 00000000..9e93aafd --- /dev/null +++ b/packages/server/src/watch.ts @@ -0,0 +1,84 @@ +import { join } from "node:path"; +import chokidar, { type EmitArgs } from "chokidar"; +import { + createChannel, + createSignal, + each, + race, + resource, + sleep, + spawn, + type Stream, + until, +} from "effection"; +import picomatch, { type Matcher } from "picomatch"; + +export function debounce( + ms: number +): (stream: Stream) => Stream { + return (stream) => ({ + *[Symbol.iterator]() { + let subscription = yield* stream; + return { + *next() { + let next = yield* subscription.next(); + while (true) { + let result = yield* race([sleep(ms), subscription.next()]); + if (!result) { + return next; + } else { + next = result; + } + } + }, + }; + }, + }); +} + +export function useWatcher() { + return resource(function* (provide) { + const changes = createSignal(); + const serviceUpdates = createChannel<{ service: string; path: string }>(); + const serviceList = new Map(); + + const watcher = chokidar.watch([], { + ignoreInitial: true, + }); + + watcher.on("change", (...args) => { + changes.send(args); + }); + + function add(service: string, paths: string[]) { + // Convert directory paths into recursive globs so that picomatch will + // match any files under those directories. Include the original path + // as well so exact matches still work. + const globs = paths.flatMap((p) => [p, join(p, "**")]); + const matchers = globs.map((g) => picomatch(g)); + serviceList.set(service, matchers); + watcher.add(paths); + } + + yield* spawn(function* () { + for (let args of yield* each(changes)) { + const [path] = args as EmitArgs; + for (let [service, matchers] of serviceList.entries()) { + const isAffected = matchers.some((matcher) => { + return matcher(path); + }); + if (isAffected) { + yield* serviceUpdates.send({ service, path }); + } + } + yield* each.next(); + } + }); + + try { + yield* provide({ serviceUpdates, add }); + } finally { + yield* until(watcher.close()); + } + }); +} diff --git a/packages/server/test/child-simulation.test.ts b/packages/server/test/child-simulation.test.ts new file mode 100644 index 00000000..079f8b0a --- /dev/null +++ b/packages/server/test/child-simulation.test.ts @@ -0,0 +1,21 @@ +import { it } from "node:test"; +import assert from "node:assert"; +import { run, sleep } from "effection"; +import { useChildSimulation } from "../src/simulation.ts"; + +it("useChildSimulation starts a child and returns port", async () => { + await run(function* () { + const listening = yield* useChildSimulation( + "child-test", + "./test/fixtures/simple-sim.ts", + [0] + ); + assert(typeof listening.port === "number"); + + // Verify we received a port and the child reported ready. + assert(typeof listening.port === "number", "port should be a number"); + + // allow a moment before teardown + yield* sleep(20); + }); +}); diff --git a/packages/server/test/data-sharing.test.ts b/packages/server/test/data-sharing.test.ts deleted file mode 100644 index 088eb0b5..00000000 --- a/packages/server/test/data-sharing.test.ts +++ /dev/null @@ -1,222 +0,0 @@ -import { it } from "node:test"; -import { run, spawn, sleep, suspend, until, createScope } from "effection"; -import { services as dataServices } from "../example/operation/data-sharing.ts"; - -import { useServiceGraph } from "../src/services.ts"; -import http from "node:http"; - -// log any uncaught errors so we can debug failing teardown -// eslint-disable-next-line no-console -process.on("uncaughtException", (err) => - console.error("uncaughtException in test:", err) -); -// eslint-disable-next-line no-console -process.on("unhandledRejection", (reason) => - console.error("unhandledRejection in test:", reason) -); - -it("data sharing: short-lived provider and dependent services", async () => { - // create scope at test level so we can control shutdown after assertions - const [scope, destroy] = createScope(); - - try { - const results = await run(function* () { - // start the graph in the provided scope - const runner = dataServices as unknown as any; // runner - scope.run(function* () { - yield* runner(); - // keep the graph task alive until the scope is destroyed - yield* suspend(); - }); - - // yield so spawned children get scheduled and exportsOperations are available - yield* sleep(0); - - const svcMap: any = (dataServices as any).services; - const res: any = {}; - res.data = yield* svcMap.data.exportsOperation; - res.a = yield* svcMap.serviceA.exportsOperation; - res.b = yield* svcMap.serviceB.exportsOperation; - res.c = yield* svcMap.serviceC.exportsOperation; - - // ensure the simulator is reachable while still in the run scope - res.simulatorSeed = undefined as number | undefined; - for (let i = 0; i < 100; i++) { - try { - const fetched = yield* until( - new Promise((resolve, reject) => { - const req = http.get( - { - hostname: "127.0.0.1", - port: res.a.port, - path: "/info", - agent: false, - }, - (r: any) => { - let body = ""; - r.on("data", (c: any) => (body += c)); - r.on("end", () => { - try { - const json = JSON.parse(body); - if (typeof json.seed === "number") { - resolve(json.seed as number); - return; - } - reject(new Error("no seed")); - } catch (err) { - reject(err); - } - }); - } - ); - req.on("error", reject); - }) - ); - res.simulatorSeed = fetched; - break; - } catch (err) { - yield* sleep(10); - } - } - - return res; - }); - - if (results.data.seed !== 42) throw new Error("data seed mismatch"); - if (results.a.handledWith !== 42) - throw new Error("serviceA did not get data"); - if (results.b.used !== 42) - throw new Error("serviceB did not get serviceA's export"); - if (results.c.dataMessage !== "hello from data") - throw new Error("serviceC did not get data message"); - - // verify that the foundation simulator created by serviceA is reachable - if (typeof results.a.port !== "number") - throw new Error("serviceA did not expose port"); - - // ensure the simulator returned the expected seed while the scope was active - if (results.simulatorSeed !== 42) - throw new Error("simulator /info did not return expected seed"); - } catch (err) { - console.error("data-sharing test error:", err); - throw err; - } finally { - console.log("data-sharing test cleanup: starting"); - - // best-effort: close any Server handles first so servers stop accepting - // connections and their finalizers can complete during destroy. - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const preHandles: any[] = (process as any)._getActiveHandles - ? (process as any)._getActiveHandles() - : []; - for (const h of preHandles) { - try { - const name = h && h.constructor && h.constructor.name; - if (name === "Server" && typeof h.close === "function") { - try { - h.close(); - } catch (e) {} - } - } catch (e) {} - } - - // give servers a moment to close - await new Promise((r) => setTimeout(r, 50)); - console.log("data-sharing test cleanup: closed preHandles"); - - // ensure destroy completes within 10s for debugging - await Promise.race([destroy(), new Promise((r) => setTimeout(r, 10000))]); - console.log("data-sharing test cleanup: destroy completed (or timed out)"); - - // best-effort: close any remaining socket handles so the test process exits - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const handles: any[] = (process as any)._getActiveHandles - ? (process as any)._getActiveHandles() - : []; - // log of handles removed; we attempt to close/destroy remaining handles below - for (const h of handles) { - try { - if (typeof h.destroy === "function") h.destroy(); - else if (typeof h.end === "function") h.end(); - } catch (e) { - // ignore - } - } - // allow handles to close - await new Promise((r) => setTimeout(r, 20)); - console.log("data-sharing test: completed OK"); - } -}); - -it("exportsOperation rejects when provider throws", async () => { - console.log("exportsOperation rejects test: start"); - const services = { - data: { - operation: (function* () { - throw new Error("boom"); - })(), - }, - dependent: { - deps: ["data"], - operation: (function* () { - return (yield* (services as any).data.exportsOperation) as any; - })(), - }, - } as any; - - await run(function* () { - yield* spawn(function* () { - try { - const run = useServiceGraph(services as any); - // start only the 'data' service so dependents don't consume the rejection - yield* run(["data"]); - } catch (err) { - // swallow; the test will observe rejection via exportsOperation - } - }); - - yield* sleep(0); - - const op = services.data.exportsOperation; - // eslint-disable-next-line no-console - console.log( - "exportsOperation typeof", - typeof op, - "isIterator", - !!(op && typeof op.next === "function"), - "op", - op - ); - // timebox waiting for exportsOperation to reject to accommodate scheduler timing - let caught = false; - // eslint-disable-next-line no-console - console.log("exportsOperation waiting for rejection (timeboxed)"); - for (let i = 0; i < 100; i++) { - try { - yield* (services as any).data.exportsOperation; - // if it resolves, that's unexpected; break to the final check - // eslint-disable-next-line no-console - console.log("exportsOperation unexpectedly resolved"); - break; - } catch (err: any) { - // eslint-disable-next-line no-console - console.log( - "exportsOperation attempt rejected with", - err && err.message - ); - if (err && err.message === "boom") { - caught = true; - break; - } - throw err; - } finally { - if (!caught) { - // give scheduler a bit of time to make progress - yield* sleep(1); - } - } - } - - if (!caught) throw new Error("expected exportsOperation to reject"); - }); -}); diff --git a/packages/server/test/examples-smoke.test.ts b/packages/server/test/examples-smoke.test.ts index e4676f58..be2f4392 100644 --- a/packages/server/test/examples-smoke.test.ts +++ b/packages/server/test/examples-smoke.test.ts @@ -19,30 +19,42 @@ function checkStatus(port: number): Promise { }); } -import type { ServicesMap } from "../src/services.ts"; +import type { ServiceGraphValue } from "../src/services.ts"; +import type { Operation } from "effection"; it("basic example imports and runs", async () => { - const runner = basicServices as unknown as any; // runner + const runner = basicServices as unknown as () => Operation< + ServiceGraphValue> + >; // runner + let provided: any; // start the graph and await exported ports in a single run operation await run(function* () { const [scope, destroy] = createScope(); + // start operation-style graph and capture provided resource synchronously + try { + provided = yield* runner(); + } catch (err) { + console.error( + "example runner threw:", + err instanceof Error ? err.stack : err + ); + throw err; + } + // keep the graph task alive until the scope is destroyed scope.run(function* () { - yield* runner(); - // keep the graph task alive until the scope is destroyed yield* suspend(); }); - // allow spawned graph to attach `exportsOperation` properties + // allow spawned graph to settle and for services to register their ports yield* sleep(0); - const svcMap: ServicesMap = (basicServices as any).services; + const svcMap = provided!.services; + const ports = provided!.servicePorts; const ps: number[] = []; for (const name of Object.keys(svcMap)) { - const exportsOp = svcMap[name].exportsOperation; - if (!exportsOp) throw new Error(`no exportsOperation on ${name}`); - const val = yield* exportsOp; - ps.push(val); + const port = ports.get(name); + if (typeof port === "number") ps.push(port); } // keep the graph alive briefly to allow HTTP checks @@ -69,17 +81,22 @@ it("basic example imports and runs", async () => { } // best-effort: close any Server handles before requesting shutdown - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const preHandles: any[] = (process as any)._getActiveHandles - ? (process as any)._getActiveHandles() - : []; + const _getActiveHandles = ( + process as unknown as { _getActiveHandles?: () => unknown[] } + )._getActiveHandles; + const preHandles: unknown[] = _getActiveHandles ? _getActiveHandles() : []; + for (const h of preHandles) { try { - const name = h && h.constructor && h.constructor.name; - if (name === "Server" && typeof h.close === "function") { - try { - h.close(); - } catch (e) {} + const name = (h as { constructor?: { name?: string } })?.constructor + ?.name; + if (name === "Server") { + const maybeClose = (h as { close?: unknown }).close; + if (typeof maybeClose === "function") { + try { + (maybeClose as () => void)(); + } catch (e) {} + } } } catch (e) {} } @@ -89,19 +106,25 @@ it("basic example imports and runs", async () => { // request the graph be shut down and wait for up to 1s for cleanup const tb = yield* timebox(1000, () => until(destroy())); if (tb.timeout) { - // eslint-disable-next-line no-console console.warn("cleanup timed out for example graph"); } // best-effort: close any remaining socket handles so tests don't hang - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const handles: any[] = (process as any)._getActiveHandles - ? (process as any)._getActiveHandles() - : []; + const _getActiveHandles2 = ( + process as unknown as { _getActiveHandles?: () => unknown[] } + )._getActiveHandles; + const handles: unknown[] = _getActiveHandles2 ? _getActiveHandles2() : []; for (const h of handles) { try { - if (typeof h.destroy === "function") h.destroy(); - else if (typeof h.end === "function") h.end(); + const maybeDestroy = (h as { destroy?: unknown }).destroy; + if (typeof maybeDestroy === "function") { + (maybeDestroy as () => void)(); + continue; + } + const maybeEnd = (h as { end?: unknown }).end; + if (typeof maybeEnd === "function") { + (maybeEnd as () => void)(); + } } catch (e) { // ignore } @@ -114,25 +137,37 @@ it("basic example imports and runs", async () => { }); it("lifecycle example imports and runs", async () => { - const runner = lifecycleServices as unknown as any; // runner + const runner = lifecycleServices as unknown as () => Operation< + ServiceGraphValue> + >; // runner + let provided: ServiceGraphValue> | undefined; await run(function* () { const [scope, destroy] = createScope(); + // start operation-style graph and capture provided resource synchronously + try { + provided = yield* runner(); + } catch (err) { + console.error( + "example runner threw:", + err instanceof Error ? err.stack : err + ); + throw err; + } + // keep the graph task alive until the scope is destroyed scope.run(function* () { - yield* runner(); - // keep the graph task alive until the scope is destroyed yield* suspend(); }); - // allow spawned graph to attach `exportsOperation` properties + // allow spawned graph to settle and for services to register their ports yield* sleep(0); - const svcMap: ServicesMap = (lifecycleServices as any).services; + const svcMap = provided!.services; + const ports = provided!.servicePorts; const ps: number[] = []; for (const name of Object.keys(svcMap)) { - const exportsOp = svcMap[name].exportsOperation; - if (!exportsOp) throw new Error(`no exportsOperation on ${name}`); - ps.push(yield* exportsOp); + const port = ports.get(name); + if (typeof port === "number") ps.push(port); } yield* sleep(200); @@ -166,25 +201,37 @@ it("lifecycle example imports and runs", async () => { }); it("concurrency example imports and runs", async () => { - const runner = concurrencyServices as unknown as any; // runner + const runner = concurrencyServices as unknown as () => Operation< + ServiceGraphValue> + >; // runner + let provided: ServiceGraphValue> | undefined; await run(function* () { const [scope, destroy] = createScope(); + // start operation-style graph and capture provided resource synchronously + try { + provided = yield* runner(); + } catch (err) { + console.error( + "example runner threw:", + err instanceof Error ? err.stack : err + ); + throw err; + } + // keep the graph task alive until the scope is destroyed scope.run(function* () { - yield* runner(); - // keep the graph task alive until the scope is destroyed yield* suspend(); }); - // allow spawned graph to attach `exportsOperation` properties + // allow spawned graph to settle and for services to register their ports yield* sleep(0); - const svcMap: ServicesMap = (concurrencyServices as any).services; + const svcMap = provided!.services; + const ports = provided!.servicePorts; const ps: number[] = []; for (const name of Object.keys(svcMap)) { - const exportsOp = svcMap[name].exportsOperation; - if (!exportsOp) throw new Error(`no exportsOperation on ${name}`); - ps.push(yield* exportsOp); + const port = ports.get(name); + if (typeof port === "number") ps.push(port); } yield* sleep(200); diff --git a/packages/server/test/fixtures/simple-sim.ts b/packages/server/test/fixtures/simple-sim.ts new file mode 100644 index 00000000..6a0c6168 --- /dev/null +++ b/packages/server/test/fixtures/simple-sim.ts @@ -0,0 +1,10 @@ +import { + createFoundationSimulationServer, + type FoundationSimulator, +} from "@simulacrum/foundation-simulator"; + +export function simulation(port: number = 9999): FoundationSimulator { + return createFoundationSimulationServer({ + port, + })(); +} diff --git a/packages/server/test/service.test.ts b/packages/server/test/service.test.ts index f26a960b..78477df1 100644 --- a/packages/server/test/service.test.ts +++ b/packages/server/test/service.test.ts @@ -72,7 +72,7 @@ describe("useService with wellness check", () => { await run(function* () { yield* useService("test-service", nodeScriptWorks, { wellnessCheck: { - timeout: 700, + timeout: 1000, frequency: 200, *operation(stdio) { for (let line of yield* each(stdio)) { diff --git a/packages/server/test/services.test.ts b/packages/server/test/services.test.ts index 1b11aff9..7b965922 100644 --- a/packages/server/test/services.test.ts +++ b/packages/server/test/services.test.ts @@ -41,10 +41,12 @@ it("starts services in dependency order", async () => { }, } ), - deps: ["A"], + deps: ["A"] as const, }, }); yield* run(); + // keep spawned graph alive + yield* suspend(); }); // The graph is running; sleep a short time to let the services start yield* sleep(200); @@ -69,7 +71,7 @@ it("throws on cycles in dependency graph", async () => { "A", "node --import tsx ./test/services/service-a.ts" ), - deps: ["B"], + deps: ["B"] as const, }, B: { operation: useService( @@ -109,7 +111,7 @@ it("runs beforeStop hooks in reverse order", async () => { beforeStop() { return (function* () { stopOrder.push("A"); - })() as unknown as Operation; + })() as Operation; }, }, B: { @@ -127,15 +129,17 @@ it("runs beforeStop hooks in reverse order", async () => { }, } ), - deps: ["A"], + deps: ["A"] as const, beforeStop() { return (function* () { stopOrder.push("B"); - })() as unknown as Operation; + })() as Operation; }, }, }); yield* run(); + // keep spawned graph alive so beforeStop hooks run on teardown + yield* suspend(); }); // let them start yield* sleep(200); @@ -184,6 +188,8 @@ it("starts independent services in parallel", async () => { }, }); yield* run(); + // keep spawned graph alive so services continue to run + yield* suspend(); }); yield* sleep(250); }); @@ -241,17 +247,17 @@ it("runs subset of services with dependencies", async () => { ), }, dependent: { - deps: ["fast", "slow"], + deps: ["fast", "slow"] as const, operation: (function* () { startTimes.set("dependent", Date.now()); yield* suspend(); - })() as unknown as Operation, + })() as Operation, }, - } as any; + }; // only request dependent; fast and slow should be included as deps const run = useServiceGraph(services); - yield* run(["dependent"]); + yield* run(); // start full graph (subset-run removed) }); yield* sleep(300); }); diff --git a/packages/server/test/simulation.test.ts b/packages/server/test/simulation.test.ts new file mode 100644 index 00000000..5cd5f3b5 --- /dev/null +++ b/packages/server/test/simulation.test.ts @@ -0,0 +1,68 @@ +import { it } from "node:test"; +import assert from "node:assert"; +import { run, createScope, suspend, until, sleep } from "effection"; +import { useSimulation } from "../src/simulation.ts"; +import { simulation } from "./fixtures/simple-sim.ts"; +import { createFoundationSimulationServer } from "@simulacrum/foundation-simulator"; + +it("useSimulation returns listening info", async () => { + const port = await run(function* () { + const listening = yield* useSimulation("test", simulation); + return listening.port; + }); + assert(typeof port === "number", "port is a number"); +}); + +it("simulation closes when scope is destroyed", async () => { + await run(function* () { + const [scope, destroy] = createScope(); + + let port: number | undefined; + + // start the simulation in the scope and keep it alive until destroy() + scope.run(function* () { + const listening = yield* useSimulation( + "inline-test", + createFoundationSimulationServer({ + port: 0, + extendRouter(router) { + router.get("/info", (_req, res) => res.json({ ok: true })); + }, + }) + ); + port = listening.port; + yield* suspend(); + }); + + // wait for the scope-run to set the port + for (let i = 0; i < 100; i++) { + if (typeof port === "number") break; + yield* sleep(5); + } + + const status = yield* until( + fetch(new URL(`http://127.0.0.1:${port}/info`)) + ); + if (!status.ok) { + throw new Error(`expected 200 OK from simulation, got ${status.status}`); + } + + // now destroy the scope and ensure the server stops accepting connections + yield* until(destroy()); + + // server should no longer accept connections + let closed = false; + for (let i = 0; i < 50; i++) { + try { + yield* until(fetch(new URL(`http://127.0.0.1:${port}/info`))); + // if request succeeded, wait and retry + } catch (e) { + closed = true; + break; + } + yield* sleep(10); + } + + if (!closed) throw new Error("simulation still responds after destroy"); + }); +}); diff --git a/packages/server/test/watch.test.ts b/packages/server/test/watch.test.ts new file mode 100644 index 00000000..237266d2 --- /dev/null +++ b/packages/server/test/watch.test.ts @@ -0,0 +1,82 @@ +import { it } from "node:test"; +import assert from "node:assert"; +import { run, suspend, sleep, until, spawn } from "effection"; +import * as fs from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { useServiceGraph } from "../src/services.ts"; +import { simulation } from "./fixtures/simple-sim.ts"; +import { useSimulation } from "../src/simulation.ts"; + +it("restarts services on watched file change and restarts dependents", async () => { + const prefix = path.join(os.tmpdir(), "sim-watch-"); + // create a temporary directory to hold test files + const dir = await fs.mkdtemp(prefix); + const trigger = path.join(dir, "trigger.txt"); + + // initial trigger file + await fs.writeFile(trigger, "initial"); + + const updates: string[] = []; + await run(function* () { + yield* spawn(function* () { + // start the graph and enable watch mode + const op = useServiceGraph( + { + a: { + watch: [dir], + operation: useSimulation("test-simulation-a", () => + simulation(5500) + ), + }, + b: { + deps: ["a"], + operation: useSimulation("test-simulation-a", () => + simulation(5501) + ), + }, + }, + { watch: true, watchDebounce: 20 } + ); + + try { + const services = yield* op(); + // subscribe to the immediate serviceUpdates stream and wait for the first update + if (!services.serviceUpdates) + throw new Error("serviceUpdates not available"); + const subscription = yield* services.serviceUpdates; + + // wait for the first update (will occur after the test touches the file) + const first = yield* subscription.next(); + updates.push(String((first.value as { service: string }).service)); + } catch (e) { + throw e; + } + + yield* suspend(); + }); + + // allow initial startup and wait for bOut to appear + for (let i = 0; i < 200; i++) { + try { + yield* until(fs.readFile(trigger, "utf8")); + break; + } catch (err) { + yield* sleep(20); + } + } + + // give the spawned subscription a moment to attach + yield* sleep(50); + // touch the trigger file to cause a restart + yield* until(fs.writeFile(trigger, "changed")); + // give watcher/poller a moment + yield* sleep(100); + }); + + // remove tmp dir + await fs.rm(dir, { recursive: true, force: true }); + + assert(updates.length >= 1, "expected at least one update"); + assert(updates[0] === "a", "first update is service 'a'"); +}); From 57011264b238380fd620b4658d498555a422ea95 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Wed, 7 Jan 2026 23:43:30 -0600 Subject: [PATCH 07/38] CLI handles subset --- packages/server/example/basic-graph.ts | 2 +- packages/server/example/concurrency-layers.ts | 2 +- packages/server/example/lifecycle-hooks.ts | 2 +- .../server/example/operation/basic-graph.ts | 2 +- .../example/operation/concurrency-layers.ts | 2 +- .../example/operation/lifecycle-hooks.ts | 2 +- packages/server/src/cli.ts | 33 +++---------------- packages/server/src/index.ts | 1 + 8 files changed, 11 insertions(+), 35 deletions(-) diff --git a/packages/server/example/basic-graph.ts b/packages/server/example/basic-graph.ts index 386db224..ce1d2108 100644 --- a/packages/server/example/basic-graph.ts +++ b/packages/server/example/basic-graph.ts @@ -58,5 +58,5 @@ export function example(opts: { duration?: number } = {}) { import { fileURLToPath } from "node:url"; if (process.argv[1] === fileURLToPath(import.meta.url)) { // run via CLI when executed directly - simulationCLI(services()); + simulationCLI(services); } diff --git a/packages/server/example/concurrency-layers.ts b/packages/server/example/concurrency-layers.ts index 2dafc1c7..953260e2 100644 --- a/packages/server/example/concurrency-layers.ts +++ b/packages/server/example/concurrency-layers.ts @@ -62,7 +62,7 @@ export const services = useServiceGraph(servicesMap); import { fileURLToPath } from "node:url"; if (process.argv[1] === fileURLToPath(import.meta.url)) { - simulationCLI(services()); + simulationCLI(services); } export function example( diff --git a/packages/server/example/lifecycle-hooks.ts b/packages/server/example/lifecycle-hooks.ts index 6adec34e..9366b5ab 100644 --- a/packages/server/example/lifecycle-hooks.ts +++ b/packages/server/example/lifecycle-hooks.ts @@ -72,7 +72,7 @@ export const services = useServiceGraph(servicesMap); import { fileURLToPath } from "node:url"; if (process.argv[1] === fileURLToPath(import.meta.url)) { - simulationCLI(services()); + simulationCLI(services); } export function example(opts: { duration?: number } = {}) { diff --git a/packages/server/example/operation/basic-graph.ts b/packages/server/example/operation/basic-graph.ts index 1754e0b0..61abbe98 100644 --- a/packages/server/example/operation/basic-graph.ts +++ b/packages/server/example/operation/basic-graph.ts @@ -27,5 +27,5 @@ export function example(opts: { duration?: number } = {}) { import { fileURLToPath } from "node:url"; if (process.argv[1] === fileURLToPath(import.meta.url)) { - simulationCLI(services()); + simulationCLI(services); } diff --git a/packages/server/example/operation/concurrency-layers.ts b/packages/server/example/operation/concurrency-layers.ts index a8c49f8f..5e55c93a 100644 --- a/packages/server/example/operation/concurrency-layers.ts +++ b/packages/server/example/operation/concurrency-layers.ts @@ -22,7 +22,7 @@ export const services = useServiceGraph(servicesMap); import { fileURLToPath } from "node:url"; if (process.argv[1] === fileURLToPath(import.meta.url)) { - simulationCLI(services()); + simulationCLI(services); } export function example(opts: { duration?: number } = {}) { diff --git a/packages/server/example/operation/lifecycle-hooks.ts b/packages/server/example/operation/lifecycle-hooks.ts index a9e0940b..776b178d 100644 --- a/packages/server/example/operation/lifecycle-hooks.ts +++ b/packages/server/example/operation/lifecycle-hooks.ts @@ -48,5 +48,5 @@ export function example(opts: { duration?: number } = {}) { import { fileURLToPath } from "node:url"; if (process.argv[1] === fileURLToPath(import.meta.url)) { - simulationCLI(services()); + simulationCLI(services); } diff --git a/packages/server/src/cli.ts b/packages/server/src/cli.ts index 6880799f..9f7d8c1e 100644 --- a/packages/server/src/cli.ts +++ b/packages/server/src/cli.ts @@ -2,9 +2,8 @@ import { parseArgs } from "node:util"; import { suspend, main, type Operation } from "effection"; import type { ServiceGraphValue } from "./services.ts"; -// Internal generator operation used by both the CLI (via main) and programmatically export function* simulationCLIOp>( - runnerOp: Operation> + serviceGraph: (subset?: string[] | string) => Operation> ) { const { values } = parseArgs({ options: { @@ -20,8 +19,6 @@ export function* simulationCLIOp>( }); function* printUsage() { - // When we only have the operation form we cannot enumerate services - // without starting the graph — so show a succinct help message. console.log( `Usage: cli [-s|--services serviceName] [--watch] [--watch-debounce ms]` ); @@ -37,14 +34,6 @@ export function* simulationCLIOp>( .map((s) => s.trim()) .filter(Boolean) : undefined; - if (subset) { - // subset-run no longer supported for operation-style graphs; warn - // (consumer should construct a graph for the subset explicitly) - // eslint-disable-next-line no-console - console.warn( - "--services subset not supported with operation-style runner; starting full graph" - ); - } const runOptions: { watch?: boolean; watchDebounce?: number } = { watch: !!values.watch, @@ -53,27 +42,13 @@ export function* simulationCLIOp>( runOptions.watchDebounce = Number(values["watch-debounce"]); // Start the graph and fetch the provided info - const servicesVal = yield* runnerOp; - if (runOptions.watch) { - // eslint-disable-next-line no-console - console.log("starting in watch mode; watched paths:"); - for (const name of Object.keys(servicesVal.services)) { - const def = servicesVal.services[name]; - const watch = def.watch - ? typeof def.watch === "function" - ? (def.watch as () => string[])() - : def.watch - : undefined; - if (watch) console.log(` - ${name}: ${watch.join(", ")}`); - } - } + yield* serviceGraph(subset); yield* suspend(); } -// Public helper: call this from examples or CLI entrypoints — it will invoke effection's main() export function simulationCLI>( - runnerOp: Operation> + serviceGraph: (subset?: string[] | string) => Operation> ) { - return main(() => simulationCLIOp(runnerOp)); + return main(() => simulationCLIOp(serviceGraph)); } diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 0a5441f3..58cec33a 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -2,3 +2,4 @@ export * from "./logging.ts"; export * from "./service.ts"; export * from "./services.ts"; export * from "./simulation.ts"; +export * from "./cli.ts"; From 61ac59f8c39cce77dd252edfa1415310ef3a2597 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Fri, 9 Jan 2026 17:47:41 -0600 Subject: [PATCH 08/38] handle services through a coordination layer --- package.json | 4 +- packages/server/README.md | 27 +- packages/server/bin/run-simulation-child.ts | 15 +- packages/server/example/README.md | 2 +- packages/server/example/basic-graph.ts | 62 -- packages/server/example/concurrency-layers.ts | 72 +- packages/server/example/lifecycle-hooks.ts | 96 +-- .../example/operation/concurrency-layers.ts | 35 - .../example/operation/lifecycle-hooks.ts | 52 -- packages/server/example/process-graph.ts | 70 ++ packages/server/example/services/a.ts | 6 - packages/server/example/services/b.ts | 6 - .../server/example/services/basic-sim-1.ts | 19 + .../server/example/services/basic-sim-2.ts | 19 + packages/server/example/services/fast.ts | 6 - .../server/example/services/http-server.ts | 49 -- packages/server/example/services/slow.ts | 6 - .../basic-graph.ts => simulation-graph.ts} | 18 +- packages/server/package.json | 3 +- packages/server/src/cli.ts | 98 ++- packages/server/src/services.ts | 768 +++--------------- packages/server/src/simulation.ts | 28 +- packages/server/src/watch.ts | 10 +- packages/server/test/examples-smoke.test.ts | 6 +- packages/server/test/signal.test.ts | 62 ++ 25 files changed, 427 insertions(+), 1112 deletions(-) delete mode 100644 packages/server/example/basic-graph.ts delete mode 100644 packages/server/example/operation/concurrency-layers.ts delete mode 100644 packages/server/example/operation/lifecycle-hooks.ts create mode 100644 packages/server/example/process-graph.ts delete mode 100644 packages/server/example/services/a.ts delete mode 100644 packages/server/example/services/b.ts create mode 100644 packages/server/example/services/basic-sim-1.ts create mode 100644 packages/server/example/services/basic-sim-2.ts delete mode 100644 packages/server/example/services/fast.ts delete mode 100644 packages/server/example/services/http-server.ts delete mode 100644 packages/server/example/services/slow.ts rename packages/server/example/{operation/basic-graph.ts => simulation-graph.ts} (59%) create mode 100644 packages/server/test/signal.test.ts diff --git a/package.json b/package.json index 399ea9bd..ba00ecc2 100644 --- a/package.json +++ b/package.json @@ -33,8 +33,8 @@ "npm": ">=11" }, "volta": { - "node": "20.19.5", - "npm": "11.6.2" + "node": "20.19.6", + "npm": "11.7.0" }, "devDependencies": { "@arethetypeswrong/core": "^0.18.2", diff --git a/packages/server/README.md b/packages/server/README.md index 326ca85a..8952de86 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -128,30 +128,9 @@ npm run example:concurrency node --import tsx ./example/basic-graph.ts ``` -### Typed exports between services 💡 +### Sharing exported values between services (note) -Services may return a value from their `operation`. That value is exposed to dependent services via an `exportsOperation` on the provider. +Previously services could expose their return value via a public `exportsOperation` that consumers could await. That mechanism has been removed in this branch as we move to a child-process-focused runner model. Provider-returned values are still delivered to dependent service factories internally, but no longer exposed as an operation on the public `services` map. -```ts -const services = { - provider: { - // provider operation returns { url: string } - operation: useService<{ url: string }>( - "provider", - "node --import tsx ./example/services/provider.ts" - ), - }, - consumer: { - deps: ["provider"], - operation() { - return (function* () { - // access provider exports via the `services` variable - const providerExports = yield* services.provider.exportsOperation; - console.log("provider url:", providerExports.url); - })(); - }, - }, -}; - -// create a runner via `useServiceGraph(services)` and pass that runner to `simulationCLI` if desired, e.g. `simulationCLI(useServiceGraph(services))` +For convenience tests may use the `servicePorts` map exposed by the running graph to discover HTTP ports that services registered when they start. ```` diff --git a/packages/server/bin/run-simulation-child.ts b/packages/server/bin/run-simulation-child.ts index 6f92e420..4290d4b5 100644 --- a/packages/server/bin/run-simulation-child.ts +++ b/packages/server/bin/run-simulation-child.ts @@ -1,15 +1,19 @@ #!/usr/bin/env node import { main, suspend, until } from "effection"; import { pathToFileURL } from "node:url"; +import type { + FoundationSimulator, + FoundationSimulatorListening, +} from "@simulacrum/foundation-simulator"; main(function* () { const args = process.argv.slice(2); + console.dir({ args }); if (args.length < 1) { throw new Error("usage: run-simulation-child.js [jsonArgs]"); } const modulePath = args[0]; - const jsonArgs = args[1] ? JSON.parse(args[1]) : []; // Resolve and import module inside the operation let mod: any; @@ -38,17 +42,13 @@ main(function* () { throw new Error(`no factory function found in module: ${modulePath}`); } - // call factory with provided args - let sim = factory(...(Array.isArray(jsonArgs) ? jsonArgs : [jsonArgs])); - if (typeof sim === "function") { - sim = sim(); - } + let sim = factory() as FoundationSimulator; if (!sim || typeof sim.listen !== "function") { throw new Error("factory did not return a simulator with .listen()"); } - let listening: any; + let listening: FoundationSimulatorListening | undefined = undefined; try { listening = yield* until(sim.listen()); const out = JSON.stringify({ @@ -59,6 +59,7 @@ main(function* () { console.log(out); yield* suspend(); } finally { + console.log("shutting down gracefully..."); try { if (listening && typeof listening.ensureClose === "function") { yield* until(listening.ensureClose()); diff --git a/packages/server/example/README.md b/packages/server/example/README.md index 330da1e7..499b6907 100644 --- a/packages/server/example/README.md +++ b/packages/server/example/README.md @@ -6,7 +6,7 @@ There are two sets of examples: - **use-service** (top-level files like `basic-graph.ts`, `lifecycle-hooks.ts`, `concurrency-layers.ts`) — these spawn separate processes using `useService` (e.g. `node --import tsx ./example/services/*.ts`). Use these to exercise the process-based behavior. -- **operation** (under `operation/`) — these use the `httpServer()` operation directly and run entirely in-process. They are faster and more deterministic for tests and quick iteration. +- **operation** (under `operation/`) — these demonstrate `useChildSimulation()` which runs each service in a child process using a simulation factory. They show how to isolate simulations and start them as independent processes. Quick commands: diff --git a/packages/server/example/basic-graph.ts b/packages/server/example/basic-graph.ts deleted file mode 100644 index ce1d2108..00000000 --- a/packages/server/example/basic-graph.ts +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env node -import { sleep, each, type Stream } from "effection"; -import { useService } from "../src/service.ts"; -import { useServiceGraph } from "../src/services.ts"; -import { simulationCLI } from "../src/cli.ts"; - -const servicesMap = { - A: { - operation: useService("A", "node --import tsx ./example/services/a.ts", { - wellnessCheck: { - frequency: 10, - *operation(stdio: Stream) { - for (let line of yield* each(stdio)) { - if (line.includes("started")) { - console.log("A ready (wellnessCheck)"); - return { ok: true, value: undefined }; - } - yield* each.next(); - } - // default: return success so the result type is well-formed - return { ok: true, value: undefined }; - }, - }, - }), - }, - B: { - operation: useService("B", "node --import tsx ./example/services/b.ts", { - wellnessCheck: { - frequency: 10, - *operation(stdio: Stream) { - for (let line of yield* each(stdio)) { - if (line.includes("started")) { - console.log("B ready (wellnessCheck)"); - return { ok: true, value: undefined }; - } - yield* each.next(); - } - // default: return success so the result type is well-formed - return { ok: true, value: undefined }; - }, - }, - }), - deps: ["A"] as const, - }, -}; - -export const services = useServiceGraph(servicesMap); - -export function example(opts: { duration?: number } = {}) { - return (function* () { - const run = services; - yield* run(); - yield* sleep(opts.duration ?? 300); - console.log(`Basic example complete`); - })(); -} - -import { fileURLToPath } from "node:url"; -if (process.argv[1] === fileURLToPath(import.meta.url)) { - // run via CLI when executed directly - simulationCLI(services); -} diff --git a/packages/server/example/concurrency-layers.ts b/packages/server/example/concurrency-layers.ts index 953260e2..7c8dc88f 100644 --- a/packages/server/example/concurrency-layers.ts +++ b/packages/server/example/concurrency-layers.ts @@ -1,60 +1,29 @@ #!/usr/bin/env node -import { sleep, each, type Stream } from "effection"; +import { resource, sleep } from "effection"; import { useServiceGraph } from "../src/services.ts"; -import { useService } from "../src/service.ts"; +import { useChildSimulation } from "../src/simulation.ts"; import { simulationCLI } from "../src/cli.ts"; const servicesMap = { fast: { - operation: useService( - "fast", - "node --import tsx ./example/services/fast.ts", - { - wellnessCheck: { - frequency: 10, - *operation(stdio: Stream) { - for (let line of yield* each(stdio)) { - if (line.includes("started")) { - console.log("fast ready"); - return { ok: true, value: undefined }; - } - yield* each.next(); - } - // default success - return { ok: true, value: undefined }; - }, - }, - } - ), + operation: useChildSimulation("fast", "./example/services/basic-sim-1.ts"), + watch: ["./example/services/basic-sim-1.ts"], }, slow: { - operation: useService( - "slow", - "node --import tsx ./example/services/slow.ts", - { - wellnessCheck: { - frequency: 10, - *operation(stdio: Stream) { - for (let line of yield* each(stdio)) { - if (line.includes("started")) { - console.log("slow ready"); - return { ok: true, value: undefined }; - } - yield* each.next(); - } - // default success - return { ok: true, value: undefined }; - }, - }, - } - ), + operation: useChildSimulation("slow", "./example/services/basic-sim-2.ts"), + watch: ["./example/services/basic-sim-2.ts"], }, dependent: { - deps: ["fast", "slow"] as const, - operation: (function* () { - console.log("dependent: all deps started; running dependent logic"); - yield* sleep(50); - })(), + // deps: ["fast", "slow"] as const, + operation: resource(function* (provide) { + try { + console.log("all deps started; running dependent service"); + yield* provide(); + } finally { + console.log("stopping dependent service"); + } + }), + watch: ["./example/services/basic-sim.ts"], }, }; @@ -65,16 +34,11 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) { simulationCLI(services); } -export function example( - opts: { duration?: number; sequential?: boolean } = {} -) { +export function example(opts: { duration?: number } = {}) { return (function* () { - if (opts.sequential) { - console.log("Running concurrency example in sequential mode"); - } const run = services; yield* run(); yield* sleep(opts.duration ?? 300); - console.log(`Concurrency example complete`); + console.log(`Concurrency example (operation) complete`); })(); } diff --git a/packages/server/example/lifecycle-hooks.ts b/packages/server/example/lifecycle-hooks.ts index 9366b5ab..f3cb47df 100644 --- a/packages/server/example/lifecycle-hooks.ts +++ b/packages/server/example/lifecycle-hooks.ts @@ -1,86 +1,56 @@ #!/usr/bin/env node -import { sleep, each, type Stream } from "effection"; -import { useService } from "../src/service.ts"; +import { sleep, suspend } from "effection"; import { useServiceGraph } from "../src/services.ts"; +import { useChildSimulation } from "../src/simulation.ts"; import { simulationCLI } from "../src/cli.ts"; const servicesMap = { provider: { - operation: useService( - "provider", - "node --import tsx ./example/services/fast.ts", - { - wellnessCheck: { - frequency: 10, - *operation(stdio: Stream) { - for (let line of yield* each(stdio)) { - if (line.includes("started")) { - return { ok: true, value: undefined }; - } - yield* each.next(); - } - return { ok: true, value: undefined }; - }, - }, + operation: (function* () { + yield* useChildSimulation( + "provider", + "./example/services/basic-sim.ts", + [0, 10] + ); + console.log("provider: afterStart (operation)"); + try { + yield* suspend(); + } finally { + console.log("provider: beforeStop (operation)"); } - ), - afterStart() { - return (function* () { - console.log("provider: afterStart"); - })(); - }, - beforeStop() { - return (function* () { - console.log("provider: beforeStop"); - })(); - }, + })(), }, consumer: { deps: ["provider"] as const, - operation: useService( - "consumer", - "node --import tsx ./example/services/a.ts", - { - wellnessCheck: { - frequency: 10, - *operation(stdio: Stream) { - for (let line of yield* each(stdio)) { - if (line.includes("started")) { - return { ok: true, value: undefined }; - } - yield* each.next(); - } - return { ok: true, value: undefined }; - }, - }, + operation: (function* () { + yield* useChildSimulation( + "consumer", + "./example/services/basic-sim.ts", + [0, 10] + ); + console.log("consumer: afterStart (operation)"); + try { + yield* suspend(); + } finally { + console.log("consumer: beforeStop (operation)"); } - ), - afterStart() { - return (function* () { - console.log("consumer: afterStart"); - })(); - }, - beforeStop() { - return (function* () { - console.log("consumer: beforeStop"); - })(); - }, + })(), }, }; export const services = useServiceGraph(servicesMap); -import { fileURLToPath } from "node:url"; -if (process.argv[1] === fileURLToPath(import.meta.url)) { - simulationCLI(services); -} - export function example(opts: { duration?: number } = {}) { return (function* () { - console.log(`Starting lifecycle hooks example`); + console.log(`Starting lifecycle hooks example (operation)`); const run = services; yield* run(); yield* sleep(opts.duration ?? 150); - console.log(`Lifecycle example complete`); + console.log(`Lifecycle example (operation) complete`); })(); } + +import { fileURLToPath } from "node:url"; +if (process.argv[1] === fileURLToPath(import.meta.url)) { + simulationCLI(services); +} diff --git a/packages/server/example/operation/concurrency-layers.ts b/packages/server/example/operation/concurrency-layers.ts deleted file mode 100644 index 5e55c93a..00000000 --- a/packages/server/example/operation/concurrency-layers.ts +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env node -import { sleep } from "effection"; -import { useServiceGraph } from "../../src/services.ts"; -import { httpServer } from "../services/http-server.ts"; -import { simulationCLI } from "../../src/cli.ts"; - -const servicesMap = { - fast: { operation: httpServer({ startDelay: 10 }) }, - slow: { operation: httpServer({ startDelay: 100 }) }, - dependent: { - deps: ["fast", "slow"] as const, - operation: (function* () { - console.log( - "dependent: all deps started; running dependent logic (operation)" - ); - yield* sleep(50); - })(), - }, -}; - -export const services = useServiceGraph(servicesMap); - -import { fileURLToPath } from "node:url"; -if (process.argv[1] === fileURLToPath(import.meta.url)) { - simulationCLI(services); -} - -export function example(opts: { duration?: number } = {}) { - return (function* () { - const run = services; - yield* run(); - yield* sleep(opts.duration ?? 300); - console.log(`Concurrency example (operation) complete`); - })(); -} diff --git a/packages/server/example/operation/lifecycle-hooks.ts b/packages/server/example/operation/lifecycle-hooks.ts deleted file mode 100644 index 776b178d..00000000 --- a/packages/server/example/operation/lifecycle-hooks.ts +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env node -import { sleep } from "effection"; -import { useServiceGraph } from "../../src/services.ts"; -import { httpServer } from "../services/http-server.ts"; -import { simulationCLI } from "../../src/cli.ts"; - -const servicesMap = { - provider: { - operation: httpServer({ startDelay: 10 }), - afterStart() { - return (function* () { - console.log("provider: afterStart (operation)"); - })(); - }, - beforeStop() { - return (function* () { - console.log("provider: beforeStop (operation)"); - })(); - }, - }, - consumer: { - deps: ["provider"] as const, - operation: httpServer({ startDelay: 10 }), - afterStart() { - return (function* () { - console.log("consumer: afterStart (operation)"); - })(); - }, - beforeStop() { - return (function* () { - console.log("consumer: beforeStop (operation)"); - })(); - }, - }, -}; - -export const services = useServiceGraph(servicesMap); - -export function example(opts: { duration?: number } = {}) { - return (function* () { - console.log(`Starting lifecycle hooks example (operation)`); - const run = services; - yield* run(); - yield* sleep(opts.duration ?? 150); - console.log(`Lifecycle example (operation) complete`); - })(); -} - -import { fileURLToPath } from "node:url"; -if (process.argv[1] === fileURLToPath(import.meta.url)) { - simulationCLI(services); -} diff --git a/packages/server/example/process-graph.ts b/packages/server/example/process-graph.ts new file mode 100644 index 00000000..ffaf02d9 --- /dev/null +++ b/packages/server/example/process-graph.ts @@ -0,0 +1,70 @@ +#!/usr/bin/env node +import { sleep, each, type Stream } from "effection"; +import { useService } from "../src/service.ts"; +import { useServiceGraph } from "../src/services.ts"; +import { simulationCLI } from "../src/cli.ts"; + +const servicesMap = { + A: { + operation: useService( + "A", + "node --import tsx ./example/services/basic-sim.ts", + { + wellnessCheck: { + frequency: 10, + *operation(stdio: Stream) { + for (let line of yield* each(stdio)) { + if (line.includes("started")) { + console.log("A ready (wellnessCheck)"); + return { ok: true, value: undefined }; + } + yield* each.next(); + } + // default: return success so the result type is well-formed + return { ok: true, value: undefined }; + }, + }, + } + ), + }, + B: { + deps: ["A"] as const, + operation: useService( + "B", + "node --import tsx ./example/services/basic-sim.ts", + { + wellnessCheck: { + frequency: 10, + *operation(stdio: Stream) { + for (let line of yield* each(stdio)) { + if (line.includes("started")) { + console.log("B ready (wellnessCheck)"); + return { ok: true, value: undefined }; + } + yield* each.next(); + } + // default: return success so the result type is well-formed + return { ok: true, value: undefined }; + }, + }, + } + ), + }, +}; + +export const services = useServiceGraph(servicesMap); + +export function example(opts: { duration?: number } = {}) { + return (function* () { + const run = services; + yield* run(); + yield* sleep(opts.duration ?? 300); + console.log(`Basic example complete`); + })(); +} + +import { fileURLToPath } from "node:url"; +if (process.argv[1] === fileURLToPath(import.meta.url)) { + // run via CLI when executed directly + simulationCLI(services); +} diff --git a/packages/server/example/services/a.ts b/packages/server/example/services/a.ts deleted file mode 100644 index 9b9ce494..00000000 --- a/packages/server/example/services/a.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { main } from "effection"; -import { httpServer } from "./http-server.ts"; - -main(function* () { - yield* httpServer({ startDelay: 10 }); -}); diff --git a/packages/server/example/services/b.ts b/packages/server/example/services/b.ts deleted file mode 100644 index 47421f18..00000000 --- a/packages/server/example/services/b.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { main } from "effection"; -import { httpServer } from "./http-server.ts"; - -main(function* () { - yield* httpServer({ startDelay: 40 }); -}); diff --git a/packages/server/example/services/basic-sim-1.ts b/packages/server/example/services/basic-sim-1.ts new file mode 100644 index 00000000..f54c87b4 --- /dev/null +++ b/packages/server/example/services/basic-sim-1.ts @@ -0,0 +1,19 @@ +import { + createFoundationSimulationServer, + type FoundationSimulator, +} from "@simulacrum/foundation-simulator"; + +export function simulation( + port: number = 3301, + startDelay: number = 10 +): FoundationSimulator { + const factory = createFoundationSimulationServer({ + port, + extendRouter(router) { + router.get("/status", (_req, res) => { + res.status(200).send("ok"); + }); + }, + })(); + return factory; +} diff --git a/packages/server/example/services/basic-sim-2.ts b/packages/server/example/services/basic-sim-2.ts new file mode 100644 index 00000000..daa3569b --- /dev/null +++ b/packages/server/example/services/basic-sim-2.ts @@ -0,0 +1,19 @@ +import { + createFoundationSimulationServer, + type FoundationSimulator, +} from "@simulacrum/foundation-simulator"; + +export function simulation( + port: number = 3302, + startDelay: number = 10 +): FoundationSimulator { + const factory = createFoundationSimulationServer({ + port, + extendRouter(router) { + router.get("/status", (_req, res) => { + res.status(200).send("ok"); + }); + }, + })(); + return factory; +} diff --git a/packages/server/example/services/fast.ts b/packages/server/example/services/fast.ts deleted file mode 100644 index 9b9ce494..00000000 --- a/packages/server/example/services/fast.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { main } from "effection"; -import { httpServer } from "./http-server.ts"; - -main(function* () { - yield* httpServer({ startDelay: 10 }); -}); diff --git a/packages/server/example/services/http-server.ts b/packages/server/example/services/http-server.ts deleted file mode 100644 index 7b212af3..00000000 --- a/packages/server/example/services/http-server.ts +++ /dev/null @@ -1,49 +0,0 @@ -import http from "node:http"; -import type { AddressInfo } from "node:net"; -import type { Operation } from "effection"; -import { sleep, resource, withResolvers } from "effection"; - -export type HttpServerOptions = { - port?: number; - startDelay?: number; // ms -}; - -export function httpServer(options: HttpServerOptions = {}): Operation { - return resource(function* (provide) { - if (options.startDelay) { - yield* sleep(options.startDelay); - } - - const port = options.port ?? 0; - const server = http.createServer((req, res) => { - if (req.url === "/status") { - res.writeHead(200, { "content-type": "text/plain" }); - res.end("ok"); - return; - } - res.writeHead(404); - res.end(); - }); - - const ready = withResolvers(); - server.listen(port, () => { - const address = server.address() as AddressInfo | string | null; - const p = typeof address === "object" && address ? address.port : port; - console.log(`http server started on port ${p}`); - ready.resolve(); - }); - server.on("error", (err) => ready.reject(err as Error)); - - // wait for server to be listening - yield* ready.operation; - - const address = server.address() as AddressInfo | string | null; - const p = typeof address === "object" && address ? address.port : port; - - try { - yield* provide(p); - } finally { - server.close(); - } - }); -} diff --git a/packages/server/example/services/slow.ts b/packages/server/example/services/slow.ts deleted file mode 100644 index e038eeff..00000000 --- a/packages/server/example/services/slow.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { main } from "effection"; -import { httpServer } from "./http-server.ts"; - -main(function* () { - yield* httpServer({ startDelay: 100 }); -}); diff --git a/packages/server/example/operation/basic-graph.ts b/packages/server/example/simulation-graph.ts similarity index 59% rename from packages/server/example/operation/basic-graph.ts rename to packages/server/example/simulation-graph.ts index 61abbe98..36e6ada8 100644 --- a/packages/server/example/operation/basic-graph.ts +++ b/packages/server/example/simulation-graph.ts @@ -1,16 +1,24 @@ #!/usr/bin/env node import { sleep } from "effection"; -import { useServiceGraph } from "../../src/services.ts"; -import { httpServer } from "../services/http-server.ts"; -import { simulationCLI } from "../../src/cli.ts"; +import { useServiceGraph } from "../src/services.ts"; +import { useChildSimulation } from "../src/simulation.ts"; +import { simulationCLI } from "../src/cli.ts"; const servicesMap = { A: { - operation: httpServer({ startDelay: 10 }), + operation: useChildSimulation( + "A", + "./example/services/basic-sim.ts", + [0, 10] + ), }, B: { deps: ["A"] as const, - operation: httpServer({ startDelay: 20 }), + operation: useChildSimulation( + "B", + "./example/services/basic-sim.ts", + [0, 20] + ), }, }; diff --git a/packages/server/package.json b/packages/server/package.json index 0a11666f..f26bb392 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -32,7 +32,8 @@ "build": "tsdown", "lint": "echo noop", "tsc": "tsc --noEmit", - "example:basic": "node --import tsx ./example/basic-graph.ts", + "example:process": "node --import tsx ./example/process-graph.ts", + "example:sim": "node --import tsx ./example/simulation-graph.ts", "example:lifecycle": "node --import tsx ./example/lifecycle-hooks.ts", "example:concurrency": "node --import tsx ./example/concurrency-layers.ts" }, diff --git a/packages/server/src/cli.ts b/packages/server/src/cli.ts index 9f7d8c1e..28ddf461 100644 --- a/packages/server/src/cli.ts +++ b/packages/server/src/cli.ts @@ -1,54 +1,62 @@ import { parseArgs } from "node:util"; -import { suspend, main, type Operation } from "effection"; -import type { ServiceGraphValue } from "./services.ts"; +import { main, suspend, type Operation } from "effection"; +import type { ServiceGraph } from "./services.ts"; export function* simulationCLIOp>( - serviceGraph: (subset?: string[] | string) => Operation> + serviceGraph: (subset?: string[] | string) => Operation> ) { - const { values } = parseArgs({ - options: { - services: { type: "string", short: "s" }, - debug: { type: "boolean", short: "d" }, - help: { type: "boolean", short: "h" }, - watch: { type: "boolean" }, - "watch-debounce": { type: "string" }, - }, - allowPositionals: true, - allowNegative: true, - allowUnknown: true, - }); - - function* printUsage() { - console.log( - `Usage: cli [-s|--services serviceName] [--watch] [--watch-debounce ms]` + try { + const { values } = parseArgs({ + options: { + services: { type: "string", short: "s" }, + debug: { type: "boolean", short: "d" }, + help: { type: "boolean", short: "h" }, + watch: { type: "boolean" }, + "watch-debounce": { type: "string" }, + }, + allowPositionals: true, + allowNegative: true, + allowUnknown: true, + }); + + function* printUsage() { + console.log( + `Usage: cli [-s|--services serviceName] [--watch] [--watch-debounce ms]` + ); + } + + if (values.help) { + return yield* printUsage(); + } + + const subset = values.services + ? (values.services as string) + .split(",") + .map((s) => s.trim()) + .filter(Boolean) + : undefined; + + const runOptions: { watch?: boolean; watchDebounce?: number } = { + watch: !!values.watch, + }; + if (values["watch-debounce"]) + runOptions.watchDebounce = Number(values["watch-debounce"]); + + // Start the graph and fetch the provided info + yield* serviceGraph(subset); + + yield* suspend(); + } catch (err) { + console.error( + "simulationCLI error:", + err instanceof Error ? err.stack : err ); } - - if (values.help) { - return yield* printUsage(); - } - - const subset = values.services - ? (values.services as string) - .split(",") - .map((s) => s.trim()) - .filter(Boolean) - : undefined; - - const runOptions: { watch?: boolean; watchDebounce?: number } = { - watch: !!values.watch, - }; - if (values["watch-debounce"]) - runOptions.watchDebounce = Number(values["watch-debounce"]); - - // Start the graph and fetch the provided info - yield* serviceGraph(subset); - - yield* suspend(); } -export function simulationCLI>( - serviceGraph: (subset?: string[] | string) => Operation> -) { - return main(() => simulationCLIOp(serviceGraph)); +export async function simulationCLI< + S extends Record>, + T +>(serviceGraph: (subset?: string[] | string) => Operation>) { + return main(() => simulationCLIOp(serviceGraph)); } diff --git a/packages/server/src/services.ts b/packages/server/src/services.ts index fac60991..4c80b639 100644 --- a/packages/server/src/services.ts +++ b/packages/server/src/services.ts @@ -2,194 +2,35 @@ import { type Operation, resource, spawn, - suspend, withResolvers, - createChannel, each, - sleep, - until, type Stream, type Task, + type WithResolvers, } from "effection"; -// Types for watcher and channels -export type ServiceUpdate = { service: string; path: string }; +import { type ServiceUpdate, useWatcher } from "./watch.ts"; -type WatchesStream = Stream & { - send: (value: string) => Operation; - close: (value?: void) => Operation; -}; - -type Watcher = { - serviceUpdates: Stream; - add: (service: string, paths: string[]) => void; -}; -import * as fs from "node:fs/promises"; -import type { Stats } from "node:fs"; -import { useWatcher } from "./watch.ts"; - -export type ServiceDefinition = { - // The operation that starts the service and returns when the service is ready. - // The operation may be provided either as an `Operation` (for example the - // `Operation` returned by `useService(...)`) or as a factory that - // returns an `Operation`. The operation may return a value of any type - // which will be delivered to dependent service factories at runtime. - // Accept either an `Operation` or a factory `() => Operation`. - operation: Operation | ((...args: any[]) => Operation); +export type ServiceDefinition = { + operation: Task>; // folders/files to watch for changes which should cause a restart watch?: string[]; // debounce in milliseconds to coalesce rapid changes for this service watchDebounce?: number; - deps?: string[]; + dependsOn?: { startup: readonly S[]; restart?: readonly S[] }; options?: { // Keep an options object for future expansion or hooks; currently unused when operation is present }; - // lifecycle hooks - beforeStart?: () => Operation; - afterStart?: () => Operation; - beforeStop?: () => Operation; - afterStop?: () => Operation; }; -// helper type to extract the return type from an Operation or an operation factory -type OpReturn = T extends Operation - ? U - : T extends () => Operation - ? U - : T extends (...args: any[]) => Operation - ? U - : never; - -// Build a tuple of dependency return types for a given service key -type DepKeys = S[K] extends { deps: readonly (infer D)[] } - ? D - : []; - -type DepArgs = DepKeys extends readonly any[] - ? { - [I in keyof DepKeys]: DepKeys[I] extends keyof S - ? S[DepKeys[I]] extends { operation: infer OP } - ? OpReturn - : unknown - : unknown; - } - : []; - -// Ensure we have a tuple/array type for rest-parameter compatibility -type ArgsTuple = DepArgs extends readonly any[] - ? DepArgs - : any[]; - -// A strongly-typed service definition for use within a concrete ServicesMap S. -export type ServiceDefinitionFor< - S extends Record, - K extends keyof S, - T = any +export type ServiceGraph< + S extends Record>, + T > = { - // operation may be a simple Operation or a factory that accepts the - // exported values of the declared `deps` and returns Operation - operation: - | Operation - | ((...args: ArgsTuple) => Operation) - | ((...args: any[]) => Generator); - // folders/files to watch for changes which should cause a restart - watch?: string[]; - // debounce in milliseconds to coalesce rapid changes for this service - watchDebounce?: number; - deps?: readonly (keyof S)[]; - options?: { - // placeholder for future options + services: { + [service in keyof S]: ServiceDefinition; }; - beforeStart?: () => Operation; - afterStart?: () => Operation; - beforeStop?: () => Operation; - afterStop?: () => Operation; -}; - -// Generic Services map - callers can use this shape, and useServiceGraph will -// enforce stronger typing via its own generic parameter -export type ServicesMap = Record>; - -// Previously we exposed a public `exportsOperation` for each service; that has been removed. -// service keyed by the return type of its `operation`. -// Note: we no longer attach an `exportsOperation` on the public service map. -// Internal exported values are still resolved and passed to dependent service factories -// but are not exposed as a separate operation on the services object. - -function computeLevels(services: ServicesMap): string[][] { - const indeg: Record = {}; - const graph: Record> = {}; - for (const name of Object.keys(services)) { - indeg[name] = 0; - graph[name] = new Set(); - } - for (const [name, def] of Object.entries(services)) { - for (const dep of def.deps ?? []) { - const depKey = String(dep); - if (!(depKey in services)) { - throw new Error( - `Service '${name}' depends on unknown service '${depKey}'` - ); - } - graph[depKey].add(String(name)); - indeg[String(name)] = (indeg[String(name)] || 0) + 1; - } - } - - const levels: string[][] = []; - let q: string[] = []; - for (const [k, v] of Object.entries(indeg)) { - if (v === 0) q.push(k); - } - - let processed = 0; - while (q.length) { - const currentLayer = q.slice(); - levels.push(currentLayer); - processed += currentLayer.length; - const next: string[] = []; - for (const n of currentLayer) { - for (const m of graph[n]) { - indeg[m] -= 1; - if (indeg[m] === 0) next.push(m); - } - } - q = next; - } - if (processed !== Object.keys(services).length) { - throw new Error(`Cycle detected in services`); - } - return levels; -} - -function* waitForAllReady( - names: string[], - readyResolvers: Map< - string, - { - operation: Operation; - resolve: () => void; - reject: (err: Error) => void; - } - > -): Operation { - for (const n of names) { - const r = readyResolvers.get(n); - if (r) { - yield* r.operation; - } - } -} - -// A runner returned by `useServiceGraph` — callable to start the graph and -// also exposes properties used by the CLI and tests. -// The object provided by the service graph resource when started -export type ServiceGraphValue> = { - services: S; serviceUpdates?: Stream | undefined; - watches: WatchesStream; - // map of last known ports for services that expose a `port` in their export value - servicePorts: Map; }; /** @@ -211,555 +52,130 @@ export type ServiceGraphValue> = { * perform actions before or after each service starts or stops. */ export function useServiceGraph< - S extends Record> + S extends Record>, + T >( - services: { [K in keyof S]: ServiceDefinitionFor } & S, - options?: { sequential?: boolean; watch?: boolean; watchDebounce?: number } -): (subset?: string[] | string) => Operation> { - // Create internal export resolvers for provider-returned values so dependent - // services can obtain them during startup. - const exportResolvers = new Map< - string, - { - operation: Operation; - resolve: (v: unknown) => void; - reject: (err: Error) => void; - } - >(); - for (const name of Object.keys(services)) { - const r = withResolvers(); - exportResolvers.set(name, { - operation: r.operation, - resolve: r.resolve, - reject: (err: Error) => r.reject(err), - }); - // note: we intentionally do not expose a public `exportsOperation` value on the services map - // previously we exposed an `exportsOperation` on the public service map; - // that behavior has been removed. Internal resolvers will still be used to - // deliver provider-exported values to dependent factories. - } - - let runnerWatcher: - | { serviceUpdates: Stream } - | undefined; - const providedWatches: WatchesStream = createChannel(); - + services: S, + options?: { watch?: boolean; watchDebounce?: number } +): (subset?: string[] | string) => Operation> { // create a simple channel that emits service names when they change. // We intentionally do not buffer updates; missing the first few updates // is acceptable and sometimes desirable because they will be used to // restart services. - const setup = withResolvers(); - return (subset?: string[] | string) => resource(function* (provide) { - const sequential = options?.sequential ?? false; // when true, start services in each layer serially - // If a subset is provided, compute the closure including dependencies - let effectiveServices: ServicesMap = services; + let effectiveServices = services; // {} as typeof services; if (subset) { - const want = new Set( - (typeof subset === "string" ? subset.split(",") : subset).map((s) => - s.trim() - ) - ); - const included = new Set(); - function include(name: string) { - if (included.has(name)) return; - if (!(name in services)) - throw new Error(`Requested service '${name}' not found`); - included.add(name); - for (const dep of services[name].deps ?? []) include(String(dep)); - } - for (const name of want) include(name); - effectiveServices = {} as ServicesMap; - for (const name of included) effectiveServices[name] = services[name]; - } - - const layers = computeLevels(effectiveServices); - console.log(`runner: starting layers ${JSON.stringify(layers)}`); - - const watcher = (yield* useWatcher()) as Watcher; - runnerWatcher = watcher; + // TODO subset again + // const want = new Set( + // (typeof subset === "string" ? subset.split(",") : subset).map((s) => + // s.trim() + // ) + // ); + // const included = new Set(); + // function include(name: keyof typeof services) { + // if (included.has(name)) return; + // if (!(name in services)) + // throw new Error(`Requested service '${name}' not found`); + // included.add(name); + // for (const dep of services[name].deps ?? []) include(String(dep)); + // } + // for (const name of want) include(name); + // for (const name of included) effectiveServices[name] = services[name]; - // channel to emit file contents for watcher consumers - const watches: WatchesStream = providedWatches; - - // Register any configured watch paths and emit initial file contents - for (const name of Object.keys(effectiveServices)) { - const def = effectiveServices[name]; - if (def.watch) { - watcher.add(name, def.watch); - for (const p of def.watch) { - try { - let stat: Stats | undefined; - try { - stat = (yield* until(fs.stat(p))) as Stats; - } catch (e) { - continue; - } - if (!stat) continue; - if ( - stat && - typeof stat.isDirectory === "function" && - stat.isDirectory() - ) { - const entries: string[] = yield* until(fs.readdir(p)); - - for (const e of entries) { - const full = `${p}/${e}`; - try { - let content = String( - (yield* until(fs.readFile(full, "utf8"))) as string - ); - // if the content is empty, retry a few times in case of a race - if (content === "") { - for (let i = 0; i < 5; i++) { - yield* sleep(10); - try { - content = String( - (yield* until(fs.readFile(full, "utf8"))) as string - ); - if (content !== "") break; - } catch (e) {} - } - } - - yield* watches.send(name); - } catch (e) {} - } - } else { - try { - let content = String( - (yield* until(fs.readFile(p, "utf8"))) as string - ); - if (content === "") { - for (let i = 0; i < 5; i++) { - yield* sleep(10); - try { - content = String( - (yield* until(fs.readFile(p, "utf8"))) as string - ); - if (content !== "") break; - } catch (e) {} - } - } - yield* watches.send(name); - } catch (e) {} - } - } catch (e) { - // ignore - } - } - } - } - - // signal that we've registered watches and emitted all initial values - // so callers can start their subscriptions with a guarantee that - // initial state has already been produced. - for (const n of Object.keys(effectiveServices)) { - const d = effectiveServices[n]; console.log( - `setup: service ${n} has beforeStop=${ - typeof d.beforeStop === "function" - }` + `service graph: starting with services: ${Object.keys( + effectiveServices + ).join(", ")}` ); } - setup.resolve(); - // Map to manage per-service debounce state and worker - const state = new Map< - string, - { - lastPath?: string; - lastAt?: number; - worker?: Operation> | undefined; - } - >(); - - // track running tasks so we can halt them for restarts - const runningTasks = new Map(); - // track the last exported ports for services that expose a port - const servicePorts = new Map(); - - // build reverse dependency graph to compute dependents closure - const reverseDeps: Record> = {}; - for (const name of Object.keys(effectiveServices)) - reverseDeps[name] = new Set(); - for (const [name, def] of Object.entries(effectiveServices)) { - for (const dep of def.deps ?? []) { - reverseDeps[String(dep)].add(String(name)); - } - } - - // Spawn a listener to collect service update events and debounce per-service - yield* spawn(function* () { - for (const ev of yield* each(watcher.serviceUpdates)) { - const { service, path: p } = ev as ServiceUpdate; - const def = effectiveServices[service]; - const debounceMs = - (def && def.watchDebounce) ?? options?.watchDebounce ?? 20; - const s = state.get(service) ?? {}; - s.lastPath = p; - s.lastAt = Date.now(); - state.set(service, s); - if (!s.worker) { - // start a worker that waits for a quiet period then reads file and emits - - s.worker = spawn(function* () { - while (true) { - const elapsed = Date.now() - (s.lastAt ?? 0); - const wait = Math.max(0, debounceMs - elapsed); - if (wait > 0) yield* sleep(wait); - if (Date.now() - (s.lastAt ?? 0) >= debounceMs) { - try { - // after debounce, send the service name (we don't need the file content) - yield* watches.send(service); - } catch (e) { - // ignore send errors - } - s.worker = undefined; - break; - } - } - }); - } - // required by `each` to allow the loop to continue correctly - yield* each.next(); - } - }); + const watcher = yield* useWatcher(); - // Map of readiness resolvers returned by `withResolvers` for the - // effective services we plan to start. - const readyResolvers = new Map< + const status = new Map< string, - { - operation: Operation; - resolve: () => void; - reject: (err: Error) => void; - } + { startup: WithResolvers; running: WithResolvers } >(); + // establish watching and ready status for (const name of Object.keys(effectiveServices)) { - const r = withResolvers(); - readyResolvers.set(name, { - operation: r.operation, - resolve: r.resolve, - reject: r.reject, + const def = effectiveServices[name]; + status.set(name, { + startup: withResolvers(), + running: withResolvers(), }); + if (def.watch) { + watcher.add(name, def.watch); + } } - // Keep track of start order so we can run beforeStop hooks in reverse - const startOrder: string[] = []; + function bumpService(service: string) { + const task = status.get(service); + if (!task) throw new Error(`missing status for service '${service}'`); + // refresh the startup resolver + task.startup = withResolvers(); + // this allows the service to continue and halt itself + task.running.resolve(); + } - // Restart coordinator: listen for debounced watch events and restart affected services yield* spawn(function* () { - const pending = new Set(); - let processing = false; - - function addClosure(name: string, set: Set) { - if (set.has(name)) return; - set.add(name); - for (const dep of reverseDeps[name] ?? []) addClosure(dep, set); - } - - for (const ev of yield* each(watches)) { - const name = ev as string; - pending.add(name); - if (processing) { - yield* each.next(); - continue; - } - processing = true; - // small debounce to coalesce multiple rapid events - yield* sleep(20); - const toProcess = Array.from(pending); - pending.clear(); - - // compute closure of affected services - const affected = new Set(); - for (const n of toProcess) addClosure(n, affected); - - if (affected.size === 0) { - processing = false; - yield* each.next(); - continue; - } - - // create new export & ready resolvers for affected services - for (const n of affected) { - const r = withResolvers(); - exportResolvers.set(n, { - operation: r.operation, - resolve: r.resolve, - reject: r.reject, - }); - // previously we exposed an `exportsOperation` on the public service map; - // that behavior has been removed. Internal resolvers will still be used to - // deliver provider-exported values to dependent factories. - - const rr = withResolvers(); - readyResolvers.set(n, { - operation: rr.operation, - resolve: rr.resolve, - reject: rr.reject, - }); - } - - console.log( - `runner: restarting services ${Array.from(affected).join(",")}` - ); - - // stop in reverse start order so dependents stop before providers - const stopOrder = startOrder - .filter((s) => affected.has(s)) - .slice() - .reverse(); - for (const n of stopOrder) { - const task = runningTasks.get(n); - if (task) { - try { - console.log(`runner: halting ${n}`); - task.halt(); - } catch (e) {} - runningTasks.delete(n); - } - // wait for port to close if known - const port = servicePorts.get(n); - if (typeof port === "number") { - const net = (yield* until( - import("node:net") - )) as typeof import("node:net"); - const start = Date.now(); - while (Date.now() - start < 2000) { - try { - yield* until( - new Promise((resolve, reject) => { - const s = net.connect({ port, host: "127.0.0.1" }, () => { - s.end(); - reject(new Error("still listening")); - }); - s.on("error", () => { - s.destroy(); - resolve(); - }); - }) - ); - break; - } catch (e) { - // still listening; wait - yield* sleep(20); - } - } - } - } - - // remove affected from startOrder so they will be re-appended in the right order - for (const n of affected) { - const i = startOrder.indexOf(n); - if (i >= 0) startOrder.splice(i, 1); - } - - // start affected services in topological order (providers first) - for (const layer of layers) { - const layerAffected = layer.filter((s) => affected.has(s)); - if (layerAffected.length === 0) continue; - for (const n of layerAffected) { - // wait for deps - yield* waitDeps(n); - console.log(`runner: respawning child ${n}`); - const task = yield* startChild(n); - runningTasks.set(n, task); - } - // after spawning the layer, wait for them to be ready - yield* waitForAllReady(layerAffected, readyResolvers); - } - - processing = false; + for (let restartService of yield* each(watcher.serviceUpdates)) { + bumpService(restartService.service); + // TODO handle service.dependsOn.restart yield* each.next(); } }); - // helper to spawn and run a single service name - function startChild(name: string): Operation> { + // small helper to await a service's dependencies + function* waitDeps(name: string, startup: boolean): Operation { const def = effectiveServices[name]; - return spawn(function* () { - try { - console.log(`startChild: starting ${name}`); - try { - if (def.beforeStart) yield* def.beforeStart(); - } catch (err) { - const exportRes = exportResolvers.get(name); - if (exportRes) exportRes.reject(err as Error); - const ready = readyResolvers.get(name); - if (ready) ready.resolve(); - return; - } - - // Collect dependency exported values into an object (keyed by dep name) - const depObj: Record = {}; - if (def.deps) { - for (const dep of def.deps) { - const depKey = String(dep); - const exportRes = exportResolvers.get(depKey); - if (!exportRes) { - throw new Error( - `Service '${name}' depends on unknown service '${depKey}'` - ); - } - const val = yield* exportRes.operation; - depObj[depKey] = val; - } - } - - // Resolve the caller-supplied operation (factory or operation). - // If it's a factory, call it with the dependency object as a single arg. - let operation: Operation; - try { - operation = - typeof def.operation === "function" - ? ( - def.operation as ( - args: Record - ) => Operation - )(depObj) - : (def.operation as Operation); - } catch (err) { - const exportRes = exportResolvers.get(name); - if (exportRes) exportRes.reject(err as Error); - const ready = readyResolvers.get(name); - if (ready) ready.resolve(); - return; - } - - let exported: unknown; - try { - exported = yield* operation; - const exportRes = exportResolvers.get(name); - if (exportRes) exportRes.resolve(exported); - console.log( - `startChild: ${name} started, exported=${typeof exported}` - ); - // record port if exported value contains one - if ( - exported && - typeof exported === "object" && - "port" in exported && - typeof (exported as Record).port === "number" - ) { - const portVal = (exported as Record).port; - if (typeof portVal === "number") { - servicePorts.set(name, portVal); - } - } - } catch (err) { - const exportRes = exportResolvers.get(name); - if (exportRes) exportRes.reject(err as Error); - const ready = readyResolvers.get(name); - if (ready) ready.resolve(); - // don't rethrow here; a failing provider should reject its export resolver - // so dependents can observe the error without crashing the whole runner - return; - } - - startOrder.push(name); - const res = readyResolvers.get(name); - if (res) res.resolve(); - try { - if (def.afterStart) yield* def.afterStart(); - } catch (err) { - const exportRes = exportResolvers.get(name); - if (exportRes) exportRes.reject(err as Error); - const ready = readyResolvers.get(name); - if (ready) ready.resolve(); - return; - } - - yield* suspend(); - } finally { - if (def.afterStop) yield* def.afterStop(); - } - }); + const deps = startup + ? def.dependsOn?.startup ?? [] + : def.dependsOn?.restart ?? []; + for (const dep of deps) { + const r = status.get(dep); + if (!r) + throw new Error( + `missing readiness resolver for dependency '${dep}'` + ); + yield* r.startup.operation; + } } - // small helper to await a service's dependencies - function* waitDeps(name: string): Operation { - const def = effectiveServices[name]; - if (def.deps) { - for (const dep of def.deps) { - const depKey = String(dep); - const r = readyResolvers.get(depKey); - if (!r) - throw new Error( - `missing readiness resolver for dependency '${depKey}'` - ); - yield* r.operation; - } + function* withRestarts(service: string) { + let startup = true; + while (true) { + const start = yield* spawn(function* () { + yield* waitDeps(service, startup); + const def = effectiveServices[service]; + const task = status.get(service); + if (!task) + throw new Error(`missing status for service '${service}'`); + task.running = withResolvers(); + yield* def.operation; + task.startup.resolve(); + yield* task.running.operation; + }); + yield* start; + startup = false; } } try { - for (const layer of layers) { - if (!sequential) { - // spawn all services in this layer in parallel - for (const name of layer) { - // wait for deps to be ready (yield the underlying Promise) - yield* waitDeps(name); - - // start without waiting; we'll wait for the whole layer below - console.log(`runner: spawning child ${name}`); - const task = yield* startChild(name); - runningTasks.set(name, task); - } - // after spawning the whole layer, wait until every service in the layer is ready - yield* waitForAllReady(layer, readyResolvers); - } else { - // sequential startup within this layer - for (const name of layer) { - // wait for deps to be ready (yield the underlying Promise) - yield* waitDeps(name); - - // start and then wait for readiness before proceeding - const task = yield* startChild(name); - runningTasks.set(name, task); - - const res = readyResolvers.get(name); - if (res) yield* res.operation; - } - } + for (let service of Object.keys(effectiveServices)) { + yield* spawn(function* () { + console.log(`service graph: starting service '${service}'`); + yield* withRestarts(service); + }); } yield* provide({ services: services as S, - serviceUpdates: runnerWatcher?.serviceUpdates, - watches: providedWatches, - servicePorts, + serviceUpdates: watcher?.serviceUpdates, }); } finally { console.log("shutting down service graph"); - // Run beforeStop hooks in reverse start order - console.log( - `runner: running beforeStop hooks for ${startOrder - .slice() - .reverse() - .join(",")}` - ); - for (const name of startOrder.slice().reverse()) { - const def = services[name]; - console.log( - `runner: beforeStop def for ${name}: hasBeforeStop=${ - typeof def?.beforeStop === "function" - }` - ); - if (def?.beforeStop) { - console.log(`runner: running beforeStop for ${name}`); - try { - yield* def.beforeStop(); - console.log(`runner: beforeStop for ${name} completed`); - } catch (err) { - console.log(`runner: beforeStop for ${name} threw`, err); - } - } - } } }); } diff --git a/packages/server/src/simulation.ts b/packages/server/src/simulation.ts index d0eb1a39..e182910b 100644 --- a/packages/server/src/simulation.ts +++ b/packages/server/src/simulation.ts @@ -39,8 +39,7 @@ export function useSimulation>( // new code. The runtime uses `bin/run-simulation-child.ts`. export function useChildSimulation>( name: string, - modulePath: string, - args: unknown[] = [] + modulePath: string ): Operation> { return resource(function* (provide) { const cmd = [ @@ -49,7 +48,6 @@ export function useChildSimulation>( "tsx", "./bin/run-simulation-child.ts", modulePath, - JSON.stringify(args), ] .map((s) => (s.includes(" ") ? `'${s}'` : s)) .join(" "); @@ -67,7 +65,8 @@ export function useChildSimulation>( for (let line of yield* each(process.stdout)) { const buf = Buffer.from(line); const str = buf.toString(); - stdout(str); + console.log(`stdout: ${str}`); + yield* stdout(str); if (!listening) { try { @@ -90,12 +89,27 @@ export function useChildSimulation>( yield* spawn(function* () { for (let line of yield* each(process.stderr)) { const str = Buffer.from(line).toString(); - stderr(str); + yield* stderr(str); yield* each.next(); } }); - // wait to get the listening info from stdout + // spawn a watcher to detect if the child exits before printing the listening info + let status: unknown = undefined; + yield* spawn(function* () { + status = yield* process.join(); + if (!listening) { + ready.reject( + new Error( + `child process exited before emitting listening info: ${JSON.stringify( + status + )}` + ) + ); + } + }); + + // wait to get the listening info from stdout (or reject if the process exited) yield* ready.operation; // we know listening is defined here listening = listening!; @@ -105,7 +119,7 @@ export function useChildSimulation>( try { yield* provide(listening); } finally { - console.log(`${name} simulation closed on port ${listening.port}`); + console.log(`${name} simulation closed on port ${listening?.port}`); } }); } diff --git a/packages/server/src/watch.ts b/packages/server/src/watch.ts index 9e93aafd..b96330d5 100644 --- a/packages/server/src/watch.ts +++ b/packages/server/src/watch.ts @@ -4,6 +4,7 @@ import { createChannel, createSignal, each, + type Operation, race, resource, sleep, @@ -36,10 +37,15 @@ export function debounce( }); } -export function useWatcher() { +export type ServiceUpdate = { service: string; path: string }; + +export function useWatcher(): Operation<{ + serviceUpdates: Stream<{ service: string; path: string }, void>; + add: (service: string, paths: string[]) => void; +}> { return resource(function* (provide) { const changes = createSignal(); - const serviceUpdates = createChannel<{ service: string; path: string }>(); + const serviceUpdates = createChannel(); const serviceList = new Map(); const watcher = chokidar.watch([], { diff --git a/packages/server/test/examples-smoke.test.ts b/packages/server/test/examples-smoke.test.ts index be2f4392..3ba50482 100644 --- a/packages/server/test/examples-smoke.test.ts +++ b/packages/server/test/examples-smoke.test.ts @@ -3,9 +3,9 @@ import http from "node:http"; import { run, sleep, suspend, createScope, until } from "effection"; import { timebox } from "@effectionx/timebox"; -import { services as basicServices } from "../example/operation/basic-graph.ts"; -import { services as lifecycleServices } from "../example/operation/lifecycle-hooks.ts"; -import { services as concurrencyServices } from "../example/operation/concurrency-layers.ts"; +import { services as basicServices } from "../example/simulation-graph.ts"; +import { services as lifecycleServices } from "../example/lifecycle-hooks.ts"; +import { services as concurrencyServices } from "../example/concurrency-layers.ts"; function checkStatus(port: number): Promise { return new Promise((resolve, reject) => { diff --git a/packages/server/test/signal.test.ts b/packages/server/test/signal.test.ts new file mode 100644 index 00000000..19152bf2 --- /dev/null +++ b/packages/server/test/signal.test.ts @@ -0,0 +1,62 @@ +import { it } from "node:test"; +import assert from "node:assert"; +import { spawn as spawnChild } from "node:child_process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +it("example process shuts down cleanly on SIGINT", async () => { + const exe = process.execPath; + const script = fileURLToPath( + new URL("../example/basic-graph.ts", import.meta.url) + ); + const cwd = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + + const child = spawnChild(exe, ["--import", "tsx", script], { + cwd, + env: { ...process.env }, + stdio: ["ignore", "pipe", "pipe"], + }); + + let stdout = ""; + let stderr = ""; + child.stdout?.on("data", (d) => (stdout += String(d))); + child.stderr?.on("data", (d) => (stderr += String(d))); + + // wait for a startup marker + await new Promise((resolve, reject) => { + const to = setTimeout(() => reject(new Error("start timeout")), 3000); + child.stdout?.on("data", function ondata(d) { + const s = String(d); + if (s.includes("runner: starting layers")) { + clearTimeout(to); + child.stdout?.off("data", ondata); + resolve(); + } + }); + }); + + // send SIGINT + process.kill(child.pid!, "SIGINT"); + + const { code, signal } = await new Promise<{ + code: number | null; + signal: NodeJS.Signals | null; + }>((resolve) => { + child.on("exit", (code, signal) => resolve({ code, signal })); + }); + + // allow stderr to flush + await new Promise((r) => setTimeout(r, 50)); + + // expect no stack traces on stderr and process exited due to SIGINT + assert.strictEqual(typeof stderr, "string"); + assert( + !/uncaughtException|UnhandledPromiseRejection|Error/.test(stderr), + `stderr contained error: ${stderr}` + ); + // Accept either signal SIGINT or code 0 or code 130 (standard SIGINT exit code) + assert( + signal === "SIGINT" || code === 0 || code === 130, + `unexpected exit: code=${code} signal=${signal}` + ); +}); From 2a5322acaafc7c3ad2bec4525114cb84be3ca021 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Sun, 11 Jan 2026 02:20:31 -0600 Subject: [PATCH 09/38] edge cases and various todo items --- packages/server/README.md | 41 +-- packages/server/bin/run-simulation-child.ts | 2 +- packages/server/example/README.md | 4 +- packages/server/example/concurrency-layers.ts | 2 +- packages/server/example/lifecycle-hooks.ts | 56 ---- packages/server/example/process-graph.ts | 2 +- .../server/example/services/basic-sim-1.ts | 20 +- .../server/example/services/basic-sim-2.ts | 20 +- .../example/services/gen-sim-factory.ts | 35 +++ packages/server/example/simulation-graph.ts | 14 +- packages/server/package.json | 1 - packages/server/src/cli.ts | 2 +- packages/server/src/services.ts | 115 +++++--- packages/server/src/watch.ts | 40 ++- packages/server/test/child-simulation.test.ts | 3 +- packages/server/test/examples-smoke.test.ts | 81 +----- packages/server/test/services.test.ts | 273 +++++++++--------- packages/server/test/services/service-a.ts | 8 +- packages/server/test/services/service-b.ts | 8 +- packages/server/test/services/service-fast.ts | 8 +- packages/server/test/services/service-slow.ts | 8 +- packages/server/test/signal.test.ts | 9 +- packages/server/test/watch.test.ts | 134 ++++++++- 23 files changed, 494 insertions(+), 392 deletions(-) delete mode 100644 packages/server/example/lifecycle-hooks.ts create mode 100644 packages/server/example/services/gen-sim-factory.ts diff --git a/packages/server/README.md b/packages/server/README.md index 8952de86..2d3f9a4b 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -15,7 +15,11 @@ Key points: - `useServiceGraph(services: ServicesMap, options?: { sequential?: boolean }): ServiceRunner` — returns a _runner function_ which you call to start the graph: `const run = useServiceGraph(services, options); yield* run(subset?: string[] | string);`. By default services in the same topological layer run concurrently; pass `options.sequential = true` to run services in each layer serially. - `ServiceDefinition.operation` (required) — an `Operation` which indicates the service has started. This operation may be long-lived (e.g. `useService`) or may return once the service is ready while a background child keeps the service running. See the example below. -- `deps` — an optional list of service names this service depends on; services without dependencies in the same layer are started concurrently by default, or serially when `options.sequential` is true. +- `dependsOn` — an optional object `{ startup: string[]; restart?: string[] }` listing service names this service depends on. Use `startup` to list services that must start before this service; use `restart` to list services that should trigger a restart of this service when they are restarted (for example, due to a watched file change). Services without dependencies in the same layer are started concurrently by default, or serially when `options.sequential` is true. + +- `subset` (runner argument) — when calling the runner returned by `useServiceGraph` you may pass a subset (e.g. `yield* run(['serviceA'])` or `yield* run('serviceA')`) to start only a subset of services; any startup dependencies required by that subset are automatically included. + +- Watching & restart propagation — pass `{ watch: true }` to `useServiceGraph` and define `watch` paths in each `ServiceDefinition` to enable file watching. The watcher will precompute transitive dependents (based on `dependsOn.restart`) and automatically emit restart updates for dependents when a watched path changes, so restarts propagate efficiently and deterministically. - Lifecycle hooks: `beforeStart`, `afterStart`, `beforeStop`, `afterStop` — each is an `Operation` that runs at the appropriate time. Example: @@ -42,7 +46,7 @@ main(function* () { "B", "node --import tsx ./test/services/service-b.ts" ), - deps: ["A"], + dependsOn: { startup: ["A"] }, }, }); }); @@ -61,29 +65,28 @@ Each `ServiceDefinition` supports lifecycle hook operations. These hooks run in ```ts const services = { A: { - operation: useService( - "A", - "node --import tsx ./test/services/service-a.ts" - ), - afterStart: () => - (function* () { - // runs after the operation returns - console.log("A has started"); - })(), - beforeStop: () => - (function* () { - // runs during shutdown in reverse order + operation: (function* () { + // start the service via useService or useChildSimulation + yield* useService("A", "node --import tsx ./test/services/service-a.ts"); + // signal that the service is ready + console.log("A has started"); + try { + // keep running until cancelled + yield* suspend(); + } finally { + // cleanup runs automatically on scope cancellation console.log("A is stopping"); - })(), + } + })(), }, }; ``` Notes: -- `afterStart` runs after `operation` returns (service is ready) -- `beforeStop` runs during cleanup in reverse-order of startup -- Hooks are optional and can be used together with a passed `operation` or a custom operation +- Use a try/finally in your `operation` to run cleanup logic when the service is stopped +- This approach leverages Effection scopes and ensures cleanup runs in reverse dependency order when the graph is shut down +- Use `useService` or `useChildSimulation` inside your operation as needed to start underlying processes Try it @@ -132,5 +135,5 @@ node --import tsx ./example/basic-graph.ts Previously services could expose their return value via a public `exportsOperation` that consumers could await. That mechanism has been removed in this branch as we move to a child-process-focused runner model. Provider-returned values are still delivered to dependent service factories internally, but no longer exposed as an operation on the public `services` map. -For convenience tests may use the `servicePorts` map exposed by the running graph to discover HTTP ports that services registered when they start. +For convenience tests may use the `servicePorts` map exposed by the running graph to discover HTTP ports that services registered when they start. The `servicePorts` map is available on the object returned by the runner and contains service name => port when a service's `operation` returns an object with a `{ port: number }` property. ```` diff --git a/packages/server/bin/run-simulation-child.ts b/packages/server/bin/run-simulation-child.ts index 4290d4b5..a64d2176 100644 --- a/packages/server/bin/run-simulation-child.ts +++ b/packages/server/bin/run-simulation-child.ts @@ -10,7 +10,7 @@ main(function* () { const args = process.argv.slice(2); console.dir({ args }); if (args.length < 1) { - throw new Error("usage: run-simulation-child.js [jsonArgs]"); + throw new Error("usage: run-simulation-child.js "); } const modulePath = args[0]; diff --git a/packages/server/example/README.md b/packages/server/example/README.md index 499b6907..38c68734 100644 --- a/packages/server/example/README.md +++ b/packages/server/example/README.md @@ -4,7 +4,7 @@ This folder contains runnable examples demonstrating `useServiceGraph` and `useS There are two sets of examples: -- **use-service** (top-level files like `basic-graph.ts`, `lifecycle-hooks.ts`, `concurrency-layers.ts`) — these spawn separate processes using `useService` (e.g. `node --import tsx ./example/services/*.ts`). Use these to exercise the process-based behavior. +- **use-service** (top-level files like `basic-graph.ts`, `concurrency-layers.ts`) — these spawn separate processes using `useService` (e.g. `node --import tsx ./example/services/*.ts`). Use these to exercise the process-based behavior. - **operation** (under `operation/`) — these demonstrate `useChildSimulation()` which runs each service in a child process using a simulation factory. They show how to isolate simulations and start them as independent processes. @@ -25,3 +25,5 @@ node --import tsx ./example/operation/basic-graph.ts ``` These examples make use of the small service implementations in `./example/services`. + +Notes: the examples now use `dependsOn` with a `{ startup, restart? }` shape. To experiment with restart propagation, add a `watch` entry to a service and include dependents via `dependsOn.restart` — when a watched file changes the watcher will restart the affected service and its transitive dependents. diff --git a/packages/server/example/concurrency-layers.ts b/packages/server/example/concurrency-layers.ts index 7c8dc88f..96390255 100644 --- a/packages/server/example/concurrency-layers.ts +++ b/packages/server/example/concurrency-layers.ts @@ -14,7 +14,7 @@ const servicesMap = { watch: ["./example/services/basic-sim-2.ts"], }, dependent: { - // deps: ["fast", "slow"] as const, + dependsOn: { startup: ["fast", "slow"] as const }, operation: resource(function* (provide) { try { console.log("all deps started; running dependent service"); diff --git a/packages/server/example/lifecycle-hooks.ts b/packages/server/example/lifecycle-hooks.ts deleted file mode 100644 index f3cb47df..00000000 --- a/packages/server/example/lifecycle-hooks.ts +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env node -import { sleep, suspend } from "effection"; -import { useServiceGraph } from "../src/services.ts"; -import { useChildSimulation } from "../src/simulation.ts"; -import { simulationCLI } from "../src/cli.ts"; - -const servicesMap = { - provider: { - operation: (function* () { - yield* useChildSimulation( - "provider", - "./example/services/basic-sim.ts", - [0, 10] - ); - console.log("provider: afterStart (operation)"); - try { - yield* suspend(); - } finally { - console.log("provider: beforeStop (operation)"); - } - })(), - }, - consumer: { - deps: ["provider"] as const, - operation: (function* () { - yield* useChildSimulation( - "consumer", - "./example/services/basic-sim.ts", - [0, 10] - ); - console.log("consumer: afterStart (operation)"); - try { - yield* suspend(); - } finally { - console.log("consumer: beforeStop (operation)"); - } - })(), - }, -}; - -export const services = useServiceGraph(servicesMap); - -export function example(opts: { duration?: number } = {}) { - return (function* () { - console.log(`Starting lifecycle hooks example (operation)`); - const run = services; - yield* run(); - yield* sleep(opts.duration ?? 150); - console.log(`Lifecycle example (operation) complete`); - })(); -} - -import { fileURLToPath } from "node:url"; -if (process.argv[1] === fileURLToPath(import.meta.url)) { - simulationCLI(services); -} diff --git a/packages/server/example/process-graph.ts b/packages/server/example/process-graph.ts index ffaf02d9..c7e9a700 100644 --- a/packages/server/example/process-graph.ts +++ b/packages/server/example/process-graph.ts @@ -28,7 +28,7 @@ const servicesMap = { ), }, B: { - deps: ["A"] as const, + dependsOn: { startup: ["A"] as const }, operation: useService( "B", "node --import tsx ./example/services/basic-sim.ts", diff --git a/packages/server/example/services/basic-sim-1.ts b/packages/server/example/services/basic-sim-1.ts index f54c87b4..ede50000 100644 --- a/packages/server/example/services/basic-sim-1.ts +++ b/packages/server/example/services/basic-sim-1.ts @@ -1,19 +1,3 @@ -import { - createFoundationSimulationServer, - type FoundationSimulator, -} from "@simulacrum/foundation-simulator"; +import { simulation as genSimulation } from "./gen-sim-factory.ts"; -export function simulation( - port: number = 3301, - startDelay: number = 10 -): FoundationSimulator { - const factory = createFoundationSimulationServer({ - port, - extendRouter(router) { - router.get("/status", (_req, res) => { - res.status(200).send("ok"); - }); - }, - })(); - return factory; -} +export const simulation = genSimulation(3301, 10); diff --git a/packages/server/example/services/basic-sim-2.ts b/packages/server/example/services/basic-sim-2.ts index daa3569b..dd5679a1 100644 --- a/packages/server/example/services/basic-sim-2.ts +++ b/packages/server/example/services/basic-sim-2.ts @@ -1,19 +1,3 @@ -import { - createFoundationSimulationServer, - type FoundationSimulator, -} from "@simulacrum/foundation-simulator"; +import { simulation as genSimulation } from "./gen-sim-factory.ts"; -export function simulation( - port: number = 3302, - startDelay: number = 10 -): FoundationSimulator { - const factory = createFoundationSimulationServer({ - port, - extendRouter(router) { - router.get("/status", (_req, res) => { - res.status(200).send("ok"); - }); - }, - })(); - return factory; -} +export const simulation = genSimulation(3302, 15); diff --git a/packages/server/example/services/gen-sim-factory.ts b/packages/server/example/services/gen-sim-factory.ts new file mode 100644 index 00000000..d34a94b4 --- /dev/null +++ b/packages/server/example/services/gen-sim-factory.ts @@ -0,0 +1,35 @@ +import { + createFoundationSimulationServer, + type FoundationSimulator, +} from "@simulacrum/foundation-simulator"; + +/* + Helper to create a basic foundation simulation server with a configurable + start delay to simulate slow startups. You would export your simulator + more directly instead of wrapping it like this in a real project. +*/ +export function simulation( + port: number = 3301, + startDelay: number = 10 +): FoundationSimulator { + const factory = createFoundationSimulationServer({ + port, + extendRouter(router) { + router.get("/status", (_req, res) => { + res.status(200).send("ok"); + }); + }, + })(); + + return { + async listen( + ...args: Parameters["listen"]> + ): Promise { + if (startDelay > 0) { + await new Promise((resolve) => setTimeout(resolve, startDelay)); + } + // delegate to underlying factory listen + return factory.listen(...args); + }, + }; +} diff --git a/packages/server/example/simulation-graph.ts b/packages/server/example/simulation-graph.ts index 36e6ada8..fd47e492 100644 --- a/packages/server/example/simulation-graph.ts +++ b/packages/server/example/simulation-graph.ts @@ -6,19 +6,11 @@ import { simulationCLI } from "../src/cli.ts"; const servicesMap = { A: { - operation: useChildSimulation( - "A", - "./example/services/basic-sim.ts", - [0, 10] - ), + operation: useChildSimulation("A", "./example/services/basic-sim-1.ts"), }, B: { - deps: ["A"] as const, - operation: useChildSimulation( - "B", - "./example/services/basic-sim.ts", - [0, 20] - ), + dependsOn: { startup: ["A"] as const }, + operation: useChildSimulation("B", "./example/services/basic-sim-2.ts"), }, }; diff --git a/packages/server/package.json b/packages/server/package.json index f26bb392..7b0588dc 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -34,7 +34,6 @@ "tsc": "tsc --noEmit", "example:process": "node --import tsx ./example/process-graph.ts", "example:sim": "node --import tsx ./example/simulation-graph.ts", - "example:lifecycle": "node --import tsx ./example/lifecycle-hooks.ts", "example:concurrency": "node --import tsx ./example/concurrency-layers.ts" }, "dependencies": { diff --git a/packages/server/src/cli.ts b/packages/server/src/cli.ts index 28ddf461..03ff2d23 100644 --- a/packages/server/src/cli.ts +++ b/packages/server/src/cli.ts @@ -1,6 +1,6 @@ import { parseArgs } from "node:util"; import { main, suspend, type Operation } from "effection"; -import type { ServiceGraph } from "./services.ts"; +import type { ServiceGraph, ServiceDefinition } from "./services.ts"; export function* simulationCLIOp>( serviceGraph: (subset?: string[] | string) => Operation> diff --git a/packages/server/src/services.ts b/packages/server/src/services.ts index 4c80b639..381dda04 100644 --- a/packages/server/src/services.ts +++ b/packages/server/src/services.ts @@ -5,14 +5,16 @@ import { withResolvers, each, type Stream, - type Task, type WithResolvers, } from "effection"; import { type ServiceUpdate, useWatcher } from "./watch.ts"; -export type ServiceDefinition = { - operation: Task>; +export type ServiceDefinition< + S, + T extends void | { port?: number } | unknown +> = { + operation: Operation; // folders/files to watch for changes which should cause a restart watch?: string[]; // debounce in milliseconds to coalesce rapid changes for this service @@ -23,14 +25,18 @@ export type ServiceDefinition = { }; }; +type MaybeSimulation = void | { port?: number } | unknown; + export type ServiceGraph< S extends Record>, - T + T extends MaybeSimulation > = { services: { [service in keyof S]: ServiceDefinition; }; serviceUpdates?: Stream | undefined; + // map of service name => listening port (when the service exposes one) + servicePorts?: Map | undefined; }; /** @@ -43,7 +49,7 @@ export type ServiceGraph< * * yield* useServiceGraph({ * A: { operation: useService('A', 'node --import tsx ./test/services/service-a.ts') }, - * B: { operation: useService('B', 'node --import tsx ./test/services/service-b.ts'), deps: ['A'] } + * B: { operation: useService('B', 'node --import tsx ./test/services/service-b.ts'), dependsOn: { startup: ['A'] } } * }); * * Services within the same topological layer are started concurrently by default. @@ -53,46 +59,70 @@ export type ServiceGraph< */ export function useServiceGraph< S extends Record>, - T + T extends MaybeSimulation >( services: S, options?: { watch?: boolean; watchDebounce?: number } ): (subset?: string[] | string) => Operation> { - // create a simple channel that emits service names when they change. - // We intentionally do not buffer updates; missing the first few updates - // is acceptable and sometimes desirable because they will be used to - // restart services. - return (subset?: string[] | string) => resource(function* (provide) { - // If a subset is provided, compute the closure including dependencies + // detect cycles in the dependency graph + const nodes = Object.keys(services); + const temp = new Set(); + const perm = new Set(); + + function visit(n: string) { + if (perm.has(n)) return; + if (temp.has(n)) throw new Error("Cycle detected in services"); + temp.add(n); + const def = services[n]; + const deps: readonly string[] = def.dependsOn?.startup ?? []; + for (const d of deps) { + if (!(d in services)) continue; + visit(d); + } + temp.delete(n); + perm.add(n); + } + + for (const n of nodes) { + visit(n); + } + let effectiveServices = services; // {} as typeof services; if (subset) { - // TODO subset again - // const want = new Set( - // (typeof subset === "string" ? subset.split(",") : subset).map((s) => - // s.trim() - // ) - // ); - // const included = new Set(); - // function include(name: keyof typeof services) { - // if (included.has(name)) return; - // if (!(name in services)) - // throw new Error(`Requested service '${name}' not found`); - // included.add(name); - // for (const dep of services[name].deps ?? []) include(String(dep)); - // } - // for (const name of want) include(name); - // for (const name of included) effectiveServices[name] = services[name]; + const want = new Set( + (typeof subset === "string" ? subset.split(",") : subset).map((s) => + s.trim() + ) + ); + const included = new Set(); + function include(name: string) { + if (included.has(name)) return; + if (!(name in services)) + throw new Error(`Requested service '${name}' not found`); + included.add(name); + for (const dep of services[name].dependsOn?.startup ?? []) { + include(String(dep)); + } + } + for (const name of want) include(name); + + const picked: Partial = {}; + for (const name of included) { + picked[name as keyof typeof services] = + services[name as keyof typeof services]; + } + effectiveServices = picked as typeof services; console.log( - `service graph: starting with services: ${Object.keys( - effectiveServices - ).join(", ")}` + `service graph: starting with services: ${Array.from(included).join( + ", " + )}` ); } - const watcher = yield* useWatcher(); + const watcher = yield* useWatcher(effectiveServices); const status = new Map< string, @@ -110,19 +140,24 @@ export function useServiceGraph< } } + // track service ports (when services expose one) + const servicePorts = new Map(); + function bumpService(service: string) { const task = status.get(service); if (!task) throw new Error(`missing status for service '${service}'`); // refresh the startup resolver task.startup = withResolvers(); // this allows the service to continue and halt itself + // remove any recorded port for the service; it will be re-registered when it starts again + servicePorts.delete(service); task.running.resolve(); } yield* spawn(function* () { + // restart propagation to dependents is handled by useWatcher for (let restartService of yield* each(watcher.serviceUpdates)) { bumpService(restartService.service); - // TODO handle service.dependsOn.restart yield* each.next(); } }); @@ -153,7 +188,18 @@ export function useServiceGraph< if (!task) throw new Error(`missing status for service '${service}'`); task.running = withResolvers(); - yield* def.operation; + + // capture any returned listening info (e.g., from useChildSimulation) + const maybeProvided = yield* def.operation; + if ( + maybeProvided && + typeof maybeProvided === "object" && + "port" in maybeProvided && + typeof maybeProvided.port === "number" + ) { + servicePorts.set(service, maybeProvided.port); + } + task.startup.resolve(); yield* task.running.operation; }); @@ -173,6 +219,7 @@ export function useServiceGraph< yield* provide({ services: services as S, serviceUpdates: watcher?.serviceUpdates, + servicePorts, }); } finally { console.log("shutting down service graph"); diff --git a/packages/server/src/watch.ts b/packages/server/src/watch.ts index b96330d5..98a5f258 100644 --- a/packages/server/src/watch.ts +++ b/packages/server/src/watch.ts @@ -39,7 +39,9 @@ export function debounce( export type ServiceUpdate = { service: string; path: string }; -export function useWatcher(): Operation<{ +export function useWatcher( + services?: Record +): Operation<{ serviceUpdates: Stream<{ service: string; path: string }, void>; add: (service: string, paths: string[]) => void; }> { @@ -66,6 +68,36 @@ export function useWatcher(): Operation<{ watcher.add(paths); } + // precompute transitive dependents map if services are provided. This allows + // the watcher to emit updates not only for the changed service but also for + // any services that declare it in dependsOn.restart, transitively. + const dependentsMap = new Map(); + if (services) { + const restartAdj: Record = {}; + for (const name of Object.keys(services)) restartAdj[name] = []; + for (const name of Object.keys(services)) { + const def = services[name]; + for (const dep of def.dependsOn?.restart ?? []) { + if (!(dep in restartAdj)) continue; + restartAdj[dep].push(name); + } + } + + for (const n of Object.keys(services)) { + const seen = new Set(); + const stack = [...(restartAdj[n] ?? [])]; + while (stack.length) { + const cur = stack.pop()!; + if (seen.has(cur)) continue; + seen.add(cur); + for (const next of restartAdj[cur] ?? []) { + if (!seen.has(next)) stack.push(next); + } + } + dependentsMap.set(n, Array.from(seen)); + } + } + yield* spawn(function* () { for (let args of yield* each(changes)) { const [path] = args as EmitArgs; @@ -74,7 +106,13 @@ export function useWatcher(): Operation<{ return matcher(path); }); if (isAffected) { + // send update for the service itself yield* serviceUpdates.send({ service, path }); + // then also send updates for its transitive dependents (if any) + const dependents = dependentsMap.get(service) ?? []; + for (const d of dependents) { + yield* serviceUpdates.send({ service: d, path }); + } } } yield* each.next(); diff --git a/packages/server/test/child-simulation.test.ts b/packages/server/test/child-simulation.test.ts index 079f8b0a..3daa5db6 100644 --- a/packages/server/test/child-simulation.test.ts +++ b/packages/server/test/child-simulation.test.ts @@ -7,8 +7,7 @@ it("useChildSimulation starts a child and returns port", async () => { await run(function* () { const listening = yield* useChildSimulation( "child-test", - "./test/fixtures/simple-sim.ts", - [0] + "./test/fixtures/simple-sim.ts" ); assert(typeof listening.port === "number"); diff --git a/packages/server/test/examples-smoke.test.ts b/packages/server/test/examples-smoke.test.ts index 3ba50482..de8c9e53 100644 --- a/packages/server/test/examples-smoke.test.ts +++ b/packages/server/test/examples-smoke.test.ts @@ -4,7 +4,6 @@ import { run, sleep, suspend, createScope, until } from "effection"; import { timebox } from "@effectionx/timebox"; import { services as basicServices } from "../example/simulation-graph.ts"; -import { services as lifecycleServices } from "../example/lifecycle-hooks.ts"; import { services as concurrencyServices } from "../example/concurrency-layers.ts"; function checkStatus(port: number): Promise { @@ -19,12 +18,12 @@ function checkStatus(port: number): Promise { }); } -import type { ServiceGraphValue } from "../src/services.ts"; +import type { ServiceGraph } from "../src/services.ts"; import type { Operation } from "effection"; it("basic example imports and runs", async () => { const runner = basicServices as unknown as () => Operation< - ServiceGraphValue> + ServiceGraph> >; // runner let provided: any; @@ -50,10 +49,10 @@ it("basic example imports and runs", async () => { yield* sleep(0); const svcMap = provided!.services; - const ports = provided!.servicePorts; + const ports = provided!.servicePorts!; const ps: number[] = []; for (const name of Object.keys(svcMap)) { - const port = ports.get(name); + const port = ports!.get(name); if (typeof port === "number") ps.push(port); } @@ -136,75 +135,11 @@ it("basic example imports and runs", async () => { }); }); -it("lifecycle example imports and runs", async () => { - const runner = lifecycleServices as unknown as () => Operation< - ServiceGraphValue> - >; // runner - let provided: ServiceGraphValue> | undefined; - - await run(function* () { - const [scope, destroy] = createScope(); - // start operation-style graph and capture provided resource synchronously - try { - provided = yield* runner(); - } catch (err) { - console.error( - "example runner threw:", - err instanceof Error ? err.stack : err - ); - throw err; - } - // keep the graph task alive until the scope is destroyed - scope.run(function* () { - yield* suspend(); - }); - - // allow spawned graph to settle and for services to register their ports - yield* sleep(0); - - const svcMap = provided!.services; - const ports = provided!.servicePorts; - const ps: number[] = []; - for (const name of Object.keys(svcMap)) { - const port = ports.get(name); - if (typeof port === "number") ps.push(port); - } - - yield* sleep(200); - - // check each port while the graph is still running - for (const p of ps) { - let ok = false; - for (let i = 0; i < 100; i++) { - try { - const status = yield* until(checkStatus(p)); - if (status === 200) { - ok = true; - break; - } - } catch (_) {} - yield* sleep(10); - } - if (!ok) { - throw new Error( - `(examples-smoke lifecycle) port ${p} did not return 200 while graph was running` - ); - } - } - - // shut down the graph to avoid hanging the test process - yield* until(destroy()); - return ps as number[]; - }); - - // nothing to check here; checks already happened while graph was running -}); - it("concurrency example imports and runs", async () => { const runner = concurrencyServices as unknown as () => Operation< - ServiceGraphValue> + ServiceGraph> >; // runner - let provided: ServiceGraphValue> | undefined; + let provided: ServiceGraph> | undefined; await run(function* () { const [scope, destroy] = createScope(); @@ -227,10 +162,10 @@ it("concurrency example imports and runs", async () => { yield* sleep(0); const svcMap = provided!.services; - const ports = provided!.servicePorts; + const ports = provided!.servicePorts!; const ps: number[] = []; for (const name of Object.keys(svcMap)) { - const port = ports.get(name); + const port = ports!.get(name); if (typeof port === "number") ps.push(port); } diff --git a/packages/server/test/services.test.ts b/packages/server/test/services.test.ts index 7b965922..c2da4b60 100644 --- a/packages/server/test/services.test.ts +++ b/packages/server/test/services.test.ts @@ -1,6 +1,6 @@ import { it } from "node:test"; import assert from "node:assert"; -import { run, sleep, spawn, Ok, suspend, type Operation } from "effection"; +import { resource, run, sleep, spawn, suspend } from "effection"; import { useServiceGraph } from "../src/services.ts"; import { useService } from "../src/service.ts"; @@ -11,37 +11,19 @@ it("starts services in dependency order", async () => { yield* spawn(function* () { const run = useServiceGraph({ A: { - operation: useService( - "A", - "node --import tsx ./test/services/service-a.ts", - { - wellnessCheck: { - frequency: 10, - *operation(_stdio) { - yield* sleep(20); - startTimes.set("A", Date.now()); - return Ok(void 0); - }, - }, - } - ), + operation: resource(function* (provide) { + yield* sleep(20); + startTimes.set("A", Date.now()); + yield* provide(); + }), }, B: { - operation: useService( - "B", - "node --import tsx ./test/services/service-b.ts", - { - wellnessCheck: { - frequency: 10, - *operation(_stdio) { - yield* sleep(40); - startTimes.set("B", Date.now()); - return Ok(void 0); - }, - }, - } - ), - deps: ["A"] as const, + operation: resource(function* (provide) { + yield* sleep(40); + startTimes.set("B", Date.now()); + yield* provide(); + }), + dependsOn: { startup: ["A"] as const }, }, }); yield* run(); @@ -71,14 +53,14 @@ it("throws on cycles in dependency graph", async () => { "A", "node --import tsx ./test/services/service-a.ts" ), - deps: ["B"] as const, + dependsOn: { startup: ["B"] as const }, }, B: { operation: useService( "B", "node --import tsx ./test/services/service-b.ts" ), - deps: ["A"], + dependsOn: { startup: ["A"] as const }, }, }); yield* runGraph(); @@ -94,47 +76,27 @@ it("runs beforeStop hooks in reverse order", async () => { yield* spawn(function* () { const run = useServiceGraph({ A: { - operation: useService( - "A", - "node --import tsx ./test/services/service-a.ts", - { - wellnessCheck: { - frequency: 10, - *operation(_stdio) { - yield* sleep(20); - startedOrder.push("A"); - return Ok(void 0); - }, - }, - } - ), - beforeStop() { - return (function* () { + operation: resource(function* (provide) { + try { + yield* sleep(20); + startedOrder.push("A"); + yield* provide(); + } finally { stopOrder.push("A"); - })() as Operation; - }, + } + }), }, B: { - operation: useService( - "B", - "node --import tsx ./test/services/service-b.ts", - { - wellnessCheck: { - frequency: 10, - *operation(_stdio) { - yield* sleep(40); - startedOrder.push("B"); - return Ok(void 0); - }, - }, - } - ), - deps: ["A"] as const, - beforeStop() { - return (function* () { + operation: resource(function* (provide) { + try { + yield* sleep(40); + startedOrder.push("B"); + yield* provide(); + } finally { stopOrder.push("B"); - })() as Operation; - }, + } + }), + dependsOn: { startup: ["A"] as const }, }, }); yield* run(); @@ -155,36 +117,18 @@ it("starts independent services in parallel", async () => { yield* spawn(function* () { const run = useServiceGraph({ fast: { - operation: useService( - "fast", - "node --import tsx ./test/services/service-fast.ts", - { - wellnessCheck: { - frequency: 10, - *operation(_stdio) { - yield* sleep(20); - startTimes.set("fast", Date.now()); - return Ok(void 0); - }, - }, - } - ), + operation: resource(function* (provide) { + yield* sleep(20); + startTimes.set("fast", Date.now()); + yield* provide(); + }), }, slow: { - operation: useService( - "slow", - "node --import tsx ./test/services/service-slow.ts", - { - wellnessCheck: { - frequency: 10, - *operation(_stdio) { - yield* sleep(50); - startTimes.set("slow", Date.now()); - return Ok(void 0); - }, - }, - } - ), + operation: resource(function* (provide) { + yield* sleep(50); + startTimes.set("slow", Date.now()); + yield* provide(); + }), }, }); yield* run(); @@ -215,49 +159,43 @@ it("runs subset of services with dependencies", async () => { yield* spawn(function* () { const services = { fast: { - operation: useService( - "fast", - "node --import tsx ./test/services/service-fast.ts", - { - wellnessCheck: { - frequency: 10, - *operation(_stdio) { - yield* sleep(20); - startTimes.set("fast", Date.now()); - return Ok(void 0); - }, - }, - } - ), + operation: resource(function* (provide) { + console.log("test: fast operation starting"); + yield* sleep(20); + console.log("test: fast operation setting startTimes"); + startTimes.set("fast", Date.now()); + yield* provide(); + }), }, slow: { - operation: useService( - "slow", - "node --import tsx ./test/services/service-slow.ts", - { - wellnessCheck: { - frequency: 10, - *operation(_stdio) { - yield* sleep(50); - startTimes.set("slow", Date.now()); - return Ok(void 0); - }, - }, - } - ), + operation: resource(function* (provide) { + console.log("test: slow operation starting"); + yield* sleep(50); + console.log("test: slow operation setting startTimes"); + startTimes.set("slow", Date.now()); + yield* provide(); + }), }, dependent: { - deps: ["fast", "slow"] as const, - operation: (function* () { + dependsOn: { startup: ["fast", "slow"] as const }, + operation: resource(function* (provide) { + // wait until both dependencies have recorded their start times + while (!startTimes.has("fast") || !startTimes.has("slow")) { + yield* sleep(5); + } + console.log("test: dependent operation starting after deps"); startTimes.set("dependent", Date.now()); - yield* suspend(); - })() as Operation, + yield* provide(); + }), }, }; // only request dependent; fast and slow should be included as deps const run = useServiceGraph(services); - yield* run(); // start full graph (subset-run removed) + // request only 'dependent' — this should cause 'fast' and 'slow' to be included as dependencies + yield* run(["dependent"]); + // keep spawned graph alive so services can start and perform startup work + yield* suspend(); }); yield* sleep(300); }); @@ -271,3 +209,78 @@ it("runs subset of services with dependencies", async () => { assert(f! <= d!, "fast should start before dependent"); assert(s! <= d!, "slow should start before dependent"); }); + +it("throws when requested subset includes a missing service", async () => { + await assert.rejects(async () => { + await run(function* () { + const services = { + a: { + operation: resource(function* (provide) { + yield* sleep(10); + yield* provide(); + }), + }, + }; + + const runGraph = useServiceGraph(services); + // request a service that does not exist + yield* runGraph(["missing"]); + }); + }, /Requested service 'missing' not found/); +}); + +it("runs subset specified as a string", async () => { + const startTimes = new Map(); + await run(function* () { + yield* spawn(function* () { + const services = { + a: { + operation: resource(function* (provide) { + yield* sleep(20); + startTimes.set("a", Date.now()); + yield* provide(); + }), + }, + b: { + operation: resource(function* (provide) { + yield* sleep(50); + startTimes.set("b", Date.now()); + yield* provide(); + }), + }, + r: { + dependsOn: { startup: ["a", "b"] as const }, + operation: resource(function* (provide) { + while (!startTimes.has("a") || !startTimes.has("b")) { + yield* sleep(5); + } + startTimes.set("r", Date.now()); + yield* provide(); + }), + }, + other: { + operation: resource(function* (provide) { + yield* sleep(10); + startTimes.set("other", Date.now()); + yield* provide(); + }), + }, + }; + + const run = useServiceGraph(services); + // pass a comma-separated string + yield* run("r"); + yield* suspend(); + }); + yield* sleep(300); + }); + + const a = startTimes.get("a"); + const b = startTimes.get("b"); + const r = startTimes.get("r"); + const other = startTimes.get("other"); + assert.ok(typeof a === "number", "a should start"); + assert.ok(typeof b === "number", "b should start"); + assert.ok(typeof r === "number", "r should start"); + assert.ok(typeof other === "undefined", "other should NOT start"); +}); diff --git a/packages/server/test/services/service-a.ts b/packages/server/test/services/service-a.ts index bd25e10d..cfcd358a 100644 --- a/packages/server/test/services/service-a.ts +++ b/packages/server/test/services/service-a.ts @@ -1,6 +1,4 @@ -import { main } from "effection"; -import { httpServer } from "../../example/services/http-server.ts"; +import { simulation as genSimulation } from "../../example/services/gen-sim-factory.ts"; +import type { FoundationSimulator } from "@simulacrum/foundation-simulator"; -main(function* () { - yield* httpServer({ startDelay: 10 }); -}); +export const simulation: FoundationSimulator = genSimulation(4010, 10); diff --git a/packages/server/test/services/service-b.ts b/packages/server/test/services/service-b.ts index 9d9a261e..316c5741 100644 --- a/packages/server/test/services/service-b.ts +++ b/packages/server/test/services/service-b.ts @@ -1,6 +1,4 @@ -import { main } from "effection"; -import { httpServer } from "../../example/services/http-server.ts"; +import { simulation as genSimulation } from "../../example/services/gen-sim-factory.ts"; +import type { FoundationSimulator } from "@simulacrum/foundation-simulator"; -main(function* () { - yield* httpServer({ startDelay: 40 }); -}); +export const simulation: FoundationSimulator = genSimulation(4020, 40); diff --git a/packages/server/test/services/service-fast.ts b/packages/server/test/services/service-fast.ts index bd25e10d..4c907299 100644 --- a/packages/server/test/services/service-fast.ts +++ b/packages/server/test/services/service-fast.ts @@ -1,6 +1,4 @@ -import { main } from "effection"; -import { httpServer } from "../../example/services/http-server.ts"; +import { simulation as genSimulation } from "../../example/services/gen-sim-factory.ts"; +import type { FoundationSimulator } from "@simulacrum/foundation-simulator"; -main(function* () { - yield* httpServer({ startDelay: 10 }); -}); +export const simulation: FoundationSimulator = genSimulation(4030, 10); diff --git a/packages/server/test/services/service-slow.ts b/packages/server/test/services/service-slow.ts index 65cec31a..61614e1a 100644 --- a/packages/server/test/services/service-slow.ts +++ b/packages/server/test/services/service-slow.ts @@ -1,6 +1,4 @@ -import { main } from "effection"; -import { httpServer } from "../../example/services/http-server.ts"; +import { simulation as genSimulation } from "../../example/services/gen-sim-factory.ts"; +import type { FoundationSimulator } from "@simulacrum/foundation-simulator"; -main(function* () { - yield* httpServer({ startDelay: 200 }); -}); +export const simulation: FoundationSimulator = genSimulation(4040, 200); diff --git a/packages/server/test/signal.test.ts b/packages/server/test/signal.test.ts index 19152bf2..48bf5fe8 100644 --- a/packages/server/test/signal.test.ts +++ b/packages/server/test/signal.test.ts @@ -7,7 +7,7 @@ import { fileURLToPath } from "node:url"; it("example process shuts down cleanly on SIGINT", async () => { const exe = process.execPath; const script = fileURLToPath( - new URL("../example/basic-graph.ts", import.meta.url) + new URL("../example/simulation-graph.ts", import.meta.url) ); const cwd = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); @@ -27,7 +27,12 @@ it("example process shuts down cleanly on SIGINT", async () => { const to = setTimeout(() => reject(new Error("start timeout")), 3000); child.stdout?.on("data", function ondata(d) { const s = String(d); - if (s.includes("runner: starting layers")) { + // the service runner logs when services start — accept either the old + // marker or the current log message used by the service graph implementation + if ( + s.includes("runner: starting layers") || + s.includes("service graph: starting service") + ) { clearTimeout(to); child.stdout?.off("data", ondata); resolve(); diff --git a/packages/server/test/watch.test.ts b/packages/server/test/watch.test.ts index 237266d2..7f1cba36 100644 --- a/packages/server/test/watch.test.ts +++ b/packages/server/test/watch.test.ts @@ -1,6 +1,6 @@ import { it } from "node:test"; import assert from "node:assert"; -import { run, suspend, sleep, until, spawn } from "effection"; +import { run, suspend, sleep, until, spawn, resource } from "effection"; import * as fs from "node:fs/promises"; import path from "node:path"; import os from "node:os"; @@ -30,7 +30,7 @@ it("restarts services on watched file change and restarts dependents", async () ), }, b: { - deps: ["a"], + dependsOn: { startup: ["a"] as const }, operation: useSimulation("test-simulation-a", () => simulation(5501) ), @@ -80,3 +80,133 @@ it("restarts services on watched file change and restarts dependents", async () assert(updates.length >= 1, "expected at least one update"); assert(updates[0] === "a", "first update is service 'a'"); }); + +it("restarts dependents when watched service changes", async () => { + const prefix = path.join(os.tmpdir(), "sim-watch-rt-"); + const dir = await fs.mkdtemp(prefix); + const trigger = path.join(dir, "trigger.txt"); + await fs.writeFile(trigger, "initial"); + + const startCounts: Record = { a: 0, b: 0 }; + + await run(function* () { + yield* spawn(function* () { + const op = useServiceGraph( + { + a: { + watch: [dir], + operation: resource(function* (provide) { + startCounts.a += 1; + yield* provide(); + }), + }, + b: { + dependsOn: { startup: [] as const, restart: ["a"] as const }, + operation: resource(function* (provide) { + startCounts.b += 1; + yield* provide(); + }), + }, + }, + { watch: true, watchDebounce: 20 } + ); + + try { + yield* op(); + } catch (e) { + throw e; + } + + yield* suspend(); + }); + + // wait for initial startup + for (let i = 0; i < 200; i++) { + if (startCounts.a > 0 && startCounts.b > 0) break; + yield* sleep(10); + } + + // trigger a change + yield* until(fs.writeFile(trigger, "changed")); + + // wait for restarts to occur + for (let i = 0; i < 200; i++) { + if (startCounts.a >= 2 && startCounts.b >= 2) break; + yield* sleep(10); + } + }); + + await fs.rm(dir, { recursive: true, force: true }); + + assert(startCounts.a >= 2, "a should have been restarted"); + assert(startCounts.b >= 2, "b should have been restarted as dependent"); +}); + +it("restarts transitive dependents when watched service changes", async () => { + const prefix = path.join(os.tmpdir(), "sim-watch-rt-2-"); + const dir = await fs.mkdtemp(prefix); + const trigger = path.join(dir, "trigger.txt"); + await fs.writeFile(trigger, "initial"); + + const startCounts: Record = { a: 0, b: 0, c: 0 }; + + await run(function* () { + yield* spawn(function* () { + const op = useServiceGraph( + { + a: { + watch: [dir], + operation: resource(function* (provide) { + startCounts.a += 1; + yield* provide(); + }), + }, + b: { + dependsOn: { startup: [] as const, restart: ["a"] as const }, + operation: resource(function* (provide) { + startCounts.b += 1; + yield* provide(); + }), + }, + c: { + dependsOn: { startup: [] as const, restart: ["b"] as const }, + operation: resource(function* (provide) { + startCounts.c += 1; + yield* provide(); + }), + }, + }, + { watch: true, watchDebounce: 20 } + ); + + try { + yield* op(); + } catch (e) { + throw e; + } + + yield* suspend(); + }); + + // wait for initial startup + for (let i = 0; i < 200; i++) { + if (startCounts.a > 0 && startCounts.b > 0 && startCounts.c > 0) break; + yield* sleep(10); + } + + // trigger a change + yield* until(fs.writeFile(trigger, "changed")); + + // wait for restarts to occur + for (let i = 0; i < 200; i++) { + if (startCounts.a >= 2 && startCounts.b >= 2 && startCounts.c >= 2) break; + yield* sleep(10); + } + }); + + await fs.rm(dir, { recursive: true, force: true }); + + assert(startCounts.a >= 2, "a should have been restarted"); + assert(startCounts.b >= 2, "b should have been restarted as dependent"); + assert(startCounts.c >= 2, "c should have been restarted as dependent of b"); +}); From 5d9d3cb105e8e635eb7cb3ed7a408f55e71d95e9 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Mon, 12 Jan 2026 14:09:38 -0600 Subject: [PATCH 10/38] watch debouncing --- package-lock.json | 17 ++---- packages/server/package.json | 2 +- packages/server/src/services.ts | 11 +++- packages/server/src/watch.ts | 57 +++++++++--------- packages/server/test/watch.test.ts | 93 ++++++++++++++++++++++++++++-- 5 files changed, 131 insertions(+), 49 deletions(-) diff --git a/package-lock.json b/package-lock.json index a520c417..12aaafb7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -121,7 +121,6 @@ "version": "7.28.4", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.3", @@ -411,9 +410,9 @@ "license": "MIT" }, "node_modules/@effectionx/stream-helpers": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/@effectionx/stream-helpers/-/stream-helpers-0.4.1.tgz", - "integrity": "sha512-zYnUYbKJcX5pMbMtFLlurj9KO6ZhoG1bedhoG1mU8Fh7Fz06WKIAxQ9dIOYefBP3Kczr8T1+lDcsq62n3YtBug==", + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/@effectionx/stream-helpers/-/stream-helpers-0.5.1.tgz", + "integrity": "sha512-lvXhkLPxVNtnj6XE4c4j/Rpm6VbD+WOVQU/+pL0kAC06INk56owwR6NV6QH/THEdZnz7RdbnFWLECPuXpMMxrA==", "license": "MIT", "dependencies": { "@effectionx/signals": "^0.4.0", @@ -3299,7 +3298,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001737", "electron-to-chromium": "^1.5.211", @@ -4504,7 +4502,6 @@ "node_modules/graphql": { "version": "16.11.0", "license": "MIT", - "peer": true, "engines": { "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } @@ -5950,7 +5947,6 @@ "version": "0.3.13", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@publint/pack": "^0.1.2", "package-manager-detector": "^1.3.0", @@ -6172,7 +6168,6 @@ "integrity": "sha512-xaPcckj+BbJhYLsv8gOqezc8EdMcKKe/gk8v47B0KPvgABDrQ0qmNPAiT/gh9n9Foe0bUkEv2qzj42uU5q1WRg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@oxc-project/types": "=0.94.0", "@rolldown/pluginutils": "1.0.0-beta.42", @@ -6958,7 +6953,6 @@ "version": "4.20.6", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "~0.25.0", "get-tsconfig": "^4.7.5" @@ -7000,7 +6994,6 @@ "version": "5.8.3", "devOptional": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -7156,7 +7149,6 @@ "version": "7.1.7", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.5.0", @@ -7412,7 +7404,6 @@ "version": "8.18.3", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10.0.0" }, @@ -7591,7 +7582,7 @@ "dependencies": { "@effectionx/context-api": "^0.2.1", "@effectionx/process": "^0.6.2", - "@effectionx/stream-helpers": "^0.4.1", + "@effectionx/stream-helpers": "^0.5.1", "@effectionx/timebox": "^0.3.1", "chokidar": "^5.0.0", "effection": "^4.0.0", diff --git a/packages/server/package.json b/packages/server/package.json index 7b0588dc..1a7a273e 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -40,7 +40,7 @@ "effection": "^4.0.0", "@effectionx/context-api": "^0.2.1", "@effectionx/process": "^0.6.2", - "@effectionx/stream-helpers": "^0.4.1", + "@effectionx/stream-helpers": "^0.5.1", "@effectionx/timebox": "^0.3.1", "chokidar": "^5.0.0", "picomatch": "^4.0.3" diff --git a/packages/server/src/services.ts b/packages/server/src/services.ts index 381dda04..4a80c400 100644 --- a/packages/server/src/services.ts +++ b/packages/server/src/services.ts @@ -35,6 +35,7 @@ export type ServiceGraph< [service in keyof S]: ServiceDefinition; }; serviceUpdates?: Stream | undefined; + serviceChanges?: Stream | undefined; // map of service name => listening port (when the service exposes one) servicePorts?: Map | undefined; }; @@ -122,7 +123,12 @@ export function useServiceGraph< ); } - const watcher = yield* useWatcher(effectiveServices); + const watcher = yield* useWatcher( + effectiveServices, + options?.watchDebounce + ? { watchDebounce: options.watchDebounce } + : undefined + ); const status = new Map< string, @@ -156,7 +162,7 @@ export function useServiceGraph< yield* spawn(function* () { // restart propagation to dependents is handled by useWatcher - for (let restartService of yield* each(watcher.serviceUpdates)) { + for (let restartService of yield* each(watcher.serviceChanges)) { bumpService(restartService.service); yield* each.next(); } @@ -219,6 +225,7 @@ export function useServiceGraph< yield* provide({ services: services as S, serviceUpdates: watcher?.serviceUpdates, + serviceChanges: watcher?.serviceChanges, servicePorts, }); } finally { diff --git a/packages/server/src/watch.ts b/packages/server/src/watch.ts index 98a5f258..80f78db0 100644 --- a/packages/server/src/watch.ts +++ b/packages/server/src/watch.ts @@ -5,44 +5,25 @@ import { createSignal, each, type Operation, - race, resource, - sleep, spawn, type Stream, until, } from "effection"; import picomatch, { type Matcher } from "picomatch"; - -export function debounce( - ms: number -): (stream: Stream) => Stream { - return (stream) => ({ - *[Symbol.iterator]() { - let subscription = yield* stream; - return { - *next() { - let next = yield* subscription.next(); - while (true) { - let result = yield* race([sleep(ms), subscription.next()]); - if (!result) { - return next; - } else { - next = result; - } - } - }, - }; - }, - }); -} +import { filter } from "@effectionx/stream-helpers"; export type ServiceUpdate = { service: string; path: string }; export function useWatcher( - services?: Record + services?: Record< + string, + { dependsOn?: { restart?: readonly string[] }; watchDebounce?: number } + >, + options?: { watchDebounce?: number } ): Operation<{ serviceUpdates: Stream<{ service: string; path: string }, void>; + serviceChanges: Stream<{ service: string; path: string }, void>; add: (service: string, paths: string[]) => void; }> { return resource(function* (provide) { @@ -119,8 +100,30 @@ export function useWatcher( } }); + const debounceMs = + options?.watchDebounce !== undefined ? options.watchDebounce : 250; + const serviceTimers = {} as Record; + const debouncedServiceChanges = filter(function* ( + updateStream + ) { + const now = performance.now(); + if ( + serviceTimers[updateStream.service] && + now - serviceTimers[updateStream.service] < debounceMs + ) { + return false; + } else { + serviceTimers[updateStream.service] = now; + return true; + } + }); + try { - yield* provide({ serviceUpdates, add }); + yield* provide({ + serviceUpdates, + add, + serviceChanges: debouncedServiceChanges(serviceUpdates), + }); } finally { yield* until(watcher.close()); } diff --git a/packages/server/test/watch.test.ts b/packages/server/test/watch.test.ts index 7f1cba36..fc60a6f6 100644 --- a/packages/server/test/watch.test.ts +++ b/packages/server/test/watch.test.ts @@ -1,6 +1,6 @@ import { it } from "node:test"; import assert from "node:assert"; -import { run, suspend, sleep, until, spawn, resource } from "effection"; +import { run, suspend, sleep, until, spawn, resource, ensure } from "effection"; import * as fs from "node:fs/promises"; import path from "node:path"; import os from "node:os"; @@ -41,12 +41,12 @@ it("restarts services on watched file change and restarts dependents", async () try { const services = yield* op(); - // subscribe to the immediate serviceUpdates stream and wait for the first update - if (!services.serviceUpdates) - throw new Error("serviceUpdates not available"); - const subscription = yield* services.serviceUpdates; + // subscribe to the immediate raw serviceChanges stream and wait for the first update + if (!services.serviceChanges) + throw new Error("serviceChanges not available"); + const subscription = yield* services.serviceChanges; - // wait for the first update (will occur after the test touches the file) + // wait for the first raw update (will occur after the test touches the file) const first = yield* subscription.next(); updates.push(String((first.value as { service: string }).service)); } catch (e) { @@ -210,3 +210,84 @@ it("restarts transitive dependents when watched service changes", async () => { assert(startCounts.b >= 2, "b should have been restarted as dependent"); assert(startCounts.c >= 2, "c should have been restarted as dependent of b"); }); + +it("debounces rapid changes per service", async () => { + const prefix = path.join(os.tmpdir(), "sim-watch-debounce-"); + const dir = await fs.mkdtemp(prefix); + const trigger = path.join(dir, "trigger.txt"); + await fs.writeFile(trigger, "initial"); + + const updates: string[] = []; + let rawCount = 0; + + await run(function* () { + yield* spawn(function* () { + const op = useServiceGraph( + { + a: { + watch: [dir], + operation: resource(function* (provide) { + yield* provide(); + }), + }, + }, + { watch: true, watchDebounce: 150 } + ); + + try { + const services = yield* op(); + if (!services.serviceUpdates || !services.serviceChanges) + throw new Error("service streams not available"); + const debSub = yield* services.serviceUpdates; + const rawSub = yield* services.serviceChanges; + + // collect debounced updates + yield* spawn(function* () { + while (true) { + const n = yield* debSub.next(); + if (n.done) break; + updates.push((n.value as { service: string }).service); + } + }); + + // count raw updates (should reflect every write) + yield* spawn(function* () { + while (true) { + const n = yield* rawSub.next(); + if (n.done) break; + if ((n.value as { service: string }).service === "a") rawCount++; + } + }); + } catch (e) { + throw e; + } + + yield* suspend(); + }); + + // ensure watcher attached + yield* sleep(0); + + // write multiple times rapidly + yield* until(fs.writeFile(trigger, "changed-1")); + yield* sleep(10); + yield* until(fs.writeFile(trigger, "changed-2")); + yield* sleep(10); + yield* until(fs.writeFile(trigger, "changed-3")); + + yield* ensure(() => until(fs.rm(dir, { recursive: true, force: true }))); + // wait longer than debounce window + yield* sleep(300); + }); + + // we expect the rapid writes to coalesce: there should be at least one + // raw and at least one debounced update, and debounced updates should be + // fewer than the number of writes (3) + assert(rawCount >= 1, `expected at least 1 raw update, got ${rawCount}`); + assert(updates.length >= 1, "expected at least one debounced update"); + const aCount = updates.filter((u) => u === "a").length; + assert( + aCount < 3, + `expected debounced updates to be fewer than writes (3), got ${aCount}` + ); +}); From eb337369ad572059ed8081d4e10ae82399871718 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Mon, 12 Jan 2026 15:05:07 -0600 Subject: [PATCH 11/38] shift logs to logger --- packages/server/bin/run-simulation-child.ts | 2 -- packages/server/src/cli.ts | 6 +++--- packages/server/src/services.ts | 7 ++++--- packages/server/src/simulation.ts | 17 +++++++++++------ 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/packages/server/bin/run-simulation-child.ts b/packages/server/bin/run-simulation-child.ts index a64d2176..45db4569 100644 --- a/packages/server/bin/run-simulation-child.ts +++ b/packages/server/bin/run-simulation-child.ts @@ -8,7 +8,6 @@ import type { main(function* () { const args = process.argv.slice(2); - console.dir({ args }); if (args.length < 1) { throw new Error("usage: run-simulation-child.js "); } @@ -59,7 +58,6 @@ main(function* () { console.log(out); yield* suspend(); } finally { - console.log("shutting down gracefully..."); try { if (listening && typeof listening.ensureClose === "function") { yield* until(listening.ensureClose()); diff --git a/packages/server/src/cli.ts b/packages/server/src/cli.ts index 03ff2d23..917edb81 100644 --- a/packages/server/src/cli.ts +++ b/packages/server/src/cli.ts @@ -2,8 +2,8 @@ import { parseArgs } from "node:util"; import { main, suspend, type Operation } from "effection"; import type { ServiceGraph, ServiceDefinition } from "./services.ts"; -export function* simulationCLIOp>( - serviceGraph: (subset?: string[] | string) => Operation> +export function* simulationCLIOp, T = any>( + serviceGraph: (subset?: string[] | string) => Operation> ) { try { const { values } = parseArgs({ @@ -20,7 +20,7 @@ export function* simulationCLIOp>( }); function* printUsage() { - console.log( + process.stdout.write( `Usage: cli [-s|--services serviceName] [--watch] [--watch-debounce ms]` ); } diff --git a/packages/server/src/services.ts b/packages/server/src/services.ts index 4a80c400..d0e0458d 100644 --- a/packages/server/src/services.ts +++ b/packages/server/src/services.ts @@ -9,6 +9,7 @@ import { } from "effection"; import { type ServiceUpdate, useWatcher } from "./watch.ts"; +import { stdout } from "./logging.ts"; export type ServiceDefinition< S, @@ -116,7 +117,7 @@ export function useServiceGraph< } effectiveServices = picked as typeof services; - console.log( + yield* stdout( `service graph: starting with services: ${Array.from(included).join( ", " )}` @@ -217,7 +218,7 @@ export function useServiceGraph< try { for (let service of Object.keys(effectiveServices)) { yield* spawn(function* () { - console.log(`service graph: starting service '${service}'`); + yield* stdout(`service graph: starting service '${service}'`); yield* withRestarts(service); }); } @@ -229,7 +230,7 @@ export function useServiceGraph< servicePorts, }); } finally { - console.log("shutting down service graph"); + yield* stdout("shutting down service graph"); } }); } diff --git a/packages/server/src/simulation.ts b/packages/server/src/simulation.ts index e182910b..57fc9f29 100644 --- a/packages/server/src/simulation.ts +++ b/packages/server/src/simulation.ts @@ -23,13 +23,13 @@ export function useSimulation>( createSim.listen() ); - console.log(`${name} simulation started on port ${listening.port}`); + yield* stdout(`${name} simulation started on port ${listening.port}`); try { yield* provide(listening); } finally { yield* until(listening.ensureClose()); - console.log(`${name} simulation closed on port ${listening.port}`); + yield* stdout(`${name} simulation closed on port ${listening.port}`); } }); } @@ -65,8 +65,6 @@ export function useChildSimulation>( for (let line of yield* each(process.stdout)) { const buf = Buffer.from(line); const str = buf.toString(); - console.log(`stdout: ${str}`); - yield* stdout(str); if (!listening) { try { @@ -76,10 +74,15 @@ export function useChildSimulation>( port: parsed.port, } as FoundationSimulatorListening; ready.resolve(Ok(listening)); + } else { + yield* stdout(str); } } catch (_) { // ignore lines that are not JSON + yield* stdout(str); } + } else { + yield* stdout(str); } yield* each.next(); @@ -114,12 +117,14 @@ export function useChildSimulation>( // we know listening is defined here listening = listening!; - console.log(`${name} process simulation started on port ${listening.port}`); + yield* stdout( + `${name} process simulation started on port ${listening.port}` + ); try { yield* provide(listening); } finally { - console.log(`${name} simulation closed on port ${listening?.port}`); + yield* stdout(`${name} simulation closed on port ${listening?.port}`); } }); } From 689eeff89f35a2981e9a31247c21c386411a706c Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Mon, 12 Jan 2026 15:16:06 -0600 Subject: [PATCH 12/38] lint --- packages/server/test/examples-smoke.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/test/examples-smoke.test.ts b/packages/server/test/examples-smoke.test.ts index de8c9e53..c95ea034 100644 --- a/packages/server/test/examples-smoke.test.ts +++ b/packages/server/test/examples-smoke.test.ts @@ -23,7 +23,7 @@ import type { Operation } from "effection"; it("basic example imports and runs", async () => { const runner = basicServices as unknown as () => Operation< - ServiceGraph> + ServiceGraph >; // runner let provided: any; @@ -137,9 +137,9 @@ it("basic example imports and runs", async () => { it("concurrency example imports and runs", async () => { const runner = concurrencyServices as unknown as () => Operation< - ServiceGraph> + ServiceGraph >; // runner - let provided: ServiceGraph> | undefined; + let provided: ServiceGraph | undefined; await run(function* () { const [scope, destroy] = createScope(); From 57c5717ebc4f472cee1d2957a123395c35746c66 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Wed, 14 Jan 2026 00:45:59 -0600 Subject: [PATCH 13/38] pass data to simulation from simulacrum service --- packages/server/bin/run-simulation-child.ts | 90 +++++++++++++------ .../example/services/gen-sim-factory.ts | 44 +++++---- packages/server/example/simulation-graph.ts | 4 +- packages/server/src/data-service.ts | 82 +++++++++++++++++ packages/server/src/services.ts | 27 ++++-- packages/server/src/simulation.ts | 14 ++- packages/server/test/data-service.test.ts | 31 +++++++ 7 files changed, 236 insertions(+), 56 deletions(-) create mode 100644 packages/server/src/data-service.ts create mode 100644 packages/server/test/data-service.test.ts diff --git a/packages/server/bin/run-simulation-child.ts b/packages/server/bin/run-simulation-child.ts index 45db4569..f2888466 100644 --- a/packages/server/bin/run-simulation-child.ts +++ b/packages/server/bin/run-simulation-child.ts @@ -1,11 +1,49 @@ #!/usr/bin/env node -import { main, suspend, until } from "effection"; +import { main, suspend, until, type Operation } from "effection"; import { pathToFileURL } from "node:url"; import type { FoundationSimulator, FoundationSimulatorListening, } from "@simulacrum/foundation-simulator"; +function guardedFactory( + factory: Function +): (initData?: unknown) => Promise> { + return async function startSimulation(initData?: unknown) { + const sim = await factory(initData); + if ("listen" in sim && typeof sim.listen === "function") { + return sim as FoundationSimulator; + } + throw new Error("factory did not return a simulator instance"); + }; +} + +function* normalizeSimulatorFactory(url: string) { + try { + const mod: unknown = yield* until(import(url)); + + // dynamically import module has to be an object if correctly resolved + if (mod && typeof mod === "object") { + const m = mod; + + // export default as factory + if ("default" in m && typeof m.default === "function") { + const factory = m.default; + return guardedFactory(factory); + } + + // export named 'simulation' as factory + if ("simulation" in m && typeof m.simulation === "function") { + const factory = m.simulation; + return guardedFactory(factory); + } + } + } catch (err) { + // no-op - will throw in fall through below + } + throw new Error("no factory or simulator instance found in module"); +} + main(function* () { const args = process.argv.slice(2); if (args.length < 1) { @@ -15,37 +53,37 @@ main(function* () { const modulePath = args[0]; // Resolve and import module inside the operation - let mod: any; - try { - const url = - modulePath.startsWith("./") || modulePath.startsWith("/") - ? pathToFileURL(modulePath).href - : modulePath; - mod = yield* until(import(url)); - } catch (err) { - throw new Error(`failed to import module: ${String(err)}`); - } + const url = + modulePath.startsWith("./") || modulePath.startsWith("/") + ? pathToFileURL(modulePath).href + : modulePath; + const factory = yield* normalizeSimulatorFactory(url); - const exportNames = ["default", "simulation"]; - let factory: Function | undefined = undefined; - for (const name of exportNames) { - if (name in mod && typeof mod[name] === "function") { - factory = mod[name]; - break; + let simulacrumPort: number | undefined = undefined; + // parse optional flags after modulePath + for (let i = 1; i < args.length; i++) { + if (args[i] === "--simulacrum-port") { + simulacrumPort = Number(args[i + 1]); + i++; } } - // fallback: module itself is a function - if (!factory && typeof mod === "function") factory = mod; - if (!factory) { - throw new Error(`no factory function found in module: ${modulePath}`); + // if present fetch the data chunk and pass it to the factory + let initData: JSON | undefined = undefined; + if (typeof simulacrumPort === "number" && !Number.isNaN(simulacrumPort)) { + try { + const res = yield* until( + fetch(`http://127.0.0.1:${simulacrumPort}/data`) + ); + initData = yield* until(res.json()); + } catch (err) { + // ignore fetch failures + console.error("failed to fetch simulacrum data:", err); + } } - let sim = factory() as FoundationSimulator; - - if (!sim || typeof sim.listen !== "function") { - throw new Error("factory did not return a simulator with .listen()"); - } + // invoke factory; it may return a simulator instance or a Promise thereof + const sim = yield* until(factory(initData)); let listening: FoundationSimulatorListening | undefined = undefined; try { diff --git a/packages/server/example/services/gen-sim-factory.ts b/packages/server/example/services/gen-sim-factory.ts index d34a94b4..1fa27a76 100644 --- a/packages/server/example/services/gen-sim-factory.ts +++ b/packages/server/example/services/gen-sim-factory.ts @@ -11,25 +11,31 @@ import { export function simulation( port: number = 3301, startDelay: number = 10 -): FoundationSimulator { - const factory = createFoundationSimulationServer({ - port, - extendRouter(router) { - router.get("/status", (_req, res) => { - res.status(200).send("ok"); - }); - }, - })(); +): (initData?: unknown) => FoundationSimulator { + return (initData?: unknown) => { + if (initData) console.log("simulation received init data:", initData); + const factory = createFoundationSimulationServer({ + port, + extendRouter(router) { + router.get("/status", (_req, res) => { + res.status(200).send("ok"); + }); + router.get("/init-data", (_req, res) => { + res.status(200).json({ data: initData ?? null }); + }); + }, + })(); - return { - async listen( - ...args: Parameters["listen"]> - ): Promise { - if (startDelay > 0) { - await new Promise((resolve) => setTimeout(resolve, startDelay)); - } - // delegate to underlying factory listen - return factory.listen(...args); - }, + return { + async listen( + ...args: Parameters["listen"]> + ): Promise { + if (startDelay > 0) { + await new Promise((resolve) => setTimeout(resolve, startDelay)); + } + // delegate to underlying factory listen + return factory.listen(...args); + }, + }; }; } diff --git a/packages/server/example/simulation-graph.ts b/packages/server/example/simulation-graph.ts index fd47e492..4417e220 100644 --- a/packages/server/example/simulation-graph.ts +++ b/packages/server/example/simulation-graph.ts @@ -14,7 +14,9 @@ const servicesMap = { }, }; -export const services = useServiceGraph(servicesMap); +export const services = useServiceGraph(servicesMap, { + globalData: { exampleKey: "exampleValue" }, +}); export function example(opts: { duration?: number } = {}) { return (function* () { diff --git a/packages/server/src/data-service.ts b/packages/server/src/data-service.ts new file mode 100644 index 00000000..a3d1022f --- /dev/null +++ b/packages/server/src/data-service.ts @@ -0,0 +1,82 @@ +import { call, resource, type Operation } from "effection"; +import { createServer } from "node:http"; +import { stdout } from "./logging.ts"; + +export type DataServiceOptions = Record | undefined; + +export function startDataService( + data: DataServiceOptions = {} +): Operation<{ port: number }> { + return resource(function* (provide) { + const server = createServer((req, res) => { + try { + const url = new URL(req.url ?? "", `http://127.0.0.1`); + const pathname = url.pathname; + + // GET /data -> whole object + if ( + req.method === "GET" && + (pathname === "/data" || pathname === "/") + ) { + const body = JSON.stringify(data || {}); + res.writeHead(200, { + "content-type": "application/json", + "content-length": String(Buffer.byteLength(body)), + }); + res.end(body); + return; + } + + // GET /data/ -> value or 404 + if (req.method === "GET" && pathname.startsWith("/data/")) { + const key = decodeURIComponent(pathname.replace(/^\/data\//, "")); + if (!key) { + res.writeHead(400); + res.end(); + return; + } + + const value = (data as any)?.[key]; + if (value === undefined) { + res.writeHead(404, { "content-type": "text/plain" }); + res.end("not found"); + return; + } + + const body = JSON.stringify(value); + res.writeHead(200, { + "content-type": "application/json", + "content-length": String(Buffer.byteLength(body)), + }); + res.end(body); + return; + } + + // unknown endpoint + res.writeHead(404, { "content-type": "text/plain" }); + res.end("not found"); + } catch (err) { + res.writeHead(500, { "content-type": "text/plain" }); + res.end(String(err)); + } + }); + + // listen on ephemeral port bound to localhost + yield* call(() => server.listen()); + + const address = server.address(); + const port = + typeof address === "object" && address !== null && "port" in address + ? address.port + : 0; + + yield* stdout(`data service: started on port ${port}`); + + try { + yield* provide({ port }); + } finally { + yield* call(() => server.close()); + yield* stdout(`data service: stopped on port ${port}`); + } + }); +} diff --git a/packages/server/src/services.ts b/packages/server/src/services.ts index d0e0458d..acfedc40 100644 --- a/packages/server/src/services.ts +++ b/packages/server/src/services.ts @@ -1,15 +1,19 @@ import { type Operation, + type Stream, + type WithResolvers, resource, spawn, withResolvers, each, - type Stream, - type WithResolvers, + createContext, } from "effection"; import { type ServiceUpdate, useWatcher } from "./watch.ts"; import { stdout } from "./logging.ts"; +import { startDataService } from "./data-service.ts"; + +export const SimulacrumEndpoint = createContext("SimulacrumEndpoint"); export type ServiceDefinition< S, @@ -64,7 +68,11 @@ export function useServiceGraph< T extends MaybeSimulation >( services: S, - options?: { watch?: boolean; watchDebounce?: number } + options?: { + globalData?: Record; + watch?: boolean; + watchDebounce?: number; + } ): (subset?: string[] | string) => Operation> { return (subset?: string[] | string) => resource(function* (provide) { @@ -123,6 +131,16 @@ export function useServiceGraph< )}` ); } + // track service ports (when services expose one) + const servicePorts = new Map(); + + const dataServiceProvided = yield* startDataService( + options?.globalData ?? {} + ); + servicePorts.set("simulacrum", dataServiceProvided.port); + // set the SimulacrumEndpoint in this operation scope so children started + // in this graph can access the port via context + yield* SimulacrumEndpoint.set(dataServiceProvided.port); const watcher = yield* useWatcher( effectiveServices, @@ -147,9 +165,6 @@ export function useServiceGraph< } } - // track service ports (when services expose one) - const servicePorts = new Map(); - function bumpService(service: string) { const task = status.get(service); if (!task) throw new Error(`missing status for service '${service}'`); diff --git a/packages/server/src/simulation.ts b/packages/server/src/simulation.ts index 57fc9f29..8a9ccdb1 100644 --- a/packages/server/src/simulation.ts +++ b/packages/server/src/simulation.ts @@ -6,6 +6,7 @@ import type { FoundationSimulator, FoundationSimulatorListening, } from "@simulacrum/foundation-simulator"; +import { SimulacrumEndpoint } from "./services.ts"; /** * Helper to start a foundation simulation server factory @@ -42,15 +43,20 @@ export function useChildSimulation>( modulePath: string ): Operation> { return resource(function* (provide) { - const cmd = [ + // attempt to read the simulacrum port from context; if not present, continue without it + const port = yield* SimulacrumEndpoint.get(); + + const parts = [ "node", "--import", "tsx", "./bin/run-simulation-child.ts", modulePath, - ] - .map((s) => (s.includes(" ") ? `'${s}'` : s)) - .join(" "); + ]; + if (typeof port === "number") { + parts.push("--simulacrum-port", String(port)); + } + const cmd = parts.map((s) => (s.includes(" ") ? `'${s}'` : s)).join(" "); const process = yield* exec(cmd); diff --git a/packages/server/test/data-service.test.ts b/packages/server/test/data-service.test.ts new file mode 100644 index 00000000..43cb341a --- /dev/null +++ b/packages/server/test/data-service.test.ts @@ -0,0 +1,31 @@ +import { it } from "node:test"; +import assert from "node:assert"; +import { run, sleep, until } from "effection"; +import { useServiceGraph } from "../src/services.ts"; + +it("starts data service and serves configured data", async () => { + await run(function* () { + const runGraph = yield* useServiceGraph( + {}, + { globalData: { a: 1, nested: { b: 2 } } } + )(); + + let port: string | number | undefined = undefined; + for (let i = 0; i < 50 && !port; i++) { + if (runGraph && runGraph.servicePorts) { + port = runGraph.servicePorts.get("simulacrum"); + if (typeof port === "number") break; + } + yield* sleep(10); + } + + assert.ok( + typeof port === "number", + "data service port should be registered on servicePorts" + ); + + const res = yield* until(fetch(`http://127.0.0.1:${port}/data`)); + const json = yield* until(res.json()); + assert.deepStrictEqual(json, { a: 1, nested: { b: 2 } }); + }); +}); From 57f11978697b1bb082d0f3e86151b69ff5de7aa4 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Wed, 14 Jan 2026 16:29:16 -0600 Subject: [PATCH 14/38] process uses daemon --- packages/server/example/process-graph.ts | 4 ++-- packages/server/example/services/basic-start-1.ts | 7 +++++++ packages/server/example/services/basic-start-2.ts | 7 +++++++ packages/server/src/service.ts | 9 +++++---- packages/server/src/simulation.ts | 4 ++-- 5 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 packages/server/example/services/basic-start-1.ts create mode 100644 packages/server/example/services/basic-start-2.ts diff --git a/packages/server/example/process-graph.ts b/packages/server/example/process-graph.ts index c7e9a700..8892aaef 100644 --- a/packages/server/example/process-graph.ts +++ b/packages/server/example/process-graph.ts @@ -8,7 +8,7 @@ const servicesMap = { A: { operation: useService( "A", - "node --import tsx ./example/services/basic-sim.ts", + "node --import tsx ./example/services/basic-start-1.ts", { wellnessCheck: { frequency: 10, @@ -31,7 +31,7 @@ const servicesMap = { dependsOn: { startup: ["A"] as const }, operation: useService( "B", - "node --import tsx ./example/services/basic-sim.ts", + "node --import tsx ./example/services/basic-start-2.ts", { wellnessCheck: { frequency: 10, diff --git a/packages/server/example/services/basic-start-1.ts b/packages/server/example/services/basic-start-1.ts new file mode 100644 index 00000000..ca64c183 --- /dev/null +++ b/packages/server/example/services/basic-start-1.ts @@ -0,0 +1,7 @@ +import { simulation } from "./basic-sim-1.ts"; + +simulation() + .listen(3301) + .then(() => { + console.log("Basic simulation 1 started on port 3301"); + }); diff --git a/packages/server/example/services/basic-start-2.ts b/packages/server/example/services/basic-start-2.ts new file mode 100644 index 00000000..928cad94 --- /dev/null +++ b/packages/server/example/services/basic-start-2.ts @@ -0,0 +1,7 @@ +import { simulation } from "./basic-sim-2.ts"; + +simulation() + .listen(3302) + .then(() => { + console.log("Basic simulation 2 started on port 3302"); + }); diff --git a/packages/server/src/service.ts b/packages/server/src/service.ts index c48fac19..f27fc128 100644 --- a/packages/server/src/service.ts +++ b/packages/server/src/service.ts @@ -4,13 +4,14 @@ import { type Stream, each, lift, + race, resource, scoped, sleep, spawn, } from "effection"; import { timebox } from "@effectionx/timebox"; -import { exec } from "@effectionx/process"; +import { daemon } from "@effectionx/process"; import type { ExecOptions as ProcessOptions } from "@effectionx/process"; import { stderr, stdout } from "./logging.ts"; import { createReplaySignal } from "./createReplaySignal.ts"; @@ -37,15 +38,15 @@ export function useService( _name: string, cmd: string, options: ServiceOptions = {} -): Operation { - return resource(function* (provide) { +) { + return resource(function* (provide) { if (cmd.startsWith("npm")) { // see https://github.com/npm/cli/issues/6684 throw new Error( "scripts run with npm don't respect signals to properly shutdown" ); } - const process = yield* exec(cmd, options.processOptions); + const process = yield* daemon(cmd, options.processOptions); const stdio = createReplaySignal(); const stdioAdd = lift(stdio.send); diff --git a/packages/server/src/simulation.ts b/packages/server/src/simulation.ts index 8a9ccdb1..82f7975e 100644 --- a/packages/server/src/simulation.ts +++ b/packages/server/src/simulation.ts @@ -1,6 +1,6 @@ import { resource, until, spawn, each, withResolvers, Ok } from "effection"; import type { Operation } from "effection"; -import { exec } from "@effectionx/process"; +import { daemon } from "@effectionx/process"; import { stderr, stdout } from "./logging.ts"; import type { FoundationSimulator, @@ -58,7 +58,7 @@ export function useChildSimulation>( } const cmd = parts.map((s) => (s.includes(" ") ? `'${s}'` : s)).join(" "); - const process = yield* exec(cmd); + const process = yield* daemon(cmd); // read the first stdout JSON line to get the listening info let listening: FoundationSimulatorListening | undefined = undefined; From ef4db1c6131319f7b9bca6cf144a21c4b4a5a9d3 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Wed, 14 Jan 2026 16:39:15 -0600 Subject: [PATCH 15/38] pass initData in --- packages/server/bin/run-simulation-child.ts | 2 +- packages/server/src/service.ts | 1 - packages/server/src/simulation.ts | 21 +++++++++++++++++++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/packages/server/bin/run-simulation-child.ts b/packages/server/bin/run-simulation-child.ts index f2888466..59ede4eb 100644 --- a/packages/server/bin/run-simulation-child.ts +++ b/packages/server/bin/run-simulation-child.ts @@ -1,5 +1,5 @@ #!/usr/bin/env node -import { main, suspend, until, type Operation } from "effection"; +import { main, suspend, until } from "effection"; import { pathToFileURL } from "node:url"; import type { FoundationSimulator, diff --git a/packages/server/src/service.ts b/packages/server/src/service.ts index f27fc128..ae8d8502 100644 --- a/packages/server/src/service.ts +++ b/packages/server/src/service.ts @@ -4,7 +4,6 @@ import { type Stream, each, lift, - race, resource, scoped, sleep, diff --git a/packages/server/src/simulation.ts b/packages/server/src/simulation.ts index 82f7975e..0c82c8c2 100644 --- a/packages/server/src/simulation.ts +++ b/packages/server/src/simulation.ts @@ -16,10 +16,27 @@ import { SimulacrumEndpoint } from "./services.ts"; */ export function useSimulation>( name: string, - createFactory: () => FoundationSimulator + createFactory: (initData?: unknown) => FoundationSimulator ): Operation> { return resource(function* (provide) { - const createSim = createFactory(); + // attempt to read the simulacrum port from context; if not present, continue without it + const simulacrumPort = yield* SimulacrumEndpoint.get(); + + // if present fetch the data chunk and pass it to the factory + let initData: unknown | undefined = undefined; + if (typeof simulacrumPort === "number" && !Number.isNaN(simulacrumPort)) { + try { + const res = yield* until( + fetch(`http://127.0.0.1:${simulacrumPort}/data`) + ); + initData = yield* until(res.json()); + } catch (err) { + // ignore fetch failures + console.error("failed to fetch simulacrum data:", err); + } + } + + const createSim = createFactory(initData); const listening: FoundationSimulatorListening = yield* until( createSim.listen() ); From dae9a9e2280dff41e5a112e17ee8e666f39ca979 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Wed, 14 Jan 2026 20:54:32 -0600 Subject: [PATCH 16/38] fix type errors --- packages/server/test/services/service-a.ts | 3 +-- packages/server/test/services/service-b.ts | 3 +-- packages/server/test/services/service-fast.ts | 3 +-- packages/server/test/services/service-slow.ts | 3 +-- packages/server/test/simulation.test.ts | 2 +- 5 files changed, 5 insertions(+), 9 deletions(-) diff --git a/packages/server/test/services/service-a.ts b/packages/server/test/services/service-a.ts index cfcd358a..9fa5d939 100644 --- a/packages/server/test/services/service-a.ts +++ b/packages/server/test/services/service-a.ts @@ -1,4 +1,3 @@ import { simulation as genSimulation } from "../../example/services/gen-sim-factory.ts"; -import type { FoundationSimulator } from "@simulacrum/foundation-simulator"; -export const simulation: FoundationSimulator = genSimulation(4010, 10); +export const simulation = genSimulation(4010, 10); diff --git a/packages/server/test/services/service-b.ts b/packages/server/test/services/service-b.ts index 316c5741..e47e6f49 100644 --- a/packages/server/test/services/service-b.ts +++ b/packages/server/test/services/service-b.ts @@ -1,4 +1,3 @@ import { simulation as genSimulation } from "../../example/services/gen-sim-factory.ts"; -import type { FoundationSimulator } from "@simulacrum/foundation-simulator"; -export const simulation: FoundationSimulator = genSimulation(4020, 40); +export const simulation = genSimulation(4020, 40); diff --git a/packages/server/test/services/service-fast.ts b/packages/server/test/services/service-fast.ts index 4c907299..93b1e120 100644 --- a/packages/server/test/services/service-fast.ts +++ b/packages/server/test/services/service-fast.ts @@ -1,4 +1,3 @@ import { simulation as genSimulation } from "../../example/services/gen-sim-factory.ts"; -import type { FoundationSimulator } from "@simulacrum/foundation-simulator"; -export const simulation: FoundationSimulator = genSimulation(4030, 10); +export const simulation = genSimulation(4030, 10); diff --git a/packages/server/test/services/service-slow.ts b/packages/server/test/services/service-slow.ts index 61614e1a..6c4015e5 100644 --- a/packages/server/test/services/service-slow.ts +++ b/packages/server/test/services/service-slow.ts @@ -1,4 +1,3 @@ import { simulation as genSimulation } from "../../example/services/gen-sim-factory.ts"; -import type { FoundationSimulator } from "@simulacrum/foundation-simulator"; -export const simulation: FoundationSimulator = genSimulation(4040, 200); +export const simulation = genSimulation(4040, 200); diff --git a/packages/server/test/simulation.test.ts b/packages/server/test/simulation.test.ts index 5cd5f3b5..35c7189e 100644 --- a/packages/server/test/simulation.test.ts +++ b/packages/server/test/simulation.test.ts @@ -7,7 +7,7 @@ import { createFoundationSimulationServer } from "@simulacrum/foundation-simulat it("useSimulation returns listening info", async () => { const port = await run(function* () { - const listening = yield* useSimulation("test", simulation); + const listening = yield* useSimulation("test", () => simulation(3000)); return listening.port; }); assert(typeof port === "number", "port is a number"); From e6cfde83c98e1327e9856322207447e4b03a04d1 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Wed, 14 Jan 2026 21:16:36 -0600 Subject: [PATCH 17/38] more docs --- packages/server/README.md | 39 ++++++++++++++--------- packages/server/example/README.md | 21 ++++++++---- packages/server/src/cli.ts | 17 ++++++++++ packages/server/src/createReplaySignal.ts | 7 ++++ packages/server/src/data-service.ts | 11 +++++++ packages/server/src/services.ts | 31 ++++++++++-------- packages/server/src/simulation.ts | 27 ++++++++++++++-- packages/server/src/watch.ts | 14 ++++++++ 8 files changed, 130 insertions(+), 37 deletions(-) diff --git a/packages/server/README.md b/packages/server/README.md index 2d3f9a4b..a0a97ae6 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -13,14 +13,21 @@ https://github.com/thefrontside/simulacrum Key points: -- `useServiceGraph(services: ServicesMap, options?: { sequential?: boolean }): ServiceRunner` — returns a _runner function_ which you call to start the graph: `const run = useServiceGraph(services, options); yield* run(subset?: string[] | string);`. By default services in the same topological layer run concurrently; pass `options.sequential = true` to run services in each layer serially. +- `useServiceGraph(services: ServicesMap, options?: { globalData?: Record; watch?: boolean; watchDebounce?: number }): ServiceRunner` — returns a _runner function_ which you call to start the graph: `const run = useServiceGraph(services, options); const provided = yield* run(subset?: string[] | string);`. By default services in the same topological layer run concurrently; pass `options.watch = true` and `options.watchDebounce` to enable file watching and restart propagation. - `ServiceDefinition.operation` (required) — an `Operation` which indicates the service has started. This operation may be long-lived (e.g. `useService`) or may return once the service is ready while a background child keeps the service running. See the example below. - `dependsOn` — an optional object `{ startup: string[]; restart?: string[] }` listing service names this service depends on. Use `startup` to list services that must start before this service; use `restart` to list services that should trigger a restart of this service when they are restarted (for example, due to a watched file change). Services without dependencies in the same layer are started concurrently by default, or serially when `options.sequential` is true. - `subset` (runner argument) — when calling the runner returned by `useServiceGraph` you may pass a subset (e.g. `yield* run(['serviceA'])` or `yield* run('serviceA')`) to start only a subset of services; any startup dependencies required by that subset are automatically included. - Watching & restart propagation — pass `{ watch: true }` to `useServiceGraph` and define `watch` paths in each `ServiceDefinition` to enable file watching. The watcher will precompute transitive dependents (based on `dependsOn.restart`) and automatically emit restart updates for dependents when a watched path changes, so restarts propagate efficiently and deterministically. -- Lifecycle hooks: `beforeStart`, `afterStart`, `beforeStop`, `afterStop` — each is an `Operation` that runs at the appropriate time. + +### Global data & the simulacrum gateway 🔁 + +The graph can optionally start a small local HTTP data service (the _simulacrum gateway_) to expose a `globalData` object to child simulations and tests. The gateway registers its listening port on the returned `servicePorts` map under the key `"simulacrum"`, which tests and examples can use to discover it. See the `test/child-simulation-simulacrum.test.ts` and `example/simulation-graph.ts` for examples and coverage of this flow. + +### Lifecycle hooks + +- Lifecycle hooks can be implemented by arranging operations and using try/finally within your service `operation` to perform startup and cleanup logic. You can keep the operation alive with `yield* suspend()` and perform cleanup in the `finally` block when the service is stopped. Example: @@ -100,35 +107,34 @@ npm test The `example` folder contains runnable examples demonstrating `useServiceGraph` and `useService`. -Run the basic dependency example: +Run the simulation-based example (starts simulators via child simulations): ```bash cd packages/server -npm run example:basic +npm run example:sim ``` -Run lifecycle hooks example: +Run the process-based example (spawns processes via `useService`): ```bash cd packages/server -npm run example:lifecycle +npm run example:process ``` -Run concurrency layers example: +Run the concurrency example: -````bash +```bash cd packages/server npm run example:concurrency +``` -Run examples directly (each example has its own npm script). You can also run the TypeScript module with `tsx`. +Run examples directly (each example module can be executed with `tsx`): ```bash cd packages/server -npm run example:basic -npm run example:lifecycle -npm run example:concurrency -# or run a module directly: -node --import tsx ./example/basic-graph.ts +node --import tsx ./example/simulation-graph.ts +node --import tsx ./example/process-graph.ts +node --import tsx ./example/concurrency-layers.ts ``` ### Sharing exported values between services (note) @@ -136,4 +142,7 @@ node --import tsx ./example/basic-graph.ts Previously services could expose their return value via a public `exportsOperation` that consumers could await. That mechanism has been removed in this branch as we move to a child-process-focused runner model. Provider-returned values are still delivered to dependent service factories internally, but no longer exposed as an operation on the public `services` map. For convenience tests may use the `servicePorts` map exposed by the running graph to discover HTTP ports that services registered when they start. The `servicePorts` map is available on the object returned by the runner and contains service name => port when a service's `operation` returns an object with a `{ port: number }` property. -```` + +``` + +``` diff --git a/packages/server/example/README.md b/packages/server/example/README.md index 38c68734..a9e7a541 100644 --- a/packages/server/example/README.md +++ b/packages/server/example/README.md @@ -4,26 +4,35 @@ This folder contains runnable examples demonstrating `useServiceGraph` and `useS There are two sets of examples: -- **use-service** (top-level files like `basic-graph.ts`, `concurrency-layers.ts`) — these spawn separate processes using `useService` (e.g. `node --import tsx ./example/services/*.ts`). Use these to exercise the process-based behavior. +- **use-service** (top-level files like `process-graph.ts`, `concurrency-layers.ts`) — these spawn separate processes using `useService`. Use these to exercise the process-based behavior. -- **operation** (under `operation/`) — these demonstrate `useChildSimulation()` which runs each service in a child process using a simulation factory. They show how to isolate simulations and start them as independent processes. +- **simulation / child processes** (e.g., `simulation-graph.ts`) — these demonstrate `useChildSimulation()` which runs each service in a child process using a simulation factory. They show how to isolate simulations and start them as independent processes. Quick commands: -Run the basic dependency example (use-service): +Run the simulation-based example (child simulations): ```bash cd packages/server -node --import tsx ./example/basic-graph.ts +node --import tsx ./example/simulation-graph.ts ``` -Run the basic dependency example (operation): +Run the process-based example (spawned processes): ```bash cd packages/server -node --import tsx ./example/operation/basic-graph.ts +node --import tsx ./example/process-graph.ts +``` + +Run the concurrency example: + +```bash +cd packages/server +node --import tsx ./example/concurrency-layers.ts ``` These examples make use of the small service implementations in `./example/services`. Notes: the examples now use `dependsOn` with a `{ startup, restart? }` shape. To experiment with restart propagation, add a `watch` entry to a service and include dependents via `dependsOn.restart` — when a watched file changes the watcher will restart the affected service and its transitive dependents. + +Global data: The simulation example (`simulation-graph.ts`) demonstrates the `globalData` option. When provided, the graph starts a small HTTP data service (the "simulacrum gateway") and registers its port on `servicePorts` under the key `"simulacrum"`. Child simulations may query this gateway at `/data` to obtain initialization data. diff --git a/packages/server/src/cli.ts b/packages/server/src/cli.ts index 917edb81..5f9cde01 100644 --- a/packages/server/src/cli.ts +++ b/packages/server/src/cli.ts @@ -2,6 +2,16 @@ import { parseArgs } from "node:util"; import { main, suspend, type Operation } from "effection"; import type { ServiceGraph, ServiceDefinition } from "./services.ts"; +/** + * CLI operation that parses args and runs a service graph runner. + * + * This operation accepts the runner returned by `useServiceGraph` and starts + * the requested subset of services. It supports `--services` (comma + * separated), `--watch` and `--watch-debounce` options for convenience when + * iterating on local development. + * + * @param serviceGraph - runner factory returned by `useServiceGraph` + */ export function* simulationCLIOp, T = any>( serviceGraph: (subset?: string[] | string) => Operation> ) { @@ -54,6 +64,13 @@ export function* simulationCLIOp, T = any>( } } +/** + * Run a service graph runner inside an effection main loop suitable for use + * as a Node CLI. This invokes `simulationCLIOp` under `main` and returns the + * resulting promise. + * + * @param serviceGraph - runner factory returned by `useServiceGraph` + */ export async function simulationCLI< S extends Record>, T diff --git a/packages/server/src/createReplaySignal.ts b/packages/server/src/createReplaySignal.ts index 61056b31..93fb85c8 100644 --- a/packages/server/src/createReplaySignal.ts +++ b/packages/server/src/createReplaySignal.ts @@ -1,6 +1,13 @@ import type { Resolve, Subscription } from "effection"; import { action, resource } from "effection"; +/** + * Create a replayable signal which exposes `send`/`close` helpers while + * preserving the original subscription semantics. Useful for forwarding + * child process stdio lines for wellness checks and logging. + * + * @returns a Signal-like value with `send` and `close` methods and a `next` operation + */ export function createReplaySignal() { const subscribers = new Set>(); // single shared durable queue storage diff --git a/packages/server/src/data-service.ts b/packages/server/src/data-service.ts index a3d1022f..d2411b46 100644 --- a/packages/server/src/data-service.ts +++ b/packages/server/src/data-service.ts @@ -4,6 +4,17 @@ import { stdout } from "./logging.ts"; export type DataServiceOptions = Record | undefined; +/** + * Start a simple local HTTP data service that serves a user-provided object. + * + * This is intended for local testing and to supply a small amount of + * configuration or initialization data to child simulations via the + * "simulacrum" gateway. The operation yields an object with `{ port }` once + * listening. + * + * @param data - Arbitrary JSON-serializable data to serve at `/data` + * @returns an operation that provides `{ port: number }` when ready + */ export function startDataService( data: DataServiceOptions = {} ): Operation<{ port: number }> { diff --git a/packages/server/src/services.ts b/packages/server/src/services.ts index acfedc40..f78ea3c7 100644 --- a/packages/server/src/services.ts +++ b/packages/server/src/services.ts @@ -13,6 +13,14 @@ import { type ServiceUpdate, useWatcher } from "./watch.ts"; import { stdout } from "./logging.ts"; import { startDataService } from "./data-service.ts"; +/** + * Context key for the Simulacrum gateway listening port. + * + * When `useServiceGraph` starts the optional simulacrum gateway (via the + * `globalData` option) it sets this context value to the listening port so + * operations in the graph (including `useSimulation` and + * `useChildSimulation`) can discover and fetch the `/data` payload. + */ export const SimulacrumEndpoint = createContext("SimulacrumEndpoint"); export type ServiceDefinition< @@ -46,22 +54,17 @@ export type ServiceGraph< }; /** - * useServiceGraph - * - * Start a set of services with dependencies (a DAG). Each service must provide an - * Operation that starts the service and returns once the service is ready. - * - * Example usage: + * Start a graph of services with dependency ordering and optional file + * watching/restart behavior. * - * yield* useServiceGraph({ - * A: { operation: useService('A', 'node --import tsx ./test/services/service-a.ts') }, - * B: { operation: useService('B', 'node --import tsx ./test/services/service-b.ts'), dependsOn: { startup: ['A'] } } - * }); + * Each service is defined as a `ServiceDefinition` that includes an + * `operation: Operation` which should return once the service is ready. The + * returned runner function starts the graph and returns a `ServiceGraph` object + * (which includes a `servicePorts` map) that can be inspected by tests. * - * Services within the same topological layer are started concurrently by default. - * Pass an optional `options` object with `{ sequential: true }` to force services - * within the same layer to start sequentially. Lifecycle hooks can be used to - * perform actions before or after each service starts or stops. + * @param services - a map of service names to definitions + * @param options - optional configuration: `{ globalData?, watch?, watchDebounce? }` + * @returns a runner function `(subset?: string[] | string) => Operation>` */ export function useServiceGraph< S extends Record>, diff --git a/packages/server/src/simulation.ts b/packages/server/src/simulation.ts index 0c82c8c2..838e5599 100644 --- a/packages/server/src/simulation.ts +++ b/packages/server/src/simulation.ts @@ -13,8 +13,19 @@ import { SimulacrumEndpoint } from "./services.ts"; * * This is implemented as an Effection `resource` so cleanup is handled by the * `provide` finalizer when the operation's scope is closed. - */ -export function useSimulation>( + *//** + * Start a simulator provided by a factory and return its listening info. + * + * The factory may accept initialization data (fetched from the simulacrum + * gateway when available) and should return a `FoundationSimulator` instance + * (or a Promise resolving to one). This operation yields the simulator's + * listening information (`{ port }`) once it starts. + * + * @param name - human-friendly name used for logging + * @param createFactory - factory function that returns a `FoundationSimulator` + * @returns an `Operation` that provides `FoundationSimulatorListening` when the + * simulator is listening + */export function useSimulation>( name: string, createFactory: (initData?: unknown) => FoundationSimulator ): Operation> { @@ -55,6 +66,18 @@ export function useSimulation>( // Spawn a child Node process to run a simulation factory in a fresh module // environment. This avoids sharing module cache and allows restarts to pick up // new code. The runtime uses `bin/run-simulation-child.ts`. +/** + * Spawn a child Node process to run a simulation factory. + * + * This runs `bin/run-simulation-child.ts ` in a separate Node + * process and reads the first JSON line printed to stdout to discover the + * child's listening port. Optionally the simulacrum gateway port will be + * passed to the child so it can fetch `globalData`. + * + * @param name - human-friendly name for logging + * @param modulePath - path to the module exporting a simulation factory or instance + * @returns an `Operation` that provides `FoundationSimulatorListening` from the child + */ export function useChildSimulation>( name: string, modulePath: string diff --git a/packages/server/src/watch.ts b/packages/server/src/watch.ts index 80f78db0..4657b94b 100644 --- a/packages/server/src/watch.ts +++ b/packages/server/src/watch.ts @@ -15,6 +15,20 @@ import { filter } from "@effectionx/stream-helpers"; export type ServiceUpdate = { service: string; path: string }; +/** + * Start a file watcher for services and provide streams of updates. + * + * This helper wraps `chokidar` and computes optional transitive dependents + * (based on `dependsOn.restart`) so that updates can be propagated to + * dependent services. The returned object exposes: + * + * - `serviceUpdates`: immediate updates for a service (`{service, path}`) + * - `serviceChanges`: debounced updates suitable for restart propagation + * - `add(service, paths)`: add watch paths for a service + * + * @param services - optional service map used to compute transitive dependents + * @param options - optional `{ watchDebounce?: number }` to configure debounce + */ export function useWatcher( services?: Record< string, From 0e330f0a57bf1dacab06d94b9bb2cab7e28f044f Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Thu, 15 Jan 2026 17:17:06 -0600 Subject: [PATCH 18/38] clean up, flesh out tests --- packages/server/src/data-service.ts | 2 +- packages/server/src/service.ts | 2 + packages/server/src/simulation.ts | 4 +- packages/server/test/child-simulation.test.ts | 193 ++++++++++++++++-- packages/server/test/data-service.test.ts | 59 +++++- packages/server/test/examples-smoke.test.ts | 21 +- packages/server/test/fixtures/broken-child.ts | 5 + .../server/test/fixtures/init-data-sim.ts | 12 ++ .../server/test/fixtures/json-before-ready.ts | 16 ++ .../server/test/fixtures/non-json-child.ts | 14 ++ packages/server/test/replay-signal.test.ts | 38 ++++ packages/server/test/service.test.ts | 23 ++- packages/server/test/services.test.ts | 4 +- packages/server/test/signal.test.ts | 149 +++++++++----- packages/server/test/simulation.test.ts | 23 +-- packages/server/test/utils.ts | 84 ++++++++ packages/server/test/watch.test.ts | 89 +++++--- 17 files changed, 604 insertions(+), 134 deletions(-) create mode 100644 packages/server/test/fixtures/broken-child.ts create mode 100644 packages/server/test/fixtures/init-data-sim.ts create mode 100644 packages/server/test/fixtures/json-before-ready.ts create mode 100644 packages/server/test/fixtures/non-json-child.ts create mode 100644 packages/server/test/replay-signal.test.ts create mode 100644 packages/server/test/utils.ts diff --git a/packages/server/src/data-service.ts b/packages/server/src/data-service.ts index d2411b46..3043d3d9 100644 --- a/packages/server/src/data-service.ts +++ b/packages/server/src/data-service.ts @@ -47,7 +47,7 @@ export function startDataService( return; } - const value = (data as any)?.[key]; + const value = (data as Record | undefined)?.[key]; if (value === undefined) { res.writeHead(404, { "content-type": "text/plain" }); res.end("not found"); diff --git a/packages/server/src/service.ts b/packages/server/src/service.ts index ae8d8502..559b2a8a 100644 --- a/packages/server/src/service.ts +++ b/packages/server/src/service.ts @@ -71,6 +71,8 @@ export function useService( yield* sleep(0); // allow stdio forwarding to start + // TODO if it fails to start up but has a wellness check, it seems to hang + // if supplied, wellness check to ensure it is running or timeout with result if (options.wellnessCheck) { const { operation } = options.wellnessCheck; diff --git a/packages/server/src/simulation.ts b/packages/server/src/simulation.ts index 838e5599..77ec5489 100644 --- a/packages/server/src/simulation.ts +++ b/packages/server/src/simulation.ts @@ -13,7 +13,7 @@ import { SimulacrumEndpoint } from "./services.ts"; * * This is implemented as an Effection `resource` so cleanup is handled by the * `provide` finalizer when the operation's scope is closed. - *//** + */ /** * Start a simulator provided by a factory and return its listening info. * * The factory may accept initialization data (fetched from the simulacrum @@ -25,7 +25,7 @@ import { SimulacrumEndpoint } from "./services.ts"; * @param createFactory - factory function that returns a `FoundationSimulator` * @returns an `Operation` that provides `FoundationSimulatorListening` when the * simulator is listening - */export function useSimulation>( + */ export function useSimulation>( name: string, createFactory: (initData?: unknown) => FoundationSimulator ): Operation> { diff --git a/packages/server/test/child-simulation.test.ts b/packages/server/test/child-simulation.test.ts index 3daa5db6..b9810b37 100644 --- a/packages/server/test/child-simulation.test.ts +++ b/packages/server/test/child-simulation.test.ts @@ -1,20 +1,187 @@ -import { it } from "node:test"; +import { describe, it } from "node:test"; import assert from "node:assert"; -import { run, sleep } from "effection"; +import { run, sleep, until } from "effection"; +import { useServiceGraph } from "../src/services.ts"; import { useChildSimulation } from "../src/simulation.ts"; +import { waitFor } from "./utils.ts"; -it("useChildSimulation starts a child and returns port", async () => { - await run(function* () { - const listening = yield* useChildSimulation( - "child-test", - "./test/fixtures/simple-sim.ts" - ); - assert(typeof listening.port === "number"); +describe("useChildSimulation", () => { + it("starts a child and returns port", async () => { + await run(function* () { + const listening = yield* useChildSimulation( + "child-test", + "./test/fixtures/simple-sim.ts" + ); + assert(typeof listening.port === "number"); - // Verify we received a port and the child reported ready. - assert(typeof listening.port === "number", "port should be a number"); + // Verify we received a port and the child reported ready. + assert(typeof listening.port === "number", "port should be a number"); - // allow a moment before teardown - yield* sleep(20); + // allow a moment before teardown + yield* sleep(20); + }); + }); + + it("handles non-JSON stdout before ready JSON from child", async () => { + await run(function* () { + const listening = yield* useChildSimulation( + "non-json", + "./test/fixtures/non-json-child.ts" + ); + assert(typeof listening.port === "number"); + }); + }); + + it("ignores JSON logs without ready/port until real ready JSON is emitted", async () => { + await run(function* () { + const listening = yield* useChildSimulation( + "json-before-ready", + "./test/fixtures/json-before-ready.ts" + ); + assert(typeof listening.port === "number"); + }); + }); + + describe("globalData forwarding", () => { + it("forwards nested objects as globalData to child simulations", async () => { + await run(function* () { + const data = { a: { b: { c: 3 } }, flag: true }; + + const op = useServiceGraph( + { + child: { + operation: useChildSimulation( + "child", + "./test/fixtures/init-data-sim.ts" + ), + }, + }, + { globalData: data } + ); + + const runGraph = yield* op(); + yield* waitFor( + () => typeof runGraph.servicePorts?.get("child") === "number", + 2000 + ); + const childPort = runGraph.servicePorts!.get("child")!; + + const res = yield* until(fetch(`http://127.0.0.1:${childPort}/init`)); + const json = (yield* until(res.json())) as { initData: typeof data }; + assert.deepStrictEqual(json.initData, data); + }); + }); + + it("forwards arrays as globalData to child simulations", async () => { + await run(function* () { + const data = { list: [1, 2, 3], nested: [{ x: 1 }, { x: 2 }] }; + + const op = useServiceGraph( + { + child: { + operation: useChildSimulation( + "child", + "./test/fixtures/init-data-sim.ts" + ), + }, + }, + { globalData: data } + ); + + const runGraph = yield* op(); + yield* waitFor( + () => typeof runGraph.servicePorts?.get("child") === "number", + 2000 + ); + const childPort = runGraph.servicePorts!.get("child")!; + + const res = yield* until(fetch(`http://127.0.0.1:${childPort}/init`)); + const json = (yield* until(res.json())) as { initData: typeof data }; + assert.deepStrictEqual(json.initData, data); + }); + }); + + it("forwards deeply nested values and special types to child simulations", async () => { + await run(function* () { + const data = { + users: [ + { + id: 1, + name: "alice", + prefs: { theme: "dark", tags: ["a", "b"] }, + }, + { id: 2, name: "bob", prefs: { theme: "light", tags: [] } }, + ], + meta: { + created: "2026-01-01", + count: 2, + active: true, + nothing: null, + }, + }; + + const op = useServiceGraph( + { + child: { + operation: useChildSimulation( + "child", + "./test/fixtures/init-data-sim.ts" + ), + }, + }, + { globalData: data } + ); + + const runGraph = yield* op(); + yield* waitFor( + () => typeof runGraph.servicePorts?.get("child") === "number", + 3000 + ); + const childPort = runGraph.servicePorts!.get("child")!; + + const res = yield* until(fetch(`http://127.0.0.1:${childPort}/init`)); + const json = (yield* until(res.json())) as { initData: typeof data }; + assert.deepStrictEqual(json.initData, data); + }); + }); + }); + + it("child simulation receives globalData via simulacrum gateway and registers its port", async () => { + await run(function* () { + const op = useServiceGraph( + { + child: { + operation: useChildSimulation( + "child", + "./test/fixtures/init-data-sim.ts" + ), + }, + }, + { globalData: { hello: "world" } } + ); + + const runGraph = yield* op(); + + // wait deterministically for the child port to be registered + yield* waitFor( + () => typeof runGraph.servicePorts?.get("child") === "number", + 3000 + ); + const childPort = runGraph.servicePorts!.get("child")!; + + const res = yield* until(fetch(`http://127.0.0.1:${childPort}/init`)); + const json = (yield* until(res.json())) as { + initData: { hello: string }; + }; + assert.deepStrictEqual(json.initData, { hello: "world" }); + }); + }); + + it("rejects when child exits before emitting listening info", async () => { + await assert.rejects(async () => { + await run(function* () { + yield* useChildSimulation("broken", "./test/fixtures/broken-child.ts"); + }); + }, /child process exited before emitting listening info/); }); }); diff --git a/packages/server/test/data-service.test.ts b/packages/server/test/data-service.test.ts index 43cb341a..e1514920 100644 --- a/packages/server/test/data-service.test.ts +++ b/packages/server/test/data-service.test.ts @@ -1,7 +1,8 @@ import { it } from "node:test"; import assert from "node:assert"; -import { run, sleep, until } from "effection"; +import { run, until } from "effection"; import { useServiceGraph } from "../src/services.ts"; +import { waitFor } from "./utils.ts"; it("starts data service and serves configured data", async () => { await run(function* () { @@ -10,14 +11,12 @@ it("starts data service and serves configured data", async () => { { globalData: { a: 1, nested: { b: 2 } } } )(); - let port: string | number | undefined = undefined; - for (let i = 0; i < 50 && !port; i++) { - if (runGraph && runGraph.servicePorts) { - port = runGraph.servicePorts.get("simulacrum"); - if (typeof port === "number") break; - } - yield* sleep(10); - } + // wait deterministically for the simulacrum port to be registered + yield* waitFor( + () => Boolean(runGraph?.servicePorts?.get("simulacrum")), + 2000 + ); + const port = runGraph!.servicePorts!.get("simulacrum")!; assert.ok( typeof port === "number", @@ -29,3 +28,45 @@ it("starts data service and serves configured data", async () => { assert.deepStrictEqual(json, { a: 1, nested: { b: 2 } }); }); }); + +it("serves individual keys and appropriate status codes", async () => { + await run(function* () { + const runGraph = yield* useServiceGraph( + {}, + { globalData: { a: 1, nested: { b: 2 } } } + )(); + + // wait deterministically for the simulacrum port + yield* waitFor( + () => Boolean(runGraph?.servicePorts?.get("simulacrum")), + 2000 + ); + const port = runGraph!.servicePorts!.get("simulacrum")!; + + assert.ok(typeof port === "number"); + + // existing key + const aRes = yield* until(fetch(`http://127.0.0.1:${port}/data/a`)); + assert.strictEqual(aRes.status, 200); + const aJson = yield* until(aRes.json()); + assert.deepStrictEqual(aJson, 1); + + // nested key returns object + const nestedRes = yield* until( + fetch(`http://127.0.0.1:${port}/data/nested`) + ); + assert.strictEqual(nestedRes.status, 200); + const nestedJson = yield* until(nestedRes.json()); + assert.deepStrictEqual(nestedJson, { b: 2 }); + + // missing key -> 404 + const missRes = yield* until( + fetch(`http://127.0.0.1:${port}/data/does-not-exist`) + ); + assert.strictEqual(missRes.status, 404); + + // empty key -> 400 + const emptyRes = yield* until(fetch(`http://127.0.0.1:${port}/data/`)); + assert.strictEqual(emptyRes.status, 400); + }); +}); diff --git a/packages/server/test/examples-smoke.test.ts b/packages/server/test/examples-smoke.test.ts index c95ea034..6ef2e431 100644 --- a/packages/server/test/examples-smoke.test.ts +++ b/packages/server/test/examples-smoke.test.ts @@ -2,6 +2,7 @@ import { it } from "node:test"; import http from "node:http"; import { run, sleep, suspend, createScope, until } from "effection"; import { timebox } from "@effectionx/timebox"; +import { waitForAsync } from "./utils.ts"; import { services as basicServices } from "../example/simulation-graph.ts"; import { services as concurrencyServices } from "../example/concurrency-layers.ts"; @@ -61,18 +62,16 @@ it("basic example imports and runs", async () => { // check each port while the graph is still running for (const p of ps) { - let ok = false; - for (let i = 0; i < 100; i++) { - try { - const status = yield* until(checkStatus(p)); - if (status === 200) { - ok = true; - break; + try { + yield* waitForAsync(async () => { + try { + const status = await checkStatus(p); + return status === 200; + } catch (_) { + return false; } - } catch (_) {} - yield* sleep(10); - } - if (!ok) { + }, 2000); + } catch (err) { throw new Error( `(examples-smoke basic) port ${p} did not return 200 while graph was running` ); diff --git a/packages/server/test/fixtures/broken-child.ts b/packages/server/test/fixtures/broken-child.ts new file mode 100644 index 00000000..baa44203 --- /dev/null +++ b/packages/server/test/fixtures/broken-child.ts @@ -0,0 +1,5 @@ +export default function simulation() { + // deliberately exit immediately so the parent sees a child that dies + // before it can emit its listening-info JSON line + process.exit(1); +} diff --git a/packages/server/test/fixtures/init-data-sim.ts b/packages/server/test/fixtures/init-data-sim.ts new file mode 100644 index 00000000..8f80b12b --- /dev/null +++ b/packages/server/test/fixtures/init-data-sim.ts @@ -0,0 +1,12 @@ +import { createFoundationSimulationServer } from "@simulacrum/foundation-simulator"; + +import type { FoundationSimulator } from "@simulacrum/foundation-simulator"; + +export function simulation(initData?: unknown): FoundationSimulator { + return createFoundationSimulationServer({ + port: 0, + extendRouter(router) { + router.get("/init", (_req, res) => res.json({ initData })); + }, + })(); +} diff --git a/packages/server/test/fixtures/json-before-ready.ts b/packages/server/test/fixtures/json-before-ready.ts new file mode 100644 index 00000000..88f86c5f --- /dev/null +++ b/packages/server/test/fixtures/json-before-ready.ts @@ -0,0 +1,16 @@ +import { createFoundationSimulationServer } from "@simulacrum/foundation-simulator"; + +// Print a JSON payload that does not contain `ready`/`port` fields +// This should be treated as a log line by the parent and ignored for readiness +console.log(JSON.stringify({ foo: "bar" })); + +import type { FoundationSimulator } from "@simulacrum/foundation-simulator"; + +export function simulation(): FoundationSimulator { + return createFoundationSimulationServer({ + port: 0, + extendRouter(router) { + router.get("/info", (_req, res) => res.json({ ok: true })); + }, + })(); +} diff --git a/packages/server/test/fixtures/non-json-child.ts b/packages/server/test/fixtures/non-json-child.ts new file mode 100644 index 00000000..af1ae77c --- /dev/null +++ b/packages/server/test/fixtures/non-json-child.ts @@ -0,0 +1,14 @@ +import { createFoundationSimulationServer } from "@simulacrum/foundation-simulator"; + +import type { FoundationSimulator } from "@simulacrum/foundation-simulator"; + +export function simulation(): FoundationSimulator { + // print some non-JSON diagnostic before the line the parent will JSON-parse + console.log("preflight: starting up"); + return createFoundationSimulationServer({ + port: 0, + extendRouter(router) { + router.get("/info", (_req, res) => res.json({ ok: true })); + }, + })(); +} diff --git a/packages/server/test/replay-signal.test.ts b/packages/server/test/replay-signal.test.ts new file mode 100644 index 00000000..cc795501 --- /dev/null +++ b/packages/server/test/replay-signal.test.ts @@ -0,0 +1,38 @@ +import { it } from "node:test"; +import assert from "node:assert"; +import { run } from "effection"; +import { createReplaySignal } from "../src/createReplaySignal.ts"; + +it("replays queued items to new subscribers and supports close", async () => { + await run(function* () { + const sig = createReplaySignal(); + + // send some items before subscribing + sig.send("a"); + sig.send("b"); + + // subscribe + const sub = yield* sig; + + // expect queued items in order + const first = yield* sub.next(); + assert.strictEqual(first.done, false); + assert.strictEqual(first.value, "a"); + + const second = yield* sub.next(); + assert.strictEqual(second.done, false); + assert.strictEqual(second.value, "b"); + + // send a new item and see it + sig.send("c"); + const third = yield* sub.next(); + assert.strictEqual(third.done, false); + assert.strictEqual(third.value, "c"); + + // close the signal with a value + sig.close("fin"); + const closed = yield* sub.next(); + assert.strictEqual(closed.done, true); + assert.strictEqual(closed.value, "fin"); + }); +}); diff --git a/packages/server/test/service.test.ts b/packages/server/test/service.test.ts index 78477df1..3e7f5774 100644 --- a/packages/server/test/service.test.ts +++ b/packages/server/test/service.test.ts @@ -7,13 +7,26 @@ import { each, Err, Ok, run } from "effection"; // const scriptDoesNotWork = "npm run test:service-main"; const nodeScriptWorks = "node --import tsx ./test/services/service-main.ts"; -it("test service", async () => { - let assertionCount = 0; +it("test service starts and prints expected startup message", async () => { + let sawStart = false; await run(function* () { - yield* useService("test-service", nodeScriptWorks); - assertionCount++; + yield* useService("test-service", nodeScriptWorks, { + wellnessCheck: { + timeout: 1000, + *operation(stdio) { + for (let line of yield* each(stdio)) { + if (line.includes("test service started")) { + return Ok(void 0); + } + yield* each.next(); + } + return Err(new Error("did not see startup message")); + }, + }, + }); + sawStart = true; }); - assert(assertionCount > 0); + assert.ok(sawStart, "service should have started and passed wellness check"); }); describe("useService with wellness check", () => { diff --git a/packages/server/test/services.test.ts b/packages/server/test/services.test.ts index c2da4b60..f625f9da 100644 --- a/packages/server/test/services.test.ts +++ b/packages/server/test/services.test.ts @@ -9,7 +9,7 @@ it("starts services in dependency order", async () => { try { await run(function* () { yield* spawn(function* () { - const run = useServiceGraph({ + const graph = useServiceGraph({ A: { operation: resource(function* (provide) { yield* sleep(20); @@ -26,7 +26,7 @@ it("starts services in dependency order", async () => { dependsOn: { startup: ["A"] as const }, }, }); - yield* run(); + yield* graph(); // keep spawned graph alive yield* suspend(); }); diff --git a/packages/server/test/signal.test.ts b/packages/server/test/signal.test.ts index 48bf5fe8..0f8711c5 100644 --- a/packages/server/test/signal.test.ts +++ b/packages/server/test/signal.test.ts @@ -3,65 +3,114 @@ import assert from "node:assert"; import { spawn as spawnChild } from "node:child_process"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { run, sleep, until, spawn, on, each } from "effection"; +import { timebox } from "@effectionx/timebox"; +import { emitterToEventTarget } from "./utils.ts"; it("example process shuts down cleanly on SIGINT", async () => { - const exe = process.execPath; - const script = fileURLToPath( - new URL("../example/simulation-graph.ts", import.meta.url) - ); - const cwd = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + await run(function* () { + const exe = process.execPath; + const script = fileURLToPath( + new URL("../example/simulation-graph.ts", import.meta.url) + ); + const cwd = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + ".." + ); - const child = spawnChild(exe, ["--import", "tsx", script], { - cwd, - env: { ...process.env }, - stdio: ["ignore", "pipe", "pipe"], - }); + const child = spawnChild(exe, ["--import", "tsx", script], { + cwd, + env: { ...process.env }, + stdio: ["ignore", "pipe", "pipe"], + }); - let stdout = ""; - let stderr = ""; - child.stdout?.on("data", (d) => (stdout += String(d))); - child.stderr?.on("data", (d) => (stderr += String(d))); + let stdout = ""; + let stderr = ""; - // wait for a startup marker - await new Promise((resolve, reject) => { - const to = setTimeout(() => reject(new Error("start timeout")), 3000); - child.stdout?.on("data", function ondata(d) { - const s = String(d); - // the service runner logs when services start — accept either the old - // marker or the current log message used by the service graph implementation - if ( - s.includes("runner: starting layers") || - s.includes("service graph: starting service") - ) { - clearTimeout(to); - child.stdout?.off("data", ondata); - resolve(); + // use Effection `on()` + `each()` by adapting Node emitter to EventTarget + const outTarget = emitterToEventTarget(child.stdout!); + const errTarget = emitterToEventTarget(child.stderr!); + + // spawn background tasks to accumulate stdout and stderr + yield* spawn(function* () { + for (let chunk of yield* each(on(outTarget, "data"))) { + stdout += String(chunk); + yield* each.next(); } }); - }); - // send SIGINT - process.kill(child.pid!, "SIGINT"); + yield* spawn(function* () { + for (let chunk of yield* each(on(errTarget, "data"))) { + stderr += String(chunk); + yield* each.next(); + } + }); - const { code, signal } = await new Promise<{ - code: number | null; - signal: NodeJS.Signals | null; - }>((resolve) => { - child.on("exit", (code, signal) => resolve({ code, signal })); - }); + try { + // wait for a startup marker using the stdout Stream with a timebox + const started = yield* timebox(3000, function* () { + for (let chunk of yield* each(on(outTarget, "data"))) { + const s = String(chunk); + if ( + s.includes("runner: starting layers") || + s.includes("service graph: starting service") + ) { + return { started: true }; + } + yield* each.next(); + } + return undefined; + }); - // allow stderr to flush - await new Promise((r) => setTimeout(r, 50)); + if (started && started.timeout) + throw new Error("startup marker not seen"); - // expect no stack traces on stderr and process exited due to SIGINT - assert.strictEqual(typeof stderr, "string"); - assert( - !/uncaughtException|UnhandledPromiseRejection|Error/.test(stderr), - `stderr contained error: ${stderr}` - ); - // Accept either signal SIGINT or code 0 or code 130 (standard SIGINT exit code) - assert( - signal === "SIGINT" || code === 0 || code === 130, - `unexpected exit: code=${code} signal=${signal}` - ); + // send SIGINT + process.kill(child.pid!, "SIGINT"); + + // wait for the child to exit (timeboxed) + const exitRes = yield* timebox(3000, function* () { + const p = new Promise<{ + code: number | null; + signal: NodeJS.Signals | null; + }>((resolve) => { + child.on("exit", (code, signal) => resolve({ code, signal })); + }); + return yield* until(p); + }); + + if (exitRes && exitRes.timeout) + throw new Error("child did not exit in time"); + + // timebox may return an object with `.value` or the value directly — handle both + let code: number | null = null; + let signal: NodeJS.Signals | null = null; + const maybe = exitRes as any; + const val = maybe && "value" in maybe ? maybe.value : maybe; + if (val && typeof val === "object") { + code = val.code; + signal = val.signal; + } + + // allow stderr to flush a little + yield* sleep(50); + + // expect no stack traces on stderr and process exited due to SIGINT + assert.strictEqual(typeof stderr, "string"); + assert( + !/uncaughtException|UnhandledPromiseRejection|Error/.test(stderr), + `stderr contained error: ${stderr}` + ); + // Accept either signal SIGINT or code 0 or code 130 (standard SIGINT exit code) + assert( + signal === "SIGINT" || code === 0 || code === 130, + `unexpected exit: code=${code} signal=${signal}` + ); + } finally { + // ensure process is killed + try { + child.kill("SIGKILL"); + } catch (_) {} + } + }); }); diff --git a/packages/server/test/simulation.test.ts b/packages/server/test/simulation.test.ts index 35c7189e..7c863fe8 100644 --- a/packages/server/test/simulation.test.ts +++ b/packages/server/test/simulation.test.ts @@ -1,9 +1,10 @@ import { it } from "node:test"; import assert from "node:assert"; -import { run, createScope, suspend, until, sleep } from "effection"; +import { run, createScope, suspend, until } from "effection"; import { useSimulation } from "../src/simulation.ts"; import { simulation } from "./fixtures/simple-sim.ts"; import { createFoundationSimulationServer } from "@simulacrum/foundation-simulator"; +import { waitFor, waitForFetchClosed } from "./utils.ts"; it("useSimulation returns listening info", async () => { const port = await run(function* () { @@ -20,6 +21,7 @@ it("simulation closes when scope is destroyed", async () => { let port: number | undefined; // start the simulation in the scope and keep it alive until destroy() + // where we can test it actually shutdown scope.run(function* () { const listening = yield* useSimulation( "inline-test", @@ -35,10 +37,7 @@ it("simulation closes when scope is destroyed", async () => { }); // wait for the scope-run to set the port - for (let i = 0; i < 100; i++) { - if (typeof port === "number") break; - yield* sleep(5); - } + yield* waitFor(() => typeof port === "number", 2000); const status = yield* until( fetch(new URL(`http://127.0.0.1:${port}/info`)) @@ -51,18 +50,6 @@ it("simulation closes when scope is destroyed", async () => { yield* until(destroy()); // server should no longer accept connections - let closed = false; - for (let i = 0; i < 50; i++) { - try { - yield* until(fetch(new URL(`http://127.0.0.1:${port}/info`))); - // if request succeeded, wait and retry - } catch (e) { - closed = true; - break; - } - yield* sleep(10); - } - - if (!closed) throw new Error("simulation still responds after destroy"); + yield* waitForFetchClosed(`http://127.0.0.1:${port}/info`, 2000); }); }); diff --git a/packages/server/test/utils.ts b/packages/server/test/utils.ts new file mode 100644 index 00000000..b96168c4 --- /dev/null +++ b/packages/server/test/utils.ts @@ -0,0 +1,84 @@ +import { timebox } from "@effectionx/timebox"; +import { sleep, until } from "effection"; + +/** + * Wait for `predicate` to become true with a timeboxed timeout. + * Throws on timeout. + */ +export function* waitFor( + predicate: () => boolean, + timeout = 2000 +): Generator { + const res = yield* timebox(timeout, function* () { + while (!predicate()) { + yield* sleep(10); + } + }); + + if (res && (res as any).timeout) { + throw new Error("timed out waiting for condition"); + } +} + +/** + * Cast a Node EventEmitter (e.g., `child.stdout`) to an EventTarget-like + * object with `addEventListener`/`removeEventListener`. This is useful for + * using `on()` with Node APIs that emit events. + */ +export function emitterToEventTarget(emitter: NodeJS.EventEmitter) { + return { + addEventListener(name: string, listener: (...args: any[]) => void) { + // Node's event listeners receive chunks or event args; keep signature loose + emitter.on(name as any, listener as any); + }, + removeEventListener(name: string, listener: (...args: any[]) => void) { + emitter.off(name as any, listener as any); + }, + } as EventTarget; +} + +/** + * Wait for an async predicate (returns Promise) to become true. + */ +export function* waitForAsync( + predicate: () => Promise, + timeout = 2000 +): Generator { + const res = yield* timebox(timeout, function* () { + while (true) { + try { + const ok = yield* until(predicate()); + if (ok) return; + } catch (_) { + // ignore and retry + } + yield* sleep(10); + } + }); + + if (res && (res as any).timeout) { + throw new Error("timed out waiting for async condition"); + } +} + +/** + * Wait until fetching the given url fails (connection refused or other error) + * which is commonly used to detect a server shutting down. + */ +export function* waitForFetchClosed(url: string, timeout = 2000) { + const res = yield* timebox(timeout, function* () { + while (true) { + try { + const s = yield* until(fetch(url)); + if (!s.ok) return; + } catch (_) { + return; + } + yield* sleep(10); + } + }); + + if (res && (res as any).timeout) { + throw new Error("timed out waiting for fetch to fail"); + } +} diff --git a/packages/server/test/watch.test.ts b/packages/server/test/watch.test.ts index fc60a6f6..71969d98 100644 --- a/packages/server/test/watch.test.ts +++ b/packages/server/test/watch.test.ts @@ -7,6 +7,8 @@ import os from "node:os"; import { useServiceGraph } from "../src/services.ts"; import { simulation } from "./fixtures/simple-sim.ts"; import { useSimulation } from "../src/simulation.ts"; +import { createFoundationSimulationServer } from "@simulacrum/foundation-simulator"; +import { waitFor, waitForAsync } from "./utils.ts"; it("restarts services on watched file change and restarts dependents", async () => { const prefix = path.join(os.tmpdir(), "sim-watch-"); @@ -56,15 +58,15 @@ it("restarts services on watched file change and restarts dependents", async () yield* suspend(); }); - // allow initial startup and wait for bOut to appear - for (let i = 0; i < 200; i++) { + // ensure initial trigger is readable + yield* waitForAsync(async () => { try { - yield* until(fs.readFile(trigger, "utf8")); - break; - } catch (err) { - yield* sleep(20); + await fs.readFile(trigger, "utf8"); + return true; + } catch (_) { + return false; } - } + }, 2000); // give the spawned subscription a moment to attach yield* sleep(50); @@ -121,19 +123,13 @@ it("restarts dependents when watched service changes", async () => { }); // wait for initial startup - for (let i = 0; i < 200; i++) { - if (startCounts.a > 0 && startCounts.b > 0) break; - yield* sleep(10); - } + yield* waitFor(() => startCounts.a > 0 && startCounts.b > 0, 2000); // trigger a change yield* until(fs.writeFile(trigger, "changed")); // wait for restarts to occur - for (let i = 0; i < 200; i++) { - if (startCounts.a >= 2 && startCounts.b >= 2) break; - yield* sleep(10); - } + yield* waitFor(() => startCounts.a >= 2 && startCounts.b >= 2, 3000); }); await fs.rm(dir, { recursive: true, force: true }); @@ -189,19 +185,19 @@ it("restarts transitive dependents when watched service changes", async () => { }); // wait for initial startup - for (let i = 0; i < 200; i++) { - if (startCounts.a > 0 && startCounts.b > 0 && startCounts.c > 0) break; - yield* sleep(10); - } + yield* waitFor( + () => startCounts.a > 0 && startCounts.b > 0 && startCounts.c > 0, + 2000 + ); // trigger a change yield* until(fs.writeFile(trigger, "changed")); // wait for restarts to occur - for (let i = 0; i < 200; i++) { - if (startCounts.a >= 2 && startCounts.b >= 2 && startCounts.c >= 2) break; - yield* sleep(10); - } + yield* waitFor( + () => startCounts.a >= 2 && startCounts.b >= 2 && startCounts.c >= 2, + 3000 + ); }); await fs.rm(dir, { recursive: true, force: true }); @@ -211,6 +207,53 @@ it("restarts transitive dependents when watched service changes", async () => { assert(startCounts.c >= 2, "c should have been restarted as dependent of b"); }); +it("updates servicePorts when a service restarts", async () => { + const prefix = path.join(os.tmpdir(), "sim-port-rt-"); + const dir = await fs.mkdtemp(prefix); + const trigger = path.join(dir, "trigger.txt"); + await fs.writeFile(trigger, "initial"); + + await run(function* () { + const op = useServiceGraph( + { + s: { + watch: [dir], + operation: useSimulation( + "s", + createFoundationSimulationServer({ port: 0 }) + ), + }, + }, + { watch: true, watchDebounce: 20 } + ); + + const services = yield* op(); + + // wait for initial port to appear + yield* waitFor( + () => typeof services.servicePorts?.get("s") === "number", + 2000 + ); + const initial = services.servicePorts!.get("s")!; + + // trigger restart by touching the file + yield* until(fs.writeFile(trigger, "changed")); + + // wait for new port value to be different from initial + yield* waitFor(() => { + const p = services.servicePorts?.get("s"); + return typeof p === "number" && p !== initial; + }, 3000); + const updated = services.servicePorts!.get("s")!; + + assert.ok(typeof initial === "number", "initial port should be present"); + assert.ok(typeof updated === "number", "updated port should be present"); + assert.notStrictEqual(initial, updated, "port should change after restart"); + }); + + await fs.rm(dir, { recursive: true, force: true }); +}); + it("debounces rapid changes per service", async () => { const prefix = path.join(os.tmpdir(), "sim-watch-debounce-"); const dir = await fs.mkdtemp(prefix); From eafa7cdc51e3bfa742eeec2e8bdeb1ed59c6cdca Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Thu, 15 Jan 2026 17:17:26 -0600 Subject: [PATCH 19/38] readme getting started --- packages/server/README.md | 78 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 6 deletions(-) diff --git a/packages/server/README.md b/packages/server/README.md index a0a97ae6..0fc5f24f 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -4,8 +4,78 @@ Server capable of running multiple concurrent simulations that can be controlled https://github.com/thefrontside/simulacrum -> [!WARNING] -> The server is undergoing a refactor, and this may not be required for your use case. The refactor includes allow for more simply running single simulators so this package will be primarily useful as a control plane for cases where there are many simulators under test and in use. For the previous iterations, see the `v0` branch which contain the previous functionality. +## Getting Started + +Set up a process or simulators such as this example `service-graph.ts`. + +```ts service-graph.ts +#!/usr/bin/env node +import { run } from "effection"; +import { + useServiceGraph, + simulationCLI, + useChildSimulation, + useSimulation, + useService, +} from "@simulacrum/server"; +import { simulation } from "./sim2.ts"; + +// define your "graph" that can be used through a CLI or as part of a test rig +export const services = useServiceGraph( + { + sim1: { + operation: useChildSimulation("sim-run-as-child-process", "./sim1.ts"), + }, + sim2: { + operation: useSimulation("sim-run-in-same-process", simulation), + }, + sim3: { + operation: useService( + "arbitray-child-process", + "node --import tsx ./sim3.ts" + ), + }, + }, + { globalData: { hello: "world" } } +); + +// this is a helper function which will give you a CLI around this service graph +// if you are calling this file directly +import { fileURLToPath } from "node:url"; +if (process.argv[1] === fileURLToPath(import.meta.url)) { + simulationCLI(services); +} +``` + +From this, you have two main entry points. One may start it directly from your shell. + +```bash +# start a local service graph defined in ./service-graph.ts +node --import tsx ./simulators/service-graph.ts +``` + +> [!NOTE] +> We use `--import tsx` here to automatically handle the typescript conversion. This is a separate package that you may be interested in using, but it not a hard requirement necessarily. + +Secondly, we can use this in tests. It is convenient to place in an `beforeAll()` or a `beforeEach()`. This is built on `effection`, and should handle all shutdown and clean up of the services when the function passes out of lexical scope. + +```ts +import { run } from "effection"; +import { services } from "./simulators/service-graph.ts"; + +beforeEach(async () => { + await run(function* () { + const services = yield* services(); + // or optionally pass a subset of services to run if not all are required for this test + const subsetOfServices = yield* services(["sim1"]); + yield* suspend(); + }); +}); + +test("things", async () => { + // do testing things here +}); +``` ## Operation-based service orchestration @@ -142,7 +212,3 @@ node --import tsx ./example/concurrency-layers.ts Previously services could expose their return value via a public `exportsOperation` that consumers could await. That mechanism has been removed in this branch as we move to a child-process-focused runner model. Provider-returned values are still delivered to dependent service factories internally, but no longer exposed as an operation on the public `services` map. For convenience tests may use the `servicePorts` map exposed by the running graph to discover HTTP ports that services registered when they start. The `servicePorts` map is available on the object returned by the runner and contains service name => port when a service's `operation` returns an object with a `{ port: number }` property. - -``` - -``` From 927afeb4924cab6a0d935047712255878bb855e7 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Fri, 16 Jan 2026 11:05:10 -0600 Subject: [PATCH 20/38] readme updates --- packages/server/README.md | 220 ++++++++++++++++++++++---------------- 1 file changed, 125 insertions(+), 95 deletions(-) diff --git a/packages/server/README.md b/packages/server/README.md index 0fc5f24f..6f765a43 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -60,6 +60,7 @@ node --import tsx ./simulators/service-graph.ts Secondly, we can use this in tests. It is convenient to place in an `beforeAll()` or a `beforeEach()`. This is built on `effection`, and should handle all shutdown and clean up of the services when the function passes out of lexical scope. ```ts +import { test, beforeEach } from "test-runner"; import { run } from "effection"; import { services } from "./simulators/service-graph.ts"; @@ -79,136 +80,165 @@ test("things", async () => { ## Operation-based service orchestration -`@simulacrum/server` provides operations to start and manage services with lifecycle hooks. The recommended pattern is to create `Operation` instances for each service (typically via `useService`) and pass them to `useServiceGraph` which starts the services respecting a dependency DAG and provides lifecycle hooks for startup and shutdown. +`@simulacrum/server` provides operations to start and manage services with lifecycle hooks. The recommended pattern is to create `Operation` instances for each service (typically via `useService`, `useSimulation`, or `useChildSimulation`) and pass them to `useServiceGraph` which starts the services respecting a dependency DAG and provides lifecycle hooks for startup and shutdown. -Key points: +See the `@simulacrum/foundation-simulator` for a basis to build simulator(s) for your services. -- `useServiceGraph(services: ServicesMap, options?: { globalData?: Record; watch?: boolean; watchDebounce?: number }): ServiceRunner` — returns a _runner function_ which you call to start the graph: `const run = useServiceGraph(services, options); const provided = yield* run(subset?: string[] | string);`. By default services in the same topological layer run concurrently; pass `options.watch = true` and `options.watchDebounce` to enable file watching and restart propagation. -- `ServiceDefinition.operation` (required) — an `Operation` which indicates the service has started. This operation may be long-lived (e.g. `useService`) or may return once the service is ready while a background child keeps the service running. See the example below. -- `dependsOn` — an optional object `{ startup: string[]; restart?: string[] }` listing service names this service depends on. Use `startup` to list services that must start before this service; use `restart` to list services that should trigger a restart of this service when they are restarted (for example, due to a watched file change). Services without dependencies in the same layer are started concurrently by default, or serially when `options.sequential` is true. +## API reference -- `subset` (runner argument) — when calling the runner returned by `useServiceGraph` you may pass a subset (e.g. `yield* run(['serviceA'])` or `yield* run('serviceA')`) to start only a subset of services; any startup dependencies required by that subset are automatically included. +### useServiceGraph(services, options?) -- Watching & restart propagation — pass `{ watch: true }` to `useServiceGraph` and define `watch` paths in each `ServiceDefinition` to enable file watching. The watcher will precompute transitive dependents (based on `dependsOn.restart`) and automatically emit restart updates for dependents when a watched path changes, so restarts propagate efficiently and deterministically. +`useServiceGraph(services: ServicesMap, options?: { globalData?: Record; watch?: boolean; watchDebounce?: number }): ServiceRunner` -### Global data & the simulacrum gateway 🔁 +Returns a "runner" function. Call the runner inside an Effection scope to start the graph: -The graph can optionally start a small local HTTP data service (the _simulacrum gateway_) to expose a `globalData` object to child simulations and tests. The gateway registers its listening port on the returned `servicePorts` map under the key `"simulacrum"`, which tests and examples can use to discover it. See the `test/child-simulation-simulacrum.test.ts` and `example/simulation-graph.ts` for examples and coverage of this flow. +```ts +const run = useServiceGraph(services, options); +const services = yield * run(subset); // holds while services run, subset is optional +``` -### Lifecycle hooks +File watching: pass `options.watch = true` and `options.watchDebounce` to enable watching and restart propagation across dependents. This is enabled through the CLI helper. -- Lifecycle hooks can be implemented by arranging operations and using try/finally within your service `operation` to perform startup and cleanup logic. You can keep the operation alive with `yield* suspend()` and perform cleanup in the `finally` block when the service is stopped. +#### ServiceDefinition -Example: +The `ServicesMap` passed as the first argument to `useServiceGraph`. ```ts -import { main, spawn, sleep } from "effection"; -import { useServiceGraph, useService } from "@simulacrum/server"; - -main(function* () { - yield* spawn(function* () { - // In many situations, pass `useService` directly: it returns once the - // process is spawned and, if a wellnessCheck is provided, once the - // wellnessCheck passes. The service is automatically shut down by - // effection when the operation goes out of scope. - const run = useServiceGraph({ - A: { - operation: useService( - "A", - "node --import tsx ./test/services/service-a.ts" - ), - }, - B: { - operation: useService( - "B", - "node --import tsx ./test/services/service-b.ts" - ), - dependsOn: { startup: ["A"] }, - }, - }); - }); -}); +const services: ServicesMap = { + serviceKey: { + operation, + dependsOn, + watch, + }, +}; ``` -Notes: +##### `operation` + +- Each service must provide an `operation: Operation` which signals that the service has started. +- The operation needs to be long-lived or return once a child process is started while it keeps the service running in the background (e.g., `useService` or `useChildSimulation`). +- If you are defining your own customer operation, use `try { ... yield* suspend(); } finally { ... }` inside an `operation` to run cleanup logic when the service stops. Using `resource()` from `effection` allows the service to stay in scope and continue running. See the `effection` documentation or the helper functions in this library for more information and examples. + +##### `dependsOn` + +Type: `{ startup?: string[]; restart?: string[] }` + +- `startup` lists services that must start before this one. +- `restart` lists services whose restart should trigger a restart of this service (useful when using the watcher). + +##### `watch` Watching & restart propagation -- `useServiceGraph` returns a _runner function_; calling the runner (e.g. `yield* run()`) returns an `Operation` that holds while services run and only cancels on parent scope termination. The returned runner has a `.services` property for introspection and can be passed directly to `simulationCLI`. -- If you want to start services sequentially or add more advanced concurrency control, compose operations yourself and use `spawn` to control how operations run. +To enable file-watching: pass `{ watch: true }` to `useServiceGraph` options (second argument) and add `watch` paths to `ServiceDefinition` objects. The watcher computes transitive dependents (using `dependsOn.restart`) and emits restart updates so restarts propagate deterministically. -### Lifecycle hooks +### ServiceRunner & returned values -Each `ServiceDefinition` supports lifecycle hook operations. These hooks run in the parent scope and are useful for performing orchestration tasks, logging, or writing sentinel files for integration tests. Hooks are `Operation` as well. +The runner returned by `useServiceGraph` is itself an operation. This allows it to be portable. Define it in one spot, then import it into any CLI, start scripts or test runners of your choosing at start it there. Optionally, it takes an argument, `subset`, to only start part of the graph. + +##### `subset` + +When calling the runner you may pass a subset (e.g., `yield* runner(['serviceA'])` or `yield* runner('serviceA')`, the latter being a comma separated list) to start only a subset of services. Any required startup dependencies are included automatically. This is particularly use when focusing on a specific feature / feedback loop, such as in a test. Only start the services you _actually_ need. + +##### returns + +The "runner" returns and exposes a `services` object when executed. The started graph exposes: + +- `servicePorts` — a Map of service name => listening port when a service returns `{ port: number }` from its operation. This is convenient for tests to discover HTTP endpoints. Note that these are only filled in if the `operation` supports this functionality. The `useChildSimulation` and `useSimulation` both support it. +- `services` - the object initially passed, useful for debugging +- `serviceUpdates` and `serviceChanges` - both a `Stream` (see `effection`) of updates from the watcher, useful for debugging + +### Simulation & process helpers 🔧 + +This package provides a few helpers to run simulations and external processes in common patterns: + +#### useSimulation(name, factory) + +`useSimulation(name: string, createFactory: (initData?: unknown) => FoundationSimulator)` + +Run a simulator _in-process_ via a factory that returns a `FoundationSimulator` (or a Promise resolving to one). Useful when you want the simulator instance in the same Node process as the runner. This API _will_ allow watching and restarts, but these restarts will not pick up changes in your code, see `useChildSimulation`. + +- If `globalData` is set on the runner, `useSimulation` will fetch it from the simulacrum gateway and pass it as the `initData` argument to your factory. +- The factory should return a `FoundationSimulator` (see below). `useSimulation` calls `await simulator.listen()` to obtain `{ port }` and registers that port on `servicePorts`. + +Example: ```ts -const services = { - A: { - operation: (function* () { - // start the service via useService or useChildSimulation - yield* useService("A", "node --import tsx ./test/services/service-a.ts"); - // signal that the service is ready - console.log("A has started"); - try { - // keep running until cancelled - yield* suspend(); - } finally { - // cleanup runs automatically on scope cancellation - console.log("A is stopping"); - } - })(), - }, -}; +// in a service definition +operation: useSimulation("app", (initData) => { + // do something with initData and/or pass it to your simulator through the closure + return createFoundationSimulationServer({ port: 0 }); +}); ``` -Notes: +#### useChildSimulation(name, modulePath) -- Use a try/finally in your `operation` to run cleanup logic when the service is stopped -- This approach leverages Effection scopes and ensures cleanup runs in reverse dependency order when the graph is shut down -- Use `useService` or `useChildSimulation` inside your operation as needed to start underlying processes +`useChildSimulation(name: string, modulePath: string)` -Try it +Run a simulator in a fresh child Node process (isolates module cache and supports restarts). Otherwise this feels the same as using `useSimulation`. -```bash -# Run the server package tests -cd packages/server -npm test +- The child is started using a wrapper, `./bin/run-simulation-child.ts `, and, when present, the `--simulacrum-port` is passed so the child can fetch `globalData`. +- The wrapper prints a JSON line to stdout like `{ "ready": true, "port": 12345 }` as its first ready signal. `useChildSimulation` reads that line to discover the port and registers it on `servicePorts`. +- Non-JSON stdout lines are forwarded to logs; if the child exits before emitting the ready JSON, `useChildSimulation` rejects. +- If using this with a simulator created from `@simulacrum/foundation-simulator`, all this wiring will be handled for you. + +Example: + +```ts +operation: useChildSimulation( + "service-key-for-logs", + "./simulator/my-simulator.js" +); ``` -## Examples +> [!WARNING] +> This does rely on having `tsx` installed which will handle the TypeScript types when running. It will allow for a simulator defined through a `.js` file or a `.ts`, so your choosing. -The `example` folder contains runnable examples demonstrating `useServiceGraph` and `useService`. +#### About `@simulacrum/foundation-simulator` -Run the simulation-based example (starts simulators via child simulations): +- A `FoundationSimulator` is a small helper that provides two key primitives you should expect from your factory: + - `simulator.listen(): Promise<{ port: number }>` — starts the server and resolves when it is listening (the object is registered in `servicePorts`). + - `simulator.ensureClose(): Promise` — used by the runner to cleanly shut down the simulator when its containing scope is cancelled. +- Use `createFoundationSimulationServer()` to create a server that listens on an ephemeral port and returns an object compatible with `useSimulation` and `useChildSimulation`. -```bash -cd packages/server -npm run example:sim -``` +#### useService(name, cmd, options?) -Run the process-based example (spawns processes via `useService`): +Spawn an external process (via the configured command) and optionally run a wellness check. `useService` forwards stdout/stderr to the package logging and keeps the operation alive until it goes out of scope. -```bash -cd packages/server -npm run example:process -``` +- `options`: + - `wellnessCheck.operation(stdio)` — an operation, `Operation<>` that needs to return a `Result` (both from `effection`) to consider the service successfully started. It is passed the stdio from the process. You may use any `effection` semantics, and inspect the stdio or http calls, etc, to decide when your service is "ready". + - `wellnessCheck.timeout` and `wellnessCheck.frequency` can be provided to control checking behavior, most useful in repeatedly `fetch`ing a `/status` or `/healthcheck` response. -Run the concurrency example: +#### simulationCLI(serviceGraph) -```bash -cd packages/server -npm run example:concurrency -``` +- `simulationCLI` wraps the runner in a small CLI loop and provides convenience flags: `--services`, `--watch`, and `--watch-debounce`. +- Use the CLI helper for local development workflows where you want to run your graph directly from a file (see `service-graph.ts` examples above). -Run examples directly (each example module can be executed with `tsx`): +## Global data & the simulacrum gateway 🔁 -```bash -cd packages/server -node --import tsx ./example/simulation-graph.ts -node --import tsx ./example/process-graph.ts -node --import tsx ./example/concurrency-layers.ts +When you call `useServiceGraph(...)` you may pass an optional `globalData` object in the options. The runner starts a tiny local HTTP data service (the **simulacrum gateway**) that serves that object so tests and child simulations can discover configuration or shared fixtures. + +- Endpoints: `GET /data` (returns the full `globalData` JSON) and `GET /data/` (returns a single key, or a 404/400 as appropriate). +- Discovery: the gateway registers its listening port on the runner's `servicePorts` map under the key `"simulacrum"`. You can read the port from your test or harness with `const port = services.servicePorts!.get("simulacrum");` and then `fetch` `http://127.0.0.1:${port}/data`. +- Service integration: when starting child simulations via `useChildSimulation` / `simulationCLI` we pass the gateway port (if present) to the child. The child will fetch `/data` on startup and receive the `globalData` object. The simulator function you define may expect to receive that global object as the first argument to the function. Useful for passing "world-level" data to all of your simulators. + +```ts +const runner = useServiceGraph( + { + child: { operation: useChildSimulation("child", "./child-main.ts") }, + }, + { globalData: { featureFlag: true } } +); + +const services = yield * runner(); +const simulacrumPort = services.servicePorts!.get("simulacrum"); +// fetch global data in a test or helper +const res = await fetch(`http://127.0.0.1:${simulacrumPort}/data`); +const data = await res.json(); ``` -### Sharing exported values between services (note) +Notes: + +The gateway is intended for local development and tests only (it is not a production data layer). Future work around this layer may include improved logging and observability. Conceptually, it provides an "orchestration status" service. -Previously services could expose their return value via a public `exportsOperation` that consumers could await. That mechanism has been removed in this branch as we move to a child-process-focused runner model. Provider-returned values are still delivered to dependent service factories internally, but no longer exposed as an operation on the public `services` map. +## Development -For convenience tests may use the `servicePorts` map exposed by the running graph to discover HTTP ports that services registered when they start. The `servicePorts` map is available on the object returned by the runner and contains service name => port when a service's `operation` returns an object with a `{ port: number }` property. +The `example` folder contains runnable examples demonstrating `useServiceGraph`. The `test` folder includes tests based on the Node test runner which pull from the `example` folder or create their own fixtures to test the APIs. From 43ffdd110845816a40106b2a3de194146f6c6636 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Sun, 22 Mar 2026 23:25:21 -0500 Subject: [PATCH 21/38] bug fixes, logging and debugging DX --- package-lock.json | 144 ++++++----- package.json | 4 +- packages/server/README.md | 10 +- packages/server/example/concurrency-layers.ts | 16 +- packages/server/example/process-graph.ts | 8 +- .../example/services/gen-sim-factory.ts | 3 +- packages/server/package.json | 10 +- packages/server/src/cli.ts | 35 ++- packages/server/src/data-service.ts | 14 +- packages/server/src/logging.ts | 49 +++- packages/server/src/service.ts | 30 ++- packages/server/src/services.ts | 224 ++++++++++++------ packages/server/src/simulation.ts | 105 ++++---- packages/server/src/watch.ts | 41 ++-- packages/server/test/child-simulation.test.ts | 51 ++-- packages/server/test/data-service.test.ts | 31 +-- packages/server/test/examples-smoke.test.ts | 16 +- packages/server/test/services.test.ts | 24 +- packages/server/test/signal.test.ts | 116 --------- packages/server/test/watch.test.ts | 70 ++++-- 20 files changed, 570 insertions(+), 431 deletions(-) delete mode 100644 packages/server/test/signal.test.ts diff --git a/package-lock.json b/package-lock.json index 0209e7f6..a964fdba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ ], "devDependencies": { "@arethetypeswrong/core": "^0.18.2", - "effection": "^4.0.0", + "effection": "^4.0.2", "publint": "^0.3.16", "tsdown": "^0.18.4", "tsx": "^4.21.0", @@ -371,76 +371,102 @@ "license": "ISC" }, "node_modules/@effectionx/context-api": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/@effectionx/context-api/-/context-api-0.2.1.tgz", - "integrity": "sha512-UCEmk/uibrx4PvUh/tm1SWy6e6rAaq2BzpEDLu5XJZ/LaRdazv5VxHqg4F3N+G4AcxgPvosprjS0j3dDSTCzFg==", + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@effectionx/context-api/-/context-api-0.3.2.tgz", + "integrity": "sha512-/x4If4tiiTrg9pcJr7Jrs3z5teorzWc+qLCWBTnD3+8YpJy+8xUQXA8d/0/UkvTgNAzYSg297nomB7YiPiwuKA==", "license": "MIT", - "dependencies": { - "effection": "^3 || ^4.0.0-0" + "engines": { + "node": ">= 22" }, + "peerDependencies": { + "effection": "^3 || ^4" + } + }, + "node_modules/@effectionx/node": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@effectionx/node/-/node-0.2.2.tgz", + "integrity": "sha512-bUwnCqzBsVERGzKZRTc6XMZ6yDLkRPgcxSvM6eAkuc2D5A7L0+nSu4J/x60y6geSYrAVv2UZBgAktukNtB6LxA==", + "license": "MIT", "engines": { - "node": ">= 16" + "node": ">= 22" + }, + "peerDependencies": { + "effection": "^3 || ^4" } }, "node_modules/@effectionx/process": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/@effectionx/process/-/process-0.6.2.tgz", - "integrity": "sha512-U94gqTNXASRw8KBJOtSE+MaWL09Tox7la9/rmJCzUpaLWSmrSOvH28NCv++PKKy8qNCErD+QQip5q+E8lARNEQ==", + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/@effectionx/process/-/process-0.7.3.tgz", + "integrity": "sha512-ic+cv9aNeq0UgF+SO61jd8nYLiF3jFT2c6n+nOYstc4pmtUez70yFb0H5Rzppu21wql5QYCbQayVrRRUa5YRQQ==", "license": "MIT", "dependencies": { - "@types/cross-spawn": "6.0.6", - "cross-spawn": "7.0.6", - "ctrlc-windows": "2.2.0", - "effection": "^3 || ^4.0.0-0", - "shellwords": "^1.1.1" + "@effectionx/node": "0.2.2", + "cross-spawn": "^7", + "ctrlc-windows": "^2", + "shellwords-ts": "^3.0.1" }, "engines": { - "node": ">= 16" + "node": ">= 22" + }, + "peerDependencies": { + "effection": "^3 || ^4" } }, "node_modules/@effectionx/signals": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/@effectionx/signals/-/signals-0.4.1.tgz", - "integrity": "sha512-y6oJwpQwqTd2rVPgC2yMQXzQV848MJpRg4zjAL2rIH9znFakPdZN7H3OU4iuY5fnBnS/JaAp43SjHw2hqMnkyA==", + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/@effectionx/signals/-/signals-0.5.2.tgz", + "integrity": "sha512-ftWU0/+LTafPjFQ4mOfwIBHT75BtA2T44YJP7+nR/d8S2kURCzkR04a2NHVbk274XRfh354qpukBULDTchZlQw==", "license": "MIT", "dependencies": { - "effection": "^3 || ^4.0.0-0", "immutable": "^5" }, "engines": { - "node": ">= 16" + "node": ">= 22" + }, + "peerDependencies": { + "effection": "^3 || ^4" } }, "node_modules/@effectionx/signals/node_modules/immutable": { - "version": "5.1.4", - "resolved": "https://registry.npmjs.org/immutable/-/immutable-5.1.4.tgz", - "integrity": "sha512-p6u1bG3YSnINT5RQmx/yRZBpenIl30kVxkTLDyHLIMk0gict704Q9n+thfDI7lTRm9vXdDYutVzXhzcThxTnXA==", + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-5.1.5.tgz", + "integrity": "sha512-t7xcm2siw+hlUM68I+UEOK+z84RzmN59as9DZ7P1l0994DKUWV7UXBMQZVxaoMSRQ+PBZbHCOoBt7a2wxOMt+A==", "license": "MIT" }, "node_modules/@effectionx/stream-helpers": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/@effectionx/stream-helpers/-/stream-helpers-0.5.1.tgz", - "integrity": "sha512-lvXhkLPxVNtnj6XE4c4j/Rpm6VbD+WOVQU/+pL0kAC06INk56owwR6NV6QH/THEdZnz7RdbnFWLECPuXpMMxrA==", + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@effectionx/stream-helpers/-/stream-helpers-0.8.1.tgz", + "integrity": "sha512-17rocf3av2VId8uqZJxBREhQRSaowA7+MKAbu27P3ZboIhpuIu8jqoQ1+YfKWdMy/ORrApeIljgoJl+1hJ3fvQ==", "license": "MIT", "dependencies": { - "@effectionx/signals": "^0.4.0", - "@effectionx/timebox": "^0.3.0", - "effection": "^3 || ^4.0.0-0" + "@effectionx/signals": "0.5.2", + "@effectionx/timebox": "0.4.2", + "immutable": "^5", + "remeda": "^2" }, "engines": { - "node": ">= 16" + "node": ">= 22" + }, + "peerDependencies": { + "effection": "^3 || ^4" } }, + "node_modules/@effectionx/stream-helpers/node_modules/immutable": { + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-5.1.5.tgz", + "integrity": "sha512-t7xcm2siw+hlUM68I+UEOK+z84RzmN59as9DZ7P1l0994DKUWV7UXBMQZVxaoMSRQ+PBZbHCOoBt7a2wxOMt+A==", + "license": "MIT" + }, "node_modules/@effectionx/timebox": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/@effectionx/timebox/-/timebox-0.3.1.tgz", - "integrity": "sha512-Ql5PihR56QNTGjoNMqMIZC8WGDnHN/Yh+glucnRr0WpHMkt3He0soTqU0D9mzwk+2F+0hsrlfOPP8ovK6Nbkdg==", + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/@effectionx/timebox/-/timebox-0.4.2.tgz", + "integrity": "sha512-CT1XCCXmYgxFjIJmz0kzBbDG0W1Jc/i8l4r2Hecu+gYXp22DW8bAkVPKYIDdR4acAC1flqevlWA8xWaNBUfb6w==", "license": "MIT", - "dependencies": { - "effection": "^3 || ^4.0.0-0" - }, "engines": { - "node": ">= 16" + "node": ">= 22" + }, + "peerDependencies": { + "effection": "^3 || ^4" } }, "node_modules/@emnapi/core": { @@ -2716,15 +2742,6 @@ "@types/node": "*" } }, - "node_modules/@types/cross-spawn": { - "version": "6.0.6", - "resolved": "https://registry.npmjs.org/@types/cross-spawn/-/cross-spawn-6.0.6.tgz", - "integrity": "sha512-fXRhhUkG4H3TQk5dBhQ7m/JDdSNHKwR2BBia62lhwEIq9xGiQKLxd6LymNhn47SjXhsUEPmxi+PKw2OkW4LLjA==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/deep-eql": { "version": "4.0.2", "dev": true, @@ -3959,9 +3976,9 @@ "license": "MIT" }, "node_modules/effection": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/effection/-/effection-4.0.0.tgz", - "integrity": "sha512-eW2yqhyBdey4k8lkp7hpiev2FSHvJvQqvaIebI3EGikHZvfUWvNy7SmkwOnJa6WcsUtSh7VHUwdjHTbV++8M9w==", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/effection/-/effection-4.0.2.tgz", + "integrity": "sha512-O8WMGP10nPuJDwbNGILcaCNWS+CvDYjcdsUSD79nWZ+WtUQ8h1MEV7JJwCSZCSeKx8+TdEaZ/8r6qPTR2o/o8w==", "license": "MIT", "engines": { "node": ">= 16" @@ -6090,6 +6107,15 @@ "invariant": "^2.2.4" } }, + "node_modules/remeda": { + "version": "2.33.6", + "resolved": "https://registry.npmjs.org/remeda/-/remeda-2.33.6.tgz", + "integrity": "sha512-tazDGH7s75kUPGBKLvhgBEHMgW+TdDFhjUAMdQj57IoWz6HsGa5D2RX5yDUz6IIqiRRvZiaEHzCzWdTeixc/Kg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/remeda" + } + }, "node_modules/remedial": { "version": "1.0.8", "dev": true, @@ -6475,10 +6501,10 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/shellwords": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/shellwords/-/shellwords-1.1.1.tgz", - "integrity": "sha512-LzESUkEHUuFbjaE7j8uyIjKvySfSFvCF6G4WOygjwSwQj3VuX8hr+v4M252B3twEct6XTWrrNSFu74mTlx4uAQ==", + "node_modules/shellwords-ts": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shellwords-ts/-/shellwords-ts-3.0.1.tgz", + "integrity": "sha512-GabK4ApLMqHFRGlpgNqg8dmtHTnYHt0WUUJkIeMd3QaDrUUBEDXHSSNi3I0PzMimg8W+I0EN4TshQxsnHv1cwg==", "license": "MIT" }, "node_modules/side-channel": { @@ -8356,12 +8382,12 @@ "version": "0.7.2", "license": "MIT", "dependencies": { - "@effectionx/context-api": "^0.2.1", - "@effectionx/process": "^0.6.2", - "@effectionx/stream-helpers": "^0.5.1", - "@effectionx/timebox": "^0.3.1", + "@effectionx/context-api": "^0.3.2", + "@effectionx/process": "^0.7.3", + "@effectionx/stream-helpers": "^0.8.1", + "@effectionx/timebox": "^0.4.2", "chokidar": "^5.0.0", - "effection": "^4.0.0", + "effection": "4.0.2", "picomatch": "^4.0.3" }, "devDependencies": { diff --git a/package.json b/package.json index ce35e507..2bf12459 100644 --- a/package.json +++ b/package.json @@ -33,12 +33,12 @@ "npm": ">=11" }, "volta": { - "node": "20.19.6", + "node": "20.20.1", "npm": "11.7.0" }, "devDependencies": { "@arethetypeswrong/core": "^0.18.2", - "effection": "^4.0.0", + "effection": "^4.0.2", "publint": "^0.3.16", "tsdown": "^0.18.4", "tsx": "^4.21.0", diff --git a/packages/server/README.md b/packages/server/README.md index 6f765a43..606260c7 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -32,11 +32,11 @@ export const services = useServiceGraph( sim3: { operation: useService( "arbitray-child-process", - "node --import tsx ./sim3.ts" + "node --import tsx ./sim3.ts", ), }, }, - { globalData: { hello: "world" } } + { globalData: { hello: "world" } }, ); // this is a helper function which will give you a CLI around this service graph @@ -128,7 +128,7 @@ Type: `{ startup?: string[]; restart?: string[] }` ##### `watch` Watching & restart propagation -To enable file-watching: pass `{ watch: true }` to `useServiceGraph` options (second argument) and add `watch` paths to `ServiceDefinition` objects. The watcher computes transitive dependents (using `dependsOn.restart`) and emits restart updates so restarts propagate deterministically. +To enable file‑watching: pass `{ watch: true }` to the `useServiceGraph` options (second argument) and add `watch` paths to your `ServiceDefinition` objects. The watcher is only started when you explicitly request it (and when at least one service includes `watch` paths); by default no file descriptor is opened, allowing the process to exit cleanly on SIGINT. The watcher computes transitive dependents (using `dependsOn.restart`) and emits restart updates so restarts propagate deterministically. ### ServiceRunner & returned values @@ -185,7 +185,7 @@ Example: ```ts operation: useChildSimulation( "service-key-for-logs", - "./simulator/my-simulator.js" + "./simulator/my-simulator.js", ); ``` @@ -225,7 +225,7 @@ const runner = useServiceGraph( { child: { operation: useChildSimulation("child", "./child-main.ts") }, }, - { globalData: { featureFlag: true } } + { globalData: { featureFlag: true } }, ); const services = yield * runner(); diff --git a/packages/server/example/concurrency-layers.ts b/packages/server/example/concurrency-layers.ts index 96390255..1f58957a 100644 --- a/packages/server/example/concurrency-layers.ts +++ b/packages/server/example/concurrency-layers.ts @@ -5,14 +5,6 @@ import { useChildSimulation } from "../src/simulation.ts"; import { simulationCLI } from "../src/cli.ts"; const servicesMap = { - fast: { - operation: useChildSimulation("fast", "./example/services/basic-sim-1.ts"), - watch: ["./example/services/basic-sim-1.ts"], - }, - slow: { - operation: useChildSimulation("slow", "./example/services/basic-sim-2.ts"), - watch: ["./example/services/basic-sim-2.ts"], - }, dependent: { dependsOn: { startup: ["fast", "slow"] as const }, operation: resource(function* (provide) { @@ -25,6 +17,14 @@ const servicesMap = { }), watch: ["./example/services/basic-sim.ts"], }, + fast: { + operation: useChildSimulation("fast", "./example/services/basic-sim-1.ts"), + watch: ["./example/services/basic-sim-1.ts"], + }, + slow: { + operation: useChildSimulation("slow", "./example/services/basic-sim-2.ts"), + watch: ["./example/services/basic-sim-2.ts"], + }, }; export const services = useServiceGraph(servicesMap); diff --git a/packages/server/example/process-graph.ts b/packages/server/example/process-graph.ts index 8892aaef..2ed599bc 100644 --- a/packages/server/example/process-graph.ts +++ b/packages/server/example/process-graph.ts @@ -12,10 +12,12 @@ const servicesMap = { { wellnessCheck: { frequency: 10, + timeout: 15000, *operation(stdio: Stream) { for (let line of yield* each(stdio)) { if (line.includes("started")) { console.log("A ready (wellnessCheck)"); + return { ok: true, value: undefined }; } yield* each.next(); @@ -24,7 +26,7 @@ const servicesMap = { return { ok: true, value: undefined }; }, }, - } + }, ), }, B: { @@ -35,10 +37,12 @@ const servicesMap = { { wellnessCheck: { frequency: 10, + timeout: 15000, *operation(stdio: Stream) { for (let line of yield* each(stdio)) { if (line.includes("started")) { console.log("B ready (wellnessCheck)"); + return { ok: true, value: undefined }; } yield* each.next(); @@ -47,7 +51,7 @@ const servicesMap = { return { ok: true, value: undefined }; }, }, - } + }, ), }, }; diff --git a/packages/server/example/services/gen-sim-factory.ts b/packages/server/example/services/gen-sim-factory.ts index 1fa27a76..895c7919 100644 --- a/packages/server/example/services/gen-sim-factory.ts +++ b/packages/server/example/services/gen-sim-factory.ts @@ -10,10 +10,9 @@ import { */ export function simulation( port: number = 3301, - startDelay: number = 10 + startDelay: number = 10, ): (initData?: unknown) => FoundationSimulator { return (initData?: unknown) => { - if (initData) console.log("simulation received init data:", initData); const factory = createFoundationSimulationServer({ port, extendRouter(router) { diff --git a/packages/server/package.json b/packages/server/package.json index f159ec0e..1a8b0535 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -37,12 +37,12 @@ "example:concurrency": "node --import tsx ./example/concurrency-layers.ts" }, "dependencies": { - "effection": "^4.0.0", - "@effectionx/context-api": "^0.2.1", - "@effectionx/process": "^0.6.2", - "@effectionx/stream-helpers": "^0.5.1", - "@effectionx/timebox": "^0.3.1", + "@effectionx/context-api": "^0.3.2", + "@effectionx/process": "^0.7.3", + "@effectionx/stream-helpers": "^0.8.1", + "@effectionx/timebox": "^0.4.2", "chokidar": "^5.0.0", + "effection": "4.0.2", "picomatch": "^4.0.3" }, "devDependencies": { diff --git a/packages/server/src/cli.ts b/packages/server/src/cli.ts index 5f9cde01..62d3f7d1 100644 --- a/packages/server/src/cli.ts +++ b/packages/server/src/cli.ts @@ -1,6 +1,8 @@ import { parseArgs } from "node:util"; import { main, suspend, type Operation } from "effection"; +import { useAttributes } from "./logging.ts"; import type { ServiceGraph, ServiceDefinition } from "./services.ts"; +import { Debugging, logger } from "./logging.ts"; /** * CLI operation that parses args and runs a service graph runner. @@ -13,13 +15,13 @@ import type { ServiceGraph, ServiceDefinition } from "./services.ts"; * @param serviceGraph - runner factory returned by `useServiceGraph` */ export function* simulationCLIOp, T = any>( - serviceGraph: (subset?: string[] | string) => Operation> + serviceGraph: (subset?: string[] | string) => Operation>, ) { try { const { values } = parseArgs({ options: { services: { type: "string", short: "s" }, - debug: { type: "boolean", short: "d" }, + debug: { type: "boolean", short: "d", default: false }, help: { type: "boolean", short: "h" }, watch: { type: "boolean" }, "watch-debounce": { type: "string" }, @@ -31,7 +33,7 @@ export function* simulationCLIOp, T = any>( function* printUsage() { process.stdout.write( - `Usage: cli [-s|--services serviceName] [--watch] [--watch-debounce ms]` + `Usage: cli [-s|--services serviceName] [--watch] [--watch-debounce ms]`, ); } @@ -45,6 +47,13 @@ export function* simulationCLIOp, T = any>( .map((s) => s.trim()) .filter(Boolean) : undefined; + yield* useAttributes({ + name: "cli", + subset: subset ? subset.join(", ") : "", + watch: String(!!values.watch), + watchDebounce: String(values["watch-debounce"] ?? ""), + debug: String(!!values.debug), + }); const runOptions: { watch?: boolean; watchDebounce?: number } = { watch: !!values.watch, @@ -52,15 +61,19 @@ export function* simulationCLIOp, T = any>( if (values["watch-debounce"]) runOptions.watchDebounce = Number(values["watch-debounce"]); + Debugging.set(!!values.debug); + // Start the graph and fetch the provided info yield* serviceGraph(subset); yield* suspend(); } catch (err) { - console.error( - "simulationCLI error:", - err instanceof Error ? err.stack : err + yield* logger.stderr( + `simulationCLI error:`, + err instanceof Error ? err.stack : err, ); + } finally { + yield* logger.debug("simulationCLI finally"); } } @@ -73,7 +86,13 @@ export function* simulationCLIOp, T = any>( */ export async function simulationCLI< S extends Record>, - T + T, >(serviceGraph: (subset?: string[] | string) => Operation>) { - return main(() => simulationCLIOp(serviceGraph)); + return main(function* () { + try { + yield* simulationCLIOp(serviceGraph); + } finally { + yield* logger.debug("simulationCLI main finally"); + } + }); } diff --git a/packages/server/src/data-service.ts b/packages/server/src/data-service.ts index 3043d3d9..6e1bd908 100644 --- a/packages/server/src/data-service.ts +++ b/packages/server/src/data-service.ts @@ -1,6 +1,7 @@ import { call, resource, type Operation } from "effection"; +import { useAttributes } from "./logging.ts"; import { createServer } from "node:http"; -import { stdout } from "./logging.ts"; +import { logger } from "./logging.ts"; export type DataServiceOptions = Record | undefined; @@ -16,9 +17,13 @@ export type DataServiceOptions = Record | undefined; * @returns an operation that provides `{ port: number }` when ready */ export function startDataService( - data: DataServiceOptions = {} + data: DataServiceOptions = {}, ): Operation<{ port: number }> { return resource(function* (provide) { + yield* useAttributes({ + name: "dataService", + keys: Object.keys(data).join(", "), + }); const server = createServer((req, res) => { try { const url = new URL(req.url ?? "", `http://127.0.0.1`); @@ -81,13 +86,14 @@ export function startDataService( ? address.port : 0; - yield* stdout(`data service: started on port ${port}`); + yield* logger.stdout(`data service started on port ${port}`); + yield* useAttributes({ name: "dataService", port }); try { yield* provide({ port }); } finally { yield* call(() => server.close()); - yield* stdout(`data service: stopped on port ${port}`); + yield* logger.debug(`data service stopped on port ${port}`); } }); } diff --git a/packages/server/src/logging.ts b/packages/server/src/logging.ts index 0ac8c51a..1cd1b2ec 100644 --- a/packages/server/src/logging.ts +++ b/packages/server/src/logging.ts @@ -1,14 +1,53 @@ -import type { Operation } from "effection"; +import { type Operation, createContext, call } from "effection"; import { createApi } from "@effectionx/context-api"; +export const Debugging = createContext("@simulacrum/debugging", false); + export const stdio = createApi("@simulacrum/logging", { *stdout(line: string): Operation { console.log(line); }, - *stderr(line: string): Operation { - console.log(line); + *stderr(...line: Parameters): Operation { + console.error(...line); + }, + *debug(line: string): Operation { + const isDebugging = yield* Debugging.expect(); + if (isDebugging) console.debug(line); }, }); -export const { stdout } = stdio.operations; -export const { stderr } = stdio.operations; +let useAttributesImpl: + | undefined + | ((attrs: Record) => Operation) = undefined; + +function* resolveUseAttributes() { + if (typeof useAttributesImpl !== "undefined") { + return; + } + + try { + const effection = yield* call(() => import("effection")); + const maybe = (effection as { useAttributes?: any }).useAttributes; + if (typeof maybe === "function") { + useAttributesImpl = maybe; + } else { + useAttributesImpl = function* () { + return; + } as any; + } + } catch { + // no-op when useAttributes is unavailable in older effection versions + useAttributesImpl = function* () { + return; + } as any; + } +} + +export function* useAttributes(attrs: Record) { + yield* resolveUseAttributes(); + if (useAttributesImpl) { + return yield* useAttributesImpl(attrs); + } +} + +export const logger = stdio.operations; diff --git a/packages/server/src/service.ts b/packages/server/src/service.ts index 559b2a8a..9f41a00c 100644 --- a/packages/server/src/service.ts +++ b/packages/server/src/service.ts @@ -9,10 +9,11 @@ import { sleep, spawn, } from "effection"; +import { useAttributes } from "./logging.ts"; import { timebox } from "@effectionx/timebox"; import { daemon } from "@effectionx/process"; import type { ExecOptions as ProcessOptions } from "@effectionx/process"; -import { stderr, stdout } from "./logging.ts"; +import { logger } from "./logging.ts"; import { createReplaySignal } from "./createReplaySignal.ts"; type ServiceOptions = { @@ -34,15 +35,16 @@ type ServiceOptions = { * process and clean up and shut down the process. */ export function useService( - _name: string, + name: string, cmd: string, - options: ServiceOptions = {} + options: ServiceOptions = {}, ) { return resource(function* (provide) { + yield* useAttributes({ name: `useService ${name}`, cmd: String(cmd) }); if (cmd.startsWith("npm")) { // see https://github.com/npm/cli/issues/6684 throw new Error( - "scripts run with npm don't respect signals to properly shutdown" + "scripts run with npm don't respect signals to properly shutdown", ); } const process = yield* daemon(cmd, options.processOptions); @@ -51,19 +53,21 @@ export function useService( // forward raw stdout for logging in chunk form (no reassembly) yield* spawn(function* () { + yield* useAttributes({ name: "stdoutForward" }); for (let line of yield* each(process.stdout)) { const buf = Buffer.from(line); const str = buf.toString(); - stdout(str); + yield* logger.stdout(str); yield* stdioAdd(str); yield* each.next(); } }); yield* spawn(function* () { + yield* useAttributes({ name: "stderrForward" }); for (let line of yield* each(process.stderr)) { const str = Buffer.from(line).toString(); - stderr(str); + yield* logger.stderr(str); yield* stdioAdd(str); yield* each.next(); } @@ -71,13 +75,17 @@ export function useService( yield* sleep(0); // allow stdio forwarding to start - // TODO if it fails to start up but has a wellness check, it seems to hang - // if supplied, wellness check to ensure it is running or timeout with result if (options.wellnessCheck) { + yield* useAttributes({ + name: `useService ${name}`, + wellnessCheck: String(true), + frequency: String(options.wellnessCheck.frequency ?? ""), + }); const { operation } = options.wellnessCheck; const frequency = options.wellnessCheck.frequency ?? 100; function* untilWell() { + yield* useAttributes({ name: `wellnessCheck` }); while (true) { try { yield* sleep(frequency); @@ -92,9 +100,13 @@ export function useService( } if (options.wellnessCheck.timeout) { + yield* useAttributes({ + name: `useService ${name}`, + timeout: String(options.wellnessCheck.timeout), + }); const checked = yield* timebox( options.wellnessCheck.timeout, - untilWell + untilWell, ); if (checked && checked.timeout) { throw new Error("service wellness check timed out"); diff --git a/packages/server/src/services.ts b/packages/server/src/services.ts index f78ea3c7..03cbaf5a 100644 --- a/packages/server/src/services.ts +++ b/packages/server/src/services.ts @@ -9,8 +9,9 @@ import { createContext, } from "effection"; +import { useAttributes } from "./logging.ts"; import { type ServiceUpdate, useWatcher } from "./watch.ts"; -import { stdout } from "./logging.ts"; +import { logger } from "./logging.ts"; import { startDataService } from "./data-service.ts"; /** @@ -25,7 +26,7 @@ export const SimulacrumEndpoint = createContext("SimulacrumEndpoint"); export type ServiceDefinition< S, - T extends void | { port?: number } | unknown + T extends void | { port?: number } | unknown, > = { operation: Operation; // folders/files to watch for changes which should cause a restart @@ -42,15 +43,26 @@ type MaybeSimulation = void | { port?: number } | unknown; export type ServiceGraph< S extends Record>, - T extends MaybeSimulation + T extends MaybeSimulation, > = { services: { [service in keyof S]: ServiceDefinition; }; serviceUpdates?: Stream | undefined; serviceChanges?: Stream | undefined; - // map of service name => listening port (when the service exposes one) - servicePorts?: Map | undefined; + status?: Map; +}; + +export type ServiceInfo = { + port?: number | undefined; + pid?: number | undefined; +}; + +export type ServiceStatus = { + startup: WithResolvers; + running: WithResolvers; + port?: number | undefined; + pid?: number | undefined; }; /** @@ -68,19 +80,24 @@ export type ServiceGraph< */ export function useServiceGraph< S extends Record>, - T extends MaybeSimulation + T extends MaybeSimulation, >( services: S, options?: { globalData?: Record; watch?: boolean; watchDebounce?: number; - } + }, ): (subset?: string[] | string) => Operation> { return (subset?: string[] | string) => resource(function* (provide) { // detect cycles in the dependency graph const nodes = Object.keys(services); + // label the root of the service graph operation + yield* useAttributes({ + name: "serviceGraph", + totalServices: String(nodes.length), + }); const temp = new Set(); const perm = new Set(); @@ -106,8 +123,8 @@ export function useServiceGraph< if (subset) { const want = new Set( (typeof subset === "string" ? subset.split(",") : subset).map((s) => - s.trim() - ) + s.trim(), + ), ); const included = new Set(); function include(name: string) { @@ -128,115 +145,188 @@ export function useServiceGraph< } effectiveServices = picked as typeof services; - yield* stdout( - `service graph: starting with services: ${Array.from(included).join( - ", " - )}` + // annotate subset details AFTER calculations to avoid overwriting + yield* useAttributes({ + name: "serviceGraph", + requestedServices: Array.from(want).join(", "), + includedServices: Array.from(included).join(", "), + }); + yield* logger.stdout( + `simulation starting with subset of services: ${Array.from( + included, + ).join(", ")}`, ); } - // track service ports (when services expose one) - const servicePorts = new Map(); + + const status = new Map(); const dataServiceProvided = yield* startDataService( - options?.globalData ?? {} + options?.globalData ?? {}, ); - servicePorts.set("simulacrum", dataServiceProvided.port); + yield* useAttributes({ + name: "serviceGraph", + dataServicePort: String(dataServiceProvided.port), + }); + + status.set("simulacrum", { + startup: withResolvers(), + running: withResolvers(), + port: dataServiceProvided.port, + }); + // set the SimulacrumEndpoint in this operation scope so children started // in this graph can access the port via context yield* SimulacrumEndpoint.set(dataServiceProvided.port); - const watcher = yield* useWatcher( - effectiveServices, - options?.watchDebounce - ? { watchDebounce: options.watchDebounce } - : undefined - ); + // start up a watcher only when the CLI or caller explicitly asks for it + // or when at least one of the services has a `watch` configuration. by + // default we avoid spinning up chokidar when not needed since it holds an + // active file descriptor and has been observed to keep the process alive + // even after the root scope has been cancelled. + const shouldWatch = + options?.watch === true || + Object.values(effectiveServices).some((d) => Array.isArray(d.watch)); + + const watcher = shouldWatch + ? yield* useWatcher( + effectiveServices, + options?.watchDebounce + ? { watchDebounce: options.watchDebounce } + : undefined, + ) + : undefined; - const status = new Map< - string, - { startup: WithResolvers; running: WithResolvers } - >(); - // establish watching and ready status for (const name of Object.keys(effectiveServices)) { const def = effectiveServices[name]; status.set(name, { startup: withResolvers(), running: withResolvers(), }); - if (def.watch) { + if (def.watch && watcher) { watcher.add(name, def.watch); } } - function bumpService(service: string) { + function* bumpService(service: string) { + yield* useAttributes({ + name: "watcher", + reason: `restarting service ${service}`, + }); const task = status.get(service); - if (!task) throw new Error(`missing status for service '${service}'`); + if (!task) throw new Error(`missing status for service ${service}`); + // log so it is clear in the inspector output when a restart is triggered + yield* logger.stdout(`restarting service ${service}`); // refresh the startup resolver task.startup = withResolvers(); - // this allows the service to continue and halt itself - // remove any recorded port for the service; it will be re-registered when it starts again - servicePorts.delete(service); + + // remove any recorded port/pid for the service; it will be re-registered when it starts again + delete task.port; + delete task.pid; + + // signal the running operation to stop so it can clean up task.running.resolve(); } - yield* spawn(function* () { - // restart propagation to dependents is handled by useWatcher - for (let restartService of yield* each(watcher.serviceChanges)) { - bumpService(restartService.service); - yield* each.next(); - } - }); + if (watcher) { + yield* spawn(function* () { + yield* useAttributes({ + name: "watcher", + reason: "startup", + }); + // restart propagation to dependents is handled by useWatcher + for (let restartService of yield* each(watcher.serviceChanges)) { + yield* bumpService(restartService.service); + yield* each.next(); + } + }); + } // small helper to await a service's dependencies - function* waitDeps(name: string, startup: boolean): Operation { - const def = effectiveServices[name]; - const deps = startup - ? def.dependsOn?.startup ?? [] - : def.dependsOn?.restart ?? []; + function* waitDeps(name: string, restartCount: number): Operation { + const deps = + restartCount === 0 + ? (effectiveServices[name].dependsOn?.startup ?? []) + : (effectiveServices[name].dependsOn?.restart ?? []); + yield* useAttributes({ + name: `service ${name}`, + depCount: String(deps.length), + }); for (const dep of deps) { const r = status.get(dep); if (!r) throw new Error( - `missing readiness resolver for dependency '${dep}'` + `missing readiness resolver for dependency '${dep}'`, ); yield* r.startup.operation; } } function* withRestarts(service: string) { - let startup = true; + // start at -1 so the first run is "restarted 0 times" + let restartCount = -1; + yield* useAttributes({ + name: `service ${service}`, + dependencies: JSON.stringify( + effectiveServices[service].dependsOn ?? {}, + ), + }); while (true) { - const start = yield* spawn(function* () { - yield* waitDeps(service, startup); - const def = effectiveServices[service]; - const task = status.get(service); - if (!task) - throw new Error(`missing status for service '${service}'`); - task.running = withResolvers(); + yield* useAttributes({ + name: `service ${service}`, + status: `restarted ${++restartCount} times`, + }); + yield* waitDeps(service, restartCount); + + const def = effectiveServices[service]; + const task = status.get(service); + if (!task) throw new Error(`missing status for service '${service}'`); + + // each run gets its own running resolver so we can cancel it on demand + task.running = withResolvers(); + // run the service in a scoped child operation so it can be cleanly + // cancelled when a file change triggers a restart + const serviceTask = yield* spawn(function* () { // capture any returned listening info (e.g., from useChildSimulation) const maybeProvided = yield* def.operation; - if ( - maybeProvided && - typeof maybeProvided === "object" && - "port" in maybeProvided && - typeof maybeProvided.port === "number" - ) { - servicePorts.set(service, maybeProvided.port); + if (maybeProvided && typeof maybeProvided === "object") { + if ( + "port" in maybeProvided && + typeof maybeProvided.port === "number" + ) { + yield* useAttributes({ + name: `service ${service}`, + port: String(maybeProvided.port), + }); + task.port = maybeProvided.port; + } + if ( + "pid" in maybeProvided && + typeof maybeProvided.pid === "number" + ) { + task.pid = maybeProvided.pid; + yield* useAttributes({ + name: `service ${service}`, + pid: String(maybeProvided.pid), + }); + } } task.startup.resolve(); + // wait until the watcher asks for this service to be restarted yield* task.running.operation; }); - yield* start; - startup = false; + yield* serviceTask; } } try { for (let service of Object.keys(effectiveServices)) { yield* spawn(function* () { - yield* stdout(`service graph: starting service '${service}'`); + yield* useAttributes({ + name: `service ${service}`, + }); + yield* logger.debug(`service graph: spawning service ${service}`); yield* withRestarts(service); }); } @@ -245,10 +335,10 @@ export function useServiceGraph< services: services as S, serviceUpdates: watcher?.serviceUpdates, serviceChanges: watcher?.serviceChanges, - servicePorts, + status, }); } finally { - yield* stdout("shutting down service graph"); + yield* logger.debug("shutting down service graph"); } }); } diff --git a/packages/server/src/simulation.ts b/packages/server/src/simulation.ts index 77ec5489..300b5786 100644 --- a/packages/server/src/simulation.ts +++ b/packages/server/src/simulation.ts @@ -1,7 +1,8 @@ -import { resource, until, spawn, each, withResolvers, Ok } from "effection"; +import { resource, until, spawn, each, withResolvers } from "effection"; +import { useAttributes } from "./logging.ts"; import type { Operation } from "effection"; import { daemon } from "@effectionx/process"; -import { stderr, stdout } from "./logging.ts"; +import { logger } from "./logging.ts"; import type { FoundationSimulator, FoundationSimulatorListening, @@ -27,9 +28,10 @@ import { SimulacrumEndpoint } from "./services.ts"; * simulator is listening */ export function useSimulation>( name: string, - createFactory: (initData?: unknown) => FoundationSimulator + createFactory: (initData?: unknown) => FoundationSimulator, ): Operation> { return resource(function* (provide) { + yield* useAttributes({ name: `useSimulation ${name}` }); // attempt to read the simulacrum port from context; if not present, continue without it const simulacrumPort = yield* SimulacrumEndpoint.get(); @@ -38,27 +40,31 @@ import { SimulacrumEndpoint } from "./services.ts"; if (typeof simulacrumPort === "number" && !Number.isNaN(simulacrumPort)) { try { const res = yield* until( - fetch(`http://127.0.0.1:${simulacrumPort}/data`) + fetch(`http://127.0.0.1:${simulacrumPort}/data`), ); initData = yield* until(res.json()); } catch (err) { // ignore fetch failures - console.error("failed to fetch simulacrum data:", err); + yield* logger.stderr("failed to fetch simulacrum data:", err); } } const createSim = createFactory(initData); const listening: FoundationSimulatorListening = yield* until( - createSim.listen() + createSim.listen(), ); - yield* stdout(`${name} simulation started on port ${listening.port}`); + yield* logger.stdout(`${name} simulation: port ${listening.port}`); + yield* useAttributes({ + name: `useSimulation ${name}`, + port: String(listening.port), + }); try { yield* provide(listening); } finally { yield* until(listening.ensureClose()); - yield* stdout(`${name} simulation closed on port ${listening.port}`); + yield* logger.stdout(`${name} simulation: closed port ${listening.port}`); } }); } @@ -78,13 +84,14 @@ import { SimulacrumEndpoint } from "./services.ts"; * @param modulePath - path to the module exporting a simulation factory or instance * @returns an `Operation` that provides `FoundationSimulatorListening` from the child */ -export function useChildSimulation>( - name: string, - modulePath: string -): Operation> { - return resource(function* (provide) { +export function useChildSimulation(name: string, modulePath: string) { + return resource<{ port: number; pid: number }>(function* (provide) { + yield* useAttributes({ + name: `useChildSimulation ${name}`, + module: modulePath, + }); // attempt to read the simulacrum port from context; if not present, continue without it - const port = yield* SimulacrumEndpoint.get(); + const contextPort = yield* SimulacrumEndpoint.get(); const parts = [ "node", @@ -93,42 +100,49 @@ export function useChildSimulation>( "./bin/run-simulation-child.ts", modulePath, ]; - if (typeof port === "number") { - parts.push("--simulacrum-port", String(port)); + if (typeof contextPort === "number") { + parts.push("--simulacrum-port", String(contextPort)); } const cmd = parts.map((s) => (s.includes(" ") ? `'${s}'` : s)).join(" "); const process = yield* daemon(cmd); + const pid = process.pid; + yield* useAttributes({ + name: `useChildSimulation ${name}`, + cmd, + pid: String(pid), + }); // read the first stdout JSON line to get the listening info - let listening: FoundationSimulatorListening | undefined = undefined; - let ready = withResolvers( - "wait until the port is returned to signal ready" + let port = undefined as number | undefined; + let ready = withResolvers( + "wait until the port is returned to signal ready", ); // forward raw stdout for logging in chunk form (no reassembly) yield* spawn(function* () { + yield* useAttributes({ + name: "stdoutForward", + }); for (let line of yield* each(process.stdout)) { const buf = Buffer.from(line); const str = buf.toString(); - if (!listening) { + if (!port) { try { const parsed = JSON.parse(str); if (parsed && parsed.ready && typeof parsed.port === "number") { - listening = { - port: parsed.port, - } as FoundationSimulatorListening; - ready.resolve(Ok(listening)); + port = parsed.port; + ready.resolve(); } else { - yield* stdout(str); + yield* logger.stdout(str); } } catch (_) { - // ignore lines that are not JSON - yield* stdout(str); + // just log lines that are not JSON + yield* logger.stdout(str); } } else { - yield* stdout(str); + yield* logger.stdout(str); } yield* each.next(); @@ -136,9 +150,12 @@ export function useChildSimulation>( }); yield* spawn(function* () { + yield* useAttributes({ + name: "stderrForward", + }); for (let line of yield* each(process.stderr)) { const str = Buffer.from(line).toString(); - yield* stderr(str); + yield* logger.stderr(str); yield* each.next(); } }); @@ -146,31 +163,39 @@ export function useChildSimulation>( // spawn a watcher to detect if the child exits before printing the listening info let status: unknown = undefined; yield* spawn(function* () { + yield* useAttributes({ + name: "childEarlyExitWatcher", + }); status = yield* process.join(); - if (!listening) { + if (!port) { ready.reject( new Error( `child process exited before emitting listening info: ${JSON.stringify( - status - )}` - ) + status, + )}`, + ), ); } }); // wait to get the listening info from stdout (or reject if the process exited) yield* ready.operation; - // we know listening is defined here - listening = listening!; - yield* stdout( - `${name} process simulation started on port ${listening.port}` - ); + if (!port) { + throw new Error( + `failed to get listening port from child process: ${JSON.stringify({ + status, + pid, + })}`, + ); + } + + yield* logger.stdout(`${name} simulation: port ${port} pid ${pid}`); try { - yield* provide(listening); + yield* provide({ port, pid }); } finally { - yield* stdout(`${name} simulation closed on port ${listening?.port}`); + yield* logger.debug(`${name} simulation: closed on port ${port}`); } }); } diff --git a/packages/server/src/watch.ts b/packages/server/src/watch.ts index 4657b94b..9895d2e9 100644 --- a/packages/server/src/watch.ts +++ b/packages/server/src/watch.ts @@ -10,6 +10,7 @@ import { type Stream, until, } from "effection"; +import { useAttributes } from "./logging.ts"; import picomatch, { type Matcher } from "picomatch"; import { filter } from "@effectionx/stream-helpers"; @@ -34,13 +35,18 @@ export function useWatcher( string, { dependsOn?: { restart?: readonly string[] }; watchDebounce?: number } >, - options?: { watchDebounce?: number } + options?: { watchDebounce?: number }, ): Operation<{ serviceUpdates: Stream<{ service: string; path: string }, void>; serviceChanges: Stream<{ service: string; path: string }, void>; add: (service: string, paths: string[]) => void; }> { return resource(function* (provide) { + yield* useAttributes({ + name: "watcher", + serviceCount: String(services ? Object.keys(services).length : 0), + debounce: String(options?.watchDebounce ?? ""), + }); const changes = createSignal(); const serviceUpdates = createChannel(); const serviceList = new Map(); @@ -94,6 +100,7 @@ export function useWatcher( } yield* spawn(function* () { + yield* useAttributes({ name: "handleChange" }); for (let args of yield* each(changes)) { const [path] = args as EmitArgs; for (let [service, matchers] of serviceList.entries()) { @@ -117,20 +124,24 @@ export function useWatcher( const debounceMs = options?.watchDebounce !== undefined ? options.watchDebounce : 250; const serviceTimers = {} as Record; - const debouncedServiceChanges = filter(function* ( - updateStream - ) { - const now = performance.now(); - if ( - serviceTimers[updateStream.service] && - now - serviceTimers[updateStream.service] < debounceMs - ) { - return false; - } else { - serviceTimers[updateStream.service] = now; - return true; - } - }); + const debouncedServiceChanges = filter( + function* (updateStream) { + yield* useAttributes({ + name: "debounceCheck", + service: updateStream.service, + }); + const now = performance.now(); + if ( + serviceTimers[updateStream.service] && + now - serviceTimers[updateStream.service] < debounceMs + ) { + return false; + } else { + serviceTimers[updateStream.service] = now; + return true; + } + }, + ); try { yield* provide({ diff --git a/packages/server/test/child-simulation.test.ts b/packages/server/test/child-simulation.test.ts index b9810b37..f544b0c9 100644 --- a/packages/server/test/child-simulation.test.ts +++ b/packages/server/test/child-simulation.test.ts @@ -1,6 +1,6 @@ import { describe, it } from "node:test"; import assert from "node:assert"; -import { run, sleep, until } from "effection"; +import { run, until } from "effection"; import { useServiceGraph } from "../src/services.ts"; import { useChildSimulation } from "../src/simulation.ts"; import { waitFor } from "./utils.ts"; @@ -10,15 +10,12 @@ describe("useChildSimulation", () => { await run(function* () { const listening = yield* useChildSimulation( "child-test", - "./test/fixtures/simple-sim.ts" + "./test/fixtures/simple-sim.ts", ); assert(typeof listening.port === "number"); // Verify we received a port and the child reported ready. assert(typeof listening.port === "number", "port should be a number"); - - // allow a moment before teardown - yield* sleep(20); }); }); @@ -26,7 +23,7 @@ describe("useChildSimulation", () => { await run(function* () { const listening = yield* useChildSimulation( "non-json", - "./test/fixtures/non-json-child.ts" + "./test/fixtures/non-json-child.ts", ); assert(typeof listening.port === "number"); }); @@ -36,7 +33,7 @@ describe("useChildSimulation", () => { await run(function* () { const listening = yield* useChildSimulation( "json-before-ready", - "./test/fixtures/json-before-ready.ts" + "./test/fixtures/json-before-ready.ts", ); assert(typeof listening.port === "number"); }); @@ -52,19 +49,19 @@ describe("useChildSimulation", () => { child: { operation: useChildSimulation( "child", - "./test/fixtures/init-data-sim.ts" + "./test/fixtures/init-data-sim.ts", ), }, }, - { globalData: data } + { globalData: data }, ); const runGraph = yield* op(); yield* waitFor( - () => typeof runGraph.servicePorts?.get("child") === "number", - 2000 + () => typeof runGraph.status?.get("child")?.port === "number", + 2000, ); - const childPort = runGraph.servicePorts!.get("child")!; + const childPort = runGraph.status!.get("child")!.port!; const res = yield* until(fetch(`http://127.0.0.1:${childPort}/init`)); const json = (yield* until(res.json())) as { initData: typeof data }; @@ -81,19 +78,19 @@ describe("useChildSimulation", () => { child: { operation: useChildSimulation( "child", - "./test/fixtures/init-data-sim.ts" + "./test/fixtures/init-data-sim.ts", ), }, }, - { globalData: data } + { globalData: data }, ); const runGraph = yield* op(); yield* waitFor( - () => typeof runGraph.servicePorts?.get("child") === "number", - 2000 + () => typeof runGraph.status?.get("child")?.port === "number", + 2000, ); - const childPort = runGraph.servicePorts!.get("child")!; + const childPort = runGraph.status!.get("child")!.port!; const res = yield* until(fetch(`http://127.0.0.1:${childPort}/init`)); const json = (yield* until(res.json())) as { initData: typeof data }; @@ -125,19 +122,19 @@ describe("useChildSimulation", () => { child: { operation: useChildSimulation( "child", - "./test/fixtures/init-data-sim.ts" + "./test/fixtures/init-data-sim.ts", ), }, }, - { globalData: data } + { globalData: data }, ); const runGraph = yield* op(); yield* waitFor( - () => typeof runGraph.servicePorts?.get("child") === "number", - 3000 + () => typeof runGraph.status?.get("child")?.port === "number", + 3000, ); - const childPort = runGraph.servicePorts!.get("child")!; + const childPort = runGraph.status!.get("child")!.port!; const res = yield* until(fetch(`http://127.0.0.1:${childPort}/init`)); const json = (yield* until(res.json())) as { initData: typeof data }; @@ -153,21 +150,21 @@ describe("useChildSimulation", () => { child: { operation: useChildSimulation( "child", - "./test/fixtures/init-data-sim.ts" + "./test/fixtures/init-data-sim.ts", ), }, }, - { globalData: { hello: "world" } } + { globalData: { hello: "world" } }, ); const runGraph = yield* op(); // wait deterministically for the child port to be registered yield* waitFor( - () => typeof runGraph.servicePorts?.get("child") === "number", - 3000 + () => typeof runGraph.status?.get("child")?.port === "number", + 3000, ); - const childPort = runGraph.servicePorts!.get("child")!; + const childPort = runGraph.status!.get("child")!.port!; const res = yield* until(fetch(`http://127.0.0.1:${childPort}/init`)); const json = (yield* until(res.json())) as { diff --git a/packages/server/test/data-service.test.ts b/packages/server/test/data-service.test.ts index e1514920..9ffa696f 100644 --- a/packages/server/test/data-service.test.ts +++ b/packages/server/test/data-service.test.ts @@ -8,19 +8,21 @@ it("starts data service and serves configured data", async () => { await run(function* () { const runGraph = yield* useServiceGraph( {}, - { globalData: { a: 1, nested: { b: 2 } } } + { + globalData: { a: 1, nested: { b: 2 } }, + }, )(); // wait deterministically for the simulacrum port to be registered yield* waitFor( - () => Boolean(runGraph?.servicePorts?.get("simulacrum")), - 2000 + () => typeof runGraph.status?.get("simulacrum")?.port === "number", + 2000, ); - const port = runGraph!.servicePorts!.get("simulacrum")!; + const port = runGraph.status!.get("simulacrum")!.port!; assert.ok( typeof port === "number", - "data service port should be registered on servicePorts" + "data service port should be registered on serviceStatus", ); const res = yield* until(fetch(`http://127.0.0.1:${port}/data`)); @@ -33,15 +35,16 @@ it("serves individual keys and appropriate status codes", async () => { await run(function* () { const runGraph = yield* useServiceGraph( {}, - { globalData: { a: 1, nested: { b: 2 } } } + { + globalData: { a: 1, nested: { b: 2 } }, + }, )(); - // wait deterministically for the simulacrum port yield* waitFor( - () => Boolean(runGraph?.servicePorts?.get("simulacrum")), - 2000 + () => typeof runGraph.status?.get("simulacrum")?.port === "number", + 2000, ); - const port = runGraph!.servicePorts!.get("simulacrum")!; + const port = runGraph.status!.get("simulacrum")!.port!; assert.ok(typeof port === "number"); @@ -53,19 +56,19 @@ it("serves individual keys and appropriate status codes", async () => { // nested key returns object const nestedRes = yield* until( - fetch(`http://127.0.0.1:${port}/data/nested`) + fetch(`http://127.0.0.1:${port}/data/nested`), ); assert.strictEqual(nestedRes.status, 200); const nestedJson = yield* until(nestedRes.json()); assert.deepStrictEqual(nestedJson, { b: 2 }); - // missing key -> 404 + // missing key returns 404 const missRes = yield* until( - fetch(`http://127.0.0.1:${port}/data/does-not-exist`) + fetch(`http://127.0.0.1:${port}/data/does-not-exist`), ); assert.strictEqual(missRes.status, 404); - // empty key -> 400 + // empty key returns 400 const emptyRes = yield* until(fetch(`http://127.0.0.1:${port}/data/`)); assert.strictEqual(emptyRes.status, 400); }); diff --git a/packages/server/test/examples-smoke.test.ts b/packages/server/test/examples-smoke.test.ts index 6ef2e431..99250410 100644 --- a/packages/server/test/examples-smoke.test.ts +++ b/packages/server/test/examples-smoke.test.ts @@ -13,7 +13,7 @@ function checkStatus(port: number): Promise { { hostname: "127.0.0.1", port, path: "/status", agent: false }, (res) => { resolve(res.statusCode ?? 0); - } + }, ); req.on("error", reject); }); @@ -37,7 +37,7 @@ it("basic example imports and runs", async () => { } catch (err) { console.error( "example runner threw:", - err instanceof Error ? err.stack : err + err instanceof Error ? err.stack : err, ); throw err; } @@ -50,10 +50,9 @@ it("basic example imports and runs", async () => { yield* sleep(0); const svcMap = provided!.services; - const ports = provided!.servicePorts!; const ps: number[] = []; for (const name of Object.keys(svcMap)) { - const port = ports!.get(name); + const port = provided!.serviceStatus?.get(name)?.port; if (typeof port === "number") ps.push(port); } @@ -73,7 +72,7 @@ it("basic example imports and runs", async () => { }, 2000); } catch (err) { throw new Error( - `(examples-smoke basic) port ${p} did not return 200 while graph was running` + `(examples-smoke basic) port ${p} did not return 200 while graph was running`, ); } } @@ -148,7 +147,7 @@ it("concurrency example imports and runs", async () => { } catch (err) { console.error( "example runner threw:", - err instanceof Error ? err.stack : err + err instanceof Error ? err.stack : err, ); throw err; } @@ -161,10 +160,9 @@ it("concurrency example imports and runs", async () => { yield* sleep(0); const svcMap = provided!.services; - const ports = provided!.servicePorts!; const ps: number[] = []; for (const name of Object.keys(svcMap)) { - const port = ports!.get(name); + const port = provided!.status?.get(name)?.port; if (typeof port === "number") ps.push(port); } @@ -188,7 +186,7 @@ it("concurrency example imports and runs", async () => { } if (!ok) { throw new Error( - `(examples-smoke concurrency) port ${p} did not return 200 while graph was running` + `(examples-smoke concurrency) port ${p} did not return 200 while graph was running`, ); } } diff --git a/packages/server/test/services.test.ts b/packages/server/test/services.test.ts index f625f9da..82d2dd7b 100644 --- a/packages/server/test/services.test.ts +++ b/packages/server/test/services.test.ts @@ -2,6 +2,7 @@ import { it } from "node:test"; import assert from "node:assert"; import { resource, run, sleep, spawn, suspend } from "effection"; import { useServiceGraph } from "../src/services.ts"; +import { waitFor } from "./utils.ts"; import { useService } from "../src/service.ts"; it("starts services in dependency order", async () => { @@ -30,8 +31,7 @@ it("starts services in dependency order", async () => { // keep spawned graph alive yield* suspend(); }); - // The graph is running; sleep a short time to let the services start - yield* sleep(200); + yield* waitFor(() => startTimes.has("A") && startTimes.has("B"), 2000); }); } catch (err) { console.log("run error:", err instanceof Error ? err.stack : err); @@ -51,14 +51,14 @@ it("throws on cycles in dependency graph", async () => { A: { operation: useService( "A", - "node --import tsx ./test/services/service-a.ts" + "node --import tsx ./test/services/service-a.ts", ), dependsOn: { startup: ["B"] as const }, }, B: { operation: useService( "B", - "node --import tsx ./test/services/service-b.ts" + "node --import tsx ./test/services/service-b.ts", ), dependsOn: { startup: ["A"] as const }, }, @@ -104,7 +104,7 @@ it("runs beforeStop hooks in reverse order", async () => { yield* suspend(); }); // let them start - yield* sleep(200); + yield* waitFor(() => startedOrder.length === 2, 2000); }); assert.strictEqual(startedOrder.join(""), "AB"); assert.strictEqual(stopOrder.join(""), "BA"); @@ -141,11 +141,11 @@ it("starts independent services in parallel", async () => { const slowStarted = startTimes.get("slow"); assert.ok( typeof fastStarted === "number", - "fast started should be recorded" + "fast started should be recorded", ); assert.ok( typeof slowStarted === "number", - "slow started should be recorded" + "slow started should be recorded", ); assert(fastStarted! <= slowStarted!, "fast should start before slow"); } finally { @@ -197,7 +197,10 @@ it("runs subset of services with dependencies", async () => { // keep spawned graph alive so services can start and perform startup work yield* suspend(); }); - yield* sleep(300); + yield* waitFor( + () => startTimes.has("fast") && startTimes.has("slow"), + 2000, + ); }); const f = startTimes.get("fast"); @@ -272,7 +275,10 @@ it("runs subset specified as a string", async () => { yield* run("r"); yield* suspend(); }); - yield* sleep(300); + yield* waitFor( + () => startTimes.has("a") && startTimes.has("b") && startTimes.has("r"), + 2000, + ); }); const a = startTimes.get("a"); diff --git a/packages/server/test/signal.test.ts b/packages/server/test/signal.test.ts deleted file mode 100644 index 0f8711c5..00000000 --- a/packages/server/test/signal.test.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { it } from "node:test"; -import assert from "node:assert"; -import { spawn as spawnChild } from "node:child_process"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; -import { run, sleep, until, spawn, on, each } from "effection"; -import { timebox } from "@effectionx/timebox"; -import { emitterToEventTarget } from "./utils.ts"; - -it("example process shuts down cleanly on SIGINT", async () => { - await run(function* () { - const exe = process.execPath; - const script = fileURLToPath( - new URL("../example/simulation-graph.ts", import.meta.url) - ); - const cwd = path.resolve( - path.dirname(fileURLToPath(import.meta.url)), - ".." - ); - - const child = spawnChild(exe, ["--import", "tsx", script], { - cwd, - env: { ...process.env }, - stdio: ["ignore", "pipe", "pipe"], - }); - - let stdout = ""; - let stderr = ""; - - // use Effection `on()` + `each()` by adapting Node emitter to EventTarget - const outTarget = emitterToEventTarget(child.stdout!); - const errTarget = emitterToEventTarget(child.stderr!); - - // spawn background tasks to accumulate stdout and stderr - yield* spawn(function* () { - for (let chunk of yield* each(on(outTarget, "data"))) { - stdout += String(chunk); - yield* each.next(); - } - }); - - yield* spawn(function* () { - for (let chunk of yield* each(on(errTarget, "data"))) { - stderr += String(chunk); - yield* each.next(); - } - }); - - try { - // wait for a startup marker using the stdout Stream with a timebox - const started = yield* timebox(3000, function* () { - for (let chunk of yield* each(on(outTarget, "data"))) { - const s = String(chunk); - if ( - s.includes("runner: starting layers") || - s.includes("service graph: starting service") - ) { - return { started: true }; - } - yield* each.next(); - } - return undefined; - }); - - if (started && started.timeout) - throw new Error("startup marker not seen"); - - // send SIGINT - process.kill(child.pid!, "SIGINT"); - - // wait for the child to exit (timeboxed) - const exitRes = yield* timebox(3000, function* () { - const p = new Promise<{ - code: number | null; - signal: NodeJS.Signals | null; - }>((resolve) => { - child.on("exit", (code, signal) => resolve({ code, signal })); - }); - return yield* until(p); - }); - - if (exitRes && exitRes.timeout) - throw new Error("child did not exit in time"); - - // timebox may return an object with `.value` or the value directly — handle both - let code: number | null = null; - let signal: NodeJS.Signals | null = null; - const maybe = exitRes as any; - const val = maybe && "value" in maybe ? maybe.value : maybe; - if (val && typeof val === "object") { - code = val.code; - signal = val.signal; - } - - // allow stderr to flush a little - yield* sleep(50); - - // expect no stack traces on stderr and process exited due to SIGINT - assert.strictEqual(typeof stderr, "string"); - assert( - !/uncaughtException|UnhandledPromiseRejection|Error/.test(stderr), - `stderr contained error: ${stderr}` - ); - // Accept either signal SIGINT or code 0 or code 130 (standard SIGINT exit code) - assert( - signal === "SIGINT" || code === 0 || code === 130, - `unexpected exit: code=${code} signal=${signal}` - ); - } finally { - // ensure process is killed - try { - child.kill("SIGKILL"); - } catch (_) {} - } - }); -}); diff --git a/packages/server/test/watch.test.ts b/packages/server/test/watch.test.ts index 71969d98..fa3038c4 100644 --- a/packages/server/test/watch.test.ts +++ b/packages/server/test/watch.test.ts @@ -1,6 +1,15 @@ import { it } from "node:test"; import assert from "node:assert"; -import { run, suspend, sleep, until, spawn, resource, ensure } from "effection"; +import { + run, + suspend, + sleep, + until, + spawn, + resource, + ensure, + withResolvers, +} from "effection"; import * as fs from "node:fs/promises"; import path from "node:path"; import os from "node:os"; @@ -20,6 +29,8 @@ it("restarts services on watched file change and restarts dependents", async () await fs.writeFile(trigger, "initial"); const updates: string[] = []; + const subscriptionReady = withResolvers(); + await run(function* () { yield* spawn(function* () { // start the graph and enable watch mode @@ -28,17 +39,17 @@ it("restarts services on watched file change and restarts dependents", async () a: { watch: [dir], operation: useSimulation("test-simulation-a", () => - simulation(5500) + simulation(5500), ), }, b: { dependsOn: { startup: ["a"] as const }, operation: useSimulation("test-simulation-a", () => - simulation(5501) + simulation(5501), ), }, }, - { watch: true, watchDebounce: 20 } + { watch: true, watchDebounce: 20 }, ); try { @@ -47,6 +58,7 @@ it("restarts services on watched file change and restarts dependents", async () if (!services.serviceChanges) throw new Error("serviceChanges not available"); const subscription = yield* services.serviceChanges; + subscriptionReady.resolve(); // wait for the first raw update (will occur after the test touches the file) const first = yield* subscription.next(); @@ -68,12 +80,14 @@ it("restarts services on watched file change and restarts dependents", async () } }, 2000); - // give the spawned subscription a moment to attach - yield* sleep(50); + // wait for the subscription to be ready before touching the file + yield* subscriptionReady.operation; + // touch the trigger file to cause a restart yield* until(fs.writeFile(trigger, "changed")); - // give watcher/poller a moment - yield* sleep(100); + + // wait for the raw watcher update to be observed + yield* waitFor(() => updates.length >= 1, 2000); }); // remove tmp dir @@ -110,7 +124,7 @@ it("restarts dependents when watched service changes", async () => { }), }, }, - { watch: true, watchDebounce: 20 } + { watch: true, watchDebounce: 20 }, ); try { @@ -172,7 +186,7 @@ it("restarts transitive dependents when watched service changes", async () => { }), }, }, - { watch: true, watchDebounce: 20 } + { watch: true, watchDebounce: 20 }, ); try { @@ -187,7 +201,7 @@ it("restarts transitive dependents when watched service changes", async () => { // wait for initial startup yield* waitFor( () => startCounts.a > 0 && startCounts.b > 0 && startCounts.c > 0, - 2000 + 2000, ); // trigger a change @@ -196,7 +210,7 @@ it("restarts transitive dependents when watched service changes", async () => { // wait for restarts to occur yield* waitFor( () => startCounts.a >= 2 && startCounts.b >= 2 && startCounts.c >= 2, - 3000 + 3000, ); }); @@ -220,31 +234,34 @@ it("updates servicePorts when a service restarts", async () => { watch: [dir], operation: useSimulation( "s", - createFoundationSimulationServer({ port: 0 }) + createFoundationSimulationServer({ port: 0 }), ), }, }, - { watch: true, watchDebounce: 20 } + { + watch: true, + watchDebounce: 20, + }, ); const services = yield* op(); // wait for initial port to appear yield* waitFor( - () => typeof services.servicePorts?.get("s") === "number", - 2000 + () => typeof services.status?.get("s")?.port === "number", + 2000, ); - const initial = services.servicePorts!.get("s")!; + const initial = services.status!.get("s")!.port!; // trigger restart by touching the file yield* until(fs.writeFile(trigger, "changed")); // wait for new port value to be different from initial yield* waitFor(() => { - const p = services.servicePorts?.get("s"); + const p = services.status?.get("s")?.port; return typeof p === "number" && p !== initial; }, 3000); - const updated = services.servicePorts!.get("s")!; + const updated = services.status!.get("s")!.port!; assert.ok(typeof initial === "number", "initial port should be present"); assert.ok(typeof updated === "number", "updated port should be present"); @@ -262,6 +279,7 @@ it("debounces rapid changes per service", async () => { const updates: string[] = []; let rawCount = 0; + const watcherReady = withResolvers(); await run(function* () { yield* spawn(function* () { @@ -274,7 +292,7 @@ it("debounces rapid changes per service", async () => { }), }, }, - { watch: true, watchDebounce: 150 } + { watch: true, watchDebounce: 150 }, ); try { @@ -284,6 +302,8 @@ it("debounces rapid changes per service", async () => { const debSub = yield* services.serviceUpdates; const rawSub = yield* services.serviceChanges; + watcherReady.resolve(); + // collect debounced updates yield* spawn(function* () { while (true) { @@ -308,8 +328,8 @@ it("debounces rapid changes per service", async () => { yield* suspend(); }); - // ensure watcher attached - yield* sleep(0); + // ensure watcher subscribed before triggering writes + yield* watcherReady.operation; // write multiple times rapidly yield* until(fs.writeFile(trigger, "changed-1")); @@ -319,8 +339,8 @@ it("debounces rapid changes per service", async () => { yield* until(fs.writeFile(trigger, "changed-3")); yield* ensure(() => until(fs.rm(dir, { recursive: true, force: true }))); - // wait longer than debounce window - yield* sleep(300); + // wait until some raw/etc updates are observed + yield* waitFor(() => rawCount > 0 && updates.length > 0, 2000); }); // we expect the rapid writes to coalesce: there should be at least one @@ -331,6 +351,6 @@ it("debounces rapid changes per service", async () => { const aCount = updates.filter((u) => u === "a").length; assert( aCount < 3, - `expected debounced updates to be fewer than writes (3), got ${aCount}` + `expected debounced updates to be fewer than writes (3), got ${aCount}`, ); }); From c4351a53b503692ea7ab679d9ee6b640282d0b37 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Mon, 23 Mar 2026 12:12:53 -0500 Subject: [PATCH 22/38] refine tests for useServiceGraph as test rig --- packages/server/example/concurrency-layers.ts | 11 +- packages/server/example/process-graph.ts | 11 +- packages/server/example/simulation-graph.ts | 10 - packages/server/src/cli.ts | 7 +- packages/server/src/services.ts | 8 +- packages/server/test/examples-smoke.test.ts | 247 +++++------------- packages/server/test/services.test.ts | 5 +- packages/server/test/services/service-fast.ts | 3 - packages/server/test/services/service-slow.ts | 3 - packages/server/test/utils.ts | 18 +- packages/server/test/watch.test.ts | 6 +- 11 files changed, 93 insertions(+), 236 deletions(-) delete mode 100644 packages/server/test/services/service-fast.ts delete mode 100644 packages/server/test/services/service-slow.ts diff --git a/packages/server/example/concurrency-layers.ts b/packages/server/example/concurrency-layers.ts index 1f58957a..4ee22368 100644 --- a/packages/server/example/concurrency-layers.ts +++ b/packages/server/example/concurrency-layers.ts @@ -1,5 +1,5 @@ #!/usr/bin/env node -import { resource, sleep } from "effection"; +import { resource } from "effection"; import { useServiceGraph } from "../src/services.ts"; import { useChildSimulation } from "../src/simulation.ts"; import { simulationCLI } from "../src/cli.ts"; @@ -33,12 +33,3 @@ import { fileURLToPath } from "node:url"; if (process.argv[1] === fileURLToPath(import.meta.url)) { simulationCLI(services); } - -export function example(opts: { duration?: number } = {}) { - return (function* () { - const run = services; - yield* run(); - yield* sleep(opts.duration ?? 300); - console.log(`Concurrency example (operation) complete`); - })(); -} diff --git a/packages/server/example/process-graph.ts b/packages/server/example/process-graph.ts index 2ed599bc..df25792f 100644 --- a/packages/server/example/process-graph.ts +++ b/packages/server/example/process-graph.ts @@ -1,5 +1,5 @@ #!/usr/bin/env node -import { sleep, each, type Stream } from "effection"; +import { each, type Stream } from "effection"; import { useService } from "../src/service.ts"; import { useServiceGraph } from "../src/services.ts"; import { simulationCLI } from "../src/cli.ts"; @@ -58,15 +58,6 @@ const servicesMap = { export const services = useServiceGraph(servicesMap); -export function example(opts: { duration?: number } = {}) { - return (function* () { - const run = services; - yield* run(); - yield* sleep(opts.duration ?? 300); - console.log(`Basic example complete`); - })(); -} - import { fileURLToPath } from "node:url"; if (process.argv[1] === fileURLToPath(import.meta.url)) { // run via CLI when executed directly diff --git a/packages/server/example/simulation-graph.ts b/packages/server/example/simulation-graph.ts index 4417e220..5b637e66 100644 --- a/packages/server/example/simulation-graph.ts +++ b/packages/server/example/simulation-graph.ts @@ -1,5 +1,4 @@ #!/usr/bin/env node -import { sleep } from "effection"; import { useServiceGraph } from "../src/services.ts"; import { useChildSimulation } from "../src/simulation.ts"; import { simulationCLI } from "../src/cli.ts"; @@ -18,15 +17,6 @@ export const services = useServiceGraph(servicesMap, { globalData: { exampleKey: "exampleValue" }, }); -export function example(opts: { duration?: number } = {}) { - return (function* () { - const run = services; - yield* run(); - yield* sleep(opts.duration ?? 300); - console.log(`Basic (operation) example complete`); - })(); -} - import { fileURLToPath } from "node:url"; if (process.argv[1] === fileURLToPath(import.meta.url)) { simulationCLI(services); diff --git a/packages/server/src/cli.ts b/packages/server/src/cli.ts index 62d3f7d1..fecc1990 100644 --- a/packages/server/src/cli.ts +++ b/packages/server/src/cli.ts @@ -15,7 +15,7 @@ import { Debugging, logger } from "./logging.ts"; * @param serviceGraph - runner factory returned by `useServiceGraph` */ export function* simulationCLIOp, T = any>( - serviceGraph: (subset?: string[] | string) => Operation>, + serviceGraph: (subset?: Array) => Operation>, ) { try { const { values } = parseArgs({ @@ -64,7 +64,8 @@ export function* simulationCLIOp, T = any>( Debugging.set(!!values.debug); // Start the graph and fetch the provided info - yield* serviceGraph(subset); + // subset is a string array from CLI; cast to service key array for strict runner + yield* serviceGraph(subset as unknown as Array); yield* suspend(); } catch (err) { @@ -87,7 +88,7 @@ export function* simulationCLIOp, T = any>( export async function simulationCLI< S extends Record>, T, ->(serviceGraph: (subset?: string[] | string) => Operation>) { +>(serviceGraph: (subset?: Array) => Operation>) { return main(function* () { try { yield* simulationCLIOp(serviceGraph); diff --git a/packages/server/src/services.ts b/packages/server/src/services.ts index 03cbaf5a..46d09fcc 100644 --- a/packages/server/src/services.ts +++ b/packages/server/src/services.ts @@ -88,8 +88,8 @@ export function useServiceGraph< watch?: boolean; watchDebounce?: number; }, -): (subset?: string[] | string) => Operation> { - return (subset?: string[] | string) => +): (subset?: Array) => Operation> { + return (subset?: Array) => resource(function* (provide) { // detect cycles in the dependency graph const nodes = Object.keys(services); @@ -122,9 +122,7 @@ export function useServiceGraph< let effectiveServices = services; // {} as typeof services; if (subset) { const want = new Set( - (typeof subset === "string" ? subset.split(",") : subset).map((s) => - s.trim(), - ), + subset.map((s) => String(s).trim()).filter((s) => s.length > 0), ); const included = new Set(); function include(name: string) { diff --git a/packages/server/test/examples-smoke.test.ts b/packages/server/test/examples-smoke.test.ts index 99250410..ee90c9a4 100644 --- a/packages/server/test/examples-smoke.test.ts +++ b/packages/server/test/examples-smoke.test.ts @@ -1,200 +1,93 @@ -import { it } from "node:test"; -import http from "node:http"; -import { run, sleep, suspend, createScope, until } from "effection"; -import { timebox } from "@effectionx/timebox"; -import { waitForAsync } from "./utils.ts"; +import { describe, it } from "node:test"; +import assert from "node:assert"; +import { run, until } from "effection"; +import { waitFor, waitForOperation } from "./utils.ts"; import { services as basicServices } from "../example/simulation-graph.ts"; import { services as concurrencyServices } from "../example/concurrency-layers.ts"; -function checkStatus(port: number): Promise { - return new Promise((resolve, reject) => { - const req = http.get( - { hostname: "127.0.0.1", port, path: "/status", agent: false }, - (res) => { - resolve(res.statusCode ?? 0); - }, - ); - req.on("error", reject); - }); -} - -import type { ServiceGraph } from "../src/services.ts"; -import type { Operation } from "effection"; - -it("basic example imports and runs", async () => { - const runner = basicServices as unknown as () => Operation< - ServiceGraph - >; // runner - let provided: any; - - // start the graph and await exported ports in a single run operation - await run(function* () { - const [scope, destroy] = createScope(); - // start operation-style graph and capture provided resource synchronously - try { - provided = yield* runner(); - } catch (err) { - console.error( - "example runner threw:", - err instanceof Error ? err.stack : err, - ); - throw err; - } - // keep the graph task alive until the scope is destroyed - scope.run(function* () { - yield* suspend(); - }); - - // allow spawned graph to settle and for services to register their ports - yield* sleep(0); - - const svcMap = provided!.services; - const ps: number[] = []; - for (const name of Object.keys(svcMap)) { - const port = provided!.serviceStatus?.get(name)?.port; - if (typeof port === "number") ps.push(port); - } +describe("example as service rig", { concurrency: 1 }, () => { + it("basic example imports and runs", async () => { + await run(function* () { + let provided = yield* basicServices(); - // keep the graph alive briefly to allow HTTP checks - yield* sleep(200); + // wait until all declared services have registered a port + yield* waitFor(() => { + return ["A", "B"].every( + (name) => typeof provided?.status?.get(name)?.port === "number", + ); + }, 5000); - // check each port while the graph is still running - for (const p of ps) { - try { - yield* waitForAsync(async () => { - try { - const status = await checkStatus(p); - return status === 200; - } catch (_) { - return false; - } - }, 2000); - } catch (err) { + if (!provided.status) { throw new Error( - `(examples-smoke basic) port ${p} did not return 200 while graph was running`, + `expected service status to be available after services started`, ); } - } - - // best-effort: close any Server handles before requesting shutdown - const _getActiveHandles = ( - process as unknown as { _getActiveHandles?: () => unknown[] } - )._getActiveHandles; - const preHandles: unknown[] = _getActiveHandles ? _getActiveHandles() : []; - - for (const h of preHandles) { - try { - const name = (h as { constructor?: { name?: string } })?.constructor - ?.name; - if (name === "Server") { - const maybeClose = (h as { close?: unknown }).close; - if (typeof maybeClose === "function") { - try { - (maybeClose as () => void)(); - } catch (e) {} - } - } - } catch (e) {} - } - // give servers a moment to close - yield* sleep(50); - // request the graph be shut down and wait for up to 1s for cleanup - const tb = yield* timebox(1000, () => until(destroy())); - if (tb.timeout) { - console.warn("cleanup timed out for example graph"); - } - - // best-effort: close any remaining socket handles so tests don't hang - const _getActiveHandles2 = ( - process as unknown as { _getActiveHandles?: () => unknown[] } - )._getActiveHandles; - const handles: unknown[] = _getActiveHandles2 ? _getActiveHandles2() : []; - for (const h of handles) { - try { - const maybeDestroy = (h as { destroy?: unknown }).destroy; - if (typeof maybeDestroy === "function") { - (maybeDestroy as () => void)(); - continue; - } - const maybeEnd = (h as { end?: unknown }).end; - if (typeof maybeEnd === "function") { - (maybeEnd as () => void)(); - } - } catch (e) { - // ignore + const ps: number[] = []; + for (const name of Object.keys(provided.services)) { + const port = provided.status.get(name)?.port; + if (typeof port === "number") ps.push(port); } - } - // allow handles to close - yield* sleep(20); - - return ps as number[]; - }); -}); -it("concurrency example imports and runs", async () => { - const runner = concurrencyServices as unknown as () => Operation< - ServiceGraph - >; // runner - let provided: ServiceGraph | undefined; - - await run(function* () { - const [scope, destroy] = createScope(); - // start operation-style graph and capture provided resource synchronously - try { - provided = yield* runner(); - } catch (err) { - console.error( - "example runner threw:", - err instanceof Error ? err.stack : err, + assert( + ps.length > 0, + "expected at least one service port to be registered", ); - throw err; - } - // keep the graph task alive until the scope is destroyed - scope.run(function* () { - yield* suspend(); + assert.ok(ps[0], "service A should have a port registered"); + assert.ok(ps[1], "service B should have a port registered"); + + // check each tapped port for healthy status while graph is running + for (const p of ps) { + yield* waitForOperation(function* () { + const status = yield* until( + fetch("http://localhost:" + p + "/status"), + ); + return status.ok; + }, 2000); + } }); + }); - // allow spawned graph to settle and for services to register their ports - yield* sleep(0); - - const svcMap = provided!.services; - const ps: number[] = []; - for (const name of Object.keys(svcMap)) { - const port = provided!.status?.get(name)?.port; - if (typeof port === "number") ps.push(port); - } + it("concurrency example imports and runs", async () => { + await run(function* () { + let provided = yield* concurrencyServices(); - yield* sleep(200); + // wait until child simulation services have registered a port + yield* waitFor(() => { + return ["fast", "slow"].every( + (name) => typeof provided?.status?.get(name)?.port === "number", + ); + }, 5000); - // check each port while the graph is still running - for (const p of ps) { - if (typeof p !== "number") { - continue; - } - let ok = false; - for (let i = 0; i < 100; i++) { - try { - const status = yield* until(checkStatus(p)); - if (status === 200) { - ok = true; - break; - } - } catch (_) {} - yield* sleep(10); - } - if (!ok) { + if (!provided.status) { throw new Error( - `(examples-smoke concurrency) port ${p} did not return 200 while graph was running`, + `expected service status to be available after services started`, ); } - } - // shut down the graph to avoid hanging the test process - yield* until(destroy()); - return ps as number[]; - }); + const ps: number[] = []; + for (const name of Object.keys(provided.services)) { + const port = provided.status.get(name)?.port; + if (typeof port === "number") ps.push(port); + } - // nothing to check here; checks already happened while graph was running + assert( + ps.length > 0, + "expected at least one service port to be registered", + ); + assert.ok(ps[0], "service fast should have a port registered"); + assert.ok(ps[1], "service slow should have a port registered"); + + // check each tapped port for healthy status while graph is running + for (const p of ps) { + yield* waitForOperation(function* () { + const status = yield* until( + fetch("http://localhost:" + p + "/status"), + ); + return status.ok; + }, 2000); + } + }); + }); }); diff --git a/packages/server/test/services.test.ts b/packages/server/test/services.test.ts index 82d2dd7b..92240ab7 100644 --- a/packages/server/test/services.test.ts +++ b/packages/server/test/services.test.ts @@ -227,7 +227,7 @@ it("throws when requested subset includes a missing service", async () => { const runGraph = useServiceGraph(services); // request a service that does not exist - yield* runGraph(["missing"]); + yield* runGraph(["missing"] as any); }); }, /Requested service 'missing' not found/); }); @@ -271,8 +271,7 @@ it("runs subset specified as a string", async () => { }; const run = useServiceGraph(services); - // pass a comma-separated string - yield* run("r"); + yield* run(["r"]); yield* suspend(); }); yield* waitFor( diff --git a/packages/server/test/services/service-fast.ts b/packages/server/test/services/service-fast.ts deleted file mode 100644 index 93b1e120..00000000 --- a/packages/server/test/services/service-fast.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { simulation as genSimulation } from "../../example/services/gen-sim-factory.ts"; - -export const simulation = genSimulation(4030, 10); diff --git a/packages/server/test/services/service-slow.ts b/packages/server/test/services/service-slow.ts deleted file mode 100644 index 6c4015e5..00000000 --- a/packages/server/test/services/service-slow.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { simulation as genSimulation } from "../../example/services/gen-sim-factory.ts"; - -export const simulation = genSimulation(4040, 200); diff --git a/packages/server/test/utils.ts b/packages/server/test/utils.ts index b96168c4..5c7a8b5c 100644 --- a/packages/server/test/utils.ts +++ b/packages/server/test/utils.ts @@ -1,5 +1,5 @@ import { timebox } from "@effectionx/timebox"; -import { sleep, until } from "effection"; +import { sleep, until, type Operation } from "effection"; /** * Wait for `predicate` to become true with a timeboxed timeout. @@ -7,8 +7,8 @@ import { sleep, until } from "effection"; */ export function* waitFor( predicate: () => boolean, - timeout = 2000 -): Generator { + timeout = 2000, +): Operation { const res = yield* timebox(timeout, function* () { while (!predicate()) { yield* sleep(10); @@ -40,14 +40,14 @@ export function emitterToEventTarget(emitter: NodeJS.EventEmitter) { /** * Wait for an async predicate (returns Promise) to become true. */ -export function* waitForAsync( - predicate: () => Promise, - timeout = 2000 -): Generator { +export function* waitForOperation( + predicate: () => Operation, + timeout = 2000, +): Operation { const res = yield* timebox(timeout, function* () { while (true) { try { - const ok = yield* until(predicate()); + const ok = yield* predicate(); if (ok) return; } catch (_) { // ignore and retry @@ -56,7 +56,7 @@ export function* waitForAsync( } }); - if (res && (res as any).timeout) { + if (res && res.timeout) { throw new Error("timed out waiting for async condition"); } } diff --git a/packages/server/test/watch.test.ts b/packages/server/test/watch.test.ts index fa3038c4..3f6d3dd2 100644 --- a/packages/server/test/watch.test.ts +++ b/packages/server/test/watch.test.ts @@ -17,7 +17,7 @@ import { useServiceGraph } from "../src/services.ts"; import { simulation } from "./fixtures/simple-sim.ts"; import { useSimulation } from "../src/simulation.ts"; import { createFoundationSimulationServer } from "@simulacrum/foundation-simulator"; -import { waitFor, waitForAsync } from "./utils.ts"; +import { waitFor, waitForOperation } from "./utils.ts"; it("restarts services on watched file change and restarts dependents", async () => { const prefix = path.join(os.tmpdir(), "sim-watch-"); @@ -71,9 +71,9 @@ it("restarts services on watched file change and restarts dependents", async () }); // ensure initial trigger is readable - yield* waitForAsync(async () => { + yield* waitForOperation(function* () { try { - await fs.readFile(trigger, "utf8"); + yield* until(fs.readFile(trigger, "utf8")); return true; } catch (_) { return false; From 36922c34e4b092ffe8c76d86de4d84a1305fd1e0 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Mon, 23 Mar 2026 12:24:19 -0500 Subject: [PATCH 23/38] extend watcher timer --- packages/server/test/watch.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/test/watch.test.ts b/packages/server/test/watch.test.ts index 3f6d3dd2..03bbfdf4 100644 --- a/packages/server/test/watch.test.ts +++ b/packages/server/test/watch.test.ts @@ -87,7 +87,7 @@ it("restarts services on watched file change and restarts dependents", async () yield* until(fs.writeFile(trigger, "changed")); // wait for the raw watcher update to be observed - yield* waitFor(() => updates.length >= 1, 2000); + yield* waitFor(() => updates.length >= 1, 5000); }); // remove tmp dir From 3f87f0b5aae45a8e5e730db88ff120ce2b9c04d7 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Mon, 30 Mar 2026 13:19:15 -0500 Subject: [PATCH 24/38] bump effectionx packages --- package-lock.json | 77 ++++++++++++++++-------------------- packages/server/package.json | 10 ++--- 2 files changed, 39 insertions(+), 48 deletions(-) diff --git a/package-lock.json b/package-lock.json index 087c71e1..529d305c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -370,58 +370,55 @@ "license": "ISC" }, "node_modules/@effectionx/context-api": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/@effectionx/context-api/-/context-api-0.3.2.tgz", - "integrity": "sha512-/x4If4tiiTrg9pcJr7Jrs3z5teorzWc+qLCWBTnD3+8YpJy+8xUQXA8d/0/UkvTgNAzYSg297nomB7YiPiwuKA==", + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/@effectionx/context-api/-/context-api-0.5.3.tgz", + "integrity": "sha512-OqS/7RGZtIoiRsL6dwetKLvS8F3NLiVU3iKlBbqxI+NPKXs/ackKn294eGlHUHx49Y89fUVU6YPalj2UbxwBzA==", "license": "MIT", - "engines": { - "node": ">= 22" + "dependencies": { + "@effectionx/middleware": "0.1.1" }, "peerDependencies": { "effection": "^3 || ^4" } }, + "node_modules/@effectionx/middleware": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@effectionx/middleware/-/middleware-0.1.1.tgz", + "integrity": "sha512-ss/bZRkt/xzJNE59r8NR1+0K/xQcIyCm0y9n8FYC8jKdFn51SPe3m3t7EfPcK8zkdjCoTOU7k1UpIXRl26asYA==", + "license": "MIT" + }, "node_modules/@effectionx/node": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/@effectionx/node/-/node-0.2.2.tgz", - "integrity": "sha512-bUwnCqzBsVERGzKZRTc6XMZ6yDLkRPgcxSvM6eAkuc2D5A7L0+nSu4J/x60y6geSYrAVv2UZBgAktukNtB6LxA==", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@effectionx/node/-/node-0.2.4.tgz", + "integrity": "sha512-cPnp3fvfBKjGWekmBHdhZr5ScAr3Mg+x5IXpO8uKFe7AZ8EPAT9Di6skuB4kuGFJtRtS0Z1e5G4+2eJyapKhYA==", "license": "MIT", - "engines": { - "node": ">= 22" - }, "peerDependencies": { "effection": "^3 || ^4" } }, "node_modules/@effectionx/process": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/@effectionx/process/-/process-0.7.3.tgz", - "integrity": "sha512-ic+cv9aNeq0UgF+SO61jd8nYLiF3jFT2c6n+nOYstc4pmtUez70yFb0H5Rzppu21wql5QYCbQayVrRRUa5YRQQ==", + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/@effectionx/process/-/process-0.7.4.tgz", + "integrity": "sha512-QtT5knNw7zN2OTcTsVu/c+z+OoWQkKDulYuvjD7WJsBaizBhVYmHPV9AbPBimCtXEw7Pq1xNV/PvLQCWvEvrkA==", "license": "MIT", "dependencies": { - "@effectionx/node": "0.2.2", + "@effectionx/node": "0.2.4", "cross-spawn": "^7", "ctrlc-windows": "^2", "shellwords-ts": "^3.0.1" }, - "engines": { - "node": ">= 22" - }, "peerDependencies": { "effection": "^3 || ^4" } }, "node_modules/@effectionx/signals": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/@effectionx/signals/-/signals-0.5.2.tgz", - "integrity": "sha512-ftWU0/+LTafPjFQ4mOfwIBHT75BtA2T44YJP7+nR/d8S2kURCzkR04a2NHVbk274XRfh354qpukBULDTchZlQw==", + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/@effectionx/signals/-/signals-0.5.3.tgz", + "integrity": "sha512-AJvbUeCD5QHJ0Lc+kVtvAOZUJjXCJcOVVkRVTyOE3DPnUNQnbxVpB2o/lmBN7dnjNlGI78F4lWLMjoGZ98XRcQ==", "license": "MIT", "dependencies": { "immutable": "^5" }, - "engines": { - "node": ">= 22" - }, "peerDependencies": { "effection": "^3 || ^4" } @@ -433,19 +430,16 @@ "license": "MIT" }, "node_modules/@effectionx/stream-helpers": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@effectionx/stream-helpers/-/stream-helpers-0.8.1.tgz", - "integrity": "sha512-17rocf3av2VId8uqZJxBREhQRSaowA7+MKAbu27P3ZboIhpuIu8jqoQ1+YfKWdMy/ORrApeIljgoJl+1hJ3fvQ==", + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/@effectionx/stream-helpers/-/stream-helpers-0.8.2.tgz", + "integrity": "sha512-dWgK7ILXX6dQy2WkTajGrF2P2u0ZXUcOFcXOD5srmr8vnOnGSU7eENjnx7XPV13PF9BMzFV/n39VXz7R42FgaA==", "license": "MIT", "dependencies": { - "@effectionx/signals": "0.5.2", - "@effectionx/timebox": "0.4.2", + "@effectionx/signals": "0.5.3", + "@effectionx/timebox": "0.4.3", "immutable": "^5", "remeda": "^2" }, - "engines": { - "node": ">= 22" - }, "peerDependencies": { "effection": "^3 || ^4" } @@ -457,13 +451,10 @@ "license": "MIT" }, "node_modules/@effectionx/timebox": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/@effectionx/timebox/-/timebox-0.4.2.tgz", - "integrity": "sha512-CT1XCCXmYgxFjIJmz0kzBbDG0W1Jc/i8l4r2Hecu+gYXp22DW8bAkVPKYIDdR4acAC1flqevlWA8xWaNBUfb6w==", + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@effectionx/timebox/-/timebox-0.4.3.tgz", + "integrity": "sha512-cc7SLpL3svAYK8M5NS8kLQuL0lrZNoQb+Hi9NSaWOudzAW1HoewuDfUtfXLemPJnnLqLYhbghRhmpVqCm4Xg3Q==", "license": "MIT", - "engines": { - "node": ">= 22" - }, "peerDependencies": { "effection": "^3 || ^4" } @@ -8355,13 +8346,13 @@ "version": "0.8.0", "license": "MIT", "dependencies": { - "@effectionx/context-api": "^0.3.2", - "@effectionx/process": "^0.7.3", - "@effectionx/stream-helpers": "^0.8.1", - "@effectionx/timebox": "^0.4.2", + "@effectionx/context-api": "^0.5.3", + "@effectionx/process": "^0.7.4", + "@effectionx/stream-helpers": "^0.8.2", + "@effectionx/timebox": "^0.4.3", "chokidar": "^5.0.0", "effection": "4.0.2", - "picomatch": "^4.0.3" + "picomatch": "^4.0.4" }, "devDependencies": { "@types/picomatch": "^4.0.2" diff --git a/packages/server/package.json b/packages/server/package.json index 2e8fe2fa..9f8d498b 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -37,13 +37,13 @@ "example:concurrency": "node --import tsx ./example/concurrency-layers.ts" }, "dependencies": { - "@effectionx/context-api": "^0.3.2", - "@effectionx/process": "^0.7.3", - "@effectionx/stream-helpers": "^0.8.1", - "@effectionx/timebox": "^0.4.2", + "@effectionx/context-api": "^0.5.3", + "@effectionx/process": "^0.7.4", + "@effectionx/stream-helpers": "^0.8.2", + "@effectionx/timebox": "^0.4.3", "chokidar": "^5.0.0", "effection": "4.0.2", - "picomatch": "^4.0.3" + "picomatch": "^4.0.4" }, "devDependencies": { "@types/picomatch": "^4.0.2" From 11c468dee78fe97494dad7d1a51eb08c3a0259ab Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Tue, 31 Mar 2026 23:06:24 -0500 Subject: [PATCH 25/38] lint with new oxlint setup --- packages/server/bin/run-simulation-child.ts | 8 +- packages/server/src/simulation.ts | 2 +- packages/server/test/utils.ts | 4 +- packages/server/test/watch.test.ts | 90 +++++++++------------ 4 files changed, 44 insertions(+), 60 deletions(-) diff --git a/packages/server/bin/run-simulation-child.ts b/packages/server/bin/run-simulation-child.ts index afcd632d..c0a36123 100644 --- a/packages/server/bin/run-simulation-child.ts +++ b/packages/server/bin/run-simulation-child.ts @@ -38,7 +38,7 @@ function* normalizeSimulatorFactory(url: string) { return guardedFactory(factory); } } - } catch (err) { + } catch (ignore) { // no-op - will throw in fall through below } throw new Error("no factory or simulator instance found in module"); @@ -74,9 +74,9 @@ main(function* () { try { const res = yield* until(fetch(`http://127.0.0.1:${simulacrumPort}/data`)); initData = yield* until(res.json()); - } catch (err) { + } catch (ignore) { // ignore fetch failures - console.error("failed to fetch simulacrum data:", err); + console.error("failed to fetch simulacrum data:", ignore); } } @@ -98,7 +98,7 @@ main(function* () { if (listening && typeof listening.ensureClose === "function") { yield* until(listening.ensureClose()); } - } catch (err) { + } catch (ignore) { // ignore } } diff --git a/packages/server/src/simulation.ts b/packages/server/src/simulation.ts index 717ccb7f..84fe226f 100644 --- a/packages/server/src/simulation.ts +++ b/packages/server/src/simulation.ts @@ -125,7 +125,7 @@ export function useChildSimulation(name: string, modulePath: string) { } else { yield* logger.stdout(str); } - } catch (_) { + } catch (ignore) { // just log lines that are not JSON yield* logger.stdout(str); } diff --git a/packages/server/test/utils.ts b/packages/server/test/utils.ts index 296b103c..f53446dc 100644 --- a/packages/server/test/utils.ts +++ b/packages/server/test/utils.ts @@ -46,7 +46,7 @@ export function* waitForOperation( try { const ok = yield* predicate(); if (ok) return; - } catch (_) { + } catch (ignore) { // ignore and retry } yield* sleep(10); @@ -68,7 +68,7 @@ export function* waitForFetchClosed(url: string, timeout = 2000) { try { const s = yield* until(fetch(url)); if (!s.ok) return; - } catch (_) { + } catch (ignore) { return; } yield* sleep(10); diff --git a/packages/server/test/watch.test.ts b/packages/server/test/watch.test.ts index 6379f635..133cb722 100644 --- a/packages/server/test/watch.test.ts +++ b/packages/server/test/watch.test.ts @@ -39,19 +39,15 @@ it("restarts services on watched file change and restarts dependents", async () { watch: true, watchDebounce: 20 }, ); - try { - const services = yield* op(); - // subscribe to the immediate raw serviceChanges stream and wait for the first update - if (!services.serviceChanges) throw new Error("serviceChanges not available"); - const subscription = yield* services.serviceChanges; - subscriptionReady.resolve(); - - // wait for the first raw update (will occur after the test touches the file) - const first = yield* subscription.next(); - updates.push(String((first.value as { service: string }).service)); - } catch (e) { - throw e; - } + const services = yield* op(); + // subscribe to the immediate raw serviceChanges stream and wait for the first update + if (!services.serviceChanges) throw new Error("serviceChanges not available"); + const subscription = yield* services.serviceChanges; + subscriptionReady.resolve(); + + // wait for the first raw update (will occur after the test touches the file) + const first = yield* subscription.next(); + updates.push(String((first.value as { service: string }).service)); yield* suspend(); }); @@ -61,7 +57,7 @@ it("restarts services on watched file change and restarts dependents", async () try { yield* until(fs.readFile(trigger, "utf8")); return true; - } catch (_) { + } catch (ignore) { return false; } }, 2000); @@ -113,11 +109,7 @@ it("restarts dependents when watched service changes", async () => { { watch: true, watchDebounce: 20 }, ); - try { - yield* op(); - } catch (e) { - throw e; - } + yield* op(); yield* suspend(); }); @@ -175,11 +167,7 @@ it("restarts transitive dependents when watched service changes", async () => { { watch: true, watchDebounce: 20 }, ); - try { - yield* op(); - } catch (e) { - throw e; - } + yield* op(); yield* suspend(); }); @@ -269,35 +257,31 @@ it("debounces rapid changes per service", async () => { { watch: true, watchDebounce: 150 }, ); - try { - const services = yield* op(); - if (!services.serviceUpdates || !services.serviceChanges) - throw new Error("service streams not available"); - const debSub = yield* services.serviceUpdates; - const rawSub = yield* services.serviceChanges; - - watcherReady.resolve(); - - // collect debounced updates - yield* spawn(function* () { - while (true) { - const n = yield* debSub.next(); - if (n.done) break; - updates.push((n.value as { service: string }).service); - } - }); - - // count raw updates (should reflect every write) - yield* spawn(function* () { - while (true) { - const n = yield* rawSub.next(); - if (n.done) break; - if ((n.value as { service: string }).service === "a") rawCount++; - } - }); - } catch (e) { - throw e; - } + const services = yield* op(); + if (!services.serviceUpdates || !services.serviceChanges) + throw new Error("service streams not available"); + const debSub = yield* services.serviceUpdates; + const rawSub = yield* services.serviceChanges; + + watcherReady.resolve(); + + // collect debounced updates + yield* spawn(function* () { + while (true) { + const n = yield* debSub.next(); + if (n.done) break; + updates.push((n.value as { service: string }).service); + } + }); + + // count raw updates (should reflect every write) + yield* spawn(function* () { + while (true) { + const n = yield* rawSub.next(); + if (n.done) break; + if ((n.value as { service: string }).service === "a") rawCount++; + } + }); yield* suspend(); }); From 9e8cce1d9d91867565f9490ddcc0086bb259251f Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Tue, 31 Mar 2026 23:25:15 -0500 Subject: [PATCH 26/38] foundation sim as devDep --- packages/server/package.json | 1 + pnpm-lock.yaml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/packages/server/package.json b/packages/server/package.json index 518b5a5f..914dedf2 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -72,6 +72,7 @@ "picomatch": "^4.0.4" }, "devDependencies": { + "@simulacrum/foundation-simulator": "workspace:^", "@types/picomatch": "^4.0.2" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 760e49e4..436897c9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -205,6 +205,9 @@ importers: specifier: ^4.0.4 version: 4.0.4 devDependencies: + '@simulacrum/foundation-simulator': + specifier: workspace:^ + version: link:../foundation '@types/picomatch': specifier: ^4.0.2 version: 4.0.2 From 1d41e2fe434ea59d0f60ecd70414cbd3fa601caf Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Tue, 31 Mar 2026 23:47:31 -0500 Subject: [PATCH 27/38] fix test tsx and path refs --- packages/server/example/process-graph.ts | 64 +++++++++++++----------- packages/server/package.json | 2 +- packages/server/src/simulation.ts | 7 ++- packages/server/test/service.test.ts | 2 +- packages/server/test/services.test.ts | 10 +++- 5 files changed, 52 insertions(+), 33 deletions(-) diff --git a/packages/server/example/process-graph.ts b/packages/server/example/process-graph.ts index 0af0e92a..46303221 100644 --- a/packages/server/example/process-graph.ts +++ b/packages/server/example/process-graph.ts @@ -6,45 +6,53 @@ import { simulationCLI } from "../src/cli.ts"; const servicesMap = { A: { - operation: useService("A", "node --import tsx ./example/services/basic-start-1.ts", { - wellnessCheck: { - frequency: 10, - timeout: 15000, - *operation(stdio: Stream) { - for (let line of yield* each(stdio)) { - if (line.includes("started")) { - console.log("A ready (wellnessCheck)"); + operation: useService( + "A", + "node --experimental-transform-types ./example/services/basic-start-1.ts", + { + wellnessCheck: { + frequency: 10, + timeout: 15000, + *operation(stdio: Stream) { + for (let line of yield* each(stdio)) { + if (line.includes("started")) { + console.log("A ready (wellnessCheck)"); - return { ok: true, value: undefined }; + return { ok: true, value: undefined }; + } + yield* each.next(); } - yield* each.next(); - } - // default: return success so the result type is well-formed - return { ok: true, value: undefined }; + // default: return success so the result type is well-formed + return { ok: true, value: undefined }; + }, }, }, - }), + ), }, B: { dependsOn: { startup: ["A"] as const }, - operation: useService("B", "node --import tsx ./example/services/basic-start-2.ts", { - wellnessCheck: { - frequency: 10, - timeout: 15000, - *operation(stdio: Stream) { - for (let line of yield* each(stdio)) { - if (line.includes("started")) { - console.log("B ready (wellnessCheck)"); + operation: useService( + "B", + "node --experimental-transform-types ./example/services/basic-start-2.ts", + { + wellnessCheck: { + frequency: 10, + timeout: 15000, + *operation(stdio: Stream) { + for (let line of yield* each(stdio)) { + if (line.includes("started")) { + console.log("B ready (wellnessCheck)"); - return { ok: true, value: undefined }; + return { ok: true, value: undefined }; + } + yield* each.next(); } - yield* each.next(); - } - // default: return success so the result type is well-formed - return { ok: true, value: undefined }; + // default: return success so the result type is well-formed + return { ok: true, value: undefined }; + }, }, }, - }), + ), }, }; diff --git a/packages/server/package.json b/packages/server/package.json index 914dedf2..77800119 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -59,7 +59,7 @@ "lint": "oxlint", "prepack": "pnpm run build", "test": "node --test-timeout=60000 --experimental-transform-types --test test/*.test.ts", - "test:service-main": "node --experimental-transform-types ./test/service-main.ts", + "test:service-main": "node --experimental-transform-types ./test/services/service-main.ts", "tsc": "tsc --noEmit" }, "dependencies": { diff --git a/packages/server/src/simulation.ts b/packages/server/src/simulation.ts index 84fe226f..790d5a5c 100644 --- a/packages/server/src/simulation.ts +++ b/packages/server/src/simulation.ts @@ -89,7 +89,12 @@ export function useChildSimulation(name: string, modulePath: string) { // attempt to read the simulacrum port from context; if not present, continue without it const contextPort = yield* SimulacrumEndpoint.get(); - const parts = ["node", "--import", "tsx", "./bin/run-simulation-child.ts", modulePath]; + const parts = [ + "node", + "--experimental-transform-types", + "./bin/run-simulation-child.ts", + modulePath, + ]; if (typeof contextPort === "number") { parts.push("--simulacrum-port", String(contextPort)); } diff --git a/packages/server/test/service.test.ts b/packages/server/test/service.test.ts index 02d4354b..10a786ec 100644 --- a/packages/server/test/service.test.ts +++ b/packages/server/test/service.test.ts @@ -5,7 +5,7 @@ import { each, Err, Ok, run } from "effection"; // these npm scripts don't work, but this is what we are trying to run // const scriptDoesNotWork = "npm run test:service-main"; -const nodeScriptWorks = "node --experimental-transform-types ./test/service-main.ts"; +const nodeScriptWorks = "node --experimental-transform-types ./test/services/service-main.ts"; it("test service starts and prints expected startup message", async () => { let sawStart = false; diff --git a/packages/server/test/services.test.ts b/packages/server/test/services.test.ts index 4177855a..f095b2c2 100644 --- a/packages/server/test/services.test.ts +++ b/packages/server/test/services.test.ts @@ -49,11 +49,17 @@ it("throws on cycles in dependency graph", async () => { await run(function* () { const runGraph = useServiceGraph({ A: { - operation: useService("A", "node --import tsx ./test/services/service-a.ts"), + operation: useService( + "A", + "node --experimental-transform-types ./test/services/service-a.ts", + ), dependsOn: { startup: ["B"] as const }, }, B: { - operation: useService("B", "node --import tsx ./test/services/service-b.ts"), + operation: useService( + "B", + "node --experimental-transform-types ./test/services/service-b.ts", + ), dependsOn: { startup: ["A"] as const }, }, }); From bf2b1eb1e0429ad916e9815215f76fc3e20a9ff5 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Wed, 1 Apr 2026 00:41:20 -0500 Subject: [PATCH 28/38] watch startup waits --- packages/server/test/watch.test.ts | 42 +++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/packages/server/test/watch.test.ts b/packages/server/test/watch.test.ts index 133cb722..c709e63c 100644 --- a/packages/server/test/watch.test.ts +++ b/packages/server/test/watch.test.ts @@ -86,6 +86,8 @@ it("restarts dependents when watched service changes", async () => { await fs.writeFile(trigger, "initial"); const startCounts: Record = { a: 0, b: 0 }; + let provided: { status?: Map } | undefined; + const servicesReady = withResolvers(); await run(function* () { yield* spawn(function* () { @@ -109,7 +111,8 @@ it("restarts dependents when watched service changes", async () => { { watch: true, watchDebounce: 20 }, ); - yield* op(); + provided = yield* op(); + servicesReady.resolve(); yield* suspend(); }); @@ -117,11 +120,21 @@ it("restarts dependents when watched service changes", async () => { // wait for initial startup yield* waitFor(() => startCounts.a > 0 && startCounts.b > 0, 2000); + // ensure we have access to status state from the running graph + yield* servicesReady.operation; + const beforeStartupA = provided?.status?.get("a")?.startup; + + const beforeA = startCounts.a; + const beforeB = startCounts.b; + // trigger a change - yield* until(fs.writeFile(trigger, "changed")); + yield* until(fs.writeFile(trigger, `changed-${Date.now()}`)); + + // watcher restart processing rotates the service's startup resolver + yield* waitFor(() => provided?.status?.get("a")?.startup !== beforeStartupA, 5000); // wait for restarts to occur - yield* waitFor(() => startCounts.a >= 2 && startCounts.b >= 2, 3000); + yield* waitFor(() => startCounts.a > beforeA && startCounts.b > beforeB, 10000); }); await fs.rm(dir, { recursive: true, force: true }); @@ -137,6 +150,8 @@ it("restarts transitive dependents when watched service changes", async () => { await fs.writeFile(trigger, "initial"); const startCounts: Record = { a: 0, b: 0, c: 0 }; + let provided: { status?: Map } | undefined; + const servicesReady = withResolvers(); await run(function* () { yield* spawn(function* () { @@ -167,7 +182,8 @@ it("restarts transitive dependents when watched service changes", async () => { { watch: true, watchDebounce: 20 }, ); - yield* op(); + provided = yield* op(); + servicesReady.resolve(); yield* suspend(); }); @@ -175,11 +191,25 @@ it("restarts transitive dependents when watched service changes", async () => { // wait for initial startup yield* waitFor(() => startCounts.a > 0 && startCounts.b > 0 && startCounts.c > 0, 2000); + // ensure we have access to status state from the running graph + yield* servicesReady.operation; + const beforeStartupA = provided?.status?.get("a")?.startup; + + const beforeA = startCounts.a; + const beforeB = startCounts.b; + const beforeC = startCounts.c; + // trigger a change - yield* until(fs.writeFile(trigger, "changed")); + yield* until(fs.writeFile(trigger, `changed-${Date.now()}`)); + + // watcher restart processing rotates the service's startup resolver + yield* waitFor(() => provided?.status?.get("a")?.startup !== beforeStartupA, 5000); // wait for restarts to occur - yield* waitFor(() => startCounts.a >= 2 && startCounts.b >= 2 && startCounts.c >= 2, 3000); + yield* waitFor( + () => startCounts.a > beforeA && startCounts.b > beforeB && startCounts.c > beforeC, + 10000, + ); }); await fs.rm(dir, { recursive: true, force: true }); From b32c54203ea5c11610d3b26d5bf41afbf8c30a92 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Wed, 1 Apr 2026 08:55:41 -0500 Subject: [PATCH 29/38] fuzzy watch on loop --- packages/server/test/watch.test.ts | 54 +++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/packages/server/test/watch.test.ts b/packages/server/test/watch.test.ts index c709e63c..0a71ac9f 100644 --- a/packages/server/test/watch.test.ts +++ b/packages/server/test/watch.test.ts @@ -126,15 +126,26 @@ it("restarts dependents when watched service changes", async () => { const beforeA = startCounts.a; const beforeB = startCounts.b; + let observed = false; + for (let attempt = 0; attempt < 6 && !observed; attempt++) { + // trigger a change (unique content each time to ensure chokidar sees an update) + yield* until(fs.writeFile(trigger, `changed-${Date.now()}-${attempt}`)); - // trigger a change - yield* until(fs.writeFile(trigger, `changed-${Date.now()}`)); + try { + // watcher restart processing rotates the service's startup resolver + yield* waitFor(() => provided?.status?.get("a")?.startup !== beforeStartupA, 2000); - // watcher restart processing rotates the service's startup resolver - yield* waitFor(() => provided?.status?.get("a")?.startup !== beforeStartupA, 5000); + // wait for restarts to occur + yield* waitFor(() => startCounts.a > beforeA && startCounts.b > beforeB, 2000); + observed = true; + } catch (ignore) { + // retry with another write; CI can occasionally miss/coalesce single events + } + } - // wait for restarts to occur - yield* waitFor(() => startCounts.a > beforeA && startCounts.b > beforeB, 10000); + if (!observed) { + throw new Error("timed out waiting for dependent restart after file changes"); + } }); await fs.rm(dir, { recursive: true, force: true }); @@ -198,18 +209,29 @@ it("restarts transitive dependents when watched service changes", async () => { const beforeA = startCounts.a; const beforeB = startCounts.b; const beforeC = startCounts.c; + let observed = false; + for (let attempt = 0; attempt < 6 && !observed; attempt++) { + // trigger a change (unique content each time to ensure chokidar sees an update) + yield* until(fs.writeFile(trigger, `changed-${Date.now()}-${attempt}`)); - // trigger a change - yield* until(fs.writeFile(trigger, `changed-${Date.now()}`)); - - // watcher restart processing rotates the service's startup resolver - yield* waitFor(() => provided?.status?.get("a")?.startup !== beforeStartupA, 5000); + try { + // watcher restart processing rotates the service's startup resolver + yield* waitFor(() => provided?.status?.get("a")?.startup !== beforeStartupA, 2000); + + // wait for transitive restarts to occur + yield* waitFor( + () => startCounts.a > beforeA && startCounts.b > beforeB && startCounts.c > beforeC, + 2000, + ); + observed = true; + } catch (ignore) { + // retry with another write; CI can occasionally miss/coalesce single events + } + } - // wait for restarts to occur - yield* waitFor( - () => startCounts.a > beforeA && startCounts.b > beforeB && startCounts.c > beforeC, - 10000, - ); + if (!observed) { + throw new Error("timed out waiting for transitive restart after file changes"); + } }); await fs.rm(dir, { recursive: true, force: true }); From 872388edb084b65a624e2fae7c8deb8878e87965 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Thu, 2 Apr 2026 14:09:54 -0500 Subject: [PATCH 30/38] deprioritize initial store updates --- ...ize-initial-store-updates-in-foundation.md | 5 +++ packages/foundation/src/index.ts | 32 +++++++++++-------- packages/foundation/src/store/index.ts | 7 ++-- packages/server/src/simulation.ts | 3 +- 4 files changed, 29 insertions(+), 18 deletions(-) create mode 100644 .changes/deprioritize-initial-store-updates-in-foundation.md diff --git a/.changes/deprioritize-initial-store-updates-in-foundation.md b/.changes/deprioritize-initial-store-updates-in-foundation.md new file mode 100644 index 00000000..3aaf5ff9 --- /dev/null +++ b/.changes/deprioritize-initial-store-updates-in-foundation.md @@ -0,0 +1,5 @@ +--- +"@simulacrum/foundation-simulator": patch:bug +--- + +Order initial log dispatching to help avoid the race condition made more prevalent by more direct sync vs async handling in effection v4. diff --git a/packages/foundation/src/index.ts b/packages/foundation/src/index.ts index 9ab4e012..659534d7 100644 --- a/packages/foundation/src/index.ts +++ b/packages/foundation/src/index.ts @@ -42,7 +42,7 @@ import { apiProxy } from "./middleware/proxy.ts"; import { delayMiddleware } from "./middleware/delay.ts"; import { generateRoutesHTML } from "./routeTemplate.ts"; import { createAppServer } from "./server.ts"; -import type { IdProp } from "starfx"; +import { sleep, type AnyState, type IdProp } from "starfx"; // for use in the OpenAPI handler functions type SimulationHandlerFunctions = ( @@ -170,6 +170,7 @@ export function createFoundationSimulationServer< app.use(express.urlencoded({ extended: false })); let simulationStore = createSimulationStore(extendStore); + const simulationRoutes = [] as ((s: AnyState) => void)[]; app.use(delayMiddleware(delayResponses)); app.use((req, _res, next) => { @@ -193,7 +194,6 @@ export function createFoundationSimulationServer< .map((stack) => stack.route) .filter((route): route is NonNullable => Boolean(route)); - const simulationRoutes = []; for (let layer of layers) { for (let stack of layer.stack) { simulationRoutes.push( @@ -210,8 +210,6 @@ export function createFoundationSimulationServer< ); } } - - simulationStore.store.dispatch(simulationStore.actions.batchUpdater(simulationRoutes)); } } @@ -224,7 +222,6 @@ export function createFoundationSimulationServer< .sync(); if (jsonFiles.length > 0) { - const simulationRoutes = []; for (let jsonFile of jsonFiles) { const route = `/${jsonFile.slice(0, jsonFile.length - 5)}`; const filename = path.join(serveJsonFiles, jsonFile); @@ -246,8 +243,6 @@ export function createFoundationSimulationServer< }), ); } - - simulationStore.store.dispatch(simulationStore.actions.batchUpdater(simulationRoutes)); } } @@ -324,11 +319,11 @@ export function createFoundationSimulationServer< }, }); - // initalize the backend + // initialize the backend api.init().then((init) => { const router = init.router; const operations = router.getOperations(); - const simulationRoutes = operations.reduce( + const oasSimulationRoutes = operations.reduce( (routes, operation) => { const url = `${router.apiRoot === "/" ? "" : router.apiRoot}${operation.path}`; routes[`${operation.method}:${url}`] = { @@ -343,11 +338,7 @@ export function createFoundationSimulationServer< }, {} as Record, ); - simulationStore.store.dispatch( - simulationStore.actions.batchUpdater([ - simulationStore.schema.simulationRoutes.add(simulationRoutes), - ]), - ); + simulationRoutes.push(simulationStore.schema.simulationRoutes.add(oasSimulationRoutes)); return init; }); app.use((req, res, next) => { @@ -393,7 +384,20 @@ export function createFoundationSimulationServer< // if no extendRouter routes or openapi routes handle this, return 404 app.all("/{*splat}", (_req, res) => res.status(404).json({ error: "not found" })); + // wait to start passing route records to give the store a moment to register + // which is technically race-y but in practice should be fine + // as we don't have a way to wait currently + simulationStore.store + .run(function* () { + // forces some async work to allow children scopes to start, e.g. the thunks + yield* sleep(0); + }) + .then(() => { + simulationStore.store.dispatch(simulationStore.actions.batchUpdater(simulationRoutes)); + }); + const genericAppServer = createAppServer(app, protocol); + return { listen: async (...listenArgs: Parameters | undefined[]) => { // over and above the `net` listen behavior, allow setting: diff --git a/packages/foundation/src/store/index.ts b/packages/foundation/src/store/index.ts index d57358df..d2d190d0 100644 --- a/packages/foundation/src/store/index.ts +++ b/packages/foundation/src/store/index.ts @@ -178,8 +178,9 @@ export function createSimulationStore< const route = yield* select(schema.simulationRoutes.selectById, { id, }); - if (route.url !== "") + if (route.url !== "") { yield* schema.update(schema.simulationRoutes.merge({ [id]: { calls: route.calls + 1 } })); + } yield* next(); }); @@ -202,11 +203,11 @@ export function createSimulationStore< }; const userTasks = inputTasks({ createWebhook, store, schema }); - let inputedActions = inputActions({ thunks, store, schema }); + let inputtedActions = inputActions({ thunks, store, schema }); let actions = { simulationLog, batchUpdater, - ...inputedActions, + ...inputtedActions, ...userTasks.actions, }; diff --git a/packages/server/src/simulation.ts b/packages/server/src/simulation.ts index 790d5a5c..b7c5fa90 100644 --- a/packages/server/src/simulation.ts +++ b/packages/server/src/simulation.ts @@ -91,7 +91,8 @@ export function useChildSimulation(name: string, modulePath: string) { const parts = [ "node", - "--experimental-transform-types", + // safest considering current LTS of >v20 + "--experimental-strip-types", "./bin/run-simulation-child.ts", modulePath, ]; From 5513e85c9343cf1a397ba40290e5989f5866e0cc Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Fri, 17 Apr 2026 17:08:53 -0500 Subject: [PATCH 31/38] bump effectionx packages --- packages/server/package.json | 10 ++-- pnpm-lock.yaml | 93 ++++++++++++++++++++++++------------ 2 files changed, 67 insertions(+), 36 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 77800119..b33a2fd7 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -63,16 +63,16 @@ "tsc": "tsc --noEmit" }, "dependencies": { - "@effectionx/context-api": "^0.2.1", - "@effectionx/process": "^0.6.2", - "@effectionx/stream-helpers": "^0.8.2", - "@effectionx/timebox": "^0.4.3", + "@effectionx/context-api": "~0.6.0", + "@effectionx/process": "~0.8.0", + "@effectionx/stream-helpers": "~0.8.2", + "@effectionx/timebox": "~0.4.3", "chokidar": "^5.0.0", "effection": "^4.0.2", "picomatch": "^4.0.4" }, "devDependencies": { "@simulacrum/foundation-simulator": "workspace:^", - "@types/picomatch": "^4.0.2" + "@types/picomatch": "^4.0.3" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 436897c9..ddaa91b1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -184,16 +184,16 @@ importers: packages/server: dependencies: '@effectionx/context-api': - specifier: ^0.2.1 - version: 0.2.1 + specifier: ~0.6.0 + version: 0.6.0(effection@4.0.2) '@effectionx/process': - specifier: ^0.6.2 - version: 0.6.2 + specifier: ~0.8.0 + version: 0.8.0(effection@4.0.2) '@effectionx/stream-helpers': - specifier: ^0.8.2 + specifier: ~0.8.2 version: 0.8.2(effection@4.0.2) '@effectionx/timebox': - specifier: ^0.4.3 + specifier: ~0.4.3 version: 0.4.3(effection@4.0.2) chokidar: specifier: ^5.0.0 @@ -209,8 +209,8 @@ importers: specifier: workspace:^ version: link:../foundation '@types/picomatch': - specifier: ^4.0.2 - version: 4.0.2 + specifier: ^4.0.3 + version: 4.0.3 packages/ui: {} @@ -337,13 +337,33 @@ packages: '@braidai/lang@1.1.2': resolution: {integrity: sha512-qBcknbBufNHlui137Hft8xauQMTZDKdophmLFv05r2eNmdIv/MlPuP4TdUknHG68UdWLgVZwgxVe735HzJNIwA==} - '@effectionx/context-api@0.2.1': - resolution: {integrity: sha512-UCEmk/uibrx4PvUh/tm1SWy6e6rAaq2BzpEDLu5XJZ/LaRdazv5VxHqg4F3N+G4AcxgPvosprjS0j3dDSTCzFg==} - engines: {node: '>= 16'} + '@effectionx/context-api@0.5.3': + resolution: {integrity: sha512-OqS/7RGZtIoiRsL6dwetKLvS8F3NLiVU3iKlBbqxI+NPKXs/ackKn294eGlHUHx49Y89fUVU6YPalj2UbxwBzA==} + peerDependencies: + effection: ^3 || ^4 - '@effectionx/process@0.6.2': - resolution: {integrity: sha512-U94gqTNXASRw8KBJOtSE+MaWL09Tox7la9/rmJCzUpaLWSmrSOvH28NCv++PKKy8qNCErD+QQip5q+E8lARNEQ==} - engines: {node: '>= 16'} + '@effectionx/context-api@0.6.0': + resolution: {integrity: sha512-t004qvlkJDMB6EhHP1lOQ97PeIn90m7cv4+wsRPnx4YBem+pJzTL+Sm1KWbKMjMeFJz4oqllUWuBJZsCi+nuTw==} + peerDependencies: + effection: ^3 || ^4 + + '@effectionx/middleware@0.1.1': + resolution: {integrity: sha512-ss/bZRkt/xzJNE59r8NR1+0K/xQcIyCm0y9n8FYC8jKdFn51SPe3m3t7EfPcK8zkdjCoTOU7k1UpIXRl26asYA==} + + '@effectionx/node@0.2.4': + resolution: {integrity: sha512-cPnp3fvfBKjGWekmBHdhZr5ScAr3Mg+x5IXpO8uKFe7AZ8EPAT9Di6skuB4kuGFJtRtS0Z1e5G4+2eJyapKhYA==} + peerDependencies: + effection: ^3 || ^4 + + '@effectionx/process@0.8.0': + resolution: {integrity: sha512-vwbK48JJl+I5t2G+eQyoLCoTp9IU3zkhGd+LmdxIS9PZp/v/j5Kq13eN4FvrZGx/KBYgibr57RBzu3F+Uf+uNw==} + peerDependencies: + effection: ^3 || ^4 + + '@effectionx/scope-eval@0.1.3': + resolution: {integrity: sha512-Acn45lb3H94WYhNVHXYtXOZYzjpBGDPPlsyW1Talb/vYjQzCzus5lkxxOlPyphzvi7d+7mGNXiIVt4JLSZmLnQ==} + peerDependencies: + effection: ^3 || ^4 '@effectionx/signals@0.5.3': resolution: {integrity: sha512-AJvbUeCD5QHJ0Lc+kVtvAOZUJjXCJcOVVkRVTyOE3DPnUNQnbxVpB2o/lmBN7dnjNlGI78F4lWLMjoGZ98XRcQ==} @@ -1410,9 +1430,6 @@ packages: '@types/cors@2.8.19': resolution: {integrity: sha512-mFNylyeyqN93lfe/9CSxOGREz8cpzAhH+E93xJ4xWQf62V8sQ/24reV2nyzUWM6H6Xji+GGHpkbLe7pVoUEskg==} - '@types/cross-spawn@6.0.6': - resolution: {integrity: sha512-fXRhhUkG4H3TQk5dBhQ7m/JDdSNHKwR2BBia62lhwEIq9xGiQKLxd6LymNhn47SjXhsUEPmxi+PKw2OkW4LLjA==} - '@types/deep-eql@4.0.2': resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} @@ -1458,8 +1475,8 @@ packages: '@types/node@25.5.0': resolution: {integrity: sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw==} - '@types/picomatch@4.0.2': - resolution: {integrity: sha512-qHHxQ+P9PysNEGbALT8f8YOSHW0KJu6l2xU8DYY0fu/EmGxXdVnuTLvFUvBgPJMSqXq29SYHveejeAha+4AYgA==} + '@types/picomatch@4.0.3': + resolution: {integrity: sha512-iG0T6+nYJ9FAPmx9SsUlnwcq1ZVRuCXcVEvWnntoPlrOpwtSTKNDC9uVAxTsC3PUvJ+99n4RpAcNgBbHX3JSnQ==} '@types/qs@6.15.0': resolution: {integrity: sha512-JawvT8iBVWpzTrz3EGw9BTQFg3BQNmwERdKE22vlTxawwtbyUSlMppvZYKLZzB5zgACXdXxbD3m1bXaMqP/9ow==} @@ -2830,8 +2847,8 @@ packages: resolution: {integrity: sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==} engines: {node: '>= 0.4'} - shellwords@1.1.1: - resolution: {integrity: sha512-LzESUkEHUuFbjaE7j8uyIjKvySfSFvCF6G4WOygjwSwQj3VuX8hr+v4M252B3twEct6XTWrrNSFu74mTlx4uAQ==} + shellwords-ts@3.0.1: + resolution: {integrity: sha512-GabK4ApLMqHFRGlpgNqg8dmtHTnYHt0WUUJkIeMd3QaDrUUBEDXHSSNi3I0PzMimg8W+I0EN4TshQxsnHv1cwg==} side-channel-list@1.0.0: resolution: {integrity: sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==} @@ -3429,17 +3446,35 @@ snapshots: '@braidai/lang@1.1.2': optional: true - '@effectionx/context-api@0.2.1': + '@effectionx/context-api@0.5.3(effection@4.0.2)': + dependencies: + '@effectionx/middleware': 0.1.1 + effection: 4.0.2 + + '@effectionx/context-api@0.6.0(effection@4.0.2)': + dependencies: + '@effectionx/middleware': 0.1.1 + effection: 4.0.2 + + '@effectionx/middleware@0.1.1': {} + + '@effectionx/node@0.2.4(effection@4.0.2)': dependencies: effection: 4.0.2 - '@effectionx/process@0.6.2': + '@effectionx/process@0.8.0(effection@4.0.2)': dependencies: - '@types/cross-spawn': 6.0.6 + '@effectionx/context-api': 0.5.3(effection@4.0.2) + '@effectionx/node': 0.2.4(effection@4.0.2) + '@effectionx/scope-eval': 0.1.3(effection@4.0.2) cross-spawn: 7.0.6 ctrlc-windows: 2.2.0 effection: 4.0.2 - shellwords: 1.1.1 + shellwords-ts: 3.0.1 + + '@effectionx/scope-eval@0.1.3(effection@4.0.2)': + dependencies: + effection: 4.0.2 '@effectionx/signals@0.5.3(effection@4.0.2)': dependencies: @@ -4490,10 +4525,6 @@ snapshots: dependencies: '@types/node': 24.12.0 - '@types/cross-spawn@6.0.6': - dependencies: - '@types/node': 24.12.0 - '@types/deep-eql@4.0.2': {} '@types/estree@1.0.8': {} @@ -4543,7 +4574,7 @@ snapshots: undici-types: 7.18.2 optional: true - '@types/picomatch@4.0.2': {} + '@types/picomatch@4.0.3': {} '@types/qs@6.15.0': {} @@ -6016,7 +6047,7 @@ snapshots: shell-quote@1.8.3: {} - shellwords@1.1.1: {} + shellwords-ts@3.0.1: {} side-channel-list@1.0.0: dependencies: From bc603b70143482c6aa0c9dd88c50830a552b4505 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Wed, 22 Apr 2026 15:25:42 -0500 Subject: [PATCH 32/38] combine into single useSimulation --- .vscode/settings.json | 3 - packages/server/README.md | 335 +++++++++---- packages/server/example/README.md | 10 +- packages/server/example/concurrency-layers.ts | 6 +- packages/server/example/simulation-graph.ts | 6 +- packages/server/package.json | 3 + packages/server/src/index.ts | 1 + packages/server/src/operation-metadata.ts | 25 + packages/server/src/service.ts | 132 ++--- packages/server/src/services.ts | 27 +- packages/server/src/simulation.ts | 321 +++++++----- packages/server/test/child-simulation.test.ts | 28 +- packages/server/test/examples-smoke.test.ts | 18 + packages/server/test/services.test.ts | 472 ++++++++++-------- packages/server/test/watch.test.ts | 4 +- 15 files changed, 855 insertions(+), 536 deletions(-) create mode 100644 packages/server/src/operation-metadata.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index 8f7c6ef4..56929129 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,7 +1,4 @@ { - "typescript.tsdk": "node_modules/typescript/lib", - "oxc.useExecPath": true, - "oxc.fmt.configPath": ".oxfmtrc.json", "editor.defaultFormatter": "oxc.oxc-vscode", "editor.formatOnSave": false, // disable default behavior "editor.codeActionsOnSave": { diff --git a/packages/server/README.md b/packages/server/README.md index d2695881..12ae25a1 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -1,64 +1,67 @@ # @simulacrum/server -Server capable of running multiple concurrent simulations that can be controlled by test cases, preview apps, and local developer environments. +Define one graph of simulations and processes, then run that same graph from the CLI, your tests, or a local development harness. + +`@simulacrum/server` is for the case where you want to describe a simulation system once and reuse it across: + +- local developer workflows +- integration and end-to-end tests +- preview or test harness environments https://github.com/thefrontside/simulacrum ## Getting Started -Set up a process or simulators such as this example `service-graph.ts`. +Start by defining a service graph once, then choose how to run it. ```ts service-graph.ts #!/usr/bin/env node -import { run } from "effection"; -import { - useServiceGraph, - simulationCLI, - useChildSimulation, - useSimulation, - useService, -} from "@simulacrum/server"; +import { useServiceGraph, simulationCLI, useSimulation, useService } from "@simulacrum/server"; import { simulation } from "./sim2.ts"; // define your "graph" that can be used through a CLI or as part of a test rig export const services = useServiceGraph( { sim1: { - operation: useChildSimulation("sim-run-as-child-process", "./sim1.ts"), + operation: useSimulation("sim-run-as-child-process", "./sim1.ts"), }, sim2: { operation: useSimulation("sim-run-in-same-process", simulation), }, sim3: { - operation: useService("arbitray-child-process", "node --import tsx ./sim3.ts"), + operation: useService("arbitrary-child-process", "node --import tsx ./sim3.ts"), }, }, - { globalData: { hello: "world" } }, + { globalData: { hello: "world" } }, // passed readonly to each simulator ); // this is a helper function which will give you a CLI around this service graph -// if you are calling this file directly +// if you are calling this file directly, e.g. `node service-graph.ts` import { fileURLToPath } from "node:url"; if (process.argv[1] === fileURLToPath(import.meta.url)) { simulationCLI(services); } ``` -From this, you have two main entry points. One may start it directly from your shell. +Once you have that file, there are two common ways to use it. + +### Run from the shell ```bash # start a local service graph defined in ./service-graph.ts -node --import tsx ./simulators/service-graph.ts +node ./simulators/service-graph.ts ``` > [!NOTE] -> We use `--import tsx` here to automatically handle the typescript conversion. This is a separate package that you may be interested in using, but it not a hard requirement necessarily. +> We use `node --import tsx file.ts` here to automatically handle the typescript conversion. This is a separate package that you may be interested in using, but it not a hard requirement necessarily. Newer versions of node also handle this for you. The latest v24 will run a typescript file directly. -Secondly, we can use this in tests. It is convenient to place in an `beforeAll()` or a `beforeEach()`. This is built on `effection`, and should handle all shutdown and clean up of the services when the function passes out of lexical scope. +### Run from a test + +If you are already working with `effection`, you may use the operation directly. ```ts -import { test, beforeEach } from "test-runner"; -import { run } from "effection"; +import { beforeEach } from "node:test"; +import { run, suspend } from "effection"; import { services } from "./simulators/service-graph.ts"; beforeEach(async () => { @@ -71,61 +74,169 @@ beforeEach(async () => { }); test("things", async () => { - // do testing things here + // run your assertions against the graph state +}); +``` + +If you are outside an `effection` scope, we include a convenience method to use the runner's `.task` property to `await` it like a promise. + +```ts +import { beforeEach } from "node:test"; +import { services } from "./simulators/service-graph.ts"; + +let graph; +beforeEach(async () => { + graph = await services().task; +}); + +test("things", async () => { + // run your assertions against the graph state }); ``` -## Operation-based service orchestration +## Building a Service Graph + +The core building blocks are: + +- `useServiceGraph(...)` coordinates startup order, restart behavior, watcher integration, and graph lifecycle +- `useSimulation(...)` starts simulators either in-process or as child processes, see `@simulacrum/foundation-simulator` or simulators built upon it +- `useService(...)` starts arbitrary child processes and can wait for a wellness check before reporting ready + +Define a service graph with a key and each service/simulator operation. -`@simulacrum/server` provides operations to start and manage services with lifecycle hooks. The recommended pattern is to create `Operation` instances for each service (typically via `useService`, `useSimulation`, or `useChildSimulation`) and pass them to `useServiceGraph` which starts the services respecting a dependency DAG and provides lifecycle hooks for startup and shutdown. +```ts +const services = useServiceGraph({ + api: { + operation: useService("api", "node --import tsx ./api.ts"), + }, + auth: { + operation: useSimulation("auth", "./auth-simulator.ts"), + }, + app: { + dependsOn: { startup: ["api", "auth"] as const }, + operation: useService("app", "node --import tsx ./app.ts"), + }, +}); +``` -See the `@simulacrum/foundation-simulator` for a basis to build simulator(s) for your services. +That gives you a single runner that can be used in multiple places without redefining your system and it's interaction. + +Use `useSimulation(...)` when the thing you are running is a simulator built on `@simulacrum/foundation-simulator`. Use `useService(...)` when you want to spawn a regular external process. You may define any number of dev servers and service required for your workflow as separate items in the graph. + +See `@simulacrum/foundation-simulator` for a basis to build simulators for your services, or packages such as `@simulacrum/auth0-simulator` and `@simulacrum/github-api-simulator` for concrete examples. ## API reference ### useServiceGraph(services, options?) -`useServiceGraph(services: ServicesMap, options?: { globalData?: Record; watch?: boolean; watchDebounce?: number }): ServiceRunner` +```ts +useServiceGraph( + services: ServicesMap, + options?: { + globalData?: Record; + watch?: boolean; + watchDebounce?: number; + }, +): ServiceRunner +``` + +Creates a runner for a graph of services, simulators, and supporting processes. -Returns a "runner" function. Call the runner inside an Effection scope to start the graph: +###### Parameters + +- `services` - a map of service definitions keyed by service name +- `options` - optional graph-level settings for `globalData`, file watching, and watch debounce behavior + +###### Returns + +- `ServiceRunner` - a runner operation factory that starts the graph when invoked + +Call the runner inside an `effection` scope to start the graph: ```ts -const run = useServiceGraph(services, options); -const services = yield * run(subset); // holds while services run, subset is optional +import { type Operation, run, main } from "effection"; + +const graph = useServiceGraph(services, options); + +// within an Operation such as +main(function* () { + const services = yield* graph(subset); // holds while services run, subset is optional +}); +// or as a promise +const services = await run(() => graph(subset)); ``` File watching: pass `options.watch = true` and `options.watchDebounce` to enable watching and restart propagation across dependents. This is enabled through the CLI helper. -#### ServiceDefinition +#### ServiceDefinition: one service entry in the graph -The `ServicesMap` passed as the first argument to `useServiceGraph`. +Each item in the `ServicesMap` passed as the first argument to `useServiceGraph` is a `ServiceDefinition`. ```ts -const services: ServicesMap = { - serviceKey: { - operation, - dependsOn, - watch, - }, +type ServiceDefinition = { + operation: Operation; + watch?: string[]; + watchDebounce?: number; + dependsOn?: { + startup?: string[]; + restart?: string[]; + }; }; ``` ##### `operation` -- Each service must provide an `operation: Operation` which signals that the service has started. -- The operation needs to be long-lived or return once a child process is started while it keeps the service running in the background (e.g., `useService` or `useChildSimulation`). -- If you are defining your own customer operation, use `try { ... yield* suspend(); } finally { ... }` inside an `operation` to run cleanup logic when the service stops. Using `resource()` from `effection` allows the service to stay in scope and continue running. See the `effection` documentation or the helper functions in this library for more information and examples. +- In most cases, pass `useSimulation(args)` or `useService(args)`. +- Each service must provide an `operation: Operation` or another long-lived `effection` operation that resolves when the service is ready. +- The operation may also return service metadata such as `{ port: number }` or `{ port: number; pid: number }` to surface runtime information in the graph's `status` map. +- If you are defining your own custom operation, use `try { ... yield* suspend(); } finally { ... }` inside an `effection` operation or `resource()` to run cleanup logic when the service stops. ##### `dependsOn` -Type: `{ startup?: string[]; restart?: string[] }` +```ts +dependsOn?: { + startup?: string[]; + restart?: string[]; +} +``` - `startup` lists services that must start before this one. - `restart` lists services whose restart should trigger a restart of this service (useful when using the watcher). ##### `watch` Watching & restart propagation -To enable file‑watching: pass `{ watch: true }` to the `useServiceGraph` options (second argument) and add `watch` paths to your `ServiceDefinition` objects. The watcher is only started when you explicitly request it (and when at least one service includes `watch` paths); by default no file descriptor is opened, allowing the process to exit cleanly on SIGINT. The watcher computes transitive dependents (using `dependsOn.restart`) and emits restart updates so restarts propagate deterministically. +```ts +watch?: string[]; +``` + +To enable file‑watching: pass `{ watch: true }` to the `useServiceGraph` options and add `watch` paths to your `ServiceDefinition` objects. The watcher is only started when you explicitly request it (and when at least one service includes `watch` paths). The watcher computes transitive dependents (using `dependsOn.restart`) and emits restart updates so restarts propagate deterministically. + +#### `globalData`: simulacrum gateway data shared across the graph + +When you call `useServiceGraph(...)` you may pass an optional `globalData` object in the options. The runner starts an HTTP data service, the simulacrum gateway, that serves that object so tests and child simulations can discover configuration or shared fixtures. + +- Endpoints: `GET /data` returns the full `globalData` JSON and `GET /data/` returns a single key, or a `404`/`400` as appropriate. +- Discovery: the gateway registers its listening port on the graph `status` map under the key `"simulacrum"`. +- Service integration: when starting child simulations via `useSimulation` or `simulationCLI`, the runner passes the gateway port to the child so it can fetch `globalData` during startup. + +```ts +const runner = useServiceGraph( + { + child: { operation: useSimulation("child", "./child-main.ts") }, + }, + { globalData: { featureFlag: true } }, +); + +main(function* (): Operation { + const services = yield* runner(); + const simulacrumPort = services.status?.get("simulacrum")?.port; + const res = yield* until(fetch(`http://127.0.0.1:${simulacrumPort}/data`)); + const data = yield* until(res.json()); + console.log(data) +}); +``` + +The gateway is intended for local development and tests only. Conceptually, it provides a small orchestration data service for the active graph. ### ServiceRunner & returned values @@ -133,105 +244,141 @@ The runner returned by `useServiceGraph` is itself an operation. This allows it ##### `subset` -When calling the runner you may pass a subset (e.g., `yield* runner(['serviceA'])` or `yield* runner('serviceA')`, the latter being a comma separated list) to start only a subset of services. Any required startup dependencies are included automatically. This is particularly use when focusing on a specific feature / feedback loop, such as in a test. Only start the services you _actually_ need. +When calling the runner you may pass a subset (e.g. `yield* runner(["serviceA"])`). Any required startup dependencies are included automatically. This is particularly useful when focusing on a specific feature or test case. + +##### returned graph -##### returns +The runner operation returns an object with the following shape: -The "runner" returns and exposes a `services` object when executed. The started graph exposes: +- `services` — the original service definitions passed to `useServiceGraph` +- `status` — a `Map` with runtime metadata for each service, including optional `port` and `pid` when the operation returns that information +- `serviceUpdates` — a `Stream` of watcher updates, if watching is enabled +- `serviceChanges` — a `Stream` of watcher restart events, if watching is enabled -- `servicePorts` — a Map of service name => listening port when a service returns `{ port: number }` from its operation. This is convenient for tests to discover HTTP endpoints. Note that these are only filled in if the `operation` supports this functionality. The `useChildSimulation` and `useSimulation` both support it. -- `services` - the object initially passed, useful for debugging -- `serviceUpdates` and `serviceChanges` - both a `Stream` (see `effection`) of updates from the watcher, useful for debugging +If a service operation returns an object like `{ port: number }` or `{ port: number; pid: number }`, that information is recorded on `status` so tests can discover listening endpoints. + +This is still an `effection` operation. If you are not operating within an `effection` scope, make use of the `task` property. It is a `type Future` which you may `await` and use like a `Promise`. ### Simulation & process helpers 🔧 This package provides a few helpers to run simulations and external processes in common patterns: -#### useSimulation(name, factory) +#### useSimulation(name, factoryOrModulePath, options?) -`useSimulation(name: string, createFactory: (initData?: unknown) => FoundationSimulator)` +`useSimulation` is built upon two main code paths. -Run a simulator _in-process_ via a factory that returns a `FoundationSimulator` (or a Promise resolving to one). Useful when you want the simulator instance in the same Node process as the runner. This API _will_ allow watching and restarts, but these restarts will not pick up changes in your code, see `useChildSimulation`. +##### `useSimulation(name: string, modulePath: string): Operation<{ port: number; pid: number }>` -- If `globalData` is set on the runner, `useSimulation` will fetch it from the simulacrum gateway and pass it as the `initData` argument to your factory. -- The factory should return a `FoundationSimulator` (see below). `useSimulation` calls `await simulator.listen()` to obtain `{ port }` and registers that port on `servicePorts`. +Starts a simulator in a fresh child process. This is the preferred form when you want reliable watch-driven restarts and a fresh module graph on each start. -Example: +###### Parameters + +- `name` - human-readable name used in logs and graph status +- `modulePath` - path to the simulator module to execute in the child process + +###### Returns + +- `Operation<{ port: number; pid: number }>` + +```ts +operation: useSimulation("service-key-for-logs", "./simulator/my-simulator.js"); +``` + +##### `useSimulation(name: string, createFactory: (initData?: unknown) => FoundationSimulator): Operation<{ port: number }>` + +Starts a simulator in the current process. This is the simplest form when you do not need subprocess isolation or module reload semantics. If you local development setup has issues with `child_process`, this is the alternative option. + +###### Parameters + +- `name` - human-readable name used in logs and graph status +- `createFactory` - a function that returns a `FoundationSimulator` + +###### Returns + +- `Operation<{ port: number }>` ```ts -// in a service definition operation: useSimulation("app", (initData) => { // do something with initData and/or pass it to your simulator through the closure return createFoundationSimulationServer({ port: 0 }); }); ``` -#### useChildSimulation(name, modulePath) +If `globalData` is set on the graph runner, `useSimulation` fetches it from the simulacrum gateway and passes it as `initData` to your factory or child module. + +When the factory form is used, `useSimulation` calls `await simulator.listen()` to obtain `{ port }` and records that port in the graph `status` map. + +> [!WARNING] +> Watching and code reload semantics are only fully supported when the simulator runs as a subprocess. Restarting an in-process simulator does not clear the module cache. + +#### Running child-process simulations -`useChildSimulation(name: string, modulePath: string)` +When the second argument to `useSimulation` is a module path string, it runs the simulator in a fresh child process using `./bin/run-simulation-child.ts`. This mode isolates module cache and is the recommended form for watch-driven restarts. -Run a simulator in a fresh child Node process (isolates module cache and supports restarts). Otherwise this feels the same as using `useSimulation`. +The child-process flow looks like this: -- The child is started using a wrapper, `./bin/run-simulation-child.ts `, and, when present, the `--simulacrum-port` is passed so the child can fetch `globalData`. -- The wrapper prints a JSON line to stdout like `{ "ready": true, "port": 12345 }` as its first ready signal. `useChildSimulation` reads that line to discover the port and registers it on `servicePorts`. -- Non-JSON stdout lines are forwarded to logs; if the child exits before emitting the ready JSON, `useChildSimulation` rejects. -- If using this with a simulator created from `@simulacrum/foundation-simulator`, all this wiring will be handled for you. +1. `useSimulation` starts the wrapper `./bin/run-simulation-child.ts `. +2. If a simulacrum gateway is running, the wrapper also receives `--simulacrum-port` so the child can fetch `globalData`. +3. The child prints a first ready line like `{ "ready": true, "port": 12345 }` to stdout. +4. `useSimulation` reads that line, captures the port, and records it in the graph `status` map. +5. Non-JSON stdout is forwarded to logs as normal. +6. If the child exits before emitting the ready line, `useSimulation` rejects. + +If you build the simulator with `@simulacrum/foundation-simulator`, this wiring is handled for you. Example: ```ts -operation: useChildSimulation("service-key-for-logs", "./simulator/my-simulator.js"); +operation: useSimulation("service-key-for-logs", "./simulator/my-simulator.js"); ``` > [!WARNING] -> This does rely on having `tsx` installed which will handle the TypeScript types when running. It will allow for a simulator defined through a `.js` file or a `.ts`, so your choosing. +> TypeScript child modules rely on your runtime setup supporting them, for example via `tsx`. JavaScript modules work as-is. #### About `@simulacrum/foundation-simulator` -- A `FoundationSimulator` is a small helper that provides two key primitives you should expect from your factory: - - `simulator.listen(): Promise<{ port: number }>` — starts the server and resolves when it is listening (the object is registered in `servicePorts`). - - `simulator.ensureClose(): Promise` — used by the runner to cleanly shut down the simulator when its containing scope is cancelled. -- Use `createFoundationSimulationServer()` to create a server that listens on an ephemeral port and returns an object compatible with `useSimulation` and `useChildSimulation`. +Use `createFoundationSimulationServer()` to create a server that returns a `FoundationSimulator`, which is the shape expected by the factory form of `useSimulation`. #### useService(name, cmd, options?) -Spawn an external process (via the configured command) and optionally run a wellness check. `useService` forwards stdout/stderr to the package logging and keeps the operation alive until it goes out of scope. +```ts +useService( + name: string, + cmd: string, + options?: { + wellnessCheck?: { + operation: (stdio: Stream) => Operation>; + timeout?: number; + frequency?: number; + }; + }, +): Operation +``` -- `options`: - - `wellnessCheck.operation(stdio)` — an operation, `Operation<>` that needs to return a `Result` (both from `effection`) to consider the service successfully started. It is passed the stdio from the process. You may use any `effection` semantics, and inspect the stdio or http calls, etc, to decide when your service is "ready". - - `wellnessCheck.timeout` and `wellnessCheck.frequency` can be provided to control checking behavior, most useful in repeatedly `fetch`ing a `/status` or `/healthcheck` response. +Starts an external process and optionally waits for a wellness check before reporting the service as ready. -#### simulationCLI(serviceGraph) +###### Parameters -- `simulationCLI` wraps the runner in a small CLI loop and provides convenience flags: `--services`, `--watch`, and `--watch-debounce`. -- Use the CLI helper for local development workflows where you want to run your graph directly from a file (see `service-graph.ts` examples above). +- `name` - human-readable name used in logs and graph status +- `cmd` - command to execute for the service process +- `options` - optional process readiness configuration -## Global data & the simulacrum gateway 🔁 +###### Returns -When you call `useServiceGraph(...)` you may pass an optional `globalData` object in the options. The runner starts a tiny local HTTP data service (the **simulacrum gateway**) that serves that object so tests and child simulations can discover configuration or shared fixtures. +- `Operation` - a long-lived operation that stays active until the service is stopped -- Endpoints: `GET /data` (returns the full `globalData` JSON) and `GET /data/` (returns a single key, or a 404/400 as appropriate). -- Discovery: the gateway registers its listening port on the runner's `servicePorts` map under the key `"simulacrum"`. You can read the port from your test or harness with `const port = services.servicePorts!.get("simulacrum");` and then `fetch` `http://127.0.0.1:${port}/data`. -- Service integration: when starting child simulations via `useChildSimulation` / `simulationCLI` we pass the gateway port (if present) to the child. The child will fetch `/data` on startup and receive the `globalData` object. The simulator function you define may expect to receive that global object as the first argument to the function. Useful for passing "world-level" data to all of your simulators. +`useService` forwards stdout and stderr to the package logger and keeps the operation alive until it goes out of scope. -```ts -const runner = useServiceGraph( - { - child: { operation: useChildSimulation("child", "./child-main.ts") }, - }, - { globalData: { featureFlag: true } }, -); +The `options.wellnessCheck` object supports: -const services = yield * runner(); -const simulacrumPort = services.servicePorts!.get("simulacrum"); -// fetch global data in a test or helper -const res = await fetch(`http://127.0.0.1:${simulacrumPort}/data`); -const data = await res.json(); -``` +- `operation(stdio)` - an operation that inspects process output and returns an Effection `Result` when the service should be considered ready +- `timeout` - maximum time to wait for the wellness check to succeed +- `frequency` - polling or retry frequency for the wellness check -Notes: +#### simulationCLI(serviceGraph) -The gateway is intended for local development and tests only (it is not a production data layer). Future work around this layer may include improved logging and observability. Conceptually, it provides an "orchestration status" service. +- `simulationCLI` wraps the runner in a small CLI loop and provides convenience flags: `--services`, `--watch`, and `--watch-debounce`. +- Use the CLI helper for local development workflows where you want to run your graph directly from a file (see `service-graph.ts` examples above). ## Development diff --git a/packages/server/example/README.md b/packages/server/example/README.md index a9e7a541..d9edacec 100644 --- a/packages/server/example/README.md +++ b/packages/server/example/README.md @@ -1,12 +1,12 @@ # Server package examples -This folder contains runnable examples demonstrating `useServiceGraph` and `useService`. +This folder contains runnable examples demonstrating `useServiceGraph`, `useService`, and `useSimulation`. -There are two sets of examples: +The examples are: -- **use-service** (top-level files like `process-graph.ts`, `concurrency-layers.ts`) — these spawn separate processes using `useService`. Use these to exercise the process-based behavior. - -- **simulation / child processes** (e.g., `simulation-graph.ts`) — these demonstrate `useChildSimulation()` which runs each service in a child process using a simulation factory. They show how to isolate simulations and start them as independent processes. +- `process-graph.ts` — a process-based graph using `useService()` to start each service as a separate command. +- `simulation-graph.ts` — a simulation graph using `useSimulation()` to start each service in a child process with module-path-based simulator factories. +- `concurrency-layers.ts` — a simulation graph with `useSimulation()`, file watching, and restart propagation via `dependsOn.restart`. Quick commands: diff --git a/packages/server/example/concurrency-layers.ts b/packages/server/example/concurrency-layers.ts index 4ee22368..da935132 100644 --- a/packages/server/example/concurrency-layers.ts +++ b/packages/server/example/concurrency-layers.ts @@ -1,7 +1,7 @@ #!/usr/bin/env node import { resource } from "effection"; import { useServiceGraph } from "../src/services.ts"; -import { useChildSimulation } from "../src/simulation.ts"; +import { useSimulation } from "../src/simulation.ts"; import { simulationCLI } from "../src/cli.ts"; const servicesMap = { @@ -18,11 +18,11 @@ const servicesMap = { watch: ["./example/services/basic-sim.ts"], }, fast: { - operation: useChildSimulation("fast", "./example/services/basic-sim-1.ts"), + operation: useSimulation("fast", "./example/services/basic-sim-1.ts"), watch: ["./example/services/basic-sim-1.ts"], }, slow: { - operation: useChildSimulation("slow", "./example/services/basic-sim-2.ts"), + operation: useSimulation("slow", "./example/services/basic-sim-2.ts"), watch: ["./example/services/basic-sim-2.ts"], }, }; diff --git a/packages/server/example/simulation-graph.ts b/packages/server/example/simulation-graph.ts index 5b637e66..b10eb7f1 100644 --- a/packages/server/example/simulation-graph.ts +++ b/packages/server/example/simulation-graph.ts @@ -1,15 +1,15 @@ #!/usr/bin/env node import { useServiceGraph } from "../src/services.ts"; -import { useChildSimulation } from "../src/simulation.ts"; +import { useSimulation } from "../src/simulation.ts"; import { simulationCLI } from "../src/cli.ts"; const servicesMap = { A: { - operation: useChildSimulation("A", "./example/services/basic-sim-1.ts"), + operation: useSimulation("A", "./example/services/basic-sim-1.ts"), }, B: { dependsOn: { startup: ["A"] as const }, - operation: useChildSimulation("B", "./example/services/basic-sim-2.ts"), + operation: useSimulation("B", "./example/services/basic-sim-2.ts"), }, }; diff --git a/packages/server/package.json b/packages/server/package.json index b33a2fd7..bc4f0c3c 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -21,6 +21,7 @@ }, "files": [ "README.md", + "bin", "dist", "src" ], @@ -42,6 +43,7 @@ "import": "./dist/index.mjs", "require": "./dist/index.cjs" }, + "./bin/run-simulation-child.ts": "./bin/run-simulation-child.ts", "./package.json": "./package.json" }, "publishConfig": { @@ -50,6 +52,7 @@ "import": "./dist/index.mjs", "require": "./dist/index.cjs" }, + "./bin/run-simulation-child.ts": "./bin/run-simulation-child.ts", "./package.json": "./package.json" } }, diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 58cec33a..b894d430 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -2,4 +2,5 @@ export * from "./logging.ts"; export * from "./service.ts"; export * from "./services.ts"; export * from "./simulation.ts"; +export * from "./operation-metadata.ts"; export * from "./cli.ts"; diff --git a/packages/server/src/operation-metadata.ts b/packages/server/src/operation-metadata.ts new file mode 100644 index 00000000..45602090 --- /dev/null +++ b/packages/server/src/operation-metadata.ts @@ -0,0 +1,25 @@ +import type { Operation } from "effection"; + +export type OperationMetadata = { + watchSafe?: boolean; + operationName?: string; +}; + +const OPERATION_METADATA = Symbol.for("@simulacrum/simulacrum/operationMetadata"); + +export type OperationWithMetadata> = T & { + [OPERATION_METADATA]?: OperationMetadata; +}; + +export function withOperationMetadata>( + operation: T, + metadata: OperationMetadata, +): T { + const operationWithMetadata = operation as OperationWithMetadata; + operationWithMetadata[OPERATION_METADATA] = metadata; + return operationWithMetadata; +} + +export function getOperationMetadata(operation: Operation): OperationMetadata | undefined { + return (operation as OperationWithMetadata>)[OPERATION_METADATA]; +} diff --git a/packages/server/src/service.ts b/packages/server/src/service.ts index 0fa0ba3f..05a2de80 100644 --- a/packages/server/src/service.ts +++ b/packages/server/src/service.ts @@ -15,6 +15,7 @@ import { daemon } from "@effectionx/process"; import type { ExecOptions as ProcessOptions } from "@effectionx/process"; import { logger } from "./logging.ts"; import { createReplaySignal } from "./createReplaySignal.ts"; +import { withOperationMetadata } from "./operation-metadata.ts"; type ServiceOptions = { wellnessCheck?: { @@ -39,79 +40,82 @@ export function useService( cmd: string, options: ServiceOptions = {}, ): Operation { - return resource(function* (provide) { - yield* useAttributes({ name: `useService ${name}`, cmd: String(cmd) }); - if (cmd.startsWith("npm")) { - // see https://github.com/npm/cli/issues/6684 - throw new Error("scripts run with npm don't respect signals to properly shutdown"); - } - const process = yield* daemon(cmd, options.processOptions); - const stdio = createReplaySignal(); - const stdioAdd = lift(stdio.send); - - // forward raw stdout for logging in chunk form (no reassembly) - yield* spawn(function* () { - yield* useAttributes({ name: "stdoutForward" }); - for (let line of yield* each(process.stdout)) { - const buf = Buffer.from(line); - const str = buf.toString(); - yield* logger.stdout(str); - yield* stdioAdd(str); - yield* each.next(); - } - }); - - yield* spawn(function* () { - yield* useAttributes({ name: "stderrForward" }); - for (let line of yield* each(process.stderr)) { - const str = Buffer.from(line).toString(); - yield* logger.stderr(str); - yield* stdioAdd(str); - yield* each.next(); + return withOperationMetadata( + resource(function* (provide) { + yield* useAttributes({ name: `useService ${name}`, cmd: String(cmd) }); + if (cmd.startsWith("npm")) { + // see https://github.com/npm/cli/issues/6684 + throw new Error("scripts run with npm don't respect signals to properly shutdown"); } - }); - - yield* sleep(0); // allow stdio forwarding to start + const process = yield* daemon(cmd, options.processOptions); + const stdio = createReplaySignal(); + const stdioAdd = lift(stdio.send); - // if supplied, wellness check to ensure it is running or timeout with result - if (options.wellnessCheck) { - yield* useAttributes({ - name: `useService ${name}`, - wellnessCheck: String(true), - frequency: String(options.wellnessCheck.frequency ?? ""), + // forward raw stdout for logging in chunk form (no reassembly) + yield* spawn(function* () { + yield* useAttributes({ name: "stdoutForward" }); + for (let line of yield* each(process.stdout)) { + const buf = Buffer.from(line); + const str = buf.toString(); + yield* logger.stdout(str); + yield* stdioAdd(str); + yield* each.next(); + } }); - const { operation } = options.wellnessCheck; - const frequency = options.wellnessCheck.frequency ?? 100; - function* untilWell() { - yield* useAttributes({ name: `wellnessCheck` }); - while (true) { - try { - yield* sleep(frequency); - const result = yield* scoped(() => operation(stdio)); - if (result && result.ok) { - break; - } - } catch (ignore) { - // noop, try again - } + + yield* spawn(function* () { + yield* useAttributes({ name: "stderrForward" }); + for (let line of yield* each(process.stderr)) { + const str = Buffer.from(line).toString(); + yield* logger.stderr(str); + yield* stdioAdd(str); + yield* each.next(); } - } + }); - if (options.wellnessCheck.timeout) { + yield* sleep(0); // allow stdio forwarding to start + + // if supplied, wellness check to ensure it is running or timeout with result + if (options.wellnessCheck) { yield* useAttributes({ name: `useService ${name}`, - timeout: String(options.wellnessCheck.timeout), + wellnessCheck: String(true), + frequency: String(options.wellnessCheck.frequency ?? ""), }); - const checked = yield* timebox(options.wellnessCheck.timeout, untilWell); - if (checked && checked.timeout) { - throw new Error("service wellness check timed out"); + const { operation } = options.wellnessCheck; + const frequency = options.wellnessCheck.frequency ?? 100; + function* untilWell() { + yield* useAttributes({ name: `wellnessCheck` }); + while (true) { + try { + yield* sleep(frequency); + const result = yield* scoped(() => operation(stdio)); + if (result && result.ok) { + break; + } + } catch (ignore) { + // noop, try again + } + } + } + + if (options.wellnessCheck.timeout) { + yield* useAttributes({ + name: `useService ${name}`, + timeout: String(options.wellnessCheck.timeout), + }); + const checked = yield* timebox(options.wellnessCheck.timeout, untilWell); + if (checked && checked.timeout) { + throw new Error("service wellness check timed out"); + } + } else { + yield* untilWell(); } - } else { - yield* untilWell(); + yield* lift(stdio.close)(); } - yield* lift(stdio.close)(); - } - yield* provide(); - }); + yield* provide(); + }), + { watchSafe: true, operationName: "useService" }, + ); } diff --git a/packages/server/src/services.ts b/packages/server/src/services.ts index 2ca40c3a..0426cc77 100644 --- a/packages/server/src/services.ts +++ b/packages/server/src/services.ts @@ -1,26 +1,28 @@ import { type Operation, type Stream, + type Task, type WithResolvers, resource, spawn, withResolvers, each, createContext, + run, } from "effection"; import { useAttributes } from "./logging.ts"; import { type ServiceUpdate, useWatcher } from "./watch.ts"; import { logger } from "./logging.ts"; import { startDataService } from "./data-service.ts"; +import { getOperationMetadata } from "./operation-metadata.ts"; /** * Context key for the Simulacrum gateway listening port. * * When `useServiceGraph` starts the optional simulacrum gateway (via the * `globalData` option) it sets this context value to the listening port so - * operations in the graph (including `useSimulation` and - * `useChildSimulation`) can discover and fetch the `/data` payload. + * operations in the graph (including `useSimulation`) can discover and fetch the `/data` payload. */ export const SimulacrumEndpoint = createContext("SimulacrumEndpoint"); @@ -85,9 +87,9 @@ export function useServiceGraph< watch?: boolean; watchDebounce?: number; }, -): (subset?: Array) => Operation> { - return (subset?: Array) => - resource(function* (provide) { +): (subset?: Array) => Operation> & { task: Task> } { + return (subset?: Array) => { + const r = resource>(function* (provide) { // detect cycles in the dependency graph const nodes = Object.keys(services); // label the root of the service graph operation @@ -201,6 +203,15 @@ export function useServiceGraph< }); const task = status.get(service); if (!task) throw new Error(`missing status for service ${service}`); + + const metadata = getOperationMetadata(effectiveServices[service].operation); + if (metadata?.watchSafe === false) { + yield* logger.stderr( + `warning: watched service '${service}' uses ${metadata.operationName ?? "an operation"} which may not reload module cache on restart. Skipping restart for this service.`, + ); + return; + } + // log so it is clear in the inspector output when a restart is triggered yield* logger.stdout(`restarting service ${service}`); // refresh the startup resolver @@ -269,7 +280,7 @@ export function useServiceGraph< // run the service in a scoped child operation so it can be cleanly // cancelled when a file change triggers a restart const serviceTask = yield* spawn(function* () { - // capture any returned listening info (e.g., from useChildSimulation) + // capture any returned listening info (e.g., from useSimulation) const maybeProvided = yield* def.operation; if (maybeProvided && typeof maybeProvided === "object") { if ("port" in maybeProvided && typeof maybeProvided.port === "number") { @@ -317,4 +328,8 @@ export function useServiceGraph< yield* logger.debug("shutting down service graph"); } }); + const graph = r as Operation> & { task: Task> }; + graph.task = run(() => graph); + return graph; + }; } diff --git a/packages/server/src/simulation.ts b/packages/server/src/simulation.ts index b7c5fa90..f9705b70 100644 --- a/packages/server/src/simulation.ts +++ b/packages/server/src/simulation.ts @@ -8,61 +8,53 @@ import type { FoundationSimulatorListening, } from "@simulacrum/foundation-simulator"; import { SimulacrumEndpoint } from "./services.ts"; +import { fileURLToPath } from "node:url"; +import { withOperationMetadata } from "./operation-metadata.ts"; -/** - * Helper to start a foundation simulation server factory - * - * This is implemented as an Effection `resource` so cleanup is handled by the - * `provide` finalizer when the operation's scope is closed. - */ /** - * Start a simulator provided by a factory and return its listening info. - * - * The factory may accept initialization data (fetched from the simulacrum - * gateway when available) and should return a `FoundationSimulator` instance - * (or a Promise resolving to one). This operation yields the simulator's - * listening information (`{ port }`) once it starts. - * - * @param name - human-friendly name used for logging - * @param createFactory - factory function that returns a `FoundationSimulator` - * @returns an `Operation` that provides `FoundationSimulatorListening` when the - * simulator is listening - */ export function useSimulation>( +type UseSimulationOptions = { + subprocess?: boolean; +}; + +function useSimulationInProcess>( name: string, createFactory: (initData?: unknown) => FoundationSimulator, ): Operation> { - return resource(function* (provide) { - yield* useAttributes({ name: `useSimulation ${name}` }); - // attempt to read the simulacrum port from context; if not present, continue without it - const simulacrumPort = yield* SimulacrumEndpoint.get(); - - // if present fetch the data chunk and pass it to the factory - let initData: unknown | undefined = undefined; - if (typeof simulacrumPort === "number" && !Number.isNaN(simulacrumPort)) { - try { - const res = yield* until(fetch(`http://127.0.0.1:${simulacrumPort}/data`)); - initData = yield* until(res.json()); - } catch (err) { - // ignore fetch failures - yield* logger.stderr("failed to fetch simulacrum data:", err); + return withOperationMetadata( + resource(function* (provide) { + yield* useAttributes({ name: `useSimulation ${name}` }); + // attempt to read the simulacrum port from context; if not present, continue without it + const simulacrumPort = yield* SimulacrumEndpoint.get(); + + // if present fetch the data chunk and pass it to the factory + let initData: unknown | undefined = undefined; + if (typeof simulacrumPort === "number" && !Number.isNaN(simulacrumPort)) { + try { + const res = yield* until(fetch(`http://127.0.0.1:${simulacrumPort}/data`)); + initData = yield* until(res.json()); + } catch (err) { + // ignore fetch failures + yield* logger.stderr("failed to fetch simulacrum data:", err); + } } - } - const createSim = createFactory(initData); - const listening: FoundationSimulatorListening = yield* until(createSim.listen()); + const createSim = createFactory(initData); + const listening: FoundationSimulatorListening = yield* until(createSim.listen()); - yield* logger.stdout(`${name} simulation: port ${listening.port}`); - yield* useAttributes({ - name: `useSimulation ${name}`, - port: String(listening.port), - }); + yield* logger.stdout(`${name} simulation: port ${listening.port}`); + yield* useAttributes({ + name: `useSimulation ${name}`, + port: String(listening.port), + }); - try { - yield* provide(listening); - } finally { - yield* until(listening.ensureClose()); - yield* logger.stdout(`${name} simulation: closed port ${listening.port}`); - } - }); + try { + yield* provide(listening); + } finally { + yield* until(listening.ensureClose()); + yield* logger.stdout(`${name} simulation: closed port ${listening.port}`); + } + }), + { watchSafe: false, operationName: "useSimulation" }, + ); } // Spawn a child Node process to run a simulation factory in a fresh module @@ -80,114 +72,173 @@ import { SimulacrumEndpoint } from "./services.ts"; * @param modulePath - path to the module exporting a simulation factory or instance * @returns an `Operation` that provides `FoundationSimulatorListening` from the child */ -export function useChildSimulation(name: string, modulePath: string) { - return resource<{ port: number; pid: number }>(function* (provide) { - yield* useAttributes({ - name: `useChildSimulation ${name}`, - module: modulePath, - }); - // attempt to read the simulacrum port from context; if not present, continue without it - const contextPort = yield* SimulacrumEndpoint.get(); - - const parts = [ - "node", - // safest considering current LTS of >v20 - "--experimental-strip-types", - "./bin/run-simulation-child.ts", - modulePath, - ]; - if (typeof contextPort === "number") { - parts.push("--simulacrum-port", String(contextPort)); - } - const cmd = parts.map((s) => (s.includes(" ") ? `'${s}'` : s)).join(" "); - - const process = yield* daemon(cmd); - const pid = process.pid; - yield* useAttributes({ - name: `useChildSimulation ${name}`, - cmd, - pid: String(pid), - }); - - // read the first stdout JSON line to get the listening info - let port = undefined as number | undefined; - let ready = withResolvers("wait until the port is returned to signal ready"); - - // forward raw stdout for logging in chunk form (no reassembly) - yield* spawn(function* () { +export function useSimulationChildProcess(name: string, modulePath: string) { + return withOperationMetadata( + resource<{ port: number; pid: number }>(function* (provide) { yield* useAttributes({ - name: "stdoutForward", + name: `useSimulation ${name}`, + module: modulePath, }); - for (let line of yield* each(process.stdout)) { - const buf = Buffer.from(line); - const str = buf.toString(); + // attempt to read the simulacrum port from context; if not present, continue without it + const contextPort = yield* SimulacrumEndpoint.get(); - if (!port) { - try { - const parsed = JSON.parse(str); - if (parsed && parsed.ready && typeof parsed.port === "number") { - port = parsed.port; - ready.resolve(); - } else { + const runnerPath = fileURLToPath( + import.meta.resolve("@simulacrum/server/bin/run-simulation-child.ts"), + ); + const parts = [ + "node", + // safest considering current LTS of >v20 + "--experimental-strip-types", + runnerPath, + modulePath, + ]; + if (typeof contextPort === "number") { + parts.push("--simulacrum-port", String(contextPort)); + } + const cmd = parts.map((s) => (s.includes(" ") ? `'${s}'` : s)).join(" "); + + const process = yield* daemon(cmd); + const pid = process.pid; + yield* useAttributes({ + name: `useSimulation ${name}`, + cmd, + pid: String(pid), + }); + + // read the first stdout JSON line to get the listening info + let port = undefined as number | undefined; + let ready = withResolvers("wait until the port is returned to signal ready"); + + // forward raw stdout for logging in chunk form (no reassembly) + yield* spawn(function* () { + yield* useAttributes({ + name: "stdoutForward", + }); + for (let line of yield* each(process.stdout)) { + const buf = Buffer.from(line); + const str = buf.toString(); + + if (!port) { + try { + const parsed = JSON.parse(str); + if (parsed && parsed.ready && typeof parsed.port === "number") { + port = parsed.port; + ready.resolve(); + } else { + yield* logger.stdout(str); + } + } catch (ignore) { + // just log lines that are not JSON yield* logger.stdout(str); } - } catch (ignore) { - // just log lines that are not JSON + } else { yield* logger.stdout(str); } - } else { - yield* logger.stdout(str); - } - yield* each.next(); - } - }); + yield* each.next(); + } + }); - yield* spawn(function* () { - yield* useAttributes({ - name: "stderrForward", + yield* spawn(function* () { + yield* useAttributes({ + name: "stderrForward", + }); + for (let line of yield* each(process.stderr)) { + const str = Buffer.from(line).toString(); + yield* logger.stderr(str); + yield* each.next(); + } }); - for (let line of yield* each(process.stderr)) { - const str = Buffer.from(line).toString(); - yield* logger.stderr(str); - yield* each.next(); - } - }); - // spawn a watcher to detect if the child exits before printing the listening info - let status: unknown = undefined; - yield* spawn(function* () { - yield* useAttributes({ - name: "childEarlyExitWatcher", + // spawn a watcher to detect if the child exits before printing the listening info + let status: unknown = undefined; + yield* spawn(function* () { + yield* useAttributes({ + name: "childEarlyExitWatcher", + }); + status = yield* process.join(); + if (!port) { + ready.reject( + new Error( + `child process exited before emitting listening info: ${JSON.stringify(status)}`, + ), + ); + } }); - status = yield* process.join(); + + // wait to get the listening info from stdout (or reject if the process exited) + yield* ready.operation; + if (!port) { - ready.reject( - new Error( - `child process exited before emitting listening info: ${JSON.stringify(status)}`, - ), + throw new Error( + `failed to get listening port from child process: ${JSON.stringify({ + status, + pid, + })}`, ); } - }); - // wait to get the listening info from stdout (or reject if the process exited) - yield* ready.operation; + yield* logger.stdout(`${name} simulation: port ${port} pid ${pid}`); + + try { + yield* provide({ port, pid }); + } finally { + yield* logger.debug(`${name} simulation: closed on port ${port}`); + } + }), + { watchSafe: true, operationName: "useSimulation" }, + ); +} - if (!port) { +/** + * Run a simulator either in-process or in a child Node subprocess. + * + * When the second argument is a factory, `useSimulation` runs the simulator + * in-process and resolves to the simulator's listening information. + * + * When the second argument is a module path string, `useSimulation` starts the + * simulator in a fresh child process and resolves to the child's listening + * information plus PID. + * + * If `globalData` is configured on the runner, this operation fetches the + * data from the Simulacrum gateway and passes it as `initData` to the factory + * or child module. + * + * @param name - human-friendly name used for logging + * @param createFactory - factory function that returns a `FoundationSimulator` + * @param modulePath - path to a module exporting a simulator factory + * @param options - optional subprocess hint for overload resolution + */ +export function useSimulation>( + name: string, + createFactory: (initData?: unknown) => FoundationSimulator, + options?: { subprocess?: false }, +): Operation>; +export function useSimulation( + name: string, + modulePath: string, + options?: { subprocess?: true }, +): Operation<{ port: number; pid: number }>; +export function useSimulation>( + name: string, + factoryOrModulePath: ((initData?: unknown) => FoundationSimulator) | string, + options: UseSimulationOptions = {}, +) { + if (typeof factoryOrModulePath === "string") { + if (options.subprocess === false) { throw new Error( - `failed to get listening port from child process: ${JSON.stringify({ - status, - pid, - })}`, + "cannot use subprocess:false when the second argument is a module path string", ); } - yield* logger.stdout(`${name} simulation: port ${port} pid ${pid}`); + return useSimulationChildProcess(name, factoryOrModulePath); + } - try { - yield* provide({ port, pid }); - } finally { - yield* logger.debug(`${name} simulation: closed on port ${port}`); - } - }); + if (options.subprocess === true) { + throw new Error( + "subprocess:true is only supported when using a module path string as the second argument", + ); + } + + return useSimulationInProcess(name, factoryOrModulePath); } diff --git a/packages/server/test/child-simulation.test.ts b/packages/server/test/child-simulation.test.ts index c5d44088..b271caf6 100644 --- a/packages/server/test/child-simulation.test.ts +++ b/packages/server/test/child-simulation.test.ts @@ -2,13 +2,13 @@ import { describe, it } from "node:test"; import assert from "node:assert"; import { run, until } from "effection"; import { useServiceGraph } from "../src/services.ts"; -import { useChildSimulation } from "../src/simulation.ts"; +import { useSimulation } from "../src/simulation.ts"; import { waitFor } from "./utils.ts"; -describe("useChildSimulation", () => { +describe("useSimulation (child process)", () => { it("starts a child and returns port", async () => { await run(function* () { - const listening = yield* useChildSimulation("child-test", "./test/fixtures/simple-sim.ts"); + const listening = yield* useSimulation("child-test", "./test/fixtures/simple-sim.ts"); assert(typeof listening.port === "number"); // Verify we received a port and the child reported ready. @@ -16,16 +16,24 @@ describe("useChildSimulation", () => { }); }); + it("supports module path child simulations via useSimulation", async () => { + await run(function* () { + const listening = yield* useSimulation("child-test", "./test/fixtures/simple-sim.ts"); + assert(typeof listening.port === "number"); + assert(typeof listening.pid === "number"); + }); + }); + it("handles non-JSON stdout before ready JSON from child", async () => { await run(function* () { - const listening = yield* useChildSimulation("non-json", "./test/fixtures/non-json-child.ts"); + const listening = yield* useSimulation("non-json", "./test/fixtures/non-json-child.ts"); assert(typeof listening.port === "number"); }); }); it("ignores JSON logs without ready/port until real ready JSON is emitted", async () => { await run(function* () { - const listening = yield* useChildSimulation( + const listening = yield* useSimulation( "json-before-ready", "./test/fixtures/json-before-ready.ts", ); @@ -41,7 +49,7 @@ describe("useChildSimulation", () => { const op = useServiceGraph( { child: { - operation: useChildSimulation("child", "./test/fixtures/init-data-sim.ts"), + operation: useSimulation("child", "./test/fixtures/init-data-sim.ts"), }, }, { globalData: data }, @@ -64,7 +72,7 @@ describe("useChildSimulation", () => { const op = useServiceGraph( { child: { - operation: useChildSimulation("child", "./test/fixtures/init-data-sim.ts"), + operation: useSimulation("child", "./test/fixtures/init-data-sim.ts"), }, }, { globalData: data }, @@ -102,7 +110,7 @@ describe("useChildSimulation", () => { const op = useServiceGraph( { child: { - operation: useChildSimulation("child", "./test/fixtures/init-data-sim.ts"), + operation: useSimulation("child", "./test/fixtures/init-data-sim.ts"), }, }, { globalData: data }, @@ -124,7 +132,7 @@ describe("useChildSimulation", () => { const op = useServiceGraph( { child: { - operation: useChildSimulation("child", "./test/fixtures/init-data-sim.ts"), + operation: useSimulation("child", "./test/fixtures/init-data-sim.ts"), }, }, { globalData: { hello: "world" } }, @@ -147,7 +155,7 @@ describe("useChildSimulation", () => { it("rejects when child exits before emitting listening info", async () => { await assert.rejects(async () => { await run(function* () { - yield* useChildSimulation("broken", "./test/fixtures/broken-child.ts"); + yield* useSimulation("broken", "./test/fixtures/broken-child.ts"); }); }, /child process exited before emitting listening info/); }); diff --git a/packages/server/test/examples-smoke.test.ts b/packages/server/test/examples-smoke.test.ts index bc7675e4..5e9b48a1 100644 --- a/packages/server/test/examples-smoke.test.ts +++ b/packages/server/test/examples-smoke.test.ts @@ -5,6 +5,7 @@ import { waitFor, waitForOperation } from "./utils.ts"; import { services as basicServices } from "../example/simulation-graph.ts"; import { services as concurrencyServices } from "../example/concurrency-layers.ts"; +import { services as processServices } from "../example/process-graph.ts"; describe("example as service rig", { concurrency: 1 }, () => { it("basic example imports and runs", async () => { @@ -74,4 +75,21 @@ describe("example as service rig", { concurrency: 1 }, () => { } }); }); + + it("process example imports and runs", async () => { + await run(function* () { + let provided = yield* processServices(); + + yield* waitFor(() => { + return ["A", "B"].every((name) => provided?.status?.has(name)); + }, 5000); + + for (const port of [3301, 3302]) { + yield* waitForOperation(function* () { + const status = yield* until(fetch(`http://localhost:${port}/status`)); + return status.ok; + }, 5000); + } + }); + }); }); diff --git a/packages/server/test/services.test.ts b/packages/server/test/services.test.ts index f095b2c2..0df7c883 100644 --- a/packages/server/test/services.test.ts +++ b/packages/server/test/services.test.ts @@ -1,279 +1,329 @@ -import { it } from "node:test"; +import { describe, it } from "node:test"; import assert from "node:assert"; import { resource, run, sleep, spawn, suspend } from "effection"; import { useServiceGraph } from "../src/services.ts"; import { waitFor } from "./utils.ts"; import { useService } from "../src/service.ts"; -it("starts services in dependency order", async () => { - const startTimes = new Map(); - try { +describe("shutdown", () => { + it("runs beforeStop hooks in reverse order", async () => { + const stopOrder: string[] = []; + const startedOrder: string[] = []; await run(function* () { + // spawn and cancel automatically when run returns yield* spawn(function* () { - const graph = useServiceGraph({ + const run = useServiceGraph({ A: { operation: resource(function* (provide) { - yield* sleep(20); - startTimes.set("A", Date.now()); - yield* provide(); + try { + yield* sleep(20); + startedOrder.push("A"); + yield* provide(); + } finally { + stopOrder.push("A"); + } }), }, B: { operation: resource(function* (provide) { - yield* sleep(40); - startTimes.set("B", Date.now()); - yield* provide(); + try { + yield* sleep(40); + startedOrder.push("B"); + yield* provide(); + } finally { + stopOrder.push("B"); + } }), dependsOn: { startup: ["A"] as const }, }, }); - yield* graph(); - // keep spawned graph alive + yield* run(); + // keep spawned graph alive so beforeStop hooks run on teardown yield* suspend(); }); - yield* waitFor(() => startTimes.has("A") && startTimes.has("B"), 2000); + // let them start + yield* waitFor(() => startedOrder.length === 2, 2000); }); - } catch (err) { - console.log("run error:", err instanceof Error ? err.stack : err); - } - - const aStarted = startTimes.get("A"); - const bStarted = startTimes.get("B"); - assert.ok(typeof aStarted === "number", "A started should be recorded"); - assert.ok(typeof bStarted === "number", "B started should be recorded"); - assert(aStarted! <= bStarted!, "A should start before B"); + assert.strictEqual(startedOrder.join(""), "AB"); + assert.strictEqual(stopOrder.join(""), "BA"); + }); }); -it("throws on cycles in dependency graph", async () => { - await assert.rejects(async () => { - await run(function* () { - const runGraph = useServiceGraph({ - A: { - operation: useService( - "A", - "node --experimental-transform-types ./test/services/service-a.ts", - ), - dependsOn: { startup: ["B"] as const }, - }, - B: { - operation: useService( - "B", - "node --experimental-transform-types ./test/services/service-b.ts", - ), - dependsOn: { startup: ["A"] as const }, - }, +describe("dependency handling", () => { + it("starts services in dependency order", async () => { + const startTimes = new Map(); + try { + await run(function* () { + yield* spawn(function* () { + const graph = useServiceGraph({ + A: { + operation: resource(function* (provide) { + yield* sleep(20); + startTimes.set("A", Date.now()); + yield* provide(); + }), + }, + B: { + operation: resource(function* (provide) { + yield* sleep(40); + startTimes.set("B", Date.now()); + yield* provide(); + }), + dependsOn: { startup: ["A"] as const }, + }, + }); + yield* graph(); + // keep spawned graph alive + yield* suspend(); + }); + yield* waitFor(() => startTimes.has("A") && startTimes.has("B"), 2000); }); - yield* runGraph(); - }); - }, /Cycle detected in services/); -}); + } catch (err) { + console.log("run error:", err instanceof Error ? err.stack : err); + } -it("runs beforeStop hooks in reverse order", async () => { - const stopOrder: string[] = []; - const startedOrder: string[] = []; - await run(function* () { - // spawn and cancel automatically when run returns - yield* spawn(function* () { - const run = useServiceGraph({ - A: { - operation: resource(function* (provide) { - try { - yield* sleep(20); - startedOrder.push("A"); - yield* provide(); - } finally { - stopOrder.push("A"); - } - }), - }, - B: { - operation: resource(function* (provide) { - try { - yield* sleep(40); - startedOrder.push("B"); - yield* provide(); - } finally { - stopOrder.push("B"); - } - }), - dependsOn: { startup: ["A"] as const }, - }, + const aStarted = startTimes.get("A"); + const bStarted = startTimes.get("B"); + assert.ok(typeof aStarted === "number", "A started should be recorded"); + assert.ok(typeof bStarted === "number", "B started should be recorded"); + assert(aStarted! <= bStarted!, "A should start before B"); + }); + + it("starts independent services in parallel", async () => { + const startTimes = new Map(); + try { + await run(function* () { + yield* spawn(function* () { + const run = useServiceGraph({ + fast: { + operation: resource(function* (provide) { + yield* sleep(20); + startTimes.set("fast", Date.now()); + yield* provide(); + }), + }, + slow: { + operation: resource(function* (provide) { + yield* sleep(50); + startTimes.set("slow", Date.now()); + yield* provide(); + }), + }, + }); + yield* run(); + // keep spawned graph alive so services continue to run + yield* suspend(); + }); + yield* sleep(250); }); - yield* run(); - // keep spawned graph alive so beforeStop hooks run on teardown - yield* suspend(); - }); - // let them start - yield* waitFor(() => startedOrder.length === 2, 2000); + const fastStarted = startTimes.get("fast"); + const slowStarted = startTimes.get("slow"); + assert.ok(typeof fastStarted === "number", "fast started should be recorded"); + assert.ok(typeof slowStarted === "number", "slow started should be recorded"); + assert(fastStarted! <= slowStarted!, "fast should start before slow"); + } finally { + // cleanup + } }); - assert.strictEqual(startedOrder.join(""), "AB"); - assert.strictEqual(stopOrder.join(""), "BA"); -}); -it("starts independent services in parallel", async () => { - const startTimes = new Map(); - try { + it("runs subset of services with dependencies", async () => { + const startTimes = new Map(); await run(function* () { yield* spawn(function* () { - const run = useServiceGraph({ + const services = { fast: { operation: resource(function* (provide) { + console.log("test: fast operation starting"); yield* sleep(20); + console.log("test: fast operation setting startTimes"); startTimes.set("fast", Date.now()); yield* provide(); }), }, slow: { operation: resource(function* (provide) { + console.log("test: slow operation starting"); yield* sleep(50); + console.log("test: slow operation setting startTimes"); startTimes.set("slow", Date.now()); yield* provide(); }), }, - }); - yield* run(); - // keep spawned graph alive so services continue to run + dependent: { + dependsOn: { startup: ["fast", "slow"] as const }, + operation: resource(function* (provide) { + // wait until both dependencies have recorded their start times + while (!startTimes.has("fast") || !startTimes.has("slow")) { + yield* sleep(5); + } + console.log("test: dependent operation starting after deps"); + startTimes.set("dependent", Date.now()); + yield* provide(); + }), + }, + }; + + // only request dependent; fast and slow should be included as deps + const run = useServiceGraph(services); + // request only 'dependent' — this should cause 'fast' and 'slow' to be included as dependencies + yield* run(["dependent"]); + // keep spawned graph alive so services can start and perform startup work yield* suspend(); }); - yield* sleep(250); + yield* waitFor(() => startTimes.has("fast") && startTimes.has("slow"), 2000); }); - const fastStarted = startTimes.get("fast"); - const slowStarted = startTimes.get("slow"); - assert.ok(typeof fastStarted === "number", "fast started should be recorded"); - assert.ok(typeof slowStarted === "number", "slow started should be recorded"); - assert(fastStarted! <= slowStarted!, "fast should start before slow"); - } finally { - // cleanup - } -}); - -it("runs subset of services with dependencies", async () => { - const startTimes = new Map(); - await run(function* () { - yield* spawn(function* () { - const services = { - fast: { - operation: resource(function* (provide) { - console.log("test: fast operation starting"); - yield* sleep(20); - console.log("test: fast operation setting startTimes"); - startTimes.set("fast", Date.now()); - yield* provide(); - }), - }, - slow: { - operation: resource(function* (provide) { - console.log("test: slow operation starting"); - yield* sleep(50); - console.log("test: slow operation setting startTimes"); - startTimes.set("slow", Date.now()); - yield* provide(); - }), - }, - dependent: { - dependsOn: { startup: ["fast", "slow"] as const }, - operation: resource(function* (provide) { - // wait until both dependencies have recorded their start times - while (!startTimes.has("fast") || !startTimes.has("slow")) { - yield* sleep(5); - } - console.log("test: dependent operation starting after deps"); - startTimes.set("dependent", Date.now()); - yield* provide(); - }), - }, - }; - // only request dependent; fast and slow should be included as deps - const run = useServiceGraph(services); - // request only 'dependent' — this should cause 'fast' and 'slow' to be included as dependencies - yield* run(["dependent"]); - // keep spawned graph alive so services can start and perform startup work - yield* suspend(); - }); - yield* waitFor(() => startTimes.has("fast") && startTimes.has("slow"), 2000); + const f = startTimes.get("fast"); + const s = startTimes.get("slow"); + const d = startTimes.get("dependent"); + assert.ok(typeof f === "number", "fast should start"); + assert.ok(typeof s === "number", "slow should start"); + assert.ok(typeof d === "number", "dependent should start"); + assert(f! <= d!, "fast should start before dependent"); + assert(s! <= d!, "slow should start before dependent"); }); - const f = startTimes.get("fast"); - const s = startTimes.get("slow"); - const d = startTimes.get("dependent"); - assert.ok(typeof f === "number", "fast should start"); - assert.ok(typeof s === "number", "slow should start"); - assert.ok(typeof d === "number", "dependent should start"); - assert(f! <= d!, "fast should start before dependent"); - assert(s! <= d!, "slow should start before dependent"); + it("throws on cycles in dependency graph", async () => { + await assert.rejects(async () => { + await run(function* () { + const runGraph = useServiceGraph({ + A: { + operation: useService( + "A", + "node --experimental-transform-types ./test/services/service-a.ts", + ), + dependsOn: { startup: ["B"] as const }, + }, + B: { + operation: useService( + "B", + "node --experimental-transform-types ./test/services/service-b.ts", + ), + dependsOn: { startup: ["A"] as const }, + }, + }); + yield* runGraph(); + }); + }, /Cycle detected in services/); + }); }); -it("throws when requested subset includes a missing service", async () => { - await assert.rejects(async () => { +describe("service subsets", () => { + it("throws when requested subset includes a missing service", async () => { + await assert.rejects(async () => { + await run(function* () { + const services = { + a: { + operation: resource(function* (provide) { + yield* sleep(10); + yield* provide(); + }), + }, + }; + + const runGraph = useServiceGraph(services); + // request a service that does not exist + yield* runGraph(["missing"] as any); + }); + }, /Requested service 'missing' not found/); + }); + + it("runs subset specified as a string", async () => { + const startTimes = new Map(); await run(function* () { - const services = { - a: { - operation: resource(function* (provide) { - yield* sleep(10); - yield* provide(); - }), - }, - }; + yield* spawn(function* () { + const services = { + a: { + operation: resource(function* (provide) { + yield* sleep(20); + startTimes.set("a", Date.now()); + yield* provide(); + }), + }, + b: { + operation: resource(function* (provide) { + yield* sleep(50); + startTimes.set("b", Date.now()); + yield* provide(); + }), + }, + r: { + dependsOn: { startup: ["a", "b"] as const }, + operation: resource(function* (provide) { + while (!startTimes.has("a") || !startTimes.has("b")) { + yield* sleep(5); + } + startTimes.set("r", Date.now()); + yield* provide(); + }), + }, + other: { + operation: resource(function* (provide) { + yield* sleep(10); + startTimes.set("other", Date.now()); + yield* provide(); + }), + }, + }; - const runGraph = useServiceGraph(services); - // request a service that does not exist - yield* runGraph(["missing"] as any); + const run = useServiceGraph(services); + yield* run(["r"]); + yield* suspend(); + }); + yield* waitFor(() => startTimes.has("a") && startTimes.has("b") && startTimes.has("r"), 2000); }); - }, /Requested service 'missing' not found/); + + const a = startTimes.get("a"); + const b = startTimes.get("b"); + const r = startTimes.get("r"); + const other = startTimes.get("other"); + assert.ok(typeof a === "number", "a should start"); + assert.ok(typeof b === "number", "b should start"); + assert.ok(typeof r === "number", "r should start"); + assert.ok(typeof other === "undefined", "other should NOT start"); + }); }); -it("runs subset specified as a string", async () => { - const startTimes = new Map(); - await run(function* () { - yield* spawn(function* () { - const services = { +describe("tasks and promises", () => { + it("returns a task on the service graph runner", async () => { + await run(function* () { + const graphRunner = useServiceGraph({ a: { - operation: resource(function* (provide) { - yield* sleep(20); - startTimes.set("a", Date.now()); - yield* provide(); - }), - }, - b: { - operation: resource(function* (provide) { - yield* sleep(50); - startTimes.set("b", Date.now()); - yield* provide(); - }), - }, - r: { - dependsOn: { startup: ["a", "b"] as const }, - operation: resource(function* (provide) { - while (!startTimes.has("a") || !startTimes.has("b")) { - yield* sleep(5); - } - startTimes.set("r", Date.now()); - yield* provide(); - }), - }, - other: { operation: resource(function* (provide) { yield* sleep(10); - startTimes.set("other", Date.now()); yield* provide(); }), }, - }; + }); - const run = useServiceGraph(services); - yield* run(["r"]); - yield* suspend(); + const graph = graphRunner(); + assert.ok(graph.task, "graph runner should expose a task"); + assert.strictEqual(typeof graph.task.halt, "function"); + + yield* sleep(20); + yield* graph.task.halt(); + yield* graph.task; }); - yield* waitFor(() => startTimes.has("a") && startTimes.has("b") && startTimes.has("r"), 2000); }); - const a = startTimes.get("a"); - const b = startTimes.get("b"); - const r = startTimes.get("r"); - const other = startTimes.get("other"); - assert.ok(typeof a === "number", "a should start"); - assert.ok(typeof b === "number", "b should start"); - assert.ok(typeof r === "number", "r should start"); - assert.ok(typeof other === "undefined", "other should NOT start"); + it("returns a task on the service graph runner", async () => { + const graphRunner = useServiceGraph({ + a: { + operation: resource(function* (provide) { + yield* sleep(10); + yield* provide(); + }), + }, + }); + + const task = graphRunner().task; + assert.ok(task, "graph runner should expose a task"); + assert.strictEqual(typeof task.halt, "function"); + + const services = await task; + assert.notStrictEqual(services.status, undefined); + const simPort = services.status?.get("simulacrum")?.port; + assert.ok(simPort && simPort > 10000); + await task.halt(); + }); }); diff --git a/packages/server/test/watch.test.ts b/packages/server/test/watch.test.ts index 0a71ac9f..16fbb30a 100644 --- a/packages/server/test/watch.test.ts +++ b/packages/server/test/watch.test.ts @@ -241,7 +241,7 @@ it("restarts transitive dependents when watched service changes", async () => { assert(startCounts.c >= 2, "c should have been restarted as dependent of b"); }); -it("updates servicePorts when a service restarts", async () => { +it("updates status ports when a child simulation restarts", async () => { const prefix = path.join(os.tmpdir(), "sim-port-rt-"); const dir = await fs.mkdtemp(prefix); const trigger = path.join(dir, "trigger.txt"); @@ -252,7 +252,7 @@ it("updates servicePorts when a service restarts", async () => { { s: { watch: [dir], - operation: useSimulation("s", createFoundationSimulationServer({ port: 0 })), + operation: useSimulation("s", "./test/fixtures/init-data-sim.ts"), }, }, { From 4e11243cbd8019e4c1737543c23ebcd915af5f96 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Wed, 22 Apr 2026 23:24:50 -0500 Subject: [PATCH 33/38] polish off logging for new process api --- packages/server/example/process-graph.ts | 64 +++++++++++------------- packages/server/src/cli.ts | 10 +--- packages/server/src/data-service.ts | 2 +- packages/server/src/service.ts | 48 ++++++------------ packages/server/src/simulation.ts | 51 +++++++------------ 5 files changed, 65 insertions(+), 110 deletions(-) diff --git a/packages/server/example/process-graph.ts b/packages/server/example/process-graph.ts index 46303221..bf53ce52 100644 --- a/packages/server/example/process-graph.ts +++ b/packages/server/example/process-graph.ts @@ -6,53 +6,45 @@ import { simulationCLI } from "../src/cli.ts"; const servicesMap = { A: { - operation: useService( - "A", - "node --experimental-transform-types ./example/services/basic-start-1.ts", - { - wellnessCheck: { - frequency: 10, - timeout: 15000, - *operation(stdio: Stream) { - for (let line of yield* each(stdio)) { - if (line.includes("started")) { - console.log("A ready (wellnessCheck)"); + operation: useService("A", "node ./example/services/basic-start-1.ts", { + wellnessCheck: { + frequency: 10, + timeout: 15000, + *operation(stdio: Stream) { + for (let line of yield* each(stdio)) { + if (line.includes("started")) { + console.log("A ready (wellnessCheck)"); - return { ok: true, value: undefined }; - } - yield* each.next(); + return { ok: true, value: undefined }; } - // default: return success so the result type is well-formed - return { ok: true, value: undefined }; - }, + yield* each.next(); + } + // default: return success so the result type is well-formed + return { ok: true, value: undefined }; }, }, - ), + }), }, B: { dependsOn: { startup: ["A"] as const }, - operation: useService( - "B", - "node --experimental-transform-types ./example/services/basic-start-2.ts", - { - wellnessCheck: { - frequency: 10, - timeout: 15000, - *operation(stdio: Stream) { - for (let line of yield* each(stdio)) { - if (line.includes("started")) { - console.log("B ready (wellnessCheck)"); + operation: useService("B", "node ./example/services/basic-start-2.ts", { + wellnessCheck: { + frequency: 10, + timeout: 15000, + *operation(stdio: Stream) { + for (let line of yield* each(stdio)) { + if (line.includes("started")) { + console.log("B ready (wellnessCheck)"); - return { ok: true, value: undefined }; - } - yield* each.next(); + return { ok: true, value: undefined }; } - // default: return success so the result type is well-formed - return { ok: true, value: undefined }; - }, + yield* each.next(); + } + // default: return success so the result type is well-formed + return { ok: true, value: undefined }; }, }, - ), + }), }, }; diff --git a/packages/server/src/cli.ts b/packages/server/src/cli.ts index 4ad4d4dd..b5319ec4 100644 --- a/packages/server/src/cli.ts +++ b/packages/server/src/cli.ts @@ -60,7 +60,7 @@ export function* simulationCLIOp, T = any>( }; if (values["watch-debounce"]) runOptions.watchDebounce = Number(values["watch-debounce"]); - Debugging.set(!!values.debug); + yield* Debugging.set(values.debug); // Start the graph and fetch the provided info // subset is a string array from CLI; cast to service key array for strict runner @@ -84,11 +84,5 @@ export function* simulationCLIOp, T = any>( export async function simulationCLI>, T>( serviceGraph: (subset?: Array) => Operation>, ) { - return main(function* () { - try { - yield* simulationCLIOp(serviceGraph); - } finally { - yield* logger.debug("simulationCLI main finally"); - } - }); + return main(() => simulationCLIOp(serviceGraph)); } diff --git a/packages/server/src/data-service.ts b/packages/server/src/data-service.ts index 6ce673c7..9da2a4b8 100644 --- a/packages/server/src/data-service.ts +++ b/packages/server/src/data-service.ts @@ -79,7 +79,7 @@ export function startDataService(data: DataServiceOptions = {}): Operation<{ por const port = typeof address === "object" && address !== null && "port" in address ? address.port : 0; - yield* logger.stdout(`data service started on port ${port}`); + yield* logger.debug(`data service started on port ${port}`); yield* useAttributes({ name: "dataService", port }); try { diff --git a/packages/server/src/service.ts b/packages/server/src/service.ts index 05a2de80..87a751d1 100644 --- a/packages/server/src/service.ts +++ b/packages/server/src/service.ts @@ -1,19 +1,8 @@ -import { - type Operation, - type Result, - type Stream, - each, - lift, - resource, - scoped, - sleep, - spawn, -} from "effection"; +import { type Operation, type Result, type Stream, lift, resource, scoped, sleep } from "effection"; import { useAttributes } from "./logging.ts"; import { timebox } from "@effectionx/timebox"; -import { daemon } from "@effectionx/process"; +import { daemon, Stdio } from "@effectionx/process"; import type { ExecOptions as ProcessOptions } from "@effectionx/process"; -import { logger } from "./logging.ts"; import { createReplaySignal } from "./createReplaySignal.ts"; import { withOperationMetadata } from "./operation-metadata.ts"; @@ -47,33 +36,26 @@ export function useService( // see https://github.com/npm/cli/issues/6684 throw new Error("scripts run with npm don't respect signals to properly shutdown"); } - const process = yield* daemon(cmd, options.processOptions); + const stdio = createReplaySignal(); const stdioAdd = lift(stdio.send); - // forward raw stdout for logging in chunk form (no reassembly) - yield* spawn(function* () { - yield* useAttributes({ name: "stdoutForward" }); - for (let line of yield* each(process.stdout)) { - const buf = Buffer.from(line); - const str = buf.toString(); - yield* logger.stdout(str); + yield* Stdio.around({ + *stdout(line, next) { + const [bytes] = line; + const str = bytes.toString(); yield* stdioAdd(str); - yield* each.next(); - } - }); - - yield* spawn(function* () { - yield* useAttributes({ name: "stderrForward" }); - for (let line of yield* each(process.stderr)) { - const str = Buffer.from(line).toString(); - yield* logger.stderr(str); + return yield* next(bytes); + }, + *stderr(line, next) { + const [bytes] = line; + const str = Buffer.from(bytes).toString(); yield* stdioAdd(str); - yield* each.next(); - } + return yield* next(bytes); + }, }); - yield* sleep(0); // allow stdio forwarding to start + yield* daemon(cmd, options.processOptions); // if supplied, wellness check to ensure it is running or timeout with result if (options.wellnessCheck) { diff --git a/packages/server/src/simulation.ts b/packages/server/src/simulation.ts index f9705b70..efb26ad1 100644 --- a/packages/server/src/simulation.ts +++ b/packages/server/src/simulation.ts @@ -97,27 +97,12 @@ export function useSimulationChildProcess(name: string, modulePath: string) { } const cmd = parts.map((s) => (s.includes(" ") ? `'${s}'` : s)).join(" "); - const process = yield* daemon(cmd); - const pid = process.pid; - yield* useAttributes({ - name: `useSimulation ${name}`, - cmd, - pid: String(pid), - }); - // read the first stdout JSON line to get the listening info let port = undefined as number | undefined; - let ready = withResolvers("wait until the port is returned to signal ready"); - - // forward raw stdout for logging in chunk form (no reassembly) - yield* spawn(function* () { - yield* useAttributes({ - name: "stdoutForward", - }); - for (let line of yield* each(process.stdout)) { - const buf = Buffer.from(line); - const str = buf.toString(); - + yield* Stdio.around({ + *stdout(line, _next) { + const [bytes] = line; + const str = bytes.toString(); if (!port) { try { const parsed = JSON.parse(str); @@ -134,22 +119,24 @@ export function useSimulationChildProcess(name: string, modulePath: string) { } else { yield* logger.stdout(str); } - - yield* each.next(); - } + }, + *stderr(line, _next) { + const [bytes] = line; + const str = bytes.toString(); + yield* logger.stderr(str); + }, }); - yield* spawn(function* () { - yield* useAttributes({ - name: "stderrForward", - }); - for (let line of yield* each(process.stderr)) { - const str = Buffer.from(line).toString(); - yield* logger.stderr(str); - yield* each.next(); - } + const process = yield* daemon(cmd); + const pid = process.pid; + yield* useAttributes({ + name: `useSimulation ${name}`, + cmd, + pid: String(pid), }); + let ready = withResolvers("wait until the port is returned to signal ready"); + // spawn a watcher to detect if the child exits before printing the listening info let status: unknown = undefined; yield* spawn(function* () { @@ -178,7 +165,7 @@ export function useSimulationChildProcess(name: string, modulePath: string) { ); } - yield* logger.stdout(`${name} simulation: port ${port} pid ${pid}`); + yield* logger.debug(`${name} simulation: port ${port} pid ${pid}`); try { yield* provide({ port, pid }); From fc06fcae573a73684a4d6c42c99b81eb0bc2cb9b Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Wed, 22 Apr 2026 23:29:26 -0500 Subject: [PATCH 34/38] gate type strip by node version --- packages/server/src/simulation.ts | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/server/src/simulation.ts b/packages/server/src/simulation.ts index efb26ad1..17b6792c 100644 --- a/packages/server/src/simulation.ts +++ b/packages/server/src/simulation.ts @@ -1,7 +1,7 @@ import { resource, until, spawn, each, withResolvers } from "effection"; import { useAttributes } from "./logging.ts"; import type { Operation } from "effection"; -import { daemon } from "@effectionx/process"; +import { daemon, Stdio } from "@effectionx/process"; import { logger } from "./logging.ts"; import type { FoundationSimulator, @@ -9,6 +9,7 @@ import type { } from "@simulacrum/foundation-simulator"; import { SimulacrumEndpoint } from "./services.ts"; import { fileURLToPath } from "node:url"; +import { versions } from "node:process"; import { withOperationMetadata } from "./operation-metadata.ts"; type UseSimulationOptions = { @@ -85,13 +86,16 @@ export function useSimulationChildProcess(name: string, modulePath: string) { const runnerPath = fileURLToPath( import.meta.resolve("@simulacrum/server/bin/run-simulation-child.ts"), ); - const parts = [ - "node", - // safest considering current LTS of >v20 - "--experimental-strip-types", - runnerPath, - modulePath, - ]; + // TODO config to overwrite the hard coded option here + const parts = ( + Number(versions.node.split(".")[0]) >= 24 + ? ["node"] + : [ + "node", + // safest considering current LTS of >v20 + "--experimental-strip-types", + ] + ).concat([runnerPath, modulePath]); if (typeof contextPort === "number") { parts.push("--simulacrum-port", String(contextPort)); } From ace7d02f9f6ee8862c9186c4b7c261f6620125ca Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Wed, 22 Apr 2026 23:32:54 -0500 Subject: [PATCH 35/38] process provide is always filled --- packages/server/README.md | 8 ++++---- packages/server/src/services.ts | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/server/README.md b/packages/server/README.md index 12ae25a1..12fe63d4 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -229,10 +229,10 @@ const runner = useServiceGraph( main(function* (): Operation { const services = yield* runner(); - const simulacrumPort = services.status?.get("simulacrum")?.port; + const simulacrumPort = services.status.get("simulacrum")?.port; const res = yield* until(fetch(`http://127.0.0.1:${simulacrumPort}/data`)); const data = yield* until(res.json()); - console.log(data) + console.log(data); }); ``` @@ -252,8 +252,8 @@ The runner operation returns an object with the following shape: - `services` — the original service definitions passed to `useServiceGraph` - `status` — a `Map` with runtime metadata for each service, including optional `port` and `pid` when the operation returns that information -- `serviceUpdates` — a `Stream` of watcher updates, if watching is enabled -- `serviceChanges` — a `Stream` of watcher restart events, if watching is enabled +- `serviceUpdates` — a `Stream` of watcher updates when watching is enabled, otherwise `undefined` +- `serviceChanges` — a `Stream` of watcher restart events when watching is enabled, otherwise `undefined` If a service operation returns an object like `{ port: number }` or `{ port: number; pid: number }`, that information is recorded on `status` so tests can discover listening endpoints. diff --git a/packages/server/src/services.ts b/packages/server/src/services.ts index 0426cc77..e9d02c6a 100644 --- a/packages/server/src/services.ts +++ b/packages/server/src/services.ts @@ -47,9 +47,9 @@ export type ServiceGraph< services: { [service in keyof S]: ServiceDefinition; }; - serviceUpdates?: Stream | undefined; - serviceChanges?: Stream | undefined; - status?: Map; + serviceUpdates: Stream | undefined; + serviceChanges: Stream | undefined; + status: Map; }; export type ServiceInfo = { From 16782be26cbd392877564569a279877a57cd06ea Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Wed, 22 Apr 2026 23:34:35 -0500 Subject: [PATCH 36/38] lint --- packages/server/src/simulation.ts | 2 +- packages/server/test/watch.test.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/server/src/simulation.ts b/packages/server/src/simulation.ts index 17b6792c..802049d6 100644 --- a/packages/server/src/simulation.ts +++ b/packages/server/src/simulation.ts @@ -1,4 +1,4 @@ -import { resource, until, spawn, each, withResolvers } from "effection"; +import { resource, until, spawn, withResolvers } from "effection"; import { useAttributes } from "./logging.ts"; import type { Operation } from "effection"; import { daemon, Stdio } from "@effectionx/process"; diff --git a/packages/server/test/watch.test.ts b/packages/server/test/watch.test.ts index 16fbb30a..94d07daf 100644 --- a/packages/server/test/watch.test.ts +++ b/packages/server/test/watch.test.ts @@ -7,7 +7,6 @@ import os from "node:os"; import { useServiceGraph } from "../src/services.ts"; import { simulation } from "./fixtures/simple-sim.ts"; import { useSimulation } from "../src/simulation.ts"; -import { createFoundationSimulationServer } from "@simulacrum/foundation-simulator"; import { waitFor, waitForOperation } from "./utils.ts"; it("restarts services on watched file change and restarts dependents", async () => { From b3a45efeffd0325cae7eb93c491b994c84b6add5 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Thu, 23 Apr 2026 16:20:58 -0500 Subject: [PATCH 37/38] import path relative to simulation file --- packages/server/src/simulation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/simulation.ts b/packages/server/src/simulation.ts index 802049d6..cfeb8199 100644 --- a/packages/server/src/simulation.ts +++ b/packages/server/src/simulation.ts @@ -84,7 +84,7 @@ export function useSimulationChildProcess(name: string, modulePath: string) { const contextPort = yield* SimulacrumEndpoint.get(); const runnerPath = fileURLToPath( - import.meta.resolve("@simulacrum/server/bin/run-simulation-child.ts"), + new URL("../bin/run-simulation-child.ts", import.meta.url), ); // TODO config to overwrite the hard coded option here const parts = ( From bc71c0f0c0f621daf2ab34a886d4789de946eb81 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Thu, 23 Apr 2026 16:25:33 -0500 Subject: [PATCH 38/38] fmt --- packages/server/src/simulation.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/server/src/simulation.ts b/packages/server/src/simulation.ts index cfeb8199..b20646c8 100644 --- a/packages/server/src/simulation.ts +++ b/packages/server/src/simulation.ts @@ -83,9 +83,7 @@ export function useSimulationChildProcess(name: string, modulePath: string) { // attempt to read the simulacrum port from context; if not present, continue without it const contextPort = yield* SimulacrumEndpoint.get(); - const runnerPath = fileURLToPath( - new URL("../bin/run-simulation-child.ts", import.meta.url), - ); + const runnerPath = fileURLToPath(new URL("../bin/run-simulation-child.ts", import.meta.url)); // TODO config to overwrite the hard coded option here const parts = ( Number(versions.node.split(".")[0]) >= 24