diff --git a/DistFiles/localization/en/BloomMediumPriority.xlf b/DistFiles/localization/en/BloomMediumPriority.xlf index f41d179b0ef1..ba4729e798d7 100644 --- a/DistFiles/localization/en/BloomMediumPriority.xlf +++ b/DistFiles/localization/en/BloomMediumPriority.xlf @@ -39,6 +39,21 @@ ID: CollectionSettingsDialog.AdvancedTab.Experimental.AppBuilder This checkbox enables the experimental publish-to-apps workflow. + + Edit Images with AI + ID: CollectionSettingsDialog.AdvancedTab.Experimental.EditWithAI + This checkbox enables the experimental "Edit with AI" image editor. + + + Edit Images with AI + ID: Feature.AiImageEditing + Name of the AI image editing feature, used in subscription messages. + + + Edit with AI... + ID: EditTab.Image.EditWithAI + Context-menu item on an image in the Edit tab; opens the AI image editor in an overlay. The "..." indicates further UI will open, as in the sibling item "Choose image from your computer...". + Choose... diff --git a/ReadMe.md b/ReadMe.md index 2fc9eabd5e43..302b81272a79 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -1,6 +1,6 @@ # Books for every child, in every language -Bloom is an [award winning](http://allchildrenreading.org/sil-international-wins-enabling-writers-prize-for-software-solution-to-childrens-book-shortage/) software solution to the children's book shortage among most of the world's languages. It is an application for Windows and [Linux](https://bloomlibrary.org/page/create/linux) that dramatically "lowers the bar" for creating, translating, and sharing books. With Bloom, communities can do the work for themselves instead of depending on outsiders. For more information, see https://bloomlibrary.org/about. For user documentation, see https://docs.bloomlibrary.org. +Bloom is an [award winning](http://allchildrenreading.org/sil-international-wins-enabling-writers-prize-for-software-solution-to-childrens-book-shortage/) software solution to the children's book shortage among most of the world's languages. It is an application for Windows and [Linux](https://bloomlibrary.org/page/create/linux) that dramatically "lowers the bar" for creating, translating, and sharing books. With Bloom, communities can do the work for themselves instead of depending on outsiders. For more information, see [https://bloomlibrary.org/about](https://bloomlibrary.org/about). For user documentation, see [https://docs.bloomlibrary.org](https://docs.bloomlibrary.org). Internally, Bloom is a hybrid. It started as a C#/WinForms app with an embedded browser for editing documents and an embedded Adobe Acrobat for displaying PDF outputs. It is growing up to be a pure React-driven offline-capable web app, with a C# backend. In its current state, Bloom is hybrid of C#/web app in which the bits of the UI are gradually moving to html. @@ -15,10 +15,9 @@ Internally, Bloom is a hybrid. It started as a C#/WinForms app with an embedded ``` 3. Run a hot-reloading server for the front end and a "watch" run of the back end. -```bash -./go.sh -``` - + ```bash + ./go.sh + ``` # Developing @@ -29,11 +28,17 @@ We don't want developer and tester runs (and crashes) polluting our statistics. ### Set up formatting In a terminal, run `dotnet tool restore`. This will install any tools we have put in .config/dotnet-tools.json along with the correct versions. + In Visual Studio, under Extensions, install "CSharpier". The extension's version will not be the same as the CSharpier installed by 'dotnet tool restore', but that's not a problem. + In Tools/Options, under CSharpier:General, set `Reformat with CSharpier on Save` to `true`. Note that you should set it for this solution, not globally. + You should also install the CSharpier extension in VSCode. + CSharpier should be using the version specified in `.config/dotnet-tools.json`. + When upgrading to a new version of CSharpier, to format everything, run `dotnet csharpier format src/BloomExe src/BloomTests src/WebView2PdfMaker`. + To hide reformatting-only commits from git blame, add the sha of the commit to `.git-blame-ignore-revs`. For Typescript formatting, we use the Prettier extension in VSCode. @@ -50,9 +55,32 @@ It may be helpful before submitting a PR to turn off pnpm dev and run pnpm build If you have the team's agent skills installed (see the Skills section of AGENTS.md), the `/preflight` skill automates the rest of the pre-review checklist: typecheck, lint, tests, draft PR, and the bot review gauntlet. +### Developing a library alongside Bloom + +Bloom uses several libraries that we maintain in their own repositories (they behave like a loose monorepo). By default, `./go.sh` uses each library exactly as installed in `node_modules` (i.e. as last `yarn install`ed / published) — you don't need their source. + +When you're also working on one of those libraries, link your local checkout of it with the repeatable `--with` flag so your edits flow into the running Bloom: + +```bash +./go.sh --with LIBRARY # auto-discover a sibling checkout +./go.sh --with LIBRARY=PATH # or point at an explicit checkout path +./go.sh --with bloom-player --with @sillsdev/config-r # link more than one +``` + +`LIBRARY` is one of the package names listed below; `PATH` is a checkout directory. With no path, the checkout is auto-discovered as a sibling of the Bloom repo (e.g. `../bloom-player`, or `../../bloom-player` from a git worktree). `go.sh` starts and stops the library's dev processes for you and tears them down on Ctrl-C; the library's own dependencies must already be installed in its checkout (`pnpm install` or `yarn install` there, as appropriate). + +The linkable libraries (by package name) and what `--with` does for each: + +- `bloom-ai-image-tools` — `--with` runs the library's own Vite dev server (`pnpm dev`) and points Bloom at it, giving full HMR inside the editor. +- `bloom-player` — `--with` aliases the package to your checkout and runs its watch-builds (both the imported library output and the standalone player assets), so your changes appear in Bloom's UI on reload. +- `@sillsdev/config-r` — `--with` aliases it to your checkout and runs its watch-build (`yarn build:dev`). + +The list of linkable libraries lives in `src/BloomBrowserUI/scripts/devLibraries.mjs`; add an entry there to make another library linkable. + ### Windows Defender exclusions For performance reasons, you probably want to exclude at least the following in the Windows Defender settings: + - node.exe (process) - Bloom source code folder (e.g. `C:/dev/BloomDesktop`) @@ -79,6 +107,7 @@ Each time code is checked in, an automatic build begins on our [TeamCity build s ### Linux Development 5.4 is the last version for Linux until we get rid of WinForms. + See the `Version5.4` branch and ReadMe if you need to update it. See [this page](https://bloomlibrary.org/page/create/linux) for how to run it. @@ -89,9 +118,10 @@ UI localization happens on [Crowdin](https://crowdin.com/project/sil-bloom). ### Registry settings -One responsibility of Bloom desktop is to handle url's starting with "bloom://"", such as those used on bloomlibrary.org when the user clicks "Translate into _your_ language!" Making this work requires some registry settings. These are automatically created when you run Bloom. If you have multiple versions installed, just make sure that the one you ran most recently is the one you want to do the download. +One responsibility of Bloom desktop is to handle url's starting with "bloom://"", such as those used on bloomlibrary.org when the user clicks "Translate into *your* language!" Making this work requires some registry settings. These are automatically created when you run Bloom. If you have multiple versions installed, just make sure that the one you ran most recently is the one you want to do the download. # License Bloom is open source, using the [MIT License](http://sil.mit-license.org). It is Copyright SIL Global. -"Bloom" is a registered trademark of SIL Global. + +"Bloom" is a registered trademark of SIL Global. \ No newline at end of file diff --git a/src/BloomBrowserUI/bookEdit/js/bloomImages.ts b/src/BloomBrowserUI/bookEdit/js/bloomImages.ts index 9a0cd0579a6b..97a00851426f 100644 --- a/src/BloomBrowserUI/bookEdit/js/bloomImages.ts +++ b/src/BloomBrowserUI/bookEdit/js/bloomImages.ts @@ -1088,7 +1088,7 @@ function getFileLengthString(bytes): string { } // Gets the src attribute out of images, and the background-image:url() of everything else -function GetRawImageUrl(imgOrDivWithBackgroundImage): string { +export function GetRawImageUrl(imgOrDivWithBackgroundImage): string { if ($(imgOrDivWithBackgroundImage).hasAttr("src")) { return $(imgOrDivWithBackgroundImage).attr("src"); } diff --git a/src/BloomBrowserUI/bookEdit/js/canvasElementManager/CanvasElementContextControls.tsx b/src/BloomBrowserUI/bookEdit/js/canvasElementManager/CanvasElementContextControls.tsx index ec60a37290df..8f1f2bcb8a40 100644 --- a/src/BloomBrowserUI/bookEdit/js/canvasElementManager/CanvasElementContextControls.tsx +++ b/src/BloomBrowserUI/bookEdit/js/canvasElementManager/CanvasElementContextControls.tsx @@ -19,6 +19,7 @@ import { getCanvasElementManager } from "../../toolbox/canvas/canvasElementPageB import { kBackgroundImageClass } from "../../toolbox/canvas/canvasElementConstants"; import { BloomTooltip } from "../../../react_components/BloomToolTip"; import { useL10n } from "../../../react_components/l10nHooks"; +import { useGetFeatureStatus } from "../../../react_components/featureStatus"; import { kBloomDisabledOpacity } from "../../../utils/colorUtils"; import { getAsync, useApiObject } from "../../../utils/bloomApi"; import AudioRecording from "../../toolbox/talkingBook/audioRecording"; @@ -88,6 +89,10 @@ const CanvasElementContextControls: React.FunctionComponent<{ }; const menuEl = useRef(null); + // The "Edit with AI" command is an experimental, subscription-gated feature. + // Its FeatureStatus.visible reflects whether the experimental feature is on; + // we feed that into the control context so the menu item is hidden when off. + const aiImageEditingStatus = useGetFeatureStatus("AiImageEditing"); const languageNameValues = useApiObject( "settings/languageNames", { @@ -454,6 +459,7 @@ const CanvasElementContextControls: React.FunctionComponent<{ textHasAudio, hasClipboardText, languageNameValues, + aiImageEditingAvailable: aiImageEditingStatus?.visible ?? false, }; const definition = diff --git a/src/BloomBrowserUI/bookEdit/toolbox/canvas/bloom-exe-ai-editor-open.uitest.ts b/src/BloomBrowserUI/bookEdit/toolbox/canvas/bloom-exe-ai-editor-open.uitest.ts new file mode 100644 index 000000000000..c42db56bcf7e --- /dev/null +++ b/src/BloomBrowserUI/bookEdit/toolbox/canvas/bloom-exe-ai-editor-open.uitest.ts @@ -0,0 +1,134 @@ +import { test, expect } from "playwright/test"; +import { connectToBloomExe } from "../../../react_components/component-tester/bloomExeCdp"; + +// End-to-end smoke test for the "Edit with AI..." integration (see +// canvasControlRegistry.ts `editWithAi` and AiImageEditorApi.cs). It attaches to a +// real, running Bloom.exe over CDP and proves the editor can actually OPEN — the exact +// thing that silently 503'd when the editor app wasn't staged into the build. +// +// PRECONDITION: Bloom is running in the Edit tab with a book selected (the launch +// endpoint needs a current book). Run it with the app up, e.g. from src/BloomBrowserUI: +// pnpm exec playwright test --config react_components/component-tester/playwright.bloom-exe.config.ts \ +// bookEdit/toolbox/canvas/bloom-exe-ai-editor-open.uitest.ts +// +// It deliberately stops short of driving the right-click menu (that is the heavier +// "full UI" flow); instead it exercises the launch contract and confirms the editor +// iframe boots, which is what "we can open the AI editor" really means. +test.describe("Bloom exe CDP: AI Image Editor", () => { + test("launches and the editor app boots in its iframe", async () => { + const connection = await connectToBloomExe(); + + try { + // 1. The launch endpoint mints a session and returns everything the overlay + // needs, including where the editor app is served (editorUrl). This is the + // call that returned 503 "not included in this build" before staging worked. + const launch = await connection.page.evaluate(async () => { + const response = await fetch( + "/bloom/api/aiImageEditor/launch", + { method: "POST" }, + ); + return { + status: response.status, + body: response.ok + ? await response.json() + : await response.text(), + }; + }); + + expect( + launch.status, + `aiImageEditor/launch failed (is a book selected in the Edit tab?): ${JSON.stringify( + launch.body, + )}`, + ).toBe(200); + + const launchData = launch.body as { + editorUrl: string; + sessionToken: string; + bookImages?: unknown[]; + }; + expect( + launchData.editorUrl, + "launch reply had no editorUrl", + ).toBeTruthy(); + expect( + launchData.sessionToken, + "launch reply had no sessionToken", + ).toBeTruthy(); + + // 2. Prove the editor really opens: load it in an iframe exactly as the overlay + // does (mode=bloom-iframe) and wait for the editor's own "ready" handshake on + // the shared "bloom-ai-image-tools" channel. That message only fires once the + // app's HTML + JS have loaded and BloomHostedImageEditor has mounted, so it is + // the strongest signal that the editor genuinely booted rather than 404'd. + const bootResult = await connection.page.evaluate( + async (editorUrl: string) => { + const iframeUrl = new URL(editorUrl, window.location.href); + iframeUrl.searchParams.set("mode", "bloom-iframe"); + + return await new Promise<{ + ready: boolean; + reason: string; + }>((resolve) => { + const iframe = document.createElement("iframe"); + iframe.id = "ai-editor-smoke-iframe"; + // Keep it out of sight; we only care that it boots. + iframe.style.cssText = + "position:fixed;left:-99999px;top:0;width:1024px;height:768px;border:0;"; + + let settled = false; + const cleanup = () => { + window.removeEventListener("message", onMessage); + clearTimeout(timer); + iframe.remove(); + }; + const finish = (ready: boolean, reason: string) => { + if (settled) return; + settled = true; + cleanup(); + resolve({ ready, reason }); + }; + + const onMessage = (event: MessageEvent) => { + if (event.source !== iframe.contentWindow) { + return; + } + const data = event.data as { + channel?: string; + type?: string; + }; + if ( + data?.channel === "bloom-ai-image-tools" && + data?.type === "ready" + ) { + finish(true, "received ready handshake"); + } + }; + + const timer = setTimeout( + () => + finish( + false, + "timed out waiting for the editor's ready handshake", + ), + 20000, + ); + + iframe.addEventListener("error", () => + finish(false, "iframe failed to load"), + ); + + window.addEventListener("message", onMessage); + iframe.src = iframeUrl.toString(); + document.body.appendChild(iframe); + }); + }, + launchData.editorUrl, + ); + + expect(bootResult.ready, bootResult.reason).toBe(true); + } finally { + await connection.browser.close(); + } + }); +}); diff --git a/src/BloomBrowserUI/bookEdit/toolbox/canvas/buildCanvasElementControlRegistryContext.ts b/src/BloomBrowserUI/bookEdit/toolbox/canvas/buildCanvasElementControlRegistryContext.ts index 6d9ed68746ab..e52f309a663d 100644 --- a/src/BloomBrowserUI/bookEdit/toolbox/canvas/buildCanvasElementControlRegistryContext.ts +++ b/src/BloomBrowserUI/bookEdit/toolbox/canvas/buildCanvasElementControlRegistryContext.ts @@ -226,5 +226,8 @@ export const buildCanvasElementControlRegistryContext = ( language2Name: "", language2Tag: "", }, + // Default off; CanvasElementContextControls overlays the real feature + // status (which is async) when it assembles the context. + aiImageEditingAvailable: false, }; }; diff --git a/src/BloomBrowserUI/bookEdit/toolbox/canvas/canvasControlAvailabilityRules.ts b/src/BloomBrowserUI/bookEdit/toolbox/canvas/canvasControlAvailabilityRules.ts index b2cf6f2b50b9..190c799c3638 100644 --- a/src/BloomBrowserUI/bookEdit/toolbox/canvas/canvasControlAvailabilityRules.ts +++ b/src/BloomBrowserUI/bookEdit/toolbox/canvas/canvasControlAvailabilityRules.ts @@ -31,6 +31,13 @@ export const imageAvailabilityRules: AvailabilityRulesMap = { visible: (ctx) => ctx.hasImage, enabled: (ctx) => ctx.isCropped, }, + editWithAi: { + // Only offered when the AI Image Editing experimental feature is turned on. + // The AI Image Editor needs a real raster image to work on, and the user + // must be allowed to modify it. + visible: (ctx) => ctx.aiImageEditingAvailable && ctx.hasImage, + enabled: (ctx) => ctx.hasRealImage && ctx.canModifyImage, + }, missingMetadata: { surfacePolicy: { toolbar: { diff --git a/src/BloomBrowserUI/bookEdit/toolbox/canvas/canvasControlRegistry.ts b/src/BloomBrowserUI/bookEdit/toolbox/canvas/canvasControlRegistry.ts index 070f04e7fbe2..bd2c37877d8a 100644 --- a/src/BloomBrowserUI/bookEdit/toolbox/canvas/canvasControlRegistry.ts +++ b/src/BloomBrowserUI/bookEdit/toolbox/canvas/canvasControlRegistry.ts @@ -46,6 +46,7 @@ import { getImageFromContainer, getImageTransparencyMode, getImageUrlFromImageContainer, + GetRawImageUrl, HandleImageError, getOwningPageBackgroundColor, isPlaceHolderImage, @@ -55,10 +56,12 @@ import { } from "../../js/bloomImages"; import { doVideoCommand } from "../../js/bloomVideo"; import { + changeImageByElement, copySelection, GetEditor, pasteClipboard, } from "../../js/bloomEditing"; +import { post, postJson, postString } from "../../../utils/bloomApi"; import { CogIcon } from "../../js/CogIcon"; import { DuplicateIcon } from "../../js/DuplicateIcon"; import { FillSpaceIcon } from "../../js/FillSpaceIcon"; @@ -503,6 +506,463 @@ export const controlRegistry: Record = { getCanvasElementManager()?.resetCropping(); }, }, + // "Edit with AI…" — the entry point for the AI Image Editor integration. + // + // This is the front-end half of a feature whose C# half is AiImageEditorApi.cs + // (read that file's header for the full picture). The editor is a SEPARATE web app + // (the `bloom-ai-image-tools` package); we do not import it — we load it into an + //