Skip to content

Switch from automerge to loro #466

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
rroohhh wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

Switch from automerge to loro #466

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0700eb8
WIP: move automerge to loro
rroohhh Mar 1, 2025
a42af82
wip
rroohhh Nov 14, 2025
c18b29f
wip
rroohhh Nov 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions backend/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ authors = [
]

dependencies = [
"loro~=1.8.2",
"automerge @ git+https://github.com/bugbakery/automerge-py.git@ca6d8d3",
"redis~=5.0",
"fastapi~=0.115",
"uvicorn[standard]~=0.20",
Expand Down
25 changes: 25 additions & 0 deletions backend/transcribee_backend/db/migrations/versions/10e80963c5a3_automerge_loro_migration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
"""automerge -> loro migration

Revision ID: 10e80963c5a3
Revises: c88376bf4844
Create Date: 2025-11-13 22:44:21.812234

"""
from alembic import op
import sqlalchemy as sa
import sqlmodel


# revision identifiers, used by Alembic.
revision = '10e80963c5a3'
down_revision = 'c88376bf4844'
branch_labels = None
depends_on = None


def upgrade() -> None:
pass


def downgrade() -> None:
pass
24 changes: 13 additions & 11 deletions backend/transcribee_backend/helpers/sync.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import asyncio
from asyncio import Queue
from typing import Callable
from typing import Callable, Tuple
import uuid

from fastapi import WebSocket, WebSocketDisconnect
from sqlmodel import Session, select
Expand Down Expand Up @@ -45,14 +46,17 @@ def __init__(
self._can_write = can_write
self._subscribed = set()
self._msg_queue = Queue()
self._id = uuid.uuid4()

def subscribe(self, channel: str):
self._subscribed.add(channel)
sync_manager.subscribe(channel, self.handle_incoming_broadcast)

async def handle_incoming_broadcast(self, channel: str, message: bytes):
async def handle_incoming_broadcast(self, channel: str, message: Tuple[uuid.UUID, bytes]):
if channel in self._subscribed:
await self._msg_queue.put(message)
id, msg = message
if (id != self._id):
await self._msg_queue.put(msg)

async def listener(self):
while True:
Expand All @@ -73,17 +77,15 @@ async def broadcast_sender(self):
# websocket connection eventually. Since it is not touched on the way, we can pass a list of
# bytes here instead of just bytes as would be allowed by the asgi spec:
# https://asgi.readthedocs.io/en/latest/specs/www.html#send-send-event
message = [bytes([SyncMessageType.FULL_DOCUMENT])]
tag_change = bytes([SyncMessageType.CHANGE])
for update in self._session.exec(statement):
message.append(update.change_bytes)
await self._ws.send_bytes(message) # type: ignore
# END
await self._ws.send_bytes(tag_change + len(update.change_bytes).to_bytes(4) + update.change_bytes)

await self._ws.send_bytes(bytes([SyncMessageType.CHANGE_BACKLOG_COMPLETE]))
await self._ws.send_bytes(bytes([SyncMessageType.BACKLOG_COMPLETE]))

while True:
msg = await self._msg_queue.get()
await self._ws.send_bytes(bytes([SyncMessageType.CHANGE]) + msg)
msg: bytes = await self._msg_queue.get()
await self._ws.send_bytes(tag_change + len(msg).to_bytes(4) + msg)

async def run(self):
await self._ws.accept()
Expand Down Expand Up @@ -126,4 +128,4 @@ async def on_message(self, message: bytes):
self._session.add(self._doc)

self._session.commit()
await sync_manager.broadcast(str(self._doc.id), message)
await sync_manager.broadcast(str(self._doc.id), (self._id, message))
84 changes: 84 additions & 0 deletions backend/uv.lock
View file
Open in desktop

Large diffs are not rendered by default.

63 changes: 18 additions & 45 deletions frontend/package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 3 additions & 3 deletions frontend/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"version": "0.1.0",
"scripts": {
"dev": "vite",
"profile": "vite --profile",
"build": "npm-run-all --continue-on-error build:*",
"build:licenses": "node scripts/generate_licenses.mjs public/LICENSES.md",
"build:vite": "vite build",
Expand Down Expand Up @@ -57,13 +58,13 @@
},
"dependencies": {
"@audapolis/webvtt-writer": "^1.0.6",
"@automerge/automerge": "~2.2.0",
"@automerge/automerge-wasm": "^0.15.0",
"@fontsource/inter": "^4.5.15",
"@podlove/html5-audio-driver": "^2.0.3",
"@popperjs/core": "^2.11.8",
"@zip.js/zip.js": "^2.7.31",
"clsx": "^1.2.1",
"fast-equals": "^5.2.2",
"loro-crdt": "^1.9.0",
"openapi-typescript-fetch": "github:bugbakery/openapi-typescript-fetch#4b5cc33983c5a658feedd628d417599db2bd672c",
"react": "^18.2.0",
"react-base16-styling": "^0.9.1",
Expand All @@ -78,7 +79,6 @@
"react-truncate-markup": "^5.1.2",
"reconnecting-websocket": "^4.4.0",
"slate": "^0.94.1",
"slate-automerge-doc": "github:bugbakery/slate-automerge-doc#98d7a78f14cb4a16484847e0090f6b5439cc16b6",
"slate-history": "^0.93.0",
"slate-react": "^0.94.2",
"swr": "^2.2.4",
Expand Down
100 changes: 52 additions & 48 deletions frontend/src/document.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,54 +1,58 @@
import { Document } from './editor/types';
import { next as Automerge } from '@automerge/automerge';
import * as AutomergeStable from '@automerge/automerge';
import { Document, EditorDocument } from './editor/types';
// import { next as Automerge } from '@automerge/automerge';
// import * as AutomergeStable from '@automerge/automerge';

function convertString(s: string): string {
// this typecasting is a hack to avoid having multiple types for different versions for now
return new AutomergeStable.Text(s.toString()) as unknown as string;
export function documentToJSON(doc: EditorDocument): Document {
return doc.toJSON().root;
}

export function migrateDocument(doc: Automerge.Doc<Document>): Automerge.Doc<Document> {
let theDoc = doc;
const v1 = theDoc.version === 1;
const actorID = Automerge.getActorId(doc);
if (v1) {
theDoc = AutomergeStable.load(Automerge.save(doc), actorID);
}
if (theDoc.version === 2) {
return doc;
}
const migratedDoc = AutomergeStable.change(theDoc, (doc: Document) => {
switch (doc.version) {
case 1:
for (const speakerID of Object.keys(doc.speaker_names)) {
doc.speaker_names[speakerID] = convertString(doc.speaker_names[speakerID]);
}
// function convertString(s: string): string {
// // this typecasting is a hack to avoid having multiple types for different versions for now
// return new AutomergeStable.Text(s.toString()) as unknown as string;
// }

doc.children.forEach((paragraph) => {
paragraph.type = convertString(paragraph.type) as 'paragraph';
paragraph.speaker = paragraph.speaker ? convertString(paragraph.speaker) : null;
paragraph.lang = convertString(paragraph.lang.toString());
// export function migrateDocument(doc: Automerge.Doc<Document>): Automerge.Doc<Document> {
// let theDoc = doc;
// const v1 = theDoc.version === 1;
// const actorID = Automerge.getActorId(doc);
// if (v1) {
// theDoc = AutomergeStable.load(Automerge.save(doc), actorID);
// }
// if (theDoc.version === 2) {
// return doc;
// }
// const migratedDoc = AutomergeStable.change(theDoc, (doc: Document) => {
// switch (doc.version) {
// case 1:
// for (const speakerID of Object.keys(doc.speaker_names)) {
// doc.speaker_names[speakerID] = convertString(doc.speaker_names[speakerID]);
// }

paragraph.children.forEach((child) => {
const start = child.start;
child.start = start;
const end = child.end;
child.end = end;
const conf = child.conf;
child.conf = conf;
child.text = convertString(child.text.toString());
});
});
doc.version = 2;
// falls through
case 2:
break;
}
});
// doc.children.forEach((paragraph) => {
// paragraph.type = convertString(paragraph.type) as 'paragraph';
// paragraph.speaker = paragraph.speaker ? convertString(paragraph.speaker) : null;
// paragraph.lang = convertString(paragraph.lang.toString());

if (v1) {
return Automerge.load(AutomergeStable.save(migratedDoc), actorID);
} else {
return migratedDoc;
}
}
// paragraph.children.forEach((child) => {
// const start = child.start;
// child.start = start;
// const end = child.end;
// child.end = end;
// const conf = child.conf;
// child.conf = conf;
// child.text = convertString(child.text.toString());
// });
// });
// doc.version = 2;
// // falls through
// case 2:
// break;
// }
// });

// if (v1) {
// return Automerge.load(AutomergeStable.save(migratedDoc), actorID);
// } else {
// return migratedDoc;
// }
// }
Loading
Loading