diff --git a/.changeset/add-audio-in-out-widgets.md b/.changeset/add-audio-in-out-widgets.md new file mode 100644 index 0000000..abfbc70 --- /dev/null +++ b/.changeset/add-audio-in-out-widgets.md @@ -0,0 +1,5 @@ +--- +'@viamrobotics/test-widgets': minor +--- + +Add `AudioInputWidget` and `AudioOutputWidget` components for audio in/out resources diff --git a/.claude/rules/viam-context.md b/.claude/rules/viam-context.md index beec09f..d0f338e 100644 --- a/.claude/rules/viam-context.md +++ b/.claude/rules/viam-context.md @@ -4,6 +4,8 @@ When reviewing or writing code that touches Viam APIs or SDK types, use the tool Use `WebFetch` on `https://docs.viam.com/` to understand what a resource type (arm, camera, sensor, etc.) is supposed to do — its RPC semantics, method signatures, and expected behavior. +Use `WebFetch` on `https://design.viam.com/` to understand the style guide and general design conventions and available UI components from the `prime-core` library. + ## Viam source repos Use `gh api` to fetch source directly from the four Viam repos: diff --git a/package.json b/package.json index dc1a6bf..15baab4 100644 --- a/package.json +++ b/package.json @@ -85,7 +85,7 @@ "@viamrobotics/motion-tools": "^1.19.1", "@viamrobotics/prime-core": "^0.1.19", "@viamrobotics/prime-editor": "^0.0.2", - "@viamrobotics/sdk": "^0.68.2", + "@viamrobotics/sdk": "^0.69.0", "@viamrobotics/svelte-sdk": "^1.2.1", "@viamrobotics/tailwind-config": "^1.0.0", "@viamrobotics/three": "^0.0.9", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f4d4820..506558d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -91,7 +91,7 @@ importers: version: 0.183.1 '@viamrobotics/motion-tools': specifier: ^1.19.1 - version: 1.19.1(c8fb3261282b32034713889b54db2a25) + version: 1.19.1(02f874e01fe6866429e442a9e6521d2b) '@viamrobotics/prime-core': specifier: ^0.1.19 version: 0.1.19(@internationalized/date@3.12.0)(svelte@5.55.5(@typescript-eslint/types@8.59.1)) @@ -99,11 +99,11 @@ importers: specifier: ^0.0.2 version: 0.0.2(@codemirror/lang-json@6.0.2)(@codemirror/merge@6.12.1)(@codemirror/state@6.6.0)(@codemirror/view@6.40.0)(@viamrobotics/prime-core@0.1.19(@internationalized/date@3.12.0)(svelte@5.55.5(@typescript-eslint/types@8.59.1)))(classnames@2.5.1)(codemirror@6.0.2)(lodash-es@4.18.1)(svelte@5.55.5(@typescript-eslint/types@8.59.1)) '@viamrobotics/sdk': - specifier: ^0.68.2 - version: 0.68.2 + specifier: ^0.69.0 + version: 0.69.0 '@viamrobotics/svelte-sdk': specifier: ^1.2.1 - version: 1.2.1(@viamrobotics/sdk@0.68.2)(svelte@5.55.5(@typescript-eslint/types@8.59.1)) + version: 1.2.1(@viamrobotics/sdk@0.69.0)(svelte@5.55.5(@typescript-eslint/types@8.59.1)) '@viamrobotics/tailwind-config': specifier: ^1.0.0 version: 1.0.0(tailwindcss@4.2.4) @@ -1314,8 +1314,8 @@ packages: lodash-es: '>=4 <5' svelte: '>=4.0.0 <5' - '@viamrobotics/sdk@0.68.2': - resolution: {integrity: sha512-15XTY+map1xiDfZPCXLpmX1uue8Sv6G/5lib1OvhAbbsHDlZXKW5wq1kg+cGMb8R/tx9XPLmM8UaA4EjN/bxfg==} + '@viamrobotics/sdk@0.69.0': + resolution: {integrity: sha512-xyJqF+4kJEmLiGiwWjf7hdcq/FBMcCcPF4kGyz3WQ/XeBC4HrkizD05ka6eNkevMFoAseGbNgOqOochEs0XTLA==} '@viamrobotics/svelte-sdk@1.2.1': resolution: {integrity: sha512-C29m8RksUmSP78V/C9MVo7/CM4vW1PsSLw9lRjG1pCyTF/+AnREjHmewaXNC3jDk322EZZ21fGJloVaAjkaPZg==} @@ -4142,7 +4142,7 @@ snapshots: '@typescript-eslint/types': 8.59.1 eslint-visitor-keys: 5.0.1 - '@viamrobotics/motion-tools@1.19.1(c8fb3261282b32034713889b54db2a25)': + '@viamrobotics/motion-tools@1.19.1(02f874e01fe6866429e442a9e6521d2b)': dependencies: '@ag-grid-community/client-side-row-model': 32.3.9 '@ag-grid-community/core': 32.3.9 @@ -4158,8 +4158,8 @@ snapshots: '@threlte/rapier': 3.4.0(@dimforge/rapier3d-compat@0.12.0)(svelte@5.55.5(@typescript-eslint/types@8.59.1))(three@0.183.2) '@threlte/xr': 1.4.0(svelte@5.55.5(@typescript-eslint/types@8.59.1))(three@0.183.2) '@viamrobotics/prime-core': 0.1.19(@internationalized/date@3.12.0)(svelte@5.55.5(@typescript-eslint/types@8.59.1)) - '@viamrobotics/sdk': 0.68.2 - '@viamrobotics/svelte-sdk': 1.2.1(@viamrobotics/sdk@0.68.2)(svelte@5.55.5(@typescript-eslint/types@8.59.1)) + '@viamrobotics/sdk': 0.69.0 + '@viamrobotics/svelte-sdk': 1.2.1(@viamrobotics/sdk@0.69.0)(svelte@5.55.5(@typescript-eslint/types@8.59.1)) '@zag-js/collapsible': 1.18.3 '@zag-js/dialog': 1.37.0 '@zag-js/floating-panel': 1.37.0 @@ -4212,7 +4212,7 @@ snapshots: lodash-es: 4.18.1 svelte: 5.55.5(@typescript-eslint/types@8.59.1) - '@viamrobotics/sdk@0.68.2': + '@viamrobotics/sdk@0.69.0': dependencies: '@bufbuild/protobuf': 1.10.1 '@connectrpc/connect': 1.7.0(@bufbuild/protobuf@1.10.1) @@ -4220,11 +4220,11 @@ snapshots: bsonfy: 1.0.2 exponential-backoff: 3.1.3 - '@viamrobotics/svelte-sdk@1.2.1(@viamrobotics/sdk@0.68.2)(svelte@5.55.5(@typescript-eslint/types@8.59.1))': + '@viamrobotics/svelte-sdk@1.2.1(@viamrobotics/sdk@0.69.0)(svelte@5.55.5(@typescript-eslint/types@8.59.1))': dependencies: '@tanstack/svelte-query': 6.1.24(svelte@5.55.5(@typescript-eslint/types@8.59.1)) '@tanstack/svelte-query-devtools': 6.1.24(@tanstack/svelte-query@6.1.24(svelte@5.55.5(@typescript-eslint/types@8.59.1)))(svelte@5.55.5(@typescript-eslint/types@8.59.1)) - '@viamrobotics/sdk': 0.68.2 + '@viamrobotics/sdk': 0.69.0 loglayer: 9.1.0 runed: 0.29.2(svelte@5.55.5(@typescript-eslint/types@8.59.1)) svelte: 5.55.5(@typescript-eslint/types@8.59.1) diff --git a/src/lib/builtin.ts b/src/lib/builtin.ts index 59489a2..d59b7b5 100644 --- a/src/lib/builtin.ts +++ b/src/lib/builtin.ts @@ -2,6 +2,8 @@ import type { ResourceName } from '@viamrobotics/sdk' import { ArmWidget, + AudioInputWidget, + AudioOutputWidget, BaseWidget, BoardWidget, ButtonWidget, @@ -35,6 +37,8 @@ const resourceMap = // list created via `cat rdkbuiltins/viam-server-stable.json | rg "api" | sort | uniq` { 'rdk:component:arm': [clientMap['rdk:component:arm'], ArmWidget, true], + 'rdk:component:audio_in': [clientMap['rdk:component:audio_in'], AudioInputWidget, true], + 'rdk:component:audio_out': [clientMap['rdk:component:audio_out'], AudioOutputWidget, true], 'rdk:component:base': [clientMap['rdk:component:base'], BaseWidget, true], 'rdk:component:board': [clientMap['rdk:component:board'], BoardWidget, true], 'rdk:component:button': [clientMap['rdk:component:button'], ButtonWidget, true], diff --git a/src/lib/client-map.ts b/src/lib/client-map.ts index 18c6160..ffebeb1 100644 --- a/src/lib/client-map.ts +++ b/src/lib/client-map.ts @@ -1,5 +1,7 @@ import { ArmClient, + AudioInClient, + AudioOutClient, BaseClient, BoardClient, ButtonClient, @@ -33,6 +35,8 @@ import { getResourceAPI } from './resource.ts' export const clientMap = { 'rdk:component:arm': ArmClient, + 'rdk:component:audio_in': AudioInClient, + 'rdk:component:audio_out': AudioOutClient, 'rdk:component:base': BaseClient, 'rdk:component:board': BoardClient, 'rdk:component:button': ButtonClient, @@ -65,3 +69,8 @@ export const clientForBuiltinResource = (resource: ResourceName) => { const resAPI = getResourceAPI(resource) return resAPI in clientMap ? clientMap[resAPI as keyof typeof clientMap] : undefined } + +export const supportsDoCommand = (resource: ResourceName): boolean => { + const client = clientForBuiltinResource(resource) + return client !== undefined && client !== MLModelClient +} diff --git a/src/lib/components/index.ts b/src/lib/components/index.ts index aa49144..46e83ac 100644 --- a/src/lib/components/index.ts +++ b/src/lib/components/index.ts @@ -5,6 +5,10 @@ export { default as ArmMoveToJointPositionsWidget } from './widgets/arm/move-to- export { default as ArmMoveToPositionWidget } from './widgets/arm/move-to-position-widget.svelte' export { default as ArmQuickMoveWidget } from './widgets/arm/quick-move-widget.svelte' +export { default as AudioInputWidget } from './widgets/audio-input/audio-input.svelte' + +export { default as AudioOutputWidget } from './widgets/audio-output/audio-output.svelte' + export { default as BaseWidget } from './widgets/base/base.svelte' export { default as BaseMoveStraightWidget } from './widgets/base/move-straight-widget.svelte' export { default as BaseQuickMoveWidget } from './widgets/base/quick-move-widget.svelte' diff --git a/src/lib/components/section-group.svelte b/src/lib/components/section-group.svelte index 6567e7d..ec60604 100644 --- a/src/lib/components/section-group.svelte +++ b/src/lib/components/section-group.svelte @@ -32,7 +32,7 @@ A group of sections in a card. For example, "test" or "do command" >
+ {:else} + {#if capture.downloadUrl} +

+ {(capture.totalBytes / 1024).toFixed(1)} kB captured +

+ + + + {/if} + + {/if} + + + + +
+ + + {#if propertiesQuery.data !== undefined} + + {/if} + + +
+ + {/snippet} + diff --git a/src/lib/components/widgets/audio-input/create-audio-capturer.svelte.ts b/src/lib/components/widgets/audio-input/create-audio-capturer.svelte.ts new file mode 100644 index 0000000..446e2b8 --- /dev/null +++ b/src/lib/components/widgets/audio-input/create-audio-capturer.svelte.ts @@ -0,0 +1,102 @@ +import { AudioInClient } from '@viamrobotics/sdk' + +type CaptureStatus = 'idle' | 'recording' | 'done' | 'error' + +export const createAudioCapturer = (client: { current: AudioInClient | undefined }) => { + let captureStatus = $state('idle') + let captureError = $state(null) + let captureTotalBytes = $state(0) + let captureDownloadUrl = $state() + let captureAbortController = $state.raw() + + $effect(() => { + return () => { + if (captureDownloadUrl) URL.revokeObjectURL(captureDownloadUrl) + } + }) + + const start = async (codec: string, duration: number) => { + if (!client.current) return + + captureStatus = 'recording' + captureError = null + captureTotalBytes = 0 + + if (captureDownloadUrl) { + URL.revokeObjectURL(captureDownloadUrl) + captureDownloadUrl = undefined + } + + const controller = new AbortController() + captureAbortController = controller + + const chunks: Uint8Array[] = [] + + try { + const stream = client.current.getAudio( + codec, + duration, + 0n, + {}, + { ...client.current.callOptions, signal: controller.signal } + ) + + for await (const chunk of stream) { + chunks.push(chunk.audioData) + captureTotalBytes += chunk.audioData.byteLength + } + + captureStatus = 'done' + } catch (error) { + if (controller.signal.aborted) { + captureStatus = 'done' + } else { + captureStatus = 'error' + captureError = error instanceof Error ? error : new Error(String(error)) + } + } finally { + captureAbortController = undefined + } + + if (chunks.length > 0) { + const totalLength = chunks.reduce((sum, c) => sum + c.byteLength, 0) + const merged = new Uint8Array(totalLength) + let offset = 0 + for (const chunk of chunks) { + merged.set(chunk, offset) + offset += chunk.byteLength + } + + const mimeTypes: Record = { + mp3: 'audio/mpeg', + wav: 'audio/wav', + aac: 'audio/aac', + opus: 'audio/ogg', + flac: 'audio/flac', + } + const mimeType = mimeTypes[codec] ?? 'audio/octet-stream' + captureDownloadUrl = URL.createObjectURL(new Blob([merged], { type: mimeType })) + } + } + + const stop = () => { + captureAbortController?.abort() + } + + return { + get status() { + return captureStatus + }, + get error() { + return captureError + }, + get totalBytes() { + return captureTotalBytes + }, + get downloadUrl() { + return captureDownloadUrl + }, + start, + stop, + } +} diff --git a/src/lib/components/widgets/audio-input/properties.svelte b/src/lib/components/widgets/audio-input/properties.svelte new file mode 100644 index 0000000..1694cff --- /dev/null +++ b/src/lib/components/widgets/audio-input/properties.svelte @@ -0,0 +1,26 @@ + + +
+
+
Supported Codecs
+
+ {supportedCodecs.length > 0 ? supportedCodecs.join(', ') : 'None'} +
+
+
+
Sample Rate
+
{sampleRateHz} Hz
+
+
+
Channels
+
{numChannels}
+
+
diff --git a/src/lib/components/widgets/audio-output/__tests__/audio-play.svelte.spec.ts b/src/lib/components/widgets/audio-output/__tests__/audio-play.svelte.spec.ts new file mode 100644 index 0000000..0d03d9d --- /dev/null +++ b/src/lib/components/widgets/audio-output/__tests__/audio-play.svelte.spec.ts @@ -0,0 +1,62 @@ +import { flushSync } from 'svelte' +import { describe, expect, it, vi } from 'vitest' + +import { createAudioPlayer } from '../create-audio-player.svelte.ts' + +type MockAudioOutClient = { + play: ReturnType +} + +const makeClient = (mockPlay: ReturnType): { current: MockAudioOutClient } => ({ + current: { + play: mockPlay, + }, +}) + +describe('createAudioPlay', () => { + it('successful play sets status to done', async () => { + const mockPlay = vi.fn().mockResolvedValue(undefined) + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const client = makeClient(mockPlay) as any + + let playContext: ReturnType | undefined + + const cleanup = $effect.root(() => { + playContext = createAudioPlayer(client) + }) + + try { + await playContext!.play(new Uint8Array([1, 2, 3]), 'wav', 48000, 1) + flushSync() + + expect(playContext!.status).toBe('done') + expect(playContext!.error).toBeNull() + } finally { + cleanup() + } + }) + + it('failed play sets status to error with the error message', async () => { + const mockPlay = vi.fn().mockRejectedValue(new Error('playback failed')) + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const client = makeClient(mockPlay) as any + + let playContext: ReturnType | undefined + + const cleanup = $effect.root(() => { + playContext = createAudioPlayer(client) + }) + + try { + await playContext!.play(new Uint8Array([1, 2, 3]), 'wav', 48000, 1) + flushSync() + + expect(playContext!.status).toBe('error') + expect(playContext!.error?.message).toBe('playback failed') + } finally { + cleanup() + } + }) +}) diff --git a/src/lib/components/widgets/audio-output/__tests__/properties.spec.ts b/src/lib/components/widgets/audio-output/__tests__/properties.spec.ts new file mode 100644 index 0000000..54d73f5 --- /dev/null +++ b/src/lib/components/widgets/audio-output/__tests__/properties.spec.ts @@ -0,0 +1,36 @@ +import type { ComponentProps } from 'svelte' + +import { render, screen } from '@testing-library/svelte' +import { describe, expect, it } from 'vitest' + +import Subject from '../properties.svelte' + +const renderSubject = (props: Partial> = {}) => + render(Subject, { + supportedCodecs: [], + sampleRateHz: 0, + numChannels: 0, + ...props, + }) + +describe('AudioOutput Properties', () => { + it('displays supported codecs', () => { + renderSubject({ supportedCodecs: ['mp3', 'pcm16'] }) + expect(screen.getByText('mp3, pcm16')).toBeInTheDocument() + }) + + it('displays None when no codecs are supported', () => { + renderSubject({ supportedCodecs: [] }) + expect(screen.getByText('None')).toBeInTheDocument() + }) + + it('displays sample rate', () => { + renderSubject({ sampleRateHz: 44100 }) + expect(screen.getByText('44100 Hz')).toBeInTheDocument() + }) + + it('displays number of channels', () => { + renderSubject({ numChannels: 1 }) + expect(screen.getByText('1')).toBeInTheDocument() + }) +}) diff --git a/src/lib/components/widgets/audio-output/audio-output.svelte b/src/lib/components/widgets/audio-output/audio-output.svelte new file mode 100644 index 0000000..d7bc9fc --- /dev/null +++ b/src/lib/components/widgets/audio-output/audio-output.svelte @@ -0,0 +1,200 @@ + + + + {#snippet connected()} +
+ +
+ +
+
+ +
+ + {#if fileInputError} +

{fileInputError}

+ {/if} + {#if selectedFile} +

{selectedFile.name}

+ {/if} + + + +
+ +
+ +
+ +
+ +
+ + + {#if propertiesQuery.data !== undefined} + + {/if} + + +
+
+ {/snippet} + diff --git a/src/lib/components/widgets/audio-output/codec.ts b/src/lib/components/widgets/audio-output/codec.ts new file mode 100644 index 0000000..9f04065 --- /dev/null +++ b/src/lib/components/widgets/audio-output/codec.ts @@ -0,0 +1,17 @@ +export const MimeToCodec: Record = { + 'audio/mpeg': 'mp3', + 'audio/mp3': 'mp3', + 'audio/wav': 'wav', + 'audio/x-wav': 'wav', + 'audio/aac': 'aac', + 'audio/ogg': 'opus', + 'audio/flac': 'flac', + 'audio/x-flac': 'flac', +} + +// Raw PCM formats have no standard MIME type; map by file extension directly. +export const ExtToCodec: Record = { + pcm16: 'pcm16', + pcm32: 'pcm32', + pcm32_float: 'pcm32_float', +} diff --git a/src/lib/components/widgets/audio-output/create-audio-player.svelte.ts b/src/lib/components/widgets/audio-output/create-audio-player.svelte.ts new file mode 100644 index 0000000..038508e --- /dev/null +++ b/src/lib/components/widgets/audio-output/create-audio-player.svelte.ts @@ -0,0 +1,42 @@ +import { AudioOutClient, commonApi } from '@viamrobotics/sdk' + +type PlayStatus = 'idle' | 'playing' | 'done' | 'error' + +export const createAudioPlayer = (client: { current: AudioOutClient | undefined }) => { + let playStatus = $state('idle') + let playError = $state(null) + + const play = async ( + audioData: Uint8Array, + codec: string, + sampleRateHz: number, + numChannels: number + ) => { + if (!client.current) return + + playStatus = 'playing' + playError = null + + try { + await client.current.play( + audioData, + commonApi.AudioInfo.fromJson({ codec, sampleRateHz, numChannels }), + {} + ) + playStatus = 'done' + } catch (error) { + playStatus = 'error' + playError = error instanceof Error ? error : new Error(String(error)) + } + } + + return { + get status() { + return playStatus + }, + get error() { + return playError + }, + play, + } +} diff --git a/src/lib/components/widgets/audio-output/properties.svelte b/src/lib/components/widgets/audio-output/properties.svelte new file mode 100644 index 0000000..1694cff --- /dev/null +++ b/src/lib/components/widgets/audio-output/properties.svelte @@ -0,0 +1,26 @@ + + +
+
+
Supported Codecs
+
+ {supportedCodecs.length > 0 ? supportedCodecs.join(', ') : 'None'} +
+
+
+
Sample Rate
+
{sampleRateHz} Hz
+
+
+
Channels
+
{numChannels}
+
+
diff --git a/src/lib/components/widgets/do-command/create-do-command-client.svelte.ts b/src/lib/components/widgets/do-command/create-do-command-client.svelte.ts new file mode 100644 index 0000000..34fde97 --- /dev/null +++ b/src/lib/components/widgets/do-command/create-do-command-client.svelte.ts @@ -0,0 +1,38 @@ +import { MachineConnectionEvent, MLModelClient, ResourceName } from '@viamrobotics/sdk' +import { useConnectionStatus, useRobotClient } from '@viamrobotics/svelte-sdk' + +import { clientForBuiltinResource, clientMap } from '$lib/client-map' + +type DoCommandable = InstanceType< + Exclude<(typeof clientMap)[keyof typeof clientMap], typeof MLModelClient> +> + +export const createDoCommandClient = ( + resource: () => ResourceName, + partID: () => string, + resourceName: () => string +): { current: DoCommandable | undefined } => { + const robotClient = useRobotClient(partID) + const connectionStatus = useConnectionStatus(partID) + + const resourceClient = $derived.by(() => { + if (!robotClient.current) return + if (connectionStatus.current !== MachineConnectionEvent.CONNECTED) return + + const constructor = clientForBuiltinResource(resource()) + if (!constructor) return + if (constructor === MLModelClient) return + + const nextClient = new constructor(robotClient.current, resourceName()) as DoCommandable + // PartIDs are used to invalidate queries for this client + ;(nextClient as typeof nextClient & { partID: string }).partID = partID() + + return nextClient + }) + + return { + get current() { + return resourceClient + }, + } +} diff --git a/src/lib/components/widgets/do-command/do-command.svelte b/src/lib/components/widgets/do-command/do-command.svelte index c91011f..5d3c169 100644 --- a/src/lib/components/widgets/do-command/do-command.svelte +++ b/src/lib/components/widgets/do-command/do-command.svelte @@ -1,14 +1,15 @@ -{#if doCommandMutation} +{#if isSupported}
Input