feat: add audio input and audio output widgets (#47)

.changeset/add-audio-in-out-widgets.md

@@ -0,0 +1,5 @@
+---
+'@viamrobotics/test-widgets': minor
+---
+
+Add `AudioInputWidget` and `AudioOutputWidget` components for audio in/out resources

.claude/rules/viam-context.md

@@ -4,6 +4,8 @@ When reviewing or writing code that touches Viam APIs or SDK types, use the tool
 
 Use `WebFetch` on `https://docs.viam.com/` to understand what a resource type (arm, camera, sensor, etc.) is supposed to do — its RPC semantics, method signatures, and expected behavior.
 
+Use `WebFetch` on `https://design.viam.com/` to understand the style guide and general design conventions and available UI components from the `prime-core` library.
+
 ## Viam source repos
 
 Use `gh api` to fetch source directly from the four Viam repos:

package.json

@@ -85,7 +85,7 @@
 		"@viamrobotics/motion-tools": "^1.19.1",
 		"@viamrobotics/prime-core": "^0.1.19",
 		"@viamrobotics/prime-editor": "^0.0.2",
-		"@viamrobotics/sdk": "^0.68.2",
+		"@viamrobotics/sdk": "^0.69.0",
 		"@viamrobotics/svelte-sdk": "^1.2.1",
 		"@viamrobotics/tailwind-config": "^1.0.0",
 		"@viamrobotics/three": "^0.0.9",

src/lib/builtin.ts

@@ -2,6 +2,8 @@ import type { ResourceName } from '@viamrobotics/sdk'
 
 import {
 	ArmWidget,
+	AudioInputWidget,
+	AudioOutputWidget,
 	BaseWidget,
 	BoardWidget,
 	ButtonWidget,
@@ -35,6 +37,8 @@ const resourceMap =
 	// list created via `cat rdkbuiltins/viam-server-stable.json | rg "api" | sort | uniq`
 	{
 		'rdk:component:arm': [clientMap['rdk:component:arm'], ArmWidget, true],
+		'rdk:component:audio_in': [clientMap['rdk:component:audio_in'], AudioInputWidget, true],
+		'rdk:component:audio_out': [clientMap['rdk:component:audio_out'], AudioOutputWidget, true],
 		'rdk:component:base': [clientMap['rdk:component:base'], BaseWidget, true],
 		'rdk:component:board': [clientMap['rdk:component:board'], BoardWidget, true],
 		'rdk:component:button': [clientMap['rdk:component:button'], ButtonWidget, true],

src/lib/client-map.ts

@@ -1,5 +1,7 @@
 import {
 	ArmClient,
+	AudioInClient,
+	AudioOutClient,
 	BaseClient,
 	BoardClient,
 	ButtonClient,
@@ -33,6 +35,8 @@ import { getResourceAPI } from './resource.ts'
 
 export const clientMap = {
 	'rdk:component:arm': ArmClient,
+	'rdk:component:audio_in': AudioInClient,
+	'rdk:component:audio_out': AudioOutClient,
 	'rdk:component:base': BaseClient,
 	'rdk:component:board': BoardClient,
 	'rdk:component:button': ButtonClient,
@@ -65,3 +69,8 @@ export const clientForBuiltinResource = (resource: ResourceName) => {
 	const resAPI = getResourceAPI(resource)
 	return resAPI in clientMap ? clientMap[resAPI as keyof typeof clientMap] : undefined
 }
+
+export const supportsDoCommand = (resource: ResourceName): boolean => {
+	const client = clientForBuiltinResource(resource)
+	return client !== undefined && client !== MLModelClient
+}

src/lib/components/index.ts

@@ -5,6 +5,10 @@ export { default as ArmMoveToJointPositionsWidget } from './widgets/arm/move-to-
 export { default as ArmMoveToPositionWidget } from './widgets/arm/move-to-position-widget.svelte'
 export { default as ArmQuickMoveWidget } from './widgets/arm/quick-move-widget.svelte'
 
+export { default as AudioInputWidget } from './widgets/audio-input/audio-input.svelte'
+
+export { default as AudioOutputWidget } from './widgets/audio-output/audio-output.svelte'
+
 export { default as BaseWidget } from './widgets/base/base.svelte'
 export { default as BaseMoveStraightWidget } from './widgets/base/move-straight-widget.svelte'
 export { default as BaseQuickMoveWidget } from './widgets/base/quick-move-widget.svelte'

src/lib/components/section-group.svelte

@@ -32,7 +32,7 @@ A group of sections in a card. For example, "test" or "do command"
 >
 	<header class={['border-gray-3 bg-light hover:bg-medium h-7', !isCollapsed && 'border-b']}>
 		<button
-			class="group flex h-full w-full flex-row items-center gap-2 px-[11px]"
+			class="group flex h-full w-full flex-row items-center gap-2 px-2.75"
 			aria-controls={collapseID}
 			aria-expanded={!isCollapsed}
 			aria-label={`${isCollapsed ? 'expand' : 'collapse'} ${title}`}

src/lib/components/widgets/audio-input/__tests__/audio-capture.svelte.spec.ts

@@ -0,0 +1,126 @@
+import { flushSync } from 'svelte'
+import { describe, expect, it, vi } from 'vitest'
+
+import { createAudioCapturer } from '../create-audio-capturer.svelte.ts'
+
+type MockAudioInClient = {
+	callOptions: Record<string, unknown>
+	getAudio: ReturnType<typeof vi.fn>
+}
+
+const makeClient = (mockGetAudio: ReturnType<typeof vi.fn>): { current: MockAudioInClient } => ({
+	current: {
+		callOptions: {},
+		getAudio: mockGetAudio,
+	},
+})
+
+async function* makeChunkStream(chunks: Uint8Array[]) {
+	for (const chunk of chunks) {
+		yield { audioData: chunk }
+	}
+}
+
+async function* makeErrorStream(message: string) {
+	yield { audioData: new Uint8Array([1]) }
+	throw new Error(message)
+}
+
+async function* makeAbortableStream(signal: AbortSignal) {
+	yield { audioData: new Uint8Array([1, 2, 3]) }
+	await new Promise<never>((_, reject) => {
+		signal.addEventListener('abort', () => reject(signal.reason))
+	})
+}
+
+describe('createAudioCapture', () => {
+	it('successful capture sets status to done and provides downloadUrl with correct totalBytes', async () => {
+		const chunk1 = new Uint8Array([1, 2, 3])
+		const chunk2 = new Uint8Array([4, 5])
+		const mockGetAudio = vi.fn().mockReturnValue(makeChunkStream([chunk1, chunk2]))
+
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any
+		const client = makeClient(mockGetAudio) as any
+
+		let capture: ReturnType<typeof createAudioCapturer> | undefined
+
+		const cleanup = $effect.root(() => {
+			capture = createAudioCapturer(client)
+		})
+
+		try {
+			await capture!.start('wav', 3)
+			flushSync()
+
+			expect(capture!.status).toBe('done')
+			expect(capture!.downloadUrl).toBeDefined()
+			expect(capture!.totalBytes).toBe(chunk1.byteLength + chunk2.byteLength)
+			expect(capture!.error).toBeNull()
+		} finally {
+			cleanup()
+		}
+	})
+
+	it('error in stream sets status to error with the error message', async () => {
+		const mockGetAudio = vi.fn().mockReturnValue(makeErrorStream('stream failed'))
+
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any
+		const client = makeClient(mockGetAudio) as any
+
+		let capture: ReturnType<typeof createAudioCapturer> | undefined
+
+		const cleanup = $effect.root(() => {
+			capture = createAudioCapturer(client)
+		})
+
+		try {
+			await capture!.start('wav', 3)
+			flushSync()
+
+			expect(capture!.status).toBe('error')
+			expect(capture!.error?.message).toBe('stream failed')
+		} finally {
+			cleanup()
+		}
+	})
+
+	it('stop aborts the stream and sets status to done without an error', async () => {
+		const mockGetAudio = vi
+			.fn()
+			.mockImplementation(
+				(
+					_codec: string,
+					_duration: number,
+					_offset: bigint,
+					_extra: unknown,
+					callOptions: { signal: AbortSignal }
+				) => makeAbortableStream(callOptions.signal)
+			)
+
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any
+		const client = makeClient(mockGetAudio) as any
+
+		let capture: ReturnType<typeof createAudioCapturer> | undefined
+
+		const cleanup = $effect.root(() => {
+			capture = createAudioCapturer(client)
+		})
+
+		try {
+			const startPromise = capture!.start('wav', 0)
+
+			// Let the stream start and yield the first chunk
+			await new Promise((resolve) => setTimeout(resolve, 0))
+
+			capture!.stop()
+
+			await startPromise
+			flushSync()
+
+			expect(capture!.status).toBe('done')
+			expect(capture!.error).toBeNull()
+		} finally {
+			cleanup()
+		}
+	})
+})

src/lib/components/widgets/audio-input/__tests__/properties.spec.ts

@@ -0,0 +1,36 @@
+import type { ComponentProps } from 'svelte'
+
+import { render, screen } from '@testing-library/svelte'
+import { describe, expect, it } from 'vitest'
+
+import Subject from '../properties.svelte'
+
+const renderSubject = (props: Partial<ComponentProps<typeof Subject>> = {}) =>
+	render(Subject, {
+		supportedCodecs: [],
+		sampleRateHz: 0,
+		numChannels: 0,
+		...props,
+	})
+
+describe('AudioInput Properties', () => {
+	it('displays supported codecs', () => {
+		renderSubject({ supportedCodecs: ['mp3', 'pcm16'] })
+		expect(screen.getByText('mp3, pcm16')).toBeInTheDocument()
+	})
+
+	it('displays None when no codecs are supported', () => {
+		renderSubject({ supportedCodecs: [] })
+		expect(screen.getByText('None')).toBeInTheDocument()
+	})
+
+	it('displays sample rate', () => {
+		renderSubject({ sampleRateHz: 48000 })
+		expect(screen.getByText('48000 Hz')).toBeInTheDocument()
+	})
+
+	it('displays number of channels', () => {
+		renderSubject({ numChannels: 2 })
+		expect(screen.getByText('2')).toBeInTheDocument()
+	})
+})