index ↓

voice

client.voice

Voice capture and intent results for a webapp. onStateChanged reports mic mute and capture state, onActivity reports each step of a voice turn, and onIntent delivers a resolved intent.

requests

you ask, the daemon answers. await the tagged result and check .ok

stateGet(): Promise<TypedRequestResult<VoiceStateReply, never>>

push form: onStateReply (subscribe instead of awaiting)

Returns the current VoiceState.

const res = await client.voice.stateGet();
if (res.ok) {
  console.log(res.response.state);
}

type Result =
  | { ok: true; response: VoiceStateReply }
  | { ok: false; kind: 'domain'; error: never }
  | { ok: false; kind: 'protocol'; error: WireError };

commands

fire-and-forget. the promise resolves once the daemon has taken the message

cancel(): Promise<void>

await client.voice.cancel();

pushToTalk(): Promise<void>

await client.voice.pushToTalk();

release(): Promise<void>

await client.voice.release();

muteMic(MicMute): Promise<void>

await client.voice.muteMic({ preserve: true });

unmuteMic(MicUnmute): Promise<void>

await client.voice.unmuteMic({ preserve: true });

events

the daemon pushes these unprompted. subscribing returns an unsubscribe function

onStateChanged(handler: (VoiceState) => void): () => void

const off = client.voice.onStateChanged((state) => {
  console.log(state.muted);
});
// call off() to unsubscribe

onActivity(handler: (VoiceActivity) => void): () => void

const off = client.voice.onActivity((voiceActivity) => {
  console.log(voiceActivity.phase);
});
// call off() to unsubscribe

onIntent(handler: (VoiceIntent) => void): () => void

const off = client.voice.onIntent((voiceIntent) => {
  console.log(voiceIntent.intent);
});
// call off() to unsubscribe

types

shapes referenced above, as the sdk types them

type VoiceState = {
  muted: boolean;
  capturing: boolean;
  phase: VoicePhase;
};

One step of a voice turn, reported as it happens.

type VoiceActivity = {
  phase: VoicePhase;
  streamId: string | null;
  reason?: VoiceCaptureReason;
  score?: number;
  transcript?: string;
  intent?: string;
  slots: NluSlots;
  stage?: NluStage;
  target?: VoiceDispatchTarget;
  error?: VoiceActivityError;
};

type VoiceIntent = {
  intent: VoiceDisplayIntent;
  slots: NluSlots;
  transcript: string;
};

type VoiceStateReply = {
  state: VoiceState;
};

preserve keeps a voice capture that is already running.

type MicMute = {
  preserve: boolean;
};

type MicUnmute = {
  preserve: boolean;
};

type VoicePhase = 'idle' | 'listening' | 'thinking' | 'done' | 'failed';

type VoiceCaptureReason = 'pushToTalk' | 'assistant' | 'wakeWord';

type NluSlots = {
  target?: string;
  targetType?: NluTargetType;
  playlist?: string;
  genre?: string;
  mood?: string;
  era?: string;
  popularityFilter?: NluPopularityFilter;
  position?: number;
  count?: number;
  scope?: NluScope;
  enabled?: boolean;
  mute?: boolean;
  repeatMode?: NluRepeatMode;
  seconds?: number;
  speed?: NluPlaybackSpeed;
  direction?: NluDirection;
  amount?: NluAmount;
  level?: number;
  preset?: string;
  view?: NluView;
  phoneAction?: NluPhoneAction;
  webappName?: string;
  uri?: string;
  contextUri?: string;
};

type NluStage =
  | 'fastPath'
  | 'model'
  | 'rejectedNoIntent'
  | 'rejectedClarify'
  | 'noModel';

Where a recognized voice intent went.

type VoiceDispatchTarget =
  | 'playback'
  | 'device' // A device action, such as pairing mode or saving a preset.
  | 'phone'
  | 'display' // The active webapp, which renders the intent.
  | 'webappSwitch';

Why a voice turn ended early.

type VoiceActivityError = {
  code: VoiceDispatchErrorCode;
  msg: string;
};

Intents that ask the webapp to change what it shows.

type VoiceDisplayIntent = 'search' | 'showView' | 'moreLikeThis';

type NluTargetType =
  | 'artist'
  | 'track'
  | 'album'
  | 'playlist'
  | 'podcast'
  | 'episode'
  | 'station';

type NluPopularityFilter =
  | 'top5'
  | 'top10'
  | 'popular'
  | 'recent'
  | 'new'
  | 'first'
  | 'random';

type NluScope = 'previousTrack' | 'restart';

type NluRepeatMode = 'off' | 'all' | 'one';

type NluPlaybackSpeed = 'one' | 'onePointTwo' | 'onePointFive' | 'two';

type NluDirection = 'up' | 'down';

type NluAmount = 'small' | 'medium' | 'large';

type NluView =
  | 'nowPlaying'
  | 'artist'
  | 'album'
  | 'playlist'
  | 'playlists'
  | 'library'
  | 'songs'
  | 'presets'
  | 'queue'
  | 'savedEpisodes'
  | 'newEpisodes';

type NluPhoneAction =
  | 'answer'
  | 'decline'
  | 'end'
  | 'hold'
  | 'unhold'
  | 'swap'
  | 'merge'
  | 'mute'
  | 'unmute';

type VoiceDispatchErrorCode =
  | 'webappNotFound'
  | 'notDispatchable'
  | 'unsupported'
  | 'playbackFailed'
  | 'badSlots'
  | 'internal';