Connect

{ "timezone": "Europe/London", "max_session_seconds": 300 }

{ "events": [{ "type": "preparing", "id": 1792529097555620 }] }

// In the browser: join a Nuance Live conversation's room with LiveKit's own
// SDK, and send and receive its events on the `nuance_live` topic.
import { Room, RoomEvent } from "livekit-client";

const NUANCE_API = "https://api.nuancelabs.ai";
const TOPIC = "nuance_live";
/** The most one data packet may carry; a larger event goes as a text stream. */
const MAX_PACKET_BYTES = 64_000;

export type LiveEvent = { type: string } & Record<string, unknown>;

export interface LiveSession {
  room: Room;
  /** Sends one event; reliable, in order. */
  send(event: LiveEvent): Promise<void>;
}

/** POSTs or GETs one API route with the session token, and reads its JSON. */
async function api<T>(token: string, path: string, body?: unknown): Promise<T> {
  const response = await fetch(`${NUANCE_API}${path}`, {
    method: body === undefined ? "GET" : "POST",
    headers: {
      Authorization: `Bearer ${token}`,
      "Content-Type": "application/json",
    },
    body: body === undefined ? undefined : JSON.stringify(body),
  });
  if (!response.ok) {
    throw new Error(
      `${path} answered ${response.status}: ${await response.text()}`,
    );
  }
  return (await response.json()) as T;
}

interface RouteEvent {
  type: string;
  id: number;
  livekit_url?: string;
  token?: string;
}

/**
 * Follows the conversation's events route from `after`, calling `onEvent` for
 * each; returns once `ended` comes. One request is always open, which keeps
 * the conversation alive.
 */
async function follow(
  token: string,
  id: string,
  onEvent: (event: RouteEvent) => void,
): Promise<void> {
  let after = 0;
  for (;;) {
    const { events } = await api<{ events: RouteEvent[] }>(
      token,
      `/conversation/${id}/events?after=${after}`,
    );
    for (const event of events) {
      after = event.id;
      onEvent(event);
      if (event.type === "ended") return;
    }
  }
}

/**
 * Creates the conversation with `token`, joins its room with the microphone,
 * and hands each Nuance Live event to `onEvent`, a `message.part` series
 * joined into the event it carries. Call it from a click, so the browser
 * plays the persona's audio.
 */
export async function joinNuanceLive(
  token: string,
  onEvent: (event: LiveEvent) => void,
): Promise<LiveSession> {
  const { id } = await api<{ id: string }>(token, "/conversation", {
    timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
  });

  const room = new Room();
  const decoder = new TextDecoder();
  let parts: string[] = [];
  room.on(RoomEvent.DataReceived, (payload, _participant, _kind, topic) => {
    if (topic !== TOPIC) return;
    const event = JSON.parse(decoder.decode(payload)) as LiveEvent;
    if (event.type !== "message.part") {
      parts = [];
      onEvent(event);
      return;
    }
    // A part out of order drops the event being joined.
    if (event.index !== parts.length) {
      parts = [];
      if (event.index !== 0) return;
    }
    parts.push(String(event.text));
    if (event.index === Number(event.count) - 1) {
      const whole = JSON.parse(parts.join("")) as LiveEvent;
      parts = [];
      onEvent(whole);
    }
  });
  // Play the persona's audio as soon as its track arrives.
  room.on(RoomEvent.TrackSubscribed, (track) => {
    if (track.kind === "audio") document.body.append(track.attach());
  });

  const admitted = new Promise<RouteEvent>((resolve, reject) => {
    follow(token, id, (event) => {
      if (event.type === "admitted") resolve(event);
    }).then(
      () => reject(new Error("The conversation ended before it began")),
      reject,
    );
  });
  const { livekit_url, token: roomToken } = await admitted;
  await room.connect(String(livekit_url), String(roomToken));
  await room.localParticipant.setMicrophoneEnabled(true);

  const encoder = new TextEncoder();
  return {
    room,
    async send(event) {
      const text = JSON.stringify(event);
      const bytes = encoder.encode(text);
      if (bytes.length <= MAX_PACKET_BYTES) {
        await room.localParticipant.publishData(bytes, {
          reliable: true,
          topic: TOPIC,
        });
      } else {
        await room.localParticipant.sendText(text, { topic: TOPIC });
      }
    },
  };
}