JavaScript SDK

npm install @nuancelabs/js-sdk

// In the browser: start a conversation with the JS SDK, show the persona, and
// send and receive Nuance Live events.
import { createNuanceConversation } from "@nuancelabs/js-sdk";

const video = document.querySelector<HTMLVideoElement>("#avatar-video");
const audio = document.querySelector<HTMLAudioElement>("#avatar-audio");
const transcript = document.querySelector<HTMLElement>("#transcript");

const conversation = createNuanceConversation({
  // Your endpoint, which mints a session token with your API key.
  token: async () => {
    const response = await fetch("/api/nuance-token", { method: "POST" });
    if (!response.ok) throw new Error(await response.text());
    return ((await response.json()) as { token: string }).token;
  },
});

conversation.onChange((event) => {
  if (event.type === "tracks-changed" && video && audio) {
    conversation.avatar.videoTrack?.attach(video);
    conversation.avatar.audioTrack?.attach(audio);
  }
  // The persona is in view: ask it to open the conversation.
  if (event.type === "avatar-presented") {
    void conversation.sendInterfaceEvent({
      type: "response.create",
      response: { instructions: "Greet the user and ask how you can help." },
    });
  }
  // The SDK has left the room; `reason` says why it ended.
  if (event.type === "session-ended" && transcript) {
    transcript.textContent += `\n(Conversation over: ${event.reason})`;
  }
});

// Subscribe before connect() to see the session's first events.
conversation.onInterfaceEvent((event) => {
  if (event.type === "session.output_transcript.delta" && transcript) {
    transcript.textContent += String(event.delta);
  }
});

// Start from a click, so the browser plays the persona's audio.
document.querySelector("#start")?.addEventListener("click", () => {
  void conversation.connect();
});
document.querySelector("#hang-up")?.addEventListener("click", () => {
  void conversation.disconnect();
});

// In the page: run the backend's calls of your tools, change which tools it
// may call, and tell the persona what happens in the app.
import type {
  Conversation,
  NuanceLiveTool,
  ToolCall,
  ToolOutput,
} from "@nuancelabs/js-sdk";

const GET_WEATHER: NuanceLiveTool = {
  type: "function",
  name: "get_weather",
  description: "Looks up the current weather in a city.",
  parameters: {
    type: "object",
    properties: { city: { type: "string" } },
    required: ["city"],
  },
};

const BOOK_TABLE: NuanceLiveTool = {
  type: "function",
  name: "book_table",
  description: "Books a table at the user's saved restaurant.",
  parameters: {
    type: "object",
    properties: { time: { type: "string" } },
    required: ["time"],
  },
  annotations: { destructiveHint: true },
};

/** Your app's own lookup and booking. */
declare function lookUpWeather(
  city: string,
): Promise<{ sky: string; temperature_c: number }>;
declare function bookTable(time: string): Promise<{ booked_for: string }>;

/** One call's result, or an error the persona can tell the user. */
async function run(call: ToolCall): Promise<ToolOutput> {
  const args = call.arguments ?? {};
  try {
    if (call.name === "get_weather" && typeof args.city === "string") {
      return await lookUpWeather(args.city);
    }
    if (call.name === "book_table" && typeof args.time === "string") {
      return await bookTable(args.time);
    }
  } catch {
    return { error: { message: `${call.name} didn't answer` } };
  }
  return { error: { message: `Can't run ${call.name} with these arguments` } };
}

export function runTools(conversation: Conversation): void {
  // Answer every call, with an error when it can't run: the backend goes on
  // only once each call of its step has a result.
  conversation.onToolCall(async (call) => {
    await conversation.sendToolResult(call.callId, await run(call));
  });
}

/** The user signed out: booking can't run until they're back. */
export async function onSignedOut(conversation: Conversation): Promise<void> {
  await conversation.setTools([GET_WEATHER]);
  await conversation.append("thinking", "The user just signed out.");
}

/** They signed in again: send every tool that can run, exactly as minted. */
export async function onSignedIn(conversation: Conversation): Promise<void> {
  await conversation.setTools([GET_WEATHER, BOOK_TABLE]);
}

// In a React app: one conversation in a component, with the SDK's hook.
import { useNuanceConversation } from "@nuancelabs/js-sdk/react";

async function fetchToken(): Promise<string> {
  const response = await fetch("/api/nuance-token", { method: "POST" });
  if (!response.ok) throw new Error(await response.text());
  return ((await response.json()) as { token: string }).token;
}

export function AvatarCall() {
  const { state, error, videoRef, audioRef, start, end } =
    useNuanceConversation({ token: fetchToken });
  const live = state === "connecting" || state === "connected";

  return (
    <div>
      <video ref={videoRef} autoPlay playsInline />
      <audio ref={audioRef} autoPlay />
      {live ? (
        <button type="button" onClick={end}>
          Hang up
        </button>
      ) : (
        <button type="button" onClick={start}>
          Start
        </button>
      )}
      {error && <p>Couldn't start the conversation.</p>}
    </div>
  );
}