pnpm add @mascotbot/react @rive-app/react-webgl2 @rive-app/webgl2 @elevenlabs/client

"use client";
import { useState } from "react";
import { MascotProvider, useMascot } from "@mascotbot/react";
import { Mascot, MascotRive, useMascotPlayback, useLipsyncStream } from "@mascotbot/react/rive";

function App() {
  return (
    <MascotProvider apiKey="mascot_pub_…">
      <MascotProvider>
        <Mascot src="/mascot.riv">
          <MascotRive />
          <ElevenLabsAvatar />
        </Mascot>
      </MascotProvider>
    </MascotProvider>
  );
}

// app/api/elevenlabs/signed-url/route.ts
export const runtime = "nodejs";

export async function POST() {
  const key = process.env.ELEVENLABS_API_KEY;
  const agentId = process.env.ELEVENLABS_AGENT_ID;
  if (!key || !agentId) {
    return Response.json({ error: "ElevenLabs env not set" }, { status: 400 });
  }
  const url = new URL("https://api.elevenlabs.io/v1/convai/conversation/get-signed-url");
  url.searchParams.set("agent_id", agentId);
  const res = await fetch(url, { headers: { "xi-api-key": key }, cache: "no-store" });
  if (!res.ok) return Response.json({ error: `ElevenLabs ${res.status}` }, { status: 502 });
  const json = (await res.json()) as { signed_url?: string };
  return Response.json({ signedUrl: json.signed_url });
}

"use client";
import { useEffect, useRef, useState } from "react";
import { useMascot } from "@mascotbot/react";
import { useMascotPlayback, useLipsyncStream } from "@mascotbot/react/rive";

// Stable module constant — see the natural-lip-sync warning below.
const LIP_SYNC = { minVisemeInterval: 60, mergeWindow: 80 } as const;

export function ElevenLabsAvatar() {
  const { client, status } = useMascot();
  const playback = useMascotPlayback({ stream: true, enableNaturalLipSync: true, naturalLipSyncConfig: LIP_SYNC });
  const [stream, setStream] = useState<MediaStream | null>(null);
  const teardownRef = useRef<null | (() => void)>(null);

  // The SDK lip-syncs whatever this MediaStream carries.
  const { error, attached } = useLipsyncStream({
    client,
    playback,
    source: { kind: "mediaStream", stream },
  });

  useEffect(() => () => teardownRef.current?.(), []);

  const start = async () => {
    if (status !== "ready") return;
    // createElementTap from "@mascotbot/react" — create in
    // this click gesture so its AudioContext isn't suspended (Safari).
    const tap = createElementTap();
    setStream(tap.stream);
    // Capture the hidden <audio> ElevenLabs creates (its srcObject is a
    // MediaStream of the worklet output we'll tap).
    const w = window as unknown as { Audio: typeof Audio; __el?: HTMLAudioElement | null };
    const OrigAudio = w.Audio;
    w.Audio = function (...args: unknown[]) {
      const el = new OrigAudio(...(args as []));
      w.__el = el;
      return el;
    } as unknown as typeof Audio;

    const { signedUrl } = await (await fetch("/api/elevenlabs/signed-url", { method: "POST" })).json();
    const { Conversation } = await import("@elevenlabs/client");
    const convo = await Conversation.startSession({ signedUrl });

    // Live-tracks guard: on a 2nd "end → start" cycle, `w.__el` will
    // briefly still point at the previous call's <audio> element (its
    // srcObject is a MediaStream whose tracks are 'ended'). Attaching
    // to it produces silence for the entire new call. Require at
    // least one 'live' track before attaching.
    const isLive = (el: HTMLMediaElement | null | undefined) =>
      !!el && el.srcObject instanceof MediaStream &&
      el.srcObject.getAudioTracks().some((t) => t.readyState === "live");

    let tries = 0;
    const iv = window.setInterval(() => {
      const el = w.__el;
      // tap.attach(el): cross-browser tap (Safari has no captureStream).
      // See /realtime/overview#tap-a-playing-element
      if (isLive(el)) {
        tap.attach(el as HTMLMediaElement);
        window.clearInterval(iv);
      } else if (++tries > 100) {
        window.clearInterval(iv);
      }
    }, 100);

    teardownRef.current = () => {
      window.clearInterval(iv);
      w.Audio = OrigAudio;
      // Null the stash so the *next* call's poll doesn't latch onto
      // this (now-dead) element before EL has constructed its new one.
      w.__el = null;
      // Close the tap's AudioContext + MediaStream — otherwise each
      // restart leaks a worklet graph.
      tap.close();
      void convo.endSession();
    };
  };

  return (
    <div>
      <button onClick={start} disabled={status !== "ready"}>Start conversation</button>
      <span>{stream ? (attached ? "lip-sync attached" : "attaching…") : "idle"}</span>
      {error ? <p>{error.message}</p> : null}
    </div>
  );
}

// Module scope — a NEW object every render reinitializes playback and
// breaks lip sync after the first chunk.
const CONVERSATION = {
  minVisemeInterval: 60,
  mergeWindow: 80,
  keyVisemePreference: 0.7,
  preserveSilence: true,
  similarityThreshold: 0.6,
  preserveCriticalVisemes: true,
} as const;

const playback = useMascotPlayback({ enableNaturalLipSync: true, naturalLipSyncConfig: CONVERSATION });

<div className="fixed bottom-4 right-4 w-64 h-64">
  <Mascot src="/widget-mascot.riv">
    <MascotRive />
    <ElevenLabsAvatar />
  </Mascot>
</div>

import { useRef } from "react";
import { useMascotInputs } from "@mascotbot/react/rive";

function ElevenLabsAvatarWithGestures() {
  // useMascotInputs() returns a fresh object every render — capture in a
  // ref so the long-lived EL callback always reads the current handle.
  const { custom } = useMascotInputs();
  const customRef = useRef(custom);
  customRef.current = custom;

  const start = async () => {
    const { signedUrl } = await (await fetch("/api/elevenlabs/signed-url", { method: "POST" })).json();
    const { Conversation } = await import("@elevenlabs/client");
    await Conversation.startSession({
      signedUrl,
      onModeChange: ({ mode }) => {
        // Fires exactly once per agent turn start; stays silent on the
        // "listening" back-edge so the gesture only plays as the agent
        // begins speaking.
        if (mode !== "speaking") return;
        customRef.current?.gesture?.fire?.();
      },
      // ... your existing onConnect / onDisconnect / onError
    });
  };

  return <button onClick={start}>Start conversation</button>;
}

const dynamicVariables = { name: userName ?? "Guest", role: userRole ?? "user" };

const { signedUrl } = await (await fetch("/api/elevenlabs/signed-url", { method: "POST" })).json();
const { Conversation } = await import("@elevenlabs/client");
const convo = await Conversation.startSession({ signedUrl, dynamicVariables });