pnpm add @mascotbot/react @rive-app/react-webgl2 @rive-app/webgl2 @openai/agents-realtime

"use client";
import { MascotProvider } from "@mascotbot/react";
import { Mascot, MascotRive } from "@mascotbot/react/rive";

export default function App() {
  return (
    <MascotProvider apiKey="mascot_pub_…">
      <MascotProvider>
        <Mascot src="/mascot.riv">
          <MascotRive />
          <OpenAIAvatar />
        </Mascot>
      </MascotProvider>
    </MascotProvider>
  );
}

// app/api/openai/token/route.ts
export const runtime = "nodejs";

export async function POST() {
  const key = process.env.OPENAI_API_KEY;
  if (!key) return Response.json({ error: "OPENAI_API_KEY not set" }, { status: 400 });

  const model = "gpt-realtime";
  const res = await fetch("https://api.openai.com/v1/realtime/client_secrets", {
    method: "POST",
    headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json" },
    body: JSON.stringify({
      session: {
        type: "realtime",
        model,
        output_modalities: ["audio"],
        audio: {
          input: { turn_detection: { type: "server_vad", threshold: 0.5, silence_duration_ms: 500 } },
          output: { voice: "marin" },
        },
      },
    }),
  });
  if (!res.ok) return Response.json({ error: `OpenAI ${res.status}` }, { status: 502 });
  const json = (await res.json()) as { value: string };
  return Response.json({ clientSecret: json.value, model });
}

"use client";
import { useEffect, useRef, useState } from "react";
import { useMascot } from "@mascotbot/react";
import { useMascotPlayback, useLipsyncStream } from "@mascotbot/react/rive";

const LIP_SYNC = { minVisemeInterval: 60, mergeWindow: 80 } as const;

export function OpenAIAvatar() {
  const { client, status } = useMascot();
  const playback = useMascotPlayback({ stream: true, enableNaturalLipSync: true, naturalLipSyncConfig: LIP_SYNC });
  const [stream, setStream] = useState<MediaStream | null>(null);
  const teardownRef = useRef<null | (() => void)>(null);

  const { error, attached } = useLipsyncStream({ client, playback, source: { kind: "mediaStream", stream } });

  useEffect(() => () => teardownRef.current?.(), []);

  const connect = async () => {
    if (status !== "ready") return;
    const { clientSecret } = await (await fetch("/api/openai/token", { method: "POST" })).json();
    const { RealtimeAgent, RealtimeSession } = await import("@openai/agents-realtime");

    const audioEl = new Audio(); // supply our own so we can tap it
    const agent = new RealtimeAgent({ name: "Assistant", instructions: "Keep replies short." });
    const session = new RealtimeSession(agent, { transport: "webrtc" });
    await session.connect({ apiKey: clientSecret, audioElement: audioEl });

    // createElementTap from "@mascotbot/react" — cross-browser
    // tap (Safari has no captureStream). /realtime/overview#tap-a-playing-element
    const tap = createElementTap();
    setStream(tap.stream);
    tap.attach(audioEl);
    teardownRef.current = () => { tap.close(); void session.close(); };
  };

  return (
    <div>
      <button onClick={connect} disabled={status !== "ready"}>Connect</button>
      <span>{stream ? (attached ? "lip-sync attached" : "attaching…") : "idle"}</span>
      {error ? <p>{error.message}</p> : null}
    </div>
  );
}

import { createPCMStreamPlayer } from "@mascotbot/react";
import { WavRecorder } from "wavtools";

// inside connect():
const player = createPCMStreamPlayer({ sampleRate: 24000 }); // create in the click, before await
setStream(player.outputStream);

const session = new RealtimeSession(agent, { transport: "websocket", model });
session.on("audio", (e: { data: ArrayBuffer }) => player.pushPCM16(new Uint8Array(e.data)));
session.on("audio_interrupted", () => player.stop());
await session.connect({ apiKey: clientSecret });

const recorder = new WavRecorder({ sampleRate: 24000 });
await recorder.begin();
// Pass the typed-array view itself (cast) — `.buffer` is over-long and yields "empty bytes".
await recorder.record((d: { mono: Int16Array }) => session.sendAudio(d.mono as unknown as ArrayBuffer));