pnpm add @mascotbot/react @rive-app/react-webgl2 @rive-app/webgl2 @google/genai

"use client";
import { MascotProvider } from "@mascotbot/react";
import { Mascot, MascotRive } from "@mascotbot/react/rive";

export default function App() {
  return (
    <MascotProvider apiKey="mascot_pub_…">
      <MascotProvider>
        <Mascot src="/mascot.riv">
          <MascotRive />
          <GeminiAvatar />
        </Mascot>
      </MascotProvider>
    </MascotProvider>
  );
}

// app/api/gemini/token/route.ts
export const runtime = "nodejs";

export async function POST() {
  const key = process.env.GEMINI_API_KEY;
  if (!key) return Response.json({ error: "GEMINI_API_KEY not set" }, { status: 400 });

  const model = "models/gemini-3.1-flash-live-preview";
  const { GoogleGenAI, Modality } = await import("@google/genai");
  const ai = new GoogleGenAI({ apiKey: key, httpOptions: { apiVersion: "v1alpha" } });

  const token = await ai.authTokens.create({
    config: {
      uses: 1,
      newSessionExpireTime: new Date(Date.now() + 10 * 60 * 1000).toISOString(),
      liveConnectConstraints: {
        model,
        config: { responseModalities: [Modality.AUDIO] },
      },
    },
  });
  return Response.json({ ephemeralToken: token.name, model });
}

"use client";
import { useEffect, useRef, useState } from "react";
import { useMascot, createPCMStreamPlayer, type PCMStreamPlayer } from "@mascotbot/react";
import { useMascotPlayback, useLipsyncStream } from "@mascotbot/react/rive";

const LIP_SYNC = { minVisemeInterval: 60, mergeWindow: 80 } as const;

export function GeminiAvatar() {
  const { client, status } = useMascot();
  const playback = useMascotPlayback({ stream: true, enableNaturalLipSync: true, naturalLipSyncConfig: LIP_SYNC });
  const playerRef = useRef<PCMStreamPlayer | null>(null);
  const [stream, setStream] = useState<MediaStream | null>(null);
  const teardownRef = useRef<null | (() => void)>(null);

  const { error, attached } = useLipsyncStream({ client, playback, source: { kind: "mediaStream", stream } });

  useEffect(() => () => { teardownRef.current?.(); void playerRef.current?.close(); }, []);

  const connect = async () => {
    if (status !== "ready") return;
    // Create the player inside the click, before any await.
    const player = createPCMStreamPlayer({ sampleRate: 24000 });
    playerRef.current = player;
    setStream(player.outputStream);

    const { ephemeralToken, model } = await (await fetch("/api/gemini/token", { method: "POST" })).json();
    const { GoogleGenAI, Modality } = await import("@google/genai");
    const ai = new GoogleGenAI({ apiKey: ephemeralToken, httpOptions: { apiVersion: "v1alpha" } });

    // Liveness flag: the mic processor fires continuously — never send to a closed socket.
    let live = true;
    const session = await ai.live.connect({
      model, // "models/gemini-3.1-flash-live-preview"
      config: { responseModalities: [Modality.AUDIO] },
      callbacks: {
        onmessage: (msg: any) => {
          const b64 = msg?.serverContent?.modelTurn?.parts?.[0]?.inlineData?.data;
          if (typeof b64 === "string") player.pushBase64PCM16(b64);
          if (msg?.serverContent?.interrupted) player.stop();
        },
        onerror: () => { live = false; },
        onclose: () => { live = false; },
      },
    });

    // Mic: 16 kHz mono → PCM16 → sendRealtimeInput
    const mic = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, sampleRate: 16000 } });
    const Ctor = (window as any).AudioContext || (window as any).webkitAudioContext;
    const ctx = new Ctor({ sampleRate: 16000 });
    const src = ctx.createMediaStreamSource(mic);
    const proc = ctx.createScriptProcessor(4096, 1, 1);
    proc.onaudioprocess = (ev: AudioProcessingEvent) => {
      if (!live) return;
      const f32 = ev.inputBuffer.getChannelData(0);
      const pcm = new Int16Array(f32.length);
      for (let i = 0; i < f32.length; i++) {
        const s = Math.max(-1, Math.min(1, f32[i]));
        pcm[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
      }
      let bin = "";
      const bytes = new Uint8Array(pcm.buffer);
      for (let i = 0; i < bytes.length; i++) bin += String.fromCharCode(bytes[i]);
      try {
        session.sendRealtimeInput({ audio: { data: btoa(bin), mimeType: "audio/pcm;rate=16000" } });
      } catch { live = false; }
    };
    src.connect(proc);
    proc.connect(ctx.destination);
    session.sendClientContent({ turns: "Say a short friendly hello.", turnComplete: true });

    teardownRef.current = () => {
      live = false;
      proc.onaudioprocess = null;
      proc.disconnect(); src.disconnect();
      mic.getTracks().forEach((t) => t.stop());
      void ctx.close();
      session.close();
    };
  };

  return (
    <div>
      <button onClick={connect} disabled={status !== "ready"}>Connect</button>
      <span>{stream ? (attached ? "lip-sync attached" : "attaching…") : "idle"}</span>
      {error ? <p>{error.message}</p> : null}
    </div>
  );
}