Realtime AI Voice Avatars - Lip Sync for OpenAI, Gemini & ElevenLabs

useLipsyncStream({ client, playback, source: { kind: "mediaStream", stream } });

import { createElementTap } from "@mascotbot/react";

// Create inside the click that starts the call (so its AudioContext isn't
// born suspended). `stream` is usable immediately — silent until attach().
const tap = createElementTap();
useLipsyncStream({
  client,
  playback,
  source: { kind: "mediaStream", stream: tap.stream },
});

tap.attach(audioEl); // now, or later once the provider's <audio> exists
// teardown: tap.close();

import { RealtimeAgent, RealtimeSession } from "@openai/agents-realtime";

const audioEl = new Audio();
const session = new RealtimeSession(new RealtimeAgent({ name: "Assistant" }), { transport: "webrtc" });
await session.connect({ apiKey: clientSecret, audioElement: audioEl }); // clientSecret from your server route

const tap = createElementTap(); // create in the click — see "Tap a playing element"
useLipsyncStream({ client, playback, source: { kind: "mediaStream", stream: tap.stream } });
tap.attach(audioEl);

import { createPCMStreamPlayer } from "@mascotbot/core";

const player = createPCMStreamPlayer({ sampleRate: 24000 }); // both emit 24 kHz
useLipsyncStream({ client, playback, source: { kind: "mediaStream", stream: player.outputStream } });

// Gemini Live (@google/genai): modelTurn audio part
session.onmessage = (m) => {
  const b64 = m?.serverContent?.modelTurn?.parts?.[0]?.inlineData?.data;
  if (typeof b64 === "string") player.pushBase64PCM16(b64);
  if (m?.serverContent?.interrupted) player.stop();
};

// OpenAI Realtime (WebSocket): PCM16 ArrayBuffer
session.on("audio", (e) => player.pushPCM16(new Uint8Array(e.data)));
session.on("audio_interrupted", () => player.stop());

const tap = createElementTap();   // in the click, before startSession
setStream(tap.stream);            // → useLipsyncStream source: { kind: "mediaStream", stream }

const w = window as unknown as { Audio: typeof Audio; __el?: HTMLAudioElement };
const Orig = w.Audio;
w.Audio = function (...a: unknown[]) {
  const el = new Orig(...(a as []));
  w.__el = el;
  return el;
} as unknown as typeof Audio;

const { Conversation } = await import("@elevenlabs/client");
await Conversation.startSession({ signedUrl });

const iv = setInterval(() => {
  const el = w.__el;
  if (el?.srcObject instanceof MediaStream) {
    clearInterval(iv);
    w.Audio = Orig;
    tap.attach(el);               // srcObject branch → tap only; stays audible
  }
}, 100);
// teardown: tap.close();

import { useRef } from "react";
import { useMascot } from "@mascotbot/react";
import { useMascotPlayback, useMascotInputs, useLipsyncStream } from "@mascotbot/react/rive";

function AvatarReactions({ stream }: { stream: MediaStream | null }) {
  const { client } = useMascot();
  const playback = useMascotPlayback({ stream: true, enableNaturalLipSync: true });
  const { has, custom } = useMascotInputs();
  const speaking = useRef(false);

  useLipsyncStream({
    client,
    playback,
    source: { kind: "mediaStream", stream }, // createElementTap() for ElevenLabs, or player.outputStream
    onFrame: (f) => {
      const isSpeech = !f.silenceDetected;
      if (isSpeech && !speaking.current) {
        speaking.current = true;
        playback.stress([{ offset: 0, stress: 1 }]);          // emphasize while speaking
        if (has("gesture")) custom.gesture.fire?.();           // one-shot reaction
      } else if (!isSpeech && speaking.current) {
        speaking.current = false;
        playback.stress([{ offset: 0, stress: 0 }]);          // ease back to neutral
      }
    },
  });
  return null;
}