Discussions

Ask a Question
Back to all

Avatar Doesnt respond back

My avatar doesnt respond back to me, I start listening it listens it shows caption like transcription but I get 0 response.

I have tried to use both push to speak and conversational with full mode

User end speak fire yes and I wanted to fire avatar.current.message but I am sure this is not the best way because I built Heygen experience before LiveAvatar

This is my main function

 const startSession = async () => {
    console.log("Start Session")
    sessionStarted.current = true;
    setIsLoadingSession(true);
    setCurrentLoadingStep(0);

    try {
      // const quota = await getHeygenRemainingQuota();
      // if (quota < 25000) {
      //   setDebug("Not enough quota");
      //   return;
      // }
      setCurrentLoadingStep(1);
      const voiceId = language == "ar" ? "8ce46741c2f74f58a1273ba0af42ce50" : undefined;
      const token = await fetchAccessToken({
        avatarId: card.id,
        knowledgeId,
        language,
        voiceId,
      });
      console.log("token", token);
      setCurrentLoadingStep(2);
      avatar.current = new LiveAvatarSession(token, { voiceChat: true });
      console.log("avatar.current", avatar.current);

      avatar.current.on(SessionEvent.SESSION_STREAM_READY, () => {
        setIsStreamReady(true);
        EndSessionAfterAmountOfTime(heygenTimeoutPlaceHolder);
      });

      avatar.current.on(SessionEvent.SESSION_DISCONNECTED, async () => {
        await endSession();
      });

      avatar.current.on(AgentEventsEnum.AVATAR_SPEAK_STARTED, async () => {
        handleMicPermission(false);
        setIsAvatarTalking(true);
        EndSessionAfterAmountOfTime(heygenTimeoutPlaceHolder);
      });

      avatar.current.on(AgentEventsEnum.AVATAR_SPEAK_ENDED, async () => {
        setIsAvatarTalking(false);
        setAvatarIsGivingIntro(false);
        setCaptionText("");
        EndSessionAfterAmountOfTime(heygenTimeoutPlaceHolder);
      });

      avatar.current.on(AgentEventsEnum.USER_SPEAK_STARTED, () => {
        console.log("User Started Talking");
        setIsUserTalking(true);
        setCaptionText("");
      });

      avatar.current.on(AgentEventsEnum.USER_SPEAK_ENDED, () => {
        setIsUserTalking(false);
      });

      avatar.current.on(AgentEventsEnum.USER_TRANSCRIPTION, (event: any) => {
        const text = event?.data?.text ?? event?.text ?? event?.message ?? "";
        if (text) { setCaptionText(text); }
      });

      avatar.current.on(AgentEventsEnum.AVATAR_TRANSCRIPTION, (event: any) => {
        const text = event?.data?.text ?? event?.text ?? event?.message ?? "";
        if (text) setCaptionText(text);
      });
      setCurrentLoadingStep(3);
      await avatar.current.start();
      // await avatar.current.voiceChat.start();
      console.log("avatar.current.voiceChat.state", avatar.current.voiceChat.state);
      setCurrentLoadingStep(4);
      setChatMode("voice_mode");
      setCurrentLoadingStep(5);
      await handleMicPermission(false);
      setDebug("");
      setCurrentLoadingStep(6);
      setStreamIsVideo(false)
    } catch (error) {
      console.error("StartSession Error:", error);
    } finally {
      setIsLoadingSession(false);
    }
  };

This is my API/get-access-token/route.ts

import { NextResponse } from 'next/server';

const HEYGEN_API_KEY = process.env.HEYGEN_API_KEY;

export async function POST(request: Request) {
  try {
    if (!HEYGEN_API_KEY) {
      throw new Error("API key is missing from .env");
    }
    console.log(HEYGEN_API_KEY)

    const { avatarId, knowledgeId, language, voiceId } = await request.json();
    if (!avatarId || !language) {
      return NextResponse.json({ error: "Missing avatarId or language" }, { status: 400 });
    }

    const avatarPersona: Record<string, string> = {
      language,
      "interactivity_type": "CONVERSATIONAL"
    };
    if (knowledgeId) avatarPersona.context_id = "393cde79-f062-4236-b3a2-fa4801a37fef";
    if (voiceId) avatarPersona.voice_id = "df0085fc-b104-455d-9260-9f33277eb64d";


    const res = await fetch(
      "https://api.liveavatar.com/v1/sessions/token",
      {
        method: "POST",
        headers: {
          "x-api-key": HEYGEN_API_KEY,
          "accept": "application/json",
          "content-type": "application/json",
        },
        body: JSON.stringify({
          mode: "FULL",
          avatar_id: avatarId,
          avatar_persona: { avatarPersona },
        }),
      },
    );
    const data = await res.json();

    const token = data.data?.session_token ?? data.data?.token;
    if (!res.ok || !token) {
      return NextResponse.json({ error: data }, { status: res.status || 500 });
    }

    return NextResponse.json({ token }, { status: 200 });
  } catch (error) {
    console.error("Error retrieving access token:", error);

    return NextResponse.json({ error: "Failed to retrieve access token" }, {
      status: 500,
    });
  }
}