I got a 3D avatar model from Sketchfab with MouthOpen and MouthClose shape keys. I have tested the model in Blender and shape keys are working fine.

Now I have imported the model in my React project using Three.js and I am getting the avatar in react.

I am building a AI chat bot, with TTS(Text-To-Speech). For that am using Web Speech Api and that is also working fine.

My requirement is to make the 3d avatar to speak the response got from LLM.

I will get the response as audio but my 3d avatar is not speaking.(i,e) no lip sync happening.

Below is my code:

export default function Chat() {
const [message, setMessage] = useState("");
const avatarRef = useRef();

// Fetch LLM Response
const getLLMResponse = async () => {
    if (!message.trim()) return;
    try {
        const response = await fetch(";, {
            method: "POST",
            headers: {
                "Content-Type": "application/json",
            },
            body: JSON.stringify({ query: message }),
        });

        if (!response.ok) {
            throw new Error(`HTTP error! Status: ${response.status}`);
        }

        const data = await response.json();
        if (data && data.bot) {
            speak("Hi There! Welcome!");
        } else {
            console.error("No bot response received.");
        }
    } catch (error) {
        console.error("Error fetching LLM response:", error);
        return null;
    }
};

// Text-to-Speech with mouth animation
const speak = (text) => {
    const utterance = new SpeechSynthesisUtterance(text);

    if (!avatarRef.current || !avatarRef.current.morphTargetDictionary) {
        console.error("Avatar morph targets not found.");
        return;
    }

    // Get shape key index
    const mouthOpenIndex = avatarRef.current.morphTargetDictionary["mouthOpen"];
    if (mouthOpenIndex === undefined) {
        console.error(`Shape key "mouthOpen" not found.`);
        return;
    }

    let isSpeaking = true;

    const animateMouth = () => {
        if (!isSpeaking) return;

        if (avatarRef.current) {
            avatarRef.current.morphTargetInfluences[mouthOpenIndex] = Math.random() * 0.5 + 0.3;
            avatarRef.current.geometry.attributes.position.needsUpdate = true;
        }
        requestAnimationFrame(animateMouth);
    };

    utterance.onstart = () => {
        isSpeaking = true;
        animateMouth();
    };

    utterance.onend = () => {
        isSpeaking = false;
        if (avatarRef.current) {
            avatarRef.current.morphTargetInfluences[mouthOpenIndex] = 0;
            avatarRef.current.geometry.attributes.position.needsUpdate = true;
        }
    };

    speechSynthesis.speak(utterance);
};

return (
    <Container maxWidth="md">
        <Paper elevation={3} sx={{ padding: 3, marginTop: 5, textAlign: "center" }}>
            <Typography variant="h5" gutterBottom>
                3D Avatar Chat
            </Typography>

            {/* 3D Avatar */}
            <Box
                sx={{
                    width: "400px",
                    height: "400px",
                    border: "1px solid #ddd",
                    margin: "auto",
                }}
            >
                <Canvas camera={{ position: [0, 2.2, 2.8], fov: 45 }}>
                    <ambientLight intensity={0.8} />
                    <directionalLight position={[2, 2, 2]} />
                    <Avatar ref={avatarRef} />
                    <OrbitControls
                        enableZoom={false}
                        enableRotate={false}
                        enablePan={false}
                        target={[0, 1.6, 0]}
                    />
                </Canvas>
            </Box>

            {/* Input and Button */}
            <Box sx={{ display: "flex", justifyContent: "center", marginTop: 2 }}>
                <TextField
                    label="Ask something..."
                    variant="outlined"
                    value={message}
                    onChange={(e) => setMessage(e.target.value)}
                    sx={{ width: "70%" }}
                />
                <Button
                    variant="contained"
                    color="primary"
                    onClick={getLLMResponse}
                    sx={{ marginLeft: 2 }}
                >
                    Send
                </Button>
            </Box>
        </Paper>
    </Container>
);
}

// 3D Avatar Component
const Avatar = React.forwardRef((props, ref) => {
const { scene } = useGLTF("/models/avatar.glb"); 
scene.traverse((child) => {
    if (child.isMesh && child.morphTargetDictionary) {
        ref.current = child; 
        child.material.morphTargets = true;
        child.material.needsUpdate = true;
    }
});
return <primitive object={scene} scale={2} position={[0, -1, 0]} />;
});