LiveKit Quickstart

lk agent init my-agent --template agent-starter-python
cd my-agent

lk agent init my-agent --template agent-starter-node
cd my-agent

uv add livekit-plugins-ai-coustics

pnpm add @livekit/plugins-ai-coustics

uv sync

pnpm install

uv run src/agent.py download-files

npx livekit-agents download-files

from livekit.plugins import ai_coustics

session = AgentSession(
  vad=ai_coustics.VAD(),  # Add ai-coustics VAD to session setup
  # ...
)

await session.start(
    agent=Assistant(),
    room=ctx.room,
    room_options=room_io.RoomOptions(
        audio_input=room_io.AudioInputOptions(
              # Add ai-coustics audio enhancement to audio input options
              noise_cancellation=ai_coustics.audio_enhancement(
                # - EnhancerModel.QUAIL_VF_L  (best for isolating the foreground speaker)
                # - EnhancerModel.QUAIL_VF_S  (smaller, more efficient version of Quail VF)
                # - EnhancerModel.QUAIL_L     (best for multiple/far-field speakers)
                model=ai_coustics.EnhancerModel.QUAIL_VF_L,
                # auth defaults to Auth.livekit_cloud() - omit it when using LiveKit Cloud.
                # For self-hosted environments, pass your license key explicitly:
                #   auth=ai_coustics.Auth.ai_coustics_api(license_key="YOUR_LICENSE_KEY"),
                # - enhancement_level = 0.5 (conservative, foreground speech is always preserved)
                # - enhancement_level = 0.8 (balanced, optimal word error rate on challenging data)
                # - enhancement_level = 1.0 (aggressive, maximum suppression of interfering speech)
                # More info: https://docs.ai-coustics.com/models/speech-enhancement/speech-enhancement-for-voice-ai-systems
                model_parameters=ai_coustics.ModelParameters(enhancement_level=0.8),
                # VAD Parameters Info: https://docs.ai-coustics.com/models/voice-activity-detection/quail-vad
                vad_settings=ai_coustics.VadSettings(
                  # 0.0 to 1.0 seconds
                  speech_hold_duration=0.03,
                  # 1.0 to 15.0
                  sensitivity=6.0,
                  # 0.0 to 1.0 seconds
                  minimum_speech_duration=0.0,
                )
            ),
        )
    ),
)

import { ServerOptions, cli, defineAgent, inference, voice } from '@livekit/agents';
import {
  EnhancerModel,
  VadSettings,
  audioEnhancement,
} from '@livekit/plugins-ai-coustics';
import dotenv from 'dotenv';
import { fileURLToPath } from 'node:url';
import { Agent } from './agent';

// Load environment variables from a local file.
// Make sure to set LIVEKIT_URL, LIVEKIT_API_KEY, and LIVEKIT_API_SECRET
// when running locally or self-hosting your agent server.
dotenv.config({ path: '.env.local' });

export default defineAgent({
  entry: async (ctx) => {
    // Set up a voice AI pipeline using OpenAI, Cartesia, Deepgram, and the LiveKit turn detector
    const session = new voice.AgentSession({
      // Speech-to-text (STT) is your agent's ears, turning the user's speech into text that the LLM can understand
      // See all available models at https://docs.livekit.io/agents/models/stt/
      stt: new inference.STT({
        model: 'deepgram/nova-3',
        language: 'multi',
      }),

      // Text-to-speech (TTS) is your agent's voice, turning the LLM's text into speech that the user can hear
      // See all available models as well as voice selections at https://docs.livekit.io/agents/models/tts/
      tts: new inference.TTS({
        model: 'cartesia/sonic-3',
        voice: '9626c31c-bec5-4cca-baa8-f8ba9e84c8bc',
      }),

      // Turn detection determines when the user is speaking and when the agent should respond.
      // The LiveKit audio turn detector is a multimodal model that encodes the user's audio
      // directly to predict end of turn. It's built into the SDK (no extra plugin) and
      // AgentSession supplies the required VAD automatically.
      // See more at https://docs.livekit.io/agents/logic/turns/turn-detector/
      turnHandling: {
        turnDetection: new inference.TurnDetector(),
        // Allow the LLM to generate a response while waiting for the end of turn
        preemptiveGeneration: { enabled: true },
      },
    });

    // Start the session, which initializes the voice pipeline and warms up the models
    await session.start({
      agent: new Agent(),
      room: ctx.room,
      inputOptions: {
        // ai-coustics QUAIL audio enhancement for noise cancellation
        // Works for both WebRTC and telephony (SIP) participants
        noiseCancellation: audioEnhancement({
          // - EnhancerModel.QuailVfL (best for isolating the foreground speaker)
          // - EnhancerModel.QuailVfS (smaller, more efficient version of Quail VF)
          // - EnhancerModel.QuailL   (best for multiple/far-field speakers)
          model: EnhancerModel.QuailVfS,
          // auth defaults to LiveKit Cloud - omit it when using LiveKit Cloud.
          // For self-hosted environments, pass your license key explicitly.
          // See the self-hosted authentication step below.
          // - enhancementLevel = 0.5 (conservative, foreground speech is always preserved)
          // - enhancementLevel = 0.8 (balanced, optimal word error rate on challenging data)
          // - enhancementLevel = 1.0 (aggressive, maximum suppression of interfering speech)
          // More info: https://docs.ai-coustics.com/models/speech-enhancement/speech-enhancement-for-voice-ai-systems
          modelParameters: { enhancementLevel: 0.8 },
          // VAD Parameters Info: https://docs.ai-coustics.com/models/voice-activity-detection/quail-vad
          vadSettings: VadSettings.new({
            // 0.0 to 1.0 seconds
            speechHoldDuration: 0.03,
            // 1.0 to 15.0
            sensitivity: 6.0,
            // 0.0 to 1.0 seconds
            minimumSpeechDuration: 0.0,
          }),
        }),
      },
    });

    // Join the room and connect to the user
    await ctx.connect();

    // Greet the user on joining
    session.generateReply({
      instructions: 'Greet the user in a helpful and friendly manner.',
    });
  },
});

// Run the agent server
cli.runApp(
  new ServerOptions({
    agent: fileURLToPath(import.meta.url),
    agentName: 'my-agent',
  }),
);

uv run python src/agent.py console

pnpm dev

noise_cancellation=ai_coustics.audio_enhancement(
    model=ai_coustics.EnhancerModel.QUAIL_VF_L,
    auth=ai_coustics.Auth.ai_coustics_api(license_key="YOUR_LICENSE_KEY"),
)

import { Auth, EnhancerModel, audioEnhancement } from '@livekit/plugins-ai-coustics';

noiseCancellation: audioEnhancement({
  model: EnhancerModel.QuailVfS,
  auth: Auth.aiCousticsApi('YOUR_LICENSE_KEY'),
}),