anam plugin

.changeset/add-anam-plugin.md

@@ -0,0 +1,13 @@
+---
+"@livekit/agents-plugin-anam": minor
+---
+
+Add new Anam plugin for avatar integration
+
+This adds a new plugin `@livekit/agents-plugin-anam` that provides:
+- AvatarSession class for managing Anam avatar connections
+- API integration for Anam avatar services
+- Real-time avatar video streaming capabilities
+- Integration with LiveKit agents for voice-to-avatar workflows
+
+The plugin enables developers to create conversational AI agents with realistic avatar representations using Anam's avatar technology.

examples/package.json

@@ -22,15 +22,16 @@
   },
   "dependencies": {
     "@livekit/agents": "workspace:*",
-    "@livekit/agents-plugin-resemble": "workspace:*",
+    "@livekit/agents-plugin-anam": "workspace:*",
+    "@livekit/agents-plugin-cartesia": "workspace:*",
     "@livekit/agents-plugin-deepgram": "workspace:*",
     "@livekit/agents-plugin-elevenlabs": "workspace:*",
     "@livekit/agents-plugin-google": "workspace:*",
     "@livekit/agents-plugin-livekit": "workspace:*",
+    "@livekit/agents-plugin-neuphonic": "workspace:*",
     "@livekit/agents-plugin-openai": "workspace:*",
+    "@livekit/agents-plugin-resemble": "workspace:*",
     "@livekit/agents-plugin-silero": "workspace:*",
-    "@livekit/agents-plugin-cartesia": "workspace:*",
-    "@livekit/agents-plugin-neuphonic": "workspace:*",
     "@livekit/noise-cancellation-node": "^0.1.9",
     "@livekit/rtc-node": "^0.13.11",
     "livekit-server-sdk": "^2.13.3",

examples/src/anam_realtime_agent.ts

@@ -0,0 +1,71 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import {
+  type JobContext,
+  type JobProcess,
+  WorkerOptions,
+  cli,
+  defineAgent,
+  voice,
+} from '@livekit/agents';
+import * as anam from '@livekit/agents-plugin-anam';
+import * as openai from '@livekit/agents-plugin-openai';
+import * as silero from '@livekit/agents-plugin-silero';
+import { fileURLToPath } from 'node:url';
+
+// Uses OpenAI Advanced Voice (Realtime), so no separate STT/TTS/VAD.
+
+export default defineAgent({
+  prewarm: async (proc: JobProcess) => {
+    proc.userData.vad = await silero.VAD.load();
+  },
+  entry: async (ctx: JobContext) => {
+    const agent = new voice.Agent({
+      instructions: 'You are a helpful assistant. Speak clearly and concisely.',
+    });
+
+    const session = new voice.AgentSession({
+      llm: new openai.realtime.RealtimeModel(),
+      voiceOptions: {
+        // allow the model to call multiple tools in a single turn if needed
+        maxToolSteps: 3,
+      },
+    });
+
+    await session.start({
+      agent,
+      room: ctx.room,
+    });
+
+    // Join the LiveKit room first (ensures room name and identity available)
+    await ctx.connect();
+
+    // Configure the Anam avatar persona (requires avatarId)
+    const personaName = process.env.ANAM_PERSONA_NAME ?? 'Agent';
+    const avatarId = process.env.ANAM_AVATAR_ID;
+    if (!avatarId) {
+      throw new Error('ANAM_AVATAR_ID is required');
+    }
+
+    // Start the Anam avatar session and route Agent audio to the avatar
+    const avatar = new anam.AvatarSession({
+      personaConfig: { name: personaName, avatarId },
+      // Allow overriding base URL via env
+      apiUrl: process.env.ANAM_API_URL,
+      // connOptions: { maxRetry: 5, retryInterval: 2, timeout: 15 },
+    });
+    await avatar.start(session, ctx.room);
+
+    session.on(voice.AgentSessionEventTypes.MetricsCollected, (ev) => {
+      console.log('metrics_collected', ev);
+    });
+
+    // With Realtime LLM, generateReply will synthesize audio via the model
+    session.generateReply({
+      instructions: 'Greet the user briefly and confirm you are ready.',
+    });
+  },
+});
+
+cli.runApp(new WorkerOptions({ agent: fileURLToPath(import.meta.url) }));

plugins/anam/CHANGELOG.md

@@ -0,0 +1 @@
+# @livekit/agents-plugin-anam

plugins/anam/README.md

@@ -0,0 +1,17 @@
+<!--
+SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+
+SPDX-License-Identifier: Apache-2.0
+-->
+# Anam plugin for LiveKit Agents
+
+The Agents Framework is designed for building realtime, programmable
+participants that run on servers. Use it to create conversational, multi-modal
+voice agents that can see, hear, and understand.
+
+This package contains the Anam plugin, which allows for voice synthesis.
+Refer to the [documentation](https://docs.livekit.io/agents/overview/) for
+information on how to use it, or browse the [API
+reference](https://docs.livekit.io/agents-js/modules/plugins_agents_plugin_Anam.html).
+See the [repository](https://github.com/livekit/agents-js) for more information
+about the framework as a whole.

plugins/anam/api-extractor.json

@@ -0,0 +1,20 @@
+/**
+ * Config file for API Extractor.  For more info, please visit: https://api-extractor.com
+ */
+{
+  "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json",
+
+  /**
+   * Optionally specifies another JSON config file that this file extends from.  This provides a way for
+   * standard settings to be shared across multiple projects.
+   *
+   * If the path starts with "./" or "../", the path is resolved relative to the folder of the file that contains
+   * the "extends" field.  Otherwise, the first path segment is interpreted as an NPM package name, and will be
+   * resolved using NodeJS require().
+   *
+   * SUPPORTED TOKENS: none
+   * DEFAULT VALUE: ""
+   */
+  "extends": "../../api-extractor-shared.json",
+  "mainEntryPointFilePath": "./dist/index.d.ts"
+}

plugins/anam/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "@livekit/agents-plugin-anam",
+  "version": "0.0.1",
+  "description": "Anam plugin for LiveKit Node Agents",
+  "main": "dist/index.js",
+  "require": "dist/index.cjs",
+  "types": "dist/index.d.ts",
+  "exports": {
+    "import": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "require": {
+      "types": "./dist/index.d.cts",
+      "default": "./dist/index.cjs"
+    }
+  },
+  "author": "LiveKit",
+  "type": "module",
+  "repository": "git@github.com:livekit/agents-js.git",
+  "license": "Apache-2.0",
+  "files": [
+    "dist",
+    "src",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsup --onSuccess \"pnpm build:types\"",
+    "build:types": "tsc --declaration --emitDeclarationOnly && node ../../scripts/copyDeclarationOutput.js",
+    "clean": "rm -rf dist",
+    "clean:build": "pnpm clean && pnpm build",
+    "lint": "eslint -f unix \"src/**/*.{ts,js}\"",
+    "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript",
+    "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose"
+  },
+  "devDependencies": {
+    "@livekit/agents": "workspace:*",
+    "@livekit/agents-plugin-openai": "workspace:*",
+    "@livekit/agents-plugins-test": "workspace:*",
+    "@livekit/rtc-node": "^0.13.12",
+    "@microsoft/api-extractor": "^7.35.0",
+    "@types/ws": "^8.5.10",
+    "tsup": "^8.3.5",
+    "typescript": "^5.0.0"
+  },
+  "dependencies": {
+    "livekit-server-sdk": "^2.9.2",
+    "ws": "^8.16.0"
+  },
+  "peerDependencies": {
+    "@livekit/agents": "workspace:*",
+    "@livekit/rtc-node": "^0.13.12"
+  }
+}

plugins/anam/src/api.ts

@@ -0,0 +1,151 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { log } from '@livekit/agents';
+import { type APIConnectOptions, AnamException, type PersonaConfig } from './types.js';
+
+const DEFAULT_API_URL = 'https://api.anam.ai';
+
+export class AnamAPI {
+  constructor(
+    private apiKey: string,
+    private apiUrl: string = DEFAULT_API_URL,
+    private conn: APIConnectOptions = { maxRetry: 3, retryInterval: 2, timeout: 10 },
+  ) {}
+
+  private get tokenPath(): string {
+    return '/v1/auth/session-token';
+  }
+
+  private get startPath(): string {
+    return '/v1/engine/session';
+  }
+
+  private async postWithHeaders<T>(
+    path: string,
+    body: unknown,
+    headersIn: Record<string, string>,
+  ): Promise<T> {
+    const url = `${this.apiUrl}${path}`;
+    const { maxRetry = 3, retryInterval = 2 } = this.conn;
+    let lastErr: unknown;
+    const logger = log().child({ module: 'AnamAPI' });
+
+    for (let attempt = 0; attempt < maxRetry; attempt++) {
+      try {
+        const headers: Record<string, string> = {
+          'Content-Type': 'application/json',
+          ...headersIn,
+        };
+
+        const redactedHeaders: Record<string, string> = { ...headers };
+        if (redactedHeaders.Authorization) {
+          redactedHeaders.Authorization = 'Bearer ****';
+        }
+        const redactedBody = (() => {
+          if (body && typeof body === 'object') {
+            try {
+              const clone = { ...(body as Record<string, unknown>) } as Record<string, unknown>;
+              if ('livekitToken' in clone) clone.livekitToken = '****';
+              if ('sessionToken' in clone) clone.sessionToken = '****' as unknown as never;
+              return clone;
+            } catch {
+              return { note: 'unserializable body' };
+            }
+          }
+          return body;
+        })();
+
+        logger.debug(
+          {
+            url,
+            method: 'POST',
+            headers: redactedHeaders,
+            body: redactedBody,
+            attempt: attempt + 1,
+          },
+          'calling Anam API',
+        );
+
+        const res = await fetch(url, {
+          method: 'POST',
+          headers,
+          body: JSON.stringify(body),
+          // simple timeout: rely on AbortController in real impl
+        });
+        if (!res.ok) {
+          const text = await res.text();
+          logger.error(
+            {
+              url,
+              method: 'POST',
+              headers: redactedHeaders,
+              body: redactedBody,
+              status: res.status,
+              response: text,
+            },
+            'Anam API request failed',
+          );
+          throw new AnamException(`Anam ${path} failed: ${res.status} ${text}`);
+        }
+        const json = (await res.json()) as T;
+        logger.debug({ url }, 'Anam API request succeeded');
+        return json;
+      } catch (e) {
+        lastErr = e;
+        if (attempt === maxRetry - 1) break;
+        logger.warn(
+          {
+            url,
+            method: 'POST',
+            body:
+              body && typeof body === 'object'
+                ? { ...(body as Record<string, unknown>), livekitToken: '****' }
+                : body,
+            error: (e as Error)?.message,
+            nextRetrySec: retryInterval,
+          },
+          'Anam API error, retrying',
+        );
+        await new Promise((r) => setTimeout(r, retryInterval * 1000));
+      }
+    }
+    throw lastErr instanceof Error ? lastErr : new AnamException('Anam API error');
+  }
+
+  private async post<T>(path: string, body: unknown): Promise<T> {
+    return this.postWithHeaders<T>(path, body, { Authorization: `Bearer ${this.apiKey}` });
+  }
+
+  createSessionToken(params: {
+    personaConfig: PersonaConfig;
+    livekitUrl?: string;
+    livekitToken?: string;
+  }) {
+    const pc = params.personaConfig;
+    const personaPayload = {
+      type: 'ephemeral',
+      name: pc.name,
+      avatarId: pc.avatarId,
+      llmId: 'CUSTOMER_CLIENT_V1',
+    };
+
+    const payload: Record<string, unknown> = {
+      personaConfig: personaPayload,
+    };
+    payload.environment = {
+      livekitUrl: params.livekitUrl,
+      livekitToken: params.livekitToken,
+    };
+
+    return this.post<{ sessionToken: string }>(this.tokenPath, payload);
+  }
+
+  startEngineSession(params: { sessionToken: string }) {
+    return this.postWithHeaders<{ sessionId: string }>(
+      this.startPath,
+      {},
+      { Authorization: `Bearer ${params.sessionToken}` },
+    );
+  }
+}