feat(voice): start and end live voice sessions from backend functions

packages/sdk/scripts/mintlify-post-processing/types-to-delete-after-processing.json

@@ -20,5 +20,8 @@
   "ImportResult",
   "SortField",
   "UpdateManyResult",
+  "VoiceSession",
+  "VoiceSessionMessage",
+  "VoiceSessionOptions",
   "user-connectors"
 ]

packages/sdk/scripts/mintlify-post-processing/types-to-expose.json

@@ -57,5 +57,9 @@
   "ServiceRolePaymentsModule",
   "SortField",
   "SsoModule",
-  "UpdateManyResult"
+  "UpdateManyResult",
+  "VoiceModule",
+  "VoiceSession",
+  "VoiceSessionMessage",
+  "VoiceSessionOptions"
 ]

packages/sdk/src/client.ts

@@ -4,6 +4,7 @@ import { createIntegrationsModule } from "./modules/integrations.js";
 import { createAuthModule } from "./modules/auth.js";
 import { createSsoModule } from "./modules/sso.js";
 import { createBrowserModule } from "./modules/browser.js";
+import { createVoiceModule } from "./modules/voice.js";
 import {
   createConnectorsModule,
   createUserConnectorsModule,
@@ -295,6 +296,7 @@ export function createClient(config: CreateClientConfig): Base44Client {
     integrations: createIntegrationsModule(serviceRoleAxiosClient, appId),
     sso: createSsoModule(serviceRoleAxiosClient, appId),
     browser: createBrowserModule(serviceRoleAxiosClient),
+    voice: createVoiceModule(serviceRoleAxiosClient, { functionsVersion }),
     connectors: createConnectorsModule(serviceRoleAxiosClient, appId),
     functions: createFunctionsModule(serviceRoleFunctionsAxiosClient, appId, {
       getAuthHeaders: () => {

packages/sdk/src/client.types.ts

@@ -3,6 +3,7 @@ import type { IntegrationsModule } from "./modules/integrations.types.js";
 import type { AuthModule } from "./modules/auth.types.js";
 import type { SsoModule } from "./modules/sso.types.js";
 import type { BrowserModule } from "./modules/browser.types.js";
+import type { VoiceModule } from "./modules/voice.types.js";
 import type {
   PaymentsModule,
   ServiceRolePaymentsModule,
@@ -256,6 +257,8 @@ export interface Base44Client {
     payments: ServiceRolePaymentsModule;
     /** {@link SsoModule | SSO module} for generating SSO tokens. */
     sso: SsoModule;
+    /** {@link VoiceModule | Voice module} for live voice sessions. */
+    voice: VoiceModule;
     /** Cleanup function to disconnect WebSocket connections. */
     cleanup: () => void;
   };

packages/sdk/src/index.ts

@@ -177,6 +177,13 @@ export type {
   BrowserSessionOptions,
 } from "./modules/browser.types.js";
 
+export type {
+  VoiceModule,
+  VoiceSession,
+  VoiceSessionMessage,
+  VoiceSessionOptions,
+} from "./modules/voice.types.js";
+
 export { Actor, type Conn } from "./actor.js";
 
 export type {

packages/sdk/src/modules/voice.ts

@@ -0,0 +1,90 @@
+import { AxiosInstance } from "axios";
+import {
+  VoiceModule,
+  VoiceSession,
+  VoiceSessionOptions,
+} from "./voice.types.js";
+
+interface VoiceSessionResponse {
+  session_id: string;
+  sdp: string;
+  expires_at: string;
+  environment: VoiceSession["environment"];
+}
+
+/**
+ * Options for {@linkcode createVoiceModule}.
+ *
+ * @internal
+ */
+export interface VoiceModuleConfig {
+  /** Sent as `Base44-Functions-Version` so the backend can pick the draft MCP profile. */
+  functionsVersion?: string;
+}
+
+/**
+ * Creates the Voice module for the Base44 SDK.
+ *
+ * The module passes every value through unchanged. Base44 validates them, so
+ * a backend rule change never needs an SDK release.
+ *
+ * @param axios - Axios instance (should be service role client)
+ * @param config - Module configuration
+ * @returns Voice module with methods to create and end live voice sessions
+ * @internal
+ */
+export function createVoiceModule(
+  axios: AxiosInstance,
+  { functionsVersion }: VoiceModuleConfig = {}
+): VoiceModule {
+  const endSession = async (sessionId: string): Promise<void> => {
+    if (!sessionId || typeof sessionId !== "string") {
+      throw new Error("Session ID is required and must be a string");
+    }
+
+    await axios.delete(`/v1/voice-sessions/${encodeURIComponent(sessionId)}`);
+  };
+
+  return {
+    async createSession(options: VoiceSessionOptions): Promise<VoiceSession> {
+      // The app is derived from the service token, so the route isn't app-scoped.
+      const response = await axios.post<VoiceSessionResponse>(
+        "/v1/voice-sessions",
+        {
+          sdp: options.sdp,
+          mcp_profile_id: options.mcpProfileId,
+          voice_instructions: options.voiceInstructions,
+          agent_instructions: options.agentInstructions,
+          // Send optional keys only when set, so the backend applies its defaults.
+          ...(options.voiceModel !== undefined && {
+            voice_model: options.voiceModel,
+          }),
+          ...(options.agentModel !== undefined && {
+            agent_model: options.agentModel,
+          }),
+          ...(options.voice !== undefined && { voice: options.voice }),
+          ...(options.initialMessages !== undefined && {
+            initial_messages: options.initialMessages,
+          }),
+        },
+        {
+          // The backend picks the draft or published MCP profile from this
+          // header. The service-role client doesn't send it by default.
+          headers: functionsVersion
+            ? { "Base44-Functions-Version": functionsVersion }
+            : undefined,
+        }
+      );
+
+      const data = response as unknown as VoiceSessionResponse;
+      return {
+        sessionId: data.session_id,
+        sdp: data.sdp,
+        expiresAt: data.expires_at,
+        environment: data.environment,
+      };
+    },
+
+    endSession,
+  };
+}

packages/sdk/src/modules/voice.types.ts

@@ -0,0 +1,195 @@
+// This file deliberately states no backend limits, value lists, durations, or
+// prices. Base44 validates every value, so changing a backend rule must never
+// need an SDK release. Those details belong in the live voice guide.
+
+/**
+ * A text message placed in the conversation before the call starts.
+ */
+export interface VoiceSessionMessage {
+  /**
+   * Who wrote the message, such as:
+   *
+   * - `'developer'`: Context for the voice agent, such as what the current user is looking at.
+   * - `'user'`: An earlier turn by the current user.
+   * - `'assistant'`: An earlier turn by the voice agent.
+   */
+  role: string;
+  /** The message text. */
+  text: string;
+}
+
+/**
+ * Options for {@linkcode VoiceModule.createSession | createSession()}.
+ */
+export interface VoiceSessionOptions {
+  /**
+   * The browser's WebRTC SDP offer.
+   *
+   * Create it in the browser with [`RTCPeerConnection.createOffer()`](https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/createOffer) and send it to your backend function. A new offer is needed for every session.
+   */
+  sdp: string;
+  /**
+   * ID of the app's MCP profile whose tools the voice agent can use.
+   *
+   * The AI inserts this ID into your backend function when it sets up voice for the app.
+   */
+  mcpProfileId: string;
+  /**
+   * Model that talks with the current user, as an AI Gateway model ID. Defaults to Base44's default voice model.
+   */
+  voiceModel?: string;
+  /**
+   * Model behind the voice that calls the app's tools, as an AI Gateway model ID, such as `'gpt_6_luna'`. Defaults to Base44's default agent model.
+   */
+  agentModel?: string;
+  /**
+   * How the voice model speaks with the current user, such as its persona, tone, and turn-taking.
+   */
+  voiceInstructions: string;
+  /**
+   * How the agent behind the voice uses the app's tools to answer and act.
+   */
+  agentInstructions: string;
+  /**
+   * Built-in voice to speak with, such as `'cedar'`. The voice is fixed for the whole session. Defaults to the provider's default voice.
+   */
+  voice?: string;
+  /**
+   * Text conversation placed before the call starts, oldest first.
+   *
+   * Use it to give the voice agent context, or to continue a conversation the current user started in text.
+   */
+  initialMessages?: VoiceSessionMessage[];
+}
+
+/**
+ * A live voice session returned by {@linkcode VoiceModule.createSession | createSession()}.
+ */
+export interface VoiceSession {
+  /**
+   * Opaque session identifier issued to your app.
+   *
+   * Pass it unchanged to [`endSession()`](#endsession) to end the session.
+   */
+  sessionId: string;
+  /**
+   * The SDP answer.
+   *
+   * Return it to the browser and apply it with [`RTCPeerConnection.setRemoteDescription()`](https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/setRemoteDescription). Audio then flows directly between the browser and the voice model.
+   */
+  sdp: string;
+  /** ISO 8601 timestamp when the session ends at the latest. */
+  expiresAt: string;
+  /**
+   * Where the voice agent's tools come from.
+   *
+   * - `'live'`: The published MCP profile.
+   * - `'draft'`: The current draft of the MCP profile. Served when the app's owner or an editor starts the session from the app editor preview, so voice works before the app is published.
+   */
+  environment: "live" | "draft";
+}
+
+/**
+ * Voice module for live voice conversations between your app's users and an AI agent.
+ *
+ * This module creates and ends live voice sessions. The voice agent can use your app's tools through one of the app's MCP profiles, acting as the current user. Audio flows directly between the browser and the voice model over [WebRTC](https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API), so your backend only creates and ends the session.
+ *
+ * A session takes three steps:
+ * - The browser creates a WebRTC SDP offer and sends it to one of your backend functions.
+ * - The backend function calls {@linkcode createSession | createSession()} with the offer and your voice settings, and returns the SDP answer to the browser.
+ * - The browser applies the answer, and the conversation starts.
+ *
+ * Keep the instructions, models, and MCP profile ID in the backend function. Only the SDP offer should come from the browser.
+ *
+ * This module is in beta and may change.
+ *
+ * ## Authentication modes
+ *
+ * This module is only available to use with a client in [service role authentication mode](/developers/references/sdk/getting-started/client#service-role) (`base44.asServiceRole.voice`), which means it can only be used in backend environments. The app's tools run as the current user of the request that created the client. Without a current user, they run as a guest, which only works in apps that allow anonymous access.
+ */
+export interface VoiceModule {
+  /**
+   * Creates a live voice session and returns the SDP answer for the browser.
+   *
+   * Each call creates a new session, which uses credits from the workspace that owns the app. The session ends on its own when it's idle or reaches its maximum duration. To end it sooner, such as when the current user hangs up, call {@linkcode endSession | endSession()} from a later request.
+   *
+   * @param options - The browser's SDP offer and the voice settings.
+   * @returns Promise resolving to the new voice session. Rejects with a [`Base44Error`](/developers/references/sdk/getting-started/work-with-sdk#error-handling) whose `code` is one of the values under [Throws](#throws).
+   * @throws `insufficient_credits`: The workspace is out of credits.
+   * @throws `authentication_required`: The app requires a current user, and the request had none.
+   * @throws `unsupported_voice_model`: The voice model isn't supported.
+   * @throws `unsupported_agent_model`: The agent model isn't supported.
+   * @throws `voice_settings_rejected`: The voice provider rejected the settings, for example because the instructions are too long.
+   * @throws `invalid_request`: A value in the options is invalid.
+   * @throws `rate_limited`: You created too many sessions in a short time.
+   * @throws `voice_capacity_exceeded`: Voice capacity is temporarily unavailable.
+   * @throws `voice_connection_failed`: The voice provider couldn't be reached or rejected the request.
+   *
+   * @example
+   * ```typescript
+   * // Backend function that creates a voice session for the current user
+   * import { createClientFromRequest } from 'npm:@base44/sdk';
+   *
+   * const VOICE_PROFILE_ID = '6a1f0c2b9d8e7f6a5b4c3d2e';
+   *
+   * Deno.serve(async (req) => {
+   *   const base44 = createClientFromRequest(req);
+   *   // Only the SDP offer comes from the browser.
+   *   const { sdp } = await req.json();
+   *
+   *   const session = await base44.asServiceRole.voice.createSession({
+   *     sdp,
+   *     mcpProfileId: VOICE_PROFILE_ID,
+   *     voiceInstructions: 'You are the shop assistant. Speak briefly and let the person interrupt.',
+   *     agentInstructions: 'Use the app tools to look up orders. Report only what the tools return.',
+   *   });
+   *
+   *   return Response.json({ sessionId: session.sessionId, sdp: session.sdp });
+   * });
+   * ```
+   *
+   * @example
+   * ```typescript
+   * // Pick the models and voice, and give the agent context about the current page
+   * const session = await base44.asServiceRole.voice.createSession({
+   *   sdp,
+   *   mcpProfileId: VOICE_PROFILE_ID,
+   *   voiceModel: 'gpt_live_1',
+   *   agentModel: 'gpt_6_luna',
+   *   voiceInstructions: 'You are the shop assistant.',
+   *   agentInstructions: 'Use the app tools to look up orders.',
+   *   voice: 'cedar',
+   *   initialMessages: [
+   *     { role: 'developer', text: 'The person is viewing order 1042 on the checkout page.' },
+   *   ],
+   * });
+   * ```
+   */
+  createSession(options: VoiceSessionOptions): Promise<VoiceSession>;
+
+  /**
+   * Ends a voice session created by your app.
+   *
+   * Call it from a later request, such as when the browser reports that the current user hung up. Ending a session that has already ended succeeds, so it's safe to call more than once.
+   *
+   * @param sessionId - The `sessionId` returned by {@linkcode createSession | createSession()}.
+   * @returns Promise that resolves when Base44 accepts the request. The session closes in the background. Rejects with a [`Base44Error`](/developers/references/sdk/getting-started/work-with-sdk#error-handling) whose `code` is one of the values under [Throws](#throws-2).
+   * @throws `voice_session_not_found`: The session ID wasn't issued to your app.
+   * @throws `voice_connection_failed`: Base44 couldn't reach its background processing. It closes the session directly where possible, and retrying is safe.
+   *
+   * @example
+   * ```typescript
+   * // End a session from a later request, using the ID sent in the request body
+   * import { createClientFromRequest } from 'npm:@base44/sdk';
+   *
+   * Deno.serve(async (req) => {
+   *   const base44 = createClientFromRequest(req);
+   *   const { sessionId } = await req.json();
+   *   await base44.asServiceRole.voice.endSession(sessionId);
+   *
+   *   return new Response(null, { status: 204 });
+   * });
+   * ```
+   */
+  endSession(sessionId: string): Promise<void>;
+}