feat(ai): Live API

FirebaseAI/CHANGELOG.md

@@ -11,6 +11,12 @@
 - [fixed] Fixed a decoding error when generating images with the
   `gemini-2.5-flash-image-preview` model using `generateContentStream` or
   `sendMessageStream` with the Gemini Developer API. (#15262)
+- [feature] Added support for the Live API, which allows bidirectional
+  communication with the model in realtime.
+
+  To get started with the Live API, see the Firebase docs on
+  [Bidirectional streaming using the Gemini Live API](https://firebase.google.com/docs/ai-logic/live-api).
+  (#15309)
 
 # 12.2.0
 - [feature] Added support for returning thought summaries, which are synthesized

FirebaseAI/Sources/AILog.swift

@@ -67,12 +67,25 @@ enum AILog {
     case executableCodeUnrecognizedLanguage = 3016
     case fallbackValueUsed = 3017
     case urlMetadataUnrecognizedURLRetrievalStatus = 3018
+    case liveSessionUnsupportedMessage = 3019
+    case liveSessionUnsupportedMessagePayload = 3020
+    case liveSessionFailedToEncodeClientMessage = 3021
+    case liveSessionFailedToEncodeClientMessagePayload = 3022
+    case liveSessionFailedToSendClientMessage = 3023
+    case liveSessionUnexpectedResponse = 3024
+    case liveSessionGoingAwaySoon = 3025
+    case decodedMissingProtoDurationSuffix = 3026
+    case decodedInvalidProtoDurationString = 3027
+    case decodedInvalidProtoDurationSeconds = 3028
+    case decodedInvalidProtoDurationNanoseconds = 3029
 
     // SDK State Errors
     case generateContentResponseNoCandidates = 4000
     case generateContentResponseNoText = 4001
     case appCheckTokenFetchFailed = 4002
     case generateContentResponseEmptyCandidates = 4003
+    case invalidWebsocketURL = 4004
+    case duplicateLiveSessionSetupComplete = 4005
 
     // SDK Debugging
     case loadRequestStreamResponseLine = 5000

FirebaseAI/Sources/FirebaseAI.swift

@@ -137,6 +137,46 @@ public final class FirebaseAI: Sendable {
     )
   }
 
+  /// **[Public Preview]** Initializes a ``LiveGenerativeModel`` with the given parameters.
+  ///
+  /// > Warning: Using the Firebase AI Logic SDKs with the Gemini Live API is in Public
+  /// Preview, which means that the feature is not subject to any SLA or deprecation policy and
+  /// could change in backwards-incompatible ways.
+  ///
+  /// > Important: Only models that support the Gemini Live API (typically containing `live-*` in
+  /// the name) are supported.
+  ///
+  /// - Parameters:
+  ///   - modelName: The name of the model to use, for example
+  ///     `"gemini-live-2.5-flash-preview"`;
+  ///     see [model versions](https://firebase.google.com/docs/ai-logic/live-api?api=dev#models-that-support-capability)
+  ///     for a list of supported models.
+  ///   - generationConfig: The content generation parameters your model should use.
+  ///   - tools: A list of ``Tool`` objects that the model may use to generate the next response.
+  ///   - toolConfig: Tool configuration for any ``Tool`` specified in the request.
+  ///   - systemInstruction: Instructions that direct the model to behave a certain way; currently
+  ///     only text content is supported.
+  ///   - requestOptions: Configuration parameters for sending requests to the backend.
+  @available(iOS 15.0, macOS 12.0, macCatalyst 15.0, tvOS 15.0, *)
+  @available(watchOS, unavailable)
+  public func liveModel(modelName: String,
+                        generationConfig: LiveGenerationConfig? = nil,
+                        tools: [Tool]? = nil,
+                        toolConfig: ToolConfig? = nil,
+                        systemInstruction: ModelContent? = nil,
+                        requestOptions: RequestOptions = RequestOptions()) -> LiveGenerativeModel {
+    return LiveGenerativeModel(
+      modelResourceName: modelResourceName(modelName: modelName),
+      firebaseInfo: firebaseInfo,
+      apiConfig: apiConfig,
+      generationConfig: generationConfig,
+      tools: tools,
+      toolConfig: toolConfig,
+      systemInstruction: systemInstruction,
+      requestOptions: requestOptions
+    )
+  }
+
   /// Class to enable FirebaseAI to register via the Objective-C based Firebase component system
   /// to include FirebaseAI in the userAgent.
   @objc(FIRVertexAIComponent) class FirebaseVertexAIComponent: NSObject {}

FirebaseAI/Sources/GenerativeAIService.swift

@@ -177,7 +177,10 @@ struct GenerativeAIService {
     urlRequest.setValue("application/json", forHTTPHeaderField: "Content-Type")
 
     if let appCheck = firebaseInfo.appCheck {
-      let tokenResult = try await fetchAppCheckToken(appCheck: appCheck)
+      let tokenResult = try await appCheck.fetchAppCheckToken(
+        limitedUse: firebaseInfo.useLimitedUseAppCheckTokens,
+        domain: "GenerativeAIService"
+      )
       urlRequest.setValue(tokenResult.token, forHTTPHeaderField: "X-Firebase-AppCheck")
       if let error = tokenResult.error {
         AILog.error(
@@ -207,53 +210,6 @@ struct GenerativeAIService {
     return urlRequest
   }
 
-  private func fetchAppCheckToken(appCheck: AppCheckInterop) async throws
-    -> FIRAppCheckTokenResultInterop {
-    if firebaseInfo.useLimitedUseAppCheckTokens {
-      if let token = await getLimitedUseAppCheckToken(appCheck: appCheck) {
-        return token
-      }
-
-      let errorMessage =
-        "The provided App Check token provider doesn't implement getLimitedUseToken(), but requireLimitedUseTokens was enabled."
-
-      #if Debug
-        fatalError(errorMessage)
-      #else
-        throw NSError(
-          domain: "\(Constants.baseErrorDomain).\(Self.self)",
-          code: AILog.MessageCode.appCheckTokenFetchFailed.rawValue,
-          userInfo: [NSLocalizedDescriptionKey: errorMessage]
-        )
-      #endif
-    }
-
-    return await appCheck.getToken(forcingRefresh: false)
-  }
-
-  private func getLimitedUseAppCheckToken(appCheck: AppCheckInterop) async
-    -> FIRAppCheckTokenResultInterop? {
-    // At the moment, `await` doesn’t get along with Objective-C’s optional protocol methods.
-    await withCheckedContinuation { (continuation: CheckedContinuation<
-      FIRAppCheckTokenResultInterop?,
-      Never
-    >) in
-      guard
-        firebaseInfo.useLimitedUseAppCheckTokens,
-        // `getLimitedUseToken(completion:)` is an optional protocol method. Optional binding
-        // is performed to make sure `continuation` is called even if the method’s not implemented.
-        let limitedUseTokenClosure = appCheck.getLimitedUseToken
-      else {
-        return continuation.resume(returning: nil)
-      }
-
-      limitedUseTokenClosure { tokenResult in
-        // The placeholder token should be used in the case of App Check error.
-        continuation.resume(returning: tokenResult)
-      }
-    }
-  }
-
   private func httpResponse(urlResponse: URLResponse) throws -> HTTPURLResponse {
     // The following condition should always be true: "Whenever you make HTTP URL load requests, any
     // response objects you get back from the URLSession, NSURLConnection, or NSURLDownload class

FirebaseAI/Sources/Types/Internal/APIConfig.swift

@@ -68,6 +68,7 @@ extension APIConfig {
 
 extension APIConfig.Service {
   /// Network addresses for generative AI API services.
+  // TODO: maybe remove the https:// prefix and just add it as needed? websockets use these too.
   enum Endpoint: String, Encodable {
     /// The Firebase proxy production endpoint.
     ///

FirebaseAI/Sources/Types/Internal/AppCheck.swift

@@ -0,0 +1,74 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import FirebaseAppCheckInterop
+
+/// Internal helper extension for fetching app check tokens.
+///
+/// Provides a common means for fetching limited use tokens, and falling back to standard tokens
+/// when it's disabled (or in debug mode). This also centrializes the error, since this method is
+/// used in multiple places.
+extension AppCheckInterop {
+  /// Fetch the appcheck token.
+  ///
+  /// - Parameters:
+  ///   - limitedUse: Should the token be a limited-use token, or a standard token.
+  ///   - domain: A string dictating where this method is being called from. Used in any thrown
+  ///     errors, to avoid hard-to-parse traces.
+  func fetchAppCheckToken(limitedUse: Bool,
+                          domain: String) async throws -> FIRAppCheckTokenResultInterop {
+    if limitedUse {
+      if let token = await getLimitedUseTokenAsync() {
+        return token
+      }
+
+      let errorMessage =
+        "The provided App Check token provider doesn't implement getLimitedUseToken(), but requireLimitedUseTokens was enabled."
+
+      #if Debug
+        fatalError(errorMessage)
+      #else
+        throw NSError(
+          domain: "\(Constants.baseErrorDomain).\(domain)",
+          code: AILog.MessageCode.appCheckTokenFetchFailed.rawValue,
+          userInfo: [NSLocalizedDescriptionKey: errorMessage]
+        )
+      #endif
+    }
+
+    return await getToken(forcingRefresh: false)
+  }
+
+  private func getLimitedUseTokenAsync() async
+    -> FIRAppCheckTokenResultInterop? {
+    // At the moment, `await` doesn’t get along with Objective-C’s optional protocol methods.
+    await withCheckedContinuation { (continuation: CheckedContinuation<
+      FIRAppCheckTokenResultInterop?,
+      Never
+    >) in
+      guard
+        // `getLimitedUseToken(completion:)` is an optional protocol method. Optional binding
+        // is performed to make sure `continuation` is called even if the method’s not implemented.
+        let limitedUseTokenClosure = getLimitedUseToken
+      else {
+        return continuation.resume(returning: nil)
+      }
+
+      limitedUseTokenClosure { tokenResult in
+        // The placeholder token should be used in the case of App Check error.
+        continuation.resume(returning: tokenResult)
+      }
+    }
+  }
+}

FirebaseAI/Sources/Types/Internal/InternalPart.swift

@@ -45,21 +45,25 @@ struct FileData: Codable, Equatable, Sendable {
 struct FunctionCall: Equatable, Sendable {
   let name: String
   let args: JSONObject
+  let id: String?
 
-  init(name: String, args: JSONObject) {
+  init(name: String, args: JSONObject, id: String?) {
     self.name = name
     self.args = args
+    self.id = id
   }
 }
 
 @available(iOS 15.0, macOS 12.0, macCatalyst 15.0, tvOS 15.0, watchOS 8.0, *)
 struct FunctionResponse: Codable, Equatable, Sendable {
   let name: String
   let response: JSONObject
+  let id: String?
 
-  init(name: String, response: JSONObject) {
+  init(name: String, response: JSONObject, id: String? = nil) {
     self.name = name
     self.response = response
+    self.id = id
   }
 }
 
@@ -135,6 +139,7 @@ extension FunctionCall: Codable {
     } else {
       args = JSONObject()
     }
+    id = try container.decodeIfPresent(String.self, forKey: .id)
   }
 }