firebase-ios-sdk/FirebaseAI/Sources/ModelContent.swift

// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import Foundation

extension [ModelContent] {
  // TODO: Rename and refactor this.
  /// Throws a ``GenerateContentError/internalError(underlying:)`` wrapping the error of the first
  /// part that failed to convert to model content (for example, an image that could not be
  /// converted to JPEG) so that a request is not sent with that part silently missing.
  func throwIfError() throws {
    for content in self {
      for part in content.internalParts {
        if case let .error(errorPart) = part.data {
          throw GenerateContentError.internalError(underlying: errorPart.error)
        }
      }
    }
  }
}

struct InternalPart: Equatable, Sendable {
  enum OneOfData: Equatable, Sendable {
    case text(String)
    case inlineData(InlineData)
    case fileData(FileData)
    case functionCall(FunctionCall)
    case functionResponse(FunctionResponse)
    case executableCode(ExecutableCode)
    case codeExecutionResult(CodeExecutionResult)
    /// A part that failed to convert to model content on the client (for example, an image that
    /// could not be converted to JPEG); it is never sent to or received from the backend.
    case error(ErrorPart)

    struct UnsupportedDataError: Error {
      let decodingError: DecodingError

      var localizedDescription: String {
        decodingError.localizedDescription
      }
    }
  }

  let data: OneOfData?

  let isThought: Bool?

  let thoughtSignature: String?

  let speechMetadata: SpeechMetadata?

  init(_ data: OneOfData,
       speechMetadata: SpeechMetadata? = nil,
       isThought: Bool?,
       thoughtSignature: String?) {
    self.data = data
    self.speechMetadata = speechMetadata
    self.isThought = isThought
    self.thoughtSignature = thoughtSignature
  }
}

/// A type describing data in media formats interpretable by an AI model. Each generative AI
/// request or response contains an `Array` of ``ModelContent``s, and each ``ModelContent`` value
/// may comprise multiple heterogeneous ``Part``s.
public struct ModelContent: Equatable, Sendable {
  /// The role of the entity creating the ``ModelContent``. For user-generated client requests,
  /// for example, the role is `user`.
  public let role: String?

  /// The data parts comprising this ``ModelContent`` value.
  public var parts: [any Part] {
    return internalParts.compactMap { part -> (any Part)? in
      switch part.data {
      case let .text(text):
        return TextPart(
          text,
          speechMetadata: part.speechMetadata,
          isThought: part.isThought,
          thoughtSignature: part.thoughtSignature
        )
      case let .inlineData(inlineData):
        return InlineDataPart(
          inlineData, isThought: part.isThought, thoughtSignature: part.thoughtSignature
        )
      case let .fileData(fileData):
        return FileDataPart(
          fileData, isThought: part.isThought, thoughtSignature: part.thoughtSignature
        )
      case let .functionCall(functionCall):
        return FunctionCallPart(
          functionCall, isThought: part.isThought, thoughtSignature: part.thoughtSignature
        )
      case let .functionResponse(functionResponse):
        return FunctionResponsePart(
          functionResponse, isThought: part.isThought, thoughtSignature: part.thoughtSignature
        )
      case let .executableCode(executableCode):
        return ExecutableCodePart(
          executableCode, isThought: part.isThought, thoughtSignature: part.thoughtSignature
        )
      case let .codeExecutionResult(codeExecutionResult):
        return CodeExecutionResultPart(
          codeExecutionResult: codeExecutionResult,
          isThought: part.isThought,
          thoughtSignature: part.thoughtSignature
        )
      case let .error(errorPart):
        return errorPart
      case .none:
        // Filter out parts that contain missing or unrecognized data
        return nil
      }
    }
  }

  // TODO: Refactor this
  let internalParts: [InternalPart]

  /// Creates a new value from a list of ``Part``s.
  public init(role: String? = "user", parts: [any Part]) {
    self.role = role
    var convertedParts = [InternalPart]()
    for part in parts {
      switch part {
      case let textPart as TextPart:
        convertedParts.append(InternalPart(
          .text(textPart.text),
          speechMetadata: textPart.speechMetadata,
          isThought: textPart._isThought,
          thoughtSignature: textPart.thoughtSignature
        ))
      case let inlineDataPart as InlineDataPart:
        convertedParts.append(InternalPart(
          .inlineData(inlineDataPart.inlineData),
          isThought: inlineDataPart._isThought,
          thoughtSignature: inlineDataPart.thoughtSignature
        ))
      case let fileDataPart as FileDataPart:
        convertedParts.append(InternalPart(
          .fileData(fileDataPart.fileData),
          isThought: fileDataPart._isThought,
          thoughtSignature: fileDataPart.thoughtSignature
        ))
      case let functionCallPart as FunctionCallPart:
        convertedParts.append(InternalPart(
          .functionCall(functionCallPart.functionCall),
          isThought: functionCallPart._isThought,
          thoughtSignature: functionCallPart.thoughtSignature
        ))
      case let functionResponsePart as FunctionResponsePart:
        convertedParts.append(InternalPart(
          .functionResponse(functionResponsePart.functionResponse),
          isThought: functionResponsePart._isThought,
          thoughtSignature: functionResponsePart.thoughtSignature
        ))
      case let executableCodePart as ExecutableCodePart:
        convertedParts.append(InternalPart(
          .executableCode(executableCodePart.executableCode),
          isThought: executableCodePart._isThought,
          thoughtSignature: executableCodePart.thoughtSignature
        ))
      case let codeExecutionResultPart as CodeExecutionResultPart:
        convertedParts.append(InternalPart(
          .codeExecutionResult(codeExecutionResultPart.codeExecutionResult),
          isThought: codeExecutionResultPart._isThought,
          thoughtSignature: codeExecutionResultPart.thoughtSignature
        ))
      case let errorPart as ErrorPart:
        // Keep the error so that `throwIfError()` fails the request instead of the part being
        // silently dropped.
        convertedParts.append(InternalPart(
          .error(errorPart),
          isThought: nil,
          thoughtSignature: nil
        ))
      default:
        AILog.error(
          code: .modelContentUnsupportedPartType,
          "Skipping a part with unsupported type: \(type(of: part))"
        )
      }
    }
    internalParts = convertedParts
  }

  /// Creates a new value from any data interpretable as a ``Part``.
  /// See ``PartsRepresentable`` for types that can be interpreted as `Part`s.
  public init(role: String? = "user", parts: any PartsRepresentable...) {
    let content = parts.flatMap { $0.partsValue }
    self.init(role: role, parts: content)
  }

  init(role: String?, parts: [InternalPart]) {
    self.role = role
    internalParts = parts
  }
}

// MARK: Codable Conformances

extension ModelContent: Codable {
  enum CodingKeys: String, CodingKey {
    case role
    case internalParts = "parts"
  }

  public init(from decoder: any Decoder) throws {
    let container = try decoder.container(keyedBy: CodingKeys.self)
    role = try container.decodeIfPresent(String.self, forKey: .role)
    internalParts = try container.decodeIfPresent([InternalPart].self, forKey: .internalParts) ?? []
  }
}

extension InternalPart: Codable {
  enum CodingKeys: String, CodingKey {
    case isThought = "thought"
    case thoughtSignature
    case speechMetadata
  }

  public func encode(to encoder: Encoder) throws {
    // `data` is `nil` for parts with missing or unrecognized data received from the backend (e.g.,
    // when sent back as chat history); encoding `nil` as a single value before requesting a keyed
    // container below would trap in `JSONEncoder`.
    if let data {
      try data.encode(to: encoder)
    }
    var container = encoder.container(keyedBy: CodingKeys.self)
    try container.encodeIfPresent(isThought, forKey: .isThought)
    try container.encodeIfPresent(thoughtSignature, forKey: .thoughtSignature)
    try container.encodeIfPresent(speechMetadata, forKey: .speechMetadata)
  }

  public init(from decoder: Decoder) throws {
    do {
      data = try OneOfData(from: decoder)
    } catch let error as OneOfData.UnsupportedDataError {
      AILog.error(code: .decodedUnsupportedPartData, error.localizedDescription)
      data = nil
    } catch { // Re-throw any other error types
      throw error
    }
    let container = try decoder.container(keyedBy: CodingKeys.self)
    isThought = try container.decodeIfPresent(Bool.self, forKey: .isThought)
    thoughtSignature = try container.decodeIfPresent(String.self, forKey: .thoughtSignature)
    speechMetadata = try container.decodeIfPresent(SpeechMetadata.self, forKey: .speechMetadata)
  }
}

extension InternalPart.OneOfData: Codable {
  enum CodingKeys: String, CodingKey {
    case text
    case inlineData
    case fileData
    case functionCall
    case functionResponse
    case executableCode
    case codeExecutionResult
  }

  public func encode(to encoder: Encoder) throws {
    var container = encoder.container(keyedBy: CodingKeys.self)
    switch self {
    case let .text(text):
      try container.encode(text, forKey: .text)
    case let .inlineData(inlineData):
      try container.encode(inlineData, forKey: .inlineData)
    case let .fileData(fileData):
      try container.encode(fileData, forKey: .fileData)
    case let .functionCall(functionCall):
      try container.encode(functionCall, forKey: .functionCall)
    case let .functionResponse(functionResponse):
      try container.encode(functionResponse, forKey: .functionResponse)
    case let .executableCode(executableCode):
      try container.encode(executableCode, forKey: .executableCode)
    case let .codeExecutionResult(codeExecutionResult):
      try container.encode(codeExecutionResult, forKey: .codeExecutionResult)
    case let .error(errorPart):
      // Requests reject these in `throwIfError()` before encoding; anything else that encodes one
      // (e.g., a `LiveSession`) gets an `EncodingError` instead of a crash.
      try errorPart.encode(to: encoder)
    }
  }

  public init(from decoder: Decoder) throws {
    let values = try decoder.container(keyedBy: CodingKeys.self)
    if values.contains(.text) {
      self = try .text(values.decode(String.self, forKey: .text))
    } else if values.contains(.inlineData) {
      self = try .inlineData(values.decode(InlineData.self, forKey: .inlineData))
    } else if values.contains(.fileData) {
      self = try .fileData(values.decode(FileData.self, forKey: .fileData))
    } else if values.contains(.functionCall) {
      self = try .functionCall(values.decode(FunctionCall.self, forKey: .functionCall))
    } else if values.contains(.functionResponse) {
      self = try .functionResponse(values.decode(FunctionResponse.self, forKey: .functionResponse))
    } else if values.contains(.executableCode) {
      self = try .executableCode(values.decode(ExecutableCode.self, forKey: .executableCode))
    } else if values.contains(.codeExecutionResult) {
      self = try .codeExecutionResult(
        values.decode(CodeExecutionResult.self, forKey: .codeExecutionResult)
      )
    } else {
      let unexpectedKeys = values.allKeys.map { $0.stringValue }
      throw UnsupportedDataError(decodingError: DecodingError.dataCorrupted(
        DecodingError.Context(
          codingPath: values.codingPath,
          debugDescription: "Unexpected Part type(s): \(unexpectedKeys)"
        )
      ))
    }
  }
}