firebase-ios-sdk/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/ImageUnderstandingIntegrationTests.swift

// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#if compiler(>=6.4) && (FoundationModelsFake || canImport(FoundationModels)) && canImport(ImageIO)
  import Foundation
  #if FoundationModelsFake
    import FoundationModelsFake
  #else
    import FoundationModels
  #endif
  import GeminiAPIClient
  import GeminiTestUtilities
  import ImageIO
  import Testing
  import UniformTypeIdentifiers

  @testable import GeminiLanguageModel

  /// Integration tests for image input (`vision`) using `GeminiLanguageModel`.
  ///
  /// These tests use the default sampling mode, since many Gemini 3 models don't support greedy
  /// sampling. To keep responses deterministic, the images are simple, high-contrast, and generated
  /// in code, and the prompts tightly constrain the format of the answer.
  @Suite(
    "Image Understanding Integration Tests",
    .requireFoundationModels,
    .tags(.integration)
  )
  struct ImageUnderstandingIntegrationTests {
    /// The instruction appended to prompts that expect a single color name.
    static let oneColorWordFormat =
      "Respond with exactly one lowercase English word and no punctuation."

    @Test(
      .requireIntegrationTestingBackend,
      arguments: IntegrationTestingBackend.availableBackends
    )
    @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
    func sessionRespondWithImage(backend: IntegrationTestingBackend) async throws {
      let model = try await backend.makeModel()
      let session = LanguageModelSession(model: model)
      let image = try TestImages.solid(.red)

      let response = try await session.respond {
        Attachment(image)
        "Name the color that fills this entire image. \(Self.oneColorWordFormat)"
      }

      #expect(response.content.localizedCaseInsensitiveContains("red"))
      #expect(response.usage.input.totalTokenCount > 0)
      #expect(response.usage.output.totalTokenCount > 0)
    }

    @Test(
      .requireIntegrationTestingBackend,
      arguments: IntegrationTestingBackend.availableBackends
    )
    @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
    func sessionRespondComparingTwoImages(backend: IntegrationTestingBackend) async throws {
      let model = try await backend.makeModel()
      let session = LanguageModelSession(model: model)
      let firstImage = try TestImages.solid(.red)
      let secondImage = try TestImages.solid(.blue)

      let response = try await session.respond {
        """
        Each of the following two images is filled with a single color. Respond with exactly two \
        lowercase English words separated by a single space and no punctuation: the color of the \
        first image, then the color of the second image.
        """
        Attachment(firstImage)
        Attachment(secondImage)
      }

      let words = response.content.lowercased().split(whereSeparator: { !$0.isLetter })
      #expect(words == ["red", "blue"])
    }

    @Generable(description: "The shape shown in an image")
    @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
    @available(tvOS, unavailable)
    enum ShapeLabel: String {
      /// A circle.
      case circle

      /// A square.
      case square

      /// A triangle.
      case triangle
    }

    // A top-level enum isn't used directly because top-level enum guided generation currently
    // fails to decode Gemini's JSON-quoted string, independent of image input.
    @Generable(description: "The classification of the shape shown in an image")
    @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
    @available(tvOS, unavailable)
    struct ShapeClassification {
      /// The shape shown in the image.
      var shape: ShapeLabel
    }

    @Test(
      .requireIntegrationTestingBackend,
      arguments: IntegrationTestingBackend.availableBackends, TestImages.Shape.allCases
    )
    @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
    func sessionRespondClassifyingImage(
      backend: IntegrationTestingBackend,
      shape: TestImages.Shape
    ) async throws {
      let model = try await backend.makeModel()
      let session = LanguageModelSession(model: model)
      let image = try TestImages.shape(shape)

      let response = try await session.respond(generating: ShapeClassification.self) {
        """
        The following image shows exactly one large black geometric shape on a white background. \
        Choose the label that names that shape: circle, square, or triangle.
        """
        Attachment(image)
      }

      #expect(response.content.shape.rawValue == shape.rawValue)
    }

    @Test(
      .requireIntegrationTestingBackend,
      arguments: IntegrationTestingBackend.availableBackends
    )
    @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
    func sessionRespondAppliesImageOrientation(backend: IntegrationTestingBackend) async throws {
      let model = try await backend.makeModel()
      let session = LanguageModelSession(model: model)
      // Stored with red on top; displaying it in the `.down` orientation puts blue on top.
      let image = try TestImages.halves(top: .red, bottom: .blue)

      let response = try await session.respond {
        Attachment(image, orientation: .down)
        """
        This image is split into a top half and a bottom half, each filled with a single color. \
        Name the color of the top half. \(Self.oneColorWordFormat)
        """
      }

      #expect(response.content.localizedCaseInsensitiveContains("blue"))
      #expect(!response.content.localizedCaseInsensitiveContains("red"))
    }

    @Test(
      .requireIntegrationTestingBackend,
      arguments: IntegrationTestingBackend.availableBackends
    )
    @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
    func sessionRespondWithImageFileURL(backend: IntegrationTestingBackend) async throws {
      let model = try await backend.makeModel()
      let session = LanguageModelSession(model: model)
      let fileData = try TestImages.encode(TestImages.solid(.green), as: .png)
      let url = try TestImages.writeTemporaryFile(fileData, pathExtension: "png")
      defer { try? FileManager.default.removeItem(at: url) }

      let response = try await session.respond {
        Attachment(imageURL: url)
        "Name the color that fills this entire image. \(Self.oneColorWordFormat)"
      }

      #expect(response.content.localizedCaseInsensitiveContains("green"))
    }

    @Test(
      .requireIntegrationTestingBackend,
      .enabled(if: TestImages.canEncode(.heic)),
      arguments: IntegrationTestingBackend.availableBackends
    )
    @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
    func sessionRespondWithHEICFileURL(backend: IntegrationTestingBackend) async throws {
      // Upright HEIC files are sent unchanged as `image/heic`, so this checks that the backend
      // accepts that MIME type.
      let model = try await backend.makeModel()
      let session = LanguageModelSession(model: model)
      let fileData = try TestImages.encode(TestImages.solid(.blue), as: .heic)
      let url = try TestImages.writeTemporaryFile(fileData, pathExtension: "heic")
      defer { try? FileManager.default.removeItem(at: url) }

      let response = try await session.respond {
        Attachment(imageURL: url)
        "Name the color that fills this entire image. \(Self.oneColorWordFormat)"
      }

      #expect(response.content.localizedCaseInsensitiveContains("blue"))
    }

    @Test(
      .requireIntegrationTestingBackend,
      arguments: IntegrationTestingBackend.availableBackends
    )
    @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
    func sessionStreamResponseWithImage(backend: IntegrationTestingBackend) async throws {
      let model = try await backend.makeModel()
      let session = LanguageModelSession(model: model)
      let image = try TestImages.solid(.yellow)

      let stream = session.streamResponse {
        Attachment(image)
        "Name the color that fills this entire image. \(Self.oneColorWordFormat)"
      }
      var snapshotCount = 0
      for try await _ in stream {
        snapshotCount += 1
      }
      let response = try await stream.collect()

      #expect(snapshotCount > 0)
      #expect(response.content.localizedCaseInsensitiveContains("yellow"))
    }

    @Test(
      .requireIntegrationTestingBackend,
      arguments: IntegrationTestingBackend.availableBackends
    )
    @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
    func sessionRespondMultiTurnWithImageInHistory(
      backend: IntegrationTestingBackend
    ) async throws {
      let model = try await backend.makeModel()
      let session = LanguageModelSession(model: model)
      let image = try TestImages.solid(.blue)

      _ = try await session.respond {
        Attachment(image)
        "Remember this image for my next question. Respond with exactly the word OK."
      }
      let response = try await session.respond(
        to: """
          Name the color that fills the entire image I sent in my previous message. \
          \(Self.oneColorWordFormat)
          """
      )

      #expect(response.content.localizedCaseInsensitiveContains("blue"))
    }
  }
#endif  // FoundationModels && canImport(ImageIO)