mirror of
https://github.com/firebase/firebase-ios-sdk
synced 2026-11-09 22:36:01 +05:45
Andrew Heard
bc9f9ea1da
Add FoundationModelsFake, a nested Swift package that fakes Apple's Foundation Models framework, so that GeminiLanguageModel (GLM) can build and run its unit tests on Linux and on OS versions without the real framework. It's used only for development and testing. - GLM depends on the fake through a `FoundationModelsFake` package trait, which is on by default where Darwin isn't available (e.g., Linux). With the trait off, the fake isn't built. The root `Package.swift` and podspecs can't reach it. - The fake mirrors Apple's public API and availability in the 27.2 SDKs, including `@Generable` macro expansions. It omits PrivateCloudComputeLanguageModel, OCRTool, BarcodeReaderTool, the adapter and feedback APIs, and deprecated APIs. SystemLanguageModel is always unavailable. Below Swift 6.4 it compiles to an empty module. - Decoding rejects transcript images with `file:` URLs and schemas whose references never resolve, instead of reading local files or overflowing the stack. - Without ImageIO, GLM passes PNG, JPEG, WebP, HEIC, and HEIF images through to Gemini after checking their signatures, and rejects images that would need re-encoding. - Add CI jobs that run both packages' tests against the fake: one on Linux in a digest-pinned `swift:6.4.2` container, and one on macOS with Xcode 27.2. Skip `sdk.ai` for changes that only touch the fake.
1021 lines
36 KiB
Swift
1021 lines
36 KiB
Swift
// Copyright 2026 Google LLC
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
#if compiler(>=6.4) && (FoundationModelsFake || canImport(FoundationModels))
|
|
import Foundation
|
|
#if FoundationModelsFake
|
|
import FoundationModelsFake
|
|
#else
|
|
import FoundationModels
|
|
#endif
|
|
import GeminiAPIDataModels
|
|
import GeminiTestUtilities
|
|
import Testing
|
|
#if canImport(UniformTypeIdentifiers)
|
|
import UniformTypeIdentifiers
|
|
#endif
|
|
|
|
@testable import GeminiLanguageModel
|
|
|
|
@Suite("GeminiTranscriptTranslator Tests", .requireFoundationModels)
|
|
struct GeminiTranscriptTranslatorTests {
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func translatesInstructionsAndPrompt() throws {
|
|
let transcript = Transcript(
|
|
entries: [
|
|
.instructions(
|
|
Transcript.Instructions(
|
|
segments: [.text(Transcript.TextSegment(content: "You are a helpful assistant."))],
|
|
toolDefinitions: []
|
|
)
|
|
),
|
|
.prompt(
|
|
Transcript.Prompt(
|
|
segments: [.text(Transcript.TextSegment(content: "Hello!"))]
|
|
)
|
|
),
|
|
]
|
|
)
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
let systemInstruction = try #require(result.systemInstruction)
|
|
#expect(
|
|
systemInstruction.parts?.first?.data == .text("You are a helpful assistant."))
|
|
#expect(result.contents.count == 1)
|
|
#expect(result.contents.first?.role == "user")
|
|
#expect(result.contents.first?.parts?.first?.data == .text("Hello!"))
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func coalescesReasoningAndResponseIntoSingleModelTurn() throws {
|
|
let signatureData = Data("test-sig".utf8)
|
|
let transcript = Transcript(
|
|
entries: [
|
|
.prompt(
|
|
Transcript.Prompt(
|
|
segments: [.text(Transcript.TextSegment(content: "Explain 2+2."))]
|
|
)
|
|
),
|
|
.reasoning(
|
|
Transcript.Reasoning(
|
|
segments: [.text(Transcript.TextSegment(content: "Thinking about addition..."))],
|
|
signature: signatureData
|
|
)
|
|
),
|
|
.response(
|
|
Transcript.Response(
|
|
segments: [.text(Transcript.TextSegment(content: "2+2 is 4."))]
|
|
)
|
|
),
|
|
]
|
|
)
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
#expect(result.contents.count == 2)
|
|
#expect(result.contents[0].role == "user")
|
|
#expect(result.contents[1].role == "model")
|
|
|
|
let modelParts = try #require(result.contents[1].parts)
|
|
#expect(modelParts.count == 2)
|
|
#expect(modelParts[0].thought == true)
|
|
#expect(modelParts[0].thoughtSignature == "test-sig")
|
|
#expect(modelParts[0].data == .text("Thinking about addition..."))
|
|
#expect(modelParts[1].data == .text("2+2 is 4."))
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func attachesSignatureOnlyReasoningToFollowingResponsePart() throws {
|
|
let signatureData = Data("sig-only".utf8)
|
|
let transcript = Transcript(
|
|
entries: [
|
|
.prompt(
|
|
Transcript.Prompt(
|
|
segments: [.text(Transcript.TextSegment(content: "Hi."))]
|
|
)
|
|
),
|
|
.reasoning(
|
|
Transcript.Reasoning(
|
|
segments: [],
|
|
signature: signatureData
|
|
)
|
|
),
|
|
.response(
|
|
Transcript.Response(
|
|
segments: [.text(Transcript.TextSegment(content: "Hello!"))]
|
|
)
|
|
),
|
|
]
|
|
)
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
#expect(result.contents.count == 2)
|
|
#expect(result.contents[1].role == "model")
|
|
let modelParts = try #require(result.contents[1].parts)
|
|
#expect(modelParts.count == 1)
|
|
#expect(modelParts[0].data == .text("Hello!"))
|
|
#expect(modelParts[0].thoughtSignature == "sig-only")
|
|
#expect(modelParts[0].thought == nil)
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func attachesTrailingSignatureOnlyReasoningToPrecedingResponsePart() throws {
|
|
let signatureData = Data("trailing-sig".utf8)
|
|
let transcript = Transcript(
|
|
entries: [
|
|
.prompt(
|
|
Transcript.Prompt(
|
|
segments: [.text(Transcript.TextSegment(content: "Weather in Seattle?"))]
|
|
)
|
|
),
|
|
.response(
|
|
Transcript.Response(
|
|
segments: [.text(Transcript.TextSegment(content: "It is 22°C and sunny."))]
|
|
)
|
|
),
|
|
.reasoning(
|
|
Transcript.Reasoning(
|
|
segments: [],
|
|
signature: signatureData
|
|
)
|
|
),
|
|
.prompt(
|
|
Transcript.Prompt(
|
|
segments: [.text(Transcript.TextSegment(content: "What about Miami?"))]
|
|
)
|
|
),
|
|
]
|
|
)
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
#expect(result.contents.count == 3)
|
|
#expect(result.contents[1].role == "model")
|
|
let modelParts = try #require(result.contents[1].parts)
|
|
#expect(modelParts.count == 1)
|
|
#expect(modelParts[0].data == .text("It is 22°C and sunny."))
|
|
#expect(modelParts[0].thoughtSignature == "trailing-sig")
|
|
#expect(modelParts[0].thought == nil)
|
|
#expect(result.contents[2].role == "user")
|
|
#expect(result.contents[2].parts?.first?.data == .text("What about Miami?"))
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func sequentialToolCallTurnsNeverEmitPartsWithoutData() throws {
|
|
let callSignature = Data("call-sig".utf8)
|
|
let responseSignature = Data("response-sig".utf8)
|
|
let args = try GeneratedContent(json: "{\"city\":\"Seattle\"}")
|
|
let toolCall = Transcript.ToolCall(id: "call-1", toolName: "getWeather", arguments: args)
|
|
let toolOutput = Transcript.ToolOutput(
|
|
id: "call-1",
|
|
toolName: "getWeather",
|
|
segments: [.text(Transcript.TextSegment(content: "22°C and sunny"))]
|
|
)
|
|
let transcript = Transcript(
|
|
entries: [
|
|
.prompt(
|
|
Transcript.Prompt(
|
|
segments: [.text(Transcript.TextSegment(content: "Weather in Seattle?"))]
|
|
)
|
|
),
|
|
.reasoning(Transcript.Reasoning(segments: [], signature: callSignature)),
|
|
.toolCalls(Transcript.ToolCalls(id: "calls-1", [toolCall])),
|
|
.toolOutput(toolOutput),
|
|
.response(
|
|
Transcript.Response(
|
|
segments: [.text(Transcript.TextSegment(content: "It is 22°C and sunny."))]
|
|
)
|
|
),
|
|
.reasoning(Transcript.Reasoning(segments: [], signature: responseSignature)),
|
|
.prompt(
|
|
Transcript.Prompt(
|
|
segments: [.text(Transcript.TextSegment(content: "What about Miami?"))]
|
|
)
|
|
),
|
|
]
|
|
)
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
#expect(result.contents.count == 5)
|
|
#expect(result.contents.map(\.role) == ["user", "model", "user", "model", "user"])
|
|
let allPartsHaveData = result.contents.allSatisfy { content in
|
|
content.parts?.allSatisfy { $0.data != nil } ?? true
|
|
}
|
|
#expect(allPartsHaveData)
|
|
let callParts = try #require(result.contents[1].parts)
|
|
#expect(callParts.count == 1)
|
|
guard case .functionCall(let functionCall) = callParts[0].data else {
|
|
Issue.record("Expected functionCall part data")
|
|
return
|
|
}
|
|
#expect(functionCall.name == "getWeather")
|
|
#expect(callParts[0].thoughtSignature == "call-sig")
|
|
let outputParts = try #require(result.contents[2].parts)
|
|
#expect(outputParts.count == 1)
|
|
guard case .functionResponse = outputParts[0].data else {
|
|
Issue.record("Expected functionResponse part data")
|
|
return
|
|
}
|
|
let responseParts = try #require(result.contents[3].parts)
|
|
#expect(responseParts.count == 1)
|
|
#expect(responseParts[0].data == .text("It is 22°C and sunny."))
|
|
#expect(responseParts[0].thoughtSignature == "response-sig")
|
|
#expect(responseParts[0].thought == nil)
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func trailingSignatureFallsBackToEmptyThoughtPartWhenLastPartIsSigned() throws {
|
|
let firstSignature = Data("sig-a".utf8)
|
|
let secondSignature = Data("sig-b".utf8)
|
|
let transcript = Transcript(
|
|
entries: [
|
|
.prompt(
|
|
Transcript.Prompt(
|
|
segments: [.text(Transcript.TextSegment(content: "Hi."))]
|
|
)
|
|
),
|
|
.reasoning(Transcript.Reasoning(segments: [], signature: firstSignature)),
|
|
.response(
|
|
Transcript.Response(
|
|
segments: [.text(Transcript.TextSegment(content: "Hello!"))]
|
|
)
|
|
),
|
|
.reasoning(Transcript.Reasoning(segments: [], signature: secondSignature)),
|
|
.prompt(
|
|
Transcript.Prompt(
|
|
segments: [.text(Transcript.TextSegment(content: "Again?"))]
|
|
)
|
|
),
|
|
]
|
|
)
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
#expect(result.contents.count == 3)
|
|
#expect(result.contents[1].role == "model")
|
|
let modelParts = try #require(result.contents[1].parts)
|
|
#expect(modelParts.count == 2)
|
|
#expect(modelParts[0].data == .text("Hello!"))
|
|
#expect(modelParts[0].thoughtSignature == "sig-a")
|
|
#expect(modelParts[1].data == .text(""))
|
|
#expect(modelParts[1].thought == true)
|
|
#expect(modelParts[1].thoughtSignature == "sig-b")
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func signatureOnlyReasoningWithoutModelPartEmitsEmptyThoughtPart() throws {
|
|
let signatureData = Data("orphan-sig".utf8)
|
|
let transcript = Transcript(
|
|
entries: [
|
|
.prompt(
|
|
Transcript.Prompt(
|
|
segments: [.text(Transcript.TextSegment(content: "Hi."))]
|
|
)
|
|
),
|
|
.reasoning(
|
|
Transcript.Reasoning(
|
|
segments: [],
|
|
signature: signatureData
|
|
)
|
|
),
|
|
]
|
|
)
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
#expect(result.contents.count == 2)
|
|
let modelParts = try #require(result.contents[1].parts)
|
|
#expect(modelParts.count == 1)
|
|
#expect(modelParts[0].thought == true)
|
|
#expect(modelParts[0].thoughtSignature == "orphan-sig")
|
|
#expect(modelParts[0].data == .text(""))
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func translatesMultipleInstructionsEntriesIntoMultipleParts() throws {
|
|
let transcript = Transcript(
|
|
entries: [
|
|
.instructions(
|
|
Transcript.Instructions(
|
|
segments: [.text(Transcript.TextSegment(content: "First instruction."))],
|
|
toolDefinitions: []
|
|
)
|
|
),
|
|
.instructions(
|
|
Transcript.Instructions(
|
|
segments: [.text(Transcript.TextSegment(content: "Second instruction."))],
|
|
toolDefinitions: []
|
|
)
|
|
),
|
|
.prompt(
|
|
Transcript.Prompt(
|
|
segments: [.text(Transcript.TextSegment(content: "Hello!"))]
|
|
)
|
|
),
|
|
]
|
|
)
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
let systemInstruction = try #require(result.systemInstruction)
|
|
let parts = try #require(systemInstruction.parts)
|
|
#expect(parts.count == 2)
|
|
#expect(parts[0].data == .text("First instruction."))
|
|
#expect(parts[1].data == .text("Second instruction."))
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func translatesToolCallsIntoModelTurn() throws {
|
|
let args = try GeneratedContent(json: "{\"location\":\"Paris\"}")
|
|
let toolCall = Transcript.ToolCall(
|
|
id: "call-1",
|
|
toolName: "get_weather",
|
|
arguments: args
|
|
)
|
|
let transcript = Transcript(
|
|
entries: [
|
|
.prompt(
|
|
Transcript.Prompt(
|
|
segments: [.text(Transcript.TextSegment(content: "Weather?"))]
|
|
)
|
|
),
|
|
.toolCalls(Transcript.ToolCalls(id: "calls-1", [toolCall])),
|
|
]
|
|
)
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
#expect(result.contents.count == 2)
|
|
#expect(result.contents[0].role == "user")
|
|
#expect(result.contents[1].role == "model")
|
|
let modelParts = try #require(result.contents[1].parts)
|
|
#expect(modelParts.count == 1)
|
|
guard case .functionCall(let functionCall) = modelParts[0].data else {
|
|
Issue.record("Expected functionCall part data")
|
|
return
|
|
}
|
|
#expect(functionCall.id == "call-1")
|
|
#expect(functionCall.name == "get_weather")
|
|
#expect(functionCall.args?["location"] == .string("Paris"))
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func translatesToolOutputWithTextSegmentWrappingUnderResult() throws {
|
|
let toolOutput = Transcript.ToolOutput(
|
|
id: "call-1",
|
|
toolName: "get_weather",
|
|
segments: [.text(Transcript.TextSegment(content: "Sunny, 22°C"))]
|
|
)
|
|
let transcript = Transcript(entries: [.toolOutput(toolOutput)])
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
#expect(result.contents.count == 1)
|
|
#expect(result.contents[0].role == "user")
|
|
let userParts = try #require(result.contents[0].parts)
|
|
#expect(userParts.count == 1)
|
|
guard case .functionResponse(let functionResponse) = userParts[0].data else {
|
|
Issue.record("Expected functionResponse part data")
|
|
return
|
|
}
|
|
#expect(functionResponse.id == "call-1")
|
|
#expect(functionResponse.name == "get_weather")
|
|
#expect(functionResponse.response?["result"] == .string("Sunny, 22°C"))
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func translatesToolOutputWithJSONTextPreservingStringUnderResult() throws {
|
|
let jsonString = "{\"temperature\":22,\"condition\":\"sunny\"}"
|
|
let toolOutput = Transcript.ToolOutput(
|
|
id: "call-1",
|
|
toolName: "get_weather",
|
|
segments: [
|
|
.text(Transcript.TextSegment(content: jsonString))
|
|
]
|
|
)
|
|
let transcript = Transcript(entries: [.toolOutput(toolOutput)])
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
let userParts = try #require(result.contents.first?.parts)
|
|
guard case .functionResponse(let functionResponse) = userParts.first?.data else {
|
|
Issue.record("Expected functionResponse part data")
|
|
return
|
|
}
|
|
#expect(functionResponse.id == "call-1")
|
|
#expect(functionResponse.name == "get_weather")
|
|
#expect(functionResponse.response?["result"] == .string(jsonString))
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
|
|
func translatesToolOutputWithStructuredSegmentUnpackingDirectly() throws {
|
|
let content = try GeneratedContent(json: "{\"temperature\":22}")
|
|
let structuredSegment = Transcript.StructuredSegment(
|
|
id: "seg-1",
|
|
schemaName: "Weather",
|
|
content: content
|
|
)
|
|
let toolOutput = Transcript.ToolOutput(
|
|
id: "call-1",
|
|
toolName: "get_weather",
|
|
segments: [.structure(structuredSegment)]
|
|
)
|
|
let transcript = Transcript(entries: [.toolOutput(toolOutput)])
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
let userParts = try #require(result.contents.first?.parts)
|
|
guard case .functionResponse(let functionResponse) = userParts.first?.data else {
|
|
Issue.record("Expected functionResponse part data")
|
|
return
|
|
}
|
|
#expect(functionResponse.id == "call-1")
|
|
#expect(functionResponse.name == "get_weather")
|
|
#expect(functionResponse.response?["temperature"] == .number(22))
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func coalescesParallelToolCallsAndOutputs() throws {
|
|
let call1 = Transcript.ToolCall(
|
|
id: "call-1",
|
|
toolName: "get_weather",
|
|
arguments: try GeneratedContent(json: "{\"location\":\"Paris\"}")
|
|
)
|
|
let call2 = Transcript.ToolCall(
|
|
id: "call-2",
|
|
toolName: "get_time",
|
|
arguments: try GeneratedContent(json: "{\"location\":\"Tokyo\"}")
|
|
)
|
|
let output1 = Transcript.ToolOutput(
|
|
id: "call-1",
|
|
toolName: "get_weather",
|
|
segments: [.text(Transcript.TextSegment(content: "22°C"))]
|
|
)
|
|
let output2 = Transcript.ToolOutput(
|
|
id: "call-2",
|
|
toolName: "get_time",
|
|
segments: [.text(Transcript.TextSegment(content: "15:00"))]
|
|
)
|
|
let transcript = Transcript(
|
|
entries: [
|
|
.toolCalls(Transcript.ToolCalls(id: "calls-1", [call1, call2])),
|
|
.toolOutput(output1),
|
|
.toolOutput(output2),
|
|
]
|
|
)
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
#expect(result.contents.count == 2)
|
|
#expect(result.contents[0].role == "model")
|
|
let modelParts = try #require(result.contents[0].parts)
|
|
#expect(modelParts.count == 2)
|
|
guard case .functionCall(let fc1) = modelParts[0].data,
|
|
case .functionCall(let fc2) = modelParts[1].data
|
|
else {
|
|
Issue.record("Expected two functionCall parts")
|
|
return
|
|
}
|
|
#expect(fc1.id == "call-1")
|
|
#expect(fc2.id == "call-2")
|
|
#expect(result.contents[1].role == "user")
|
|
let userParts = try #require(result.contents[1].parts)
|
|
#expect(userParts.count == 2)
|
|
guard case .functionResponse(let fr1) = userParts[0].data,
|
|
case .functionResponse(let fr2) = userParts[1].data
|
|
else {
|
|
Issue.record("Expected two functionResponse parts in single user turn")
|
|
return
|
|
}
|
|
#expect(fr1.id == "call-1")
|
|
#expect(fr2.id == "call-2")
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func translatesToolCallsWithEmptyArguments() throws {
|
|
let call = Transcript.ToolCall(
|
|
id: "call-1",
|
|
toolName: "get_time",
|
|
arguments: try GeneratedContent(json: "{}")
|
|
)
|
|
let transcript = Transcript(
|
|
entries: [.toolCalls(Transcript.ToolCalls(id: "calls-1", [call]))]
|
|
)
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
let modelParts = try #require(result.contents.first?.parts)
|
|
#expect(modelParts.count == 1)
|
|
guard case .functionCall(let functionCall) = modelParts[0].data else {
|
|
Issue.record("Expected functionCall part")
|
|
return
|
|
}
|
|
#expect(functionCall.id == "call-1")
|
|
#expect(functionCall.name == "get_time")
|
|
#expect(functionCall.args == nil)
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func coalescesReasoningAndToolCallsIntoSingleModelTurn() throws {
|
|
let signatureData = Data("test-sig".utf8)
|
|
let call = Transcript.ToolCall(
|
|
id: "call-1",
|
|
toolName: "get_weather",
|
|
arguments: try GeneratedContent(json: "{\"location\":\"Paris\"}")
|
|
)
|
|
let transcript = Transcript(
|
|
entries: [
|
|
.prompt(Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Hi."))])),
|
|
.reasoning(
|
|
Transcript.Reasoning(
|
|
segments: [.text(Transcript.TextSegment(content: "Need to check weather."))],
|
|
signature: signatureData
|
|
)
|
|
),
|
|
.toolCalls(Transcript.ToolCalls(id: "calls-1", [call])),
|
|
]
|
|
)
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
#expect(result.contents.count == 2)
|
|
#expect(result.contents[0].role == "user")
|
|
#expect(result.contents[1].role == "model")
|
|
let modelParts = try #require(result.contents[1].parts)
|
|
#expect(modelParts.count == 2)
|
|
#expect(modelParts[0].thought == true)
|
|
#expect(modelParts[0].thoughtSignature == nil)
|
|
#expect(modelParts[0].data == .text("Need to check weather."))
|
|
#expect(modelParts[1].thoughtSignature == "test-sig")
|
|
guard case .functionCall(let functionCall) = modelParts[1].data else {
|
|
Issue.record("Expected functionCall part")
|
|
return
|
|
}
|
|
#expect(functionCall.id == "call-1")
|
|
#expect(functionCall.name == "get_weather")
|
|
}
|
|
|
|
@Generable
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
@available(tvOS, unavailable)
|
|
struct City {
|
|
var name: String
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func instructionsWithToolDefinitionsTranslatesCleanly() throws {
|
|
let toolDefinition = Transcript.ToolDefinition(
|
|
name: "get_weather",
|
|
description: "Get weather",
|
|
parameters: City.generationSchema
|
|
)
|
|
let transcript = Transcript(
|
|
entries: [
|
|
.instructions(
|
|
Transcript.Instructions(
|
|
segments: [.text(Transcript.TextSegment(content: "Be concise."))],
|
|
toolDefinitions: [toolDefinition]
|
|
)
|
|
),
|
|
.prompt(Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Hello"))])),
|
|
]
|
|
)
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
let systemInstruction = try #require(result.systemInstruction)
|
|
#expect(systemInstruction.parts?.first?.data == Part.PartData.text("Be concise."))
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func translatesToolOutputWithMultipleSegmentsIntoMultipleParts() throws {
|
|
let toolOutput = Transcript.ToolOutput(
|
|
id: "call-1",
|
|
toolName: "get_weather",
|
|
segments: [
|
|
.text(Transcript.TextSegment(content: "Part 1")),
|
|
.text(Transcript.TextSegment(content: "Part 2")),
|
|
]
|
|
)
|
|
let transcript = Transcript(entries: [.toolOutput(toolOutput)])
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
let userParts = try #require(result.contents.first?.parts)
|
|
#expect(userParts.count == 2)
|
|
guard case .functionResponse(let fr1) = userParts[0].data,
|
|
case .functionResponse(let fr2) = userParts[1].data
|
|
else {
|
|
Issue.record("Expected functionResponse part data for both segments")
|
|
return
|
|
}
|
|
#expect(fr1.id == "call-1")
|
|
#expect(fr1.name == "get_weather")
|
|
#expect(fr1.response?["result"] == .string("Part 1"))
|
|
#expect(fr2.id == "call-1")
|
|
#expect(fr2.name == "get_weather")
|
|
#expect(fr2.response?["result"] == .string("Part 2"))
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func translatesToolOutputWithAttachmentThrowsUnsupportedTranscriptContent() throws {
|
|
let imageAttachment = Transcript.ImageAttachment(
|
|
imageURL: URL(fileURLWithPath: "/tmp/test.png")
|
|
)
|
|
let attachment = Transcript.Segment.attachment(
|
|
Transcript.AttachmentSegment(
|
|
id: "att-1",
|
|
content: .image(imageAttachment)
|
|
)
|
|
)
|
|
let toolOutput = Transcript.ToolOutput(
|
|
id: "call-1",
|
|
toolName: "get_weather",
|
|
segments: [
|
|
.text(Transcript.TextSegment(content: "Here is the image:")),
|
|
attachment,
|
|
]
|
|
)
|
|
let transcript = Transcript(entries: [.toolOutput(toolOutput)])
|
|
|
|
do {
|
|
_ = try GeminiTranscriptTranslator.translate(transcript)
|
|
Issue.record("Expected unsupportedTranscriptContent error")
|
|
} catch LanguageModelError.unsupportedTranscriptContent {
|
|
// Expected
|
|
} catch {
|
|
Issue.record("Unexpected error thrown: \(error)")
|
|
}
|
|
}
|
|
|
|
// `Transcript.Entry.data` was added in Xcode 27.2 (Swift 6.4.2+).
|
|
#if compiler(>=6.4.2)
|
|
@Test
|
|
@available(macOS 27.2, iOS 27.2, watchOS 27.2, visionOS 27.2, *)
|
|
@available(tvOS, unavailable)
|
|
func translatesDataEntryThrowsUnsupportedTranscriptContent() throws {
|
|
let dataEntry = Transcript.DataEntry(
|
|
id: "data-1",
|
|
contentType: .json,
|
|
content: Data(#"{"key":"value"}"#.utf8)
|
|
)
|
|
let transcript = Transcript(entries: [.data(dataEntry)])
|
|
|
|
do {
|
|
_ = try GeminiTranscriptTranslator.translate(transcript)
|
|
Issue.record("Expected unsupportedTranscriptContent error")
|
|
} catch LanguageModelError.unsupportedTranscriptContent(let context) {
|
|
#expect(context.unsupportedContent == [.data(dataEntry)])
|
|
} catch {
|
|
Issue.record("Unexpected error thrown: \(error)")
|
|
}
|
|
}
|
|
#endif // compiler(>=6.4.2)
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func emptyToolCallsDoesNotDropPendingReasoningSignature() throws {
|
|
let signatureData = Data("test-sig".utf8)
|
|
let prompt = Transcript.Prompt(
|
|
segments: [.text(Transcript.TextSegment(content: "Hi"))]
|
|
)
|
|
let reasoning = Transcript.Reasoning(
|
|
segments: [.text(Transcript.TextSegment(content: "Thinking..."))],
|
|
signature: signatureData
|
|
)
|
|
let emptyToolCalls = Transcript.ToolCalls(id: "calls-empty", [])
|
|
let finalResponse = Transcript.Response(
|
|
segments: [.text(Transcript.TextSegment(content: "Hello!"))]
|
|
)
|
|
let transcript = Transcript(
|
|
entries: [
|
|
.prompt(prompt),
|
|
.reasoning(reasoning),
|
|
.toolCalls(emptyToolCalls),
|
|
.response(finalResponse),
|
|
]
|
|
)
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
let modelTurn = try #require(result.contents.last)
|
|
let modelParts = try #require(modelTurn.parts)
|
|
#expect(modelParts.first?.thoughtSignature == "test-sig")
|
|
}
|
|
|
|
// MARK: - Image Attachments
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func emptyTextOnlyPromptStillProducesSingleTextPart() throws {
|
|
let transcript = Transcript(
|
|
entries: [.prompt(Transcript.Prompt(segments: [Self.textSegment("")]))]
|
|
)
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
let parts = try #require(result.contents.first?.parts)
|
|
#expect(parts.count == 1)
|
|
#expect(parts[0].data == .text(""))
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func multipleTextSegmentsWithoutImagesProduceSingleTextPart() throws {
|
|
let transcript = Transcript(
|
|
entries: [
|
|
.prompt(
|
|
Transcript.Prompt(segments: [Self.textSegment("Hello, "), Self.textSegment("world!")])
|
|
)
|
|
]
|
|
)
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
let parts = try #require(result.contents.first?.parts)
|
|
#expect(parts.count == 1)
|
|
#expect(parts[0].data == .text("Hello, world!"))
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func promptWithRemoteImageURLThrowsUnsupportedTranscriptContent() throws {
|
|
let url = try #require(URL(string: "gs://cloud-samples-data/generative-ai/image/scones.jpg"))
|
|
let prompt = Transcript.Prompt(
|
|
segments: [
|
|
Self.textSegment("Describe this image."),
|
|
.attachment(
|
|
Transcript.AttachmentSegment(content: .image(Transcript.ImageAttachment(imageURL: url)))
|
|
),
|
|
]
|
|
)
|
|
let transcript = Transcript(entries: [.prompt(prompt)])
|
|
|
|
let context = try Self.unsupportedTranscriptContent(translating: transcript)
|
|
|
|
#expect(context.unsupportedContent == [.prompt(prompt)])
|
|
#expect(context.debugDescription.contains("\"gs\""))
|
|
#expect(!context.debugDescription.contains(url.absoluteString))
|
|
}
|
|
|
|
#if compiler(>=6.4.2)
|
|
@Test
|
|
@available(macOS 27.2, iOS 27.2, watchOS 27.2, visionOS 27.2, *)
|
|
@available(tvOS, unavailable)
|
|
func promptWithDataAttachmentThrowsUnsupportedTranscriptContent() throws {
|
|
let dataAttachment = Transcript.DataAttachment(
|
|
contentType: .json,
|
|
content: Data(#"{"key":"value"}"#.utf8)
|
|
)
|
|
let prompt = Transcript.Prompt(
|
|
segments: [
|
|
Self.textSegment("Summarize this data."),
|
|
.attachment(Transcript.AttachmentSegment(content: .data(dataAttachment))),
|
|
]
|
|
)
|
|
let transcript = Transcript(entries: [.prompt(prompt)])
|
|
|
|
let context = try Self.unsupportedTranscriptContent(translating: transcript)
|
|
|
|
#expect(context.unsupportedContent == [.prompt(prompt)])
|
|
#expect(context.debugDescription.contains("Data attachment"))
|
|
}
|
|
#endif // compiler(>=6.4.2)
|
|
|
|
#if canImport(ImageIO)
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func translatesPromptWithImageBetweenTextPreservingOrder() throws {
|
|
let transcript = Transcript(
|
|
entries: [
|
|
.prompt(
|
|
Transcript.Prompt(
|
|
segments: [
|
|
Self.textSegment("Before"),
|
|
try Self.imageSegment(),
|
|
Self.textSegment("After"),
|
|
]
|
|
)
|
|
)
|
|
]
|
|
)
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
#expect(result.contents.count == 1)
|
|
let userTurn = try #require(result.contents.first)
|
|
#expect(userTurn.role == "user")
|
|
let parts = try #require(userTurn.parts)
|
|
#expect(parts.count == 3)
|
|
#expect(parts[0].data == .text("Before"))
|
|
#expect(Self.isJPEGInlineData(parts[1].data))
|
|
#expect(parts[2].data == .text("After"))
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func mergesAdjacentTextSegmentsAroundImages() throws {
|
|
let transcript = Transcript(
|
|
entries: [
|
|
.prompt(
|
|
Transcript.Prompt(
|
|
segments: [
|
|
Self.textSegment("Compare "),
|
|
Self.textSegment("these:"),
|
|
try Self.imageSegment(),
|
|
try Self.imageSegment(color: .blue),
|
|
Self.textSegment("Be "),
|
|
Self.textSegment("brief."),
|
|
]
|
|
)
|
|
)
|
|
]
|
|
)
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
let parts = try #require(result.contents.first?.parts)
|
|
#expect(parts.count == 4)
|
|
#expect(parts[0].data == .text("Compare these:"))
|
|
#expect(Self.isJPEGInlineData(parts[1].data))
|
|
#expect(Self.isJPEGInlineData(parts[2].data))
|
|
#expect(parts[1].data != parts[2].data)
|
|
#expect(parts[3].data == .text("Be brief."))
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func imageOnlyPromptOmitsEmptyTextParts() throws {
|
|
let transcript = Transcript(
|
|
entries: [
|
|
.prompt(
|
|
Transcript.Prompt(
|
|
segments: [
|
|
Self.textSegment(""),
|
|
try Self.imageSegment(),
|
|
Self.textSegment(""),
|
|
]
|
|
)
|
|
)
|
|
]
|
|
)
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
let parts = try #require(result.contents.first?.parts)
|
|
#expect(parts.count == 1)
|
|
#expect(Self.isJPEGInlineData(parts[0].data))
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func attachmentLabelIsNotSentToModel() throws {
|
|
let label = "barcode-image"
|
|
let transcript = Transcript(
|
|
entries: [
|
|
.prompt(Transcript.Prompt(segments: [try Self.imageSegment(label: label)]))
|
|
]
|
|
)
|
|
|
|
let result = try GeminiTranscriptTranslator.translate(transcript)
|
|
|
|
let parts = try #require(result.contents.first?.parts)
|
|
#expect(parts.count == 1)
|
|
guard case .inlineData(let blob) = parts[0].data else {
|
|
Issue.record("Expected inline data, got \(String(describing: parts[0].data)).")
|
|
return
|
|
}
|
|
#expect(blob.displayName == nil)
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func imageInInstructionsThrowsUnsupportedTranscriptContent() throws {
|
|
let instructions = Transcript.Instructions(
|
|
segments: [try Self.imageSegment()],
|
|
toolDefinitions: []
|
|
)
|
|
let transcript = Transcript(entries: [.instructions(instructions)])
|
|
|
|
let context = try Self.unsupportedTranscriptContent(translating: transcript)
|
|
|
|
#expect(context.unsupportedContent == [.instructions(instructions)])
|
|
#expect(context.debugDescription == "Attachment segments are only supported in prompts.")
|
|
}
|
|
|
|
@Test
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
func imageInResponseThrowsUnsupportedTranscriptContent() throws {
|
|
let response = Transcript.Response(segments: [try Self.imageSegment()])
|
|
let transcript = Transcript(
|
|
entries: [
|
|
.prompt(Transcript.Prompt(segments: [Self.textSegment("Draw a picture.")])),
|
|
.response(response),
|
|
]
|
|
)
|
|
|
|
let context = try Self.unsupportedTranscriptContent(translating: transcript)
|
|
|
|
#expect(context.unsupportedContent == [.response(response)])
|
|
#expect(context.debugDescription == "Attachment segments are only supported in prompts.")
|
|
}
|
|
#endif // canImport(ImageIO)
|
|
|
|
// MARK: - Helpers
|
|
|
|
/// Translates a transcript that is expected to fail with `unsupportedTranscriptContent`.
|
|
///
|
|
/// - Parameter transcript: The transcript to translate.
|
|
/// - Returns: The context of the thrown `unsupportedTranscriptContent` error.
|
|
/// - Throws: An error if translation succeeds or throws a different error.
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
private static func unsupportedTranscriptContent(
|
|
translating transcript: Transcript
|
|
) throws -> LanguageModelError.UnsupportedTranscriptContent {
|
|
let error = try #require(throws: LanguageModelError.self) {
|
|
try GeminiTranscriptTranslator.translate(transcript)
|
|
}
|
|
guard case .unsupportedTranscriptContent(let context) = error else {
|
|
Issue.record("Expected unsupportedTranscriptContent, got \(error).")
|
|
throw error
|
|
}
|
|
return context
|
|
}
|
|
|
|
/// Creates a text segment.
|
|
///
|
|
/// - Parameter content: The text of the segment.
|
|
/// - Returns: The text segment.
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
private static func textSegment(_ content: String) -> Transcript.Segment {
|
|
.text(Transcript.TextSegment(content: content))
|
|
}
|
|
|
|
#if canImport(ImageIO)
|
|
/// Creates an attachment segment containing a small solid-color image.
|
|
///
|
|
/// - Parameters:
|
|
/// - color: The color that fills the image.
|
|
/// - label: An optional label for the attachment.
|
|
/// - Returns: The attachment segment.
|
|
@available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *)
|
|
private static func imageSegment(
|
|
color: TestImages.Color = .red,
|
|
label: String? = nil
|
|
) throws -> Transcript.Segment {
|
|
let image = try TestImages.solid(color, width: 16, height: 16)
|
|
return .attachment(
|
|
Transcript.AttachmentSegment(
|
|
content: .image(Transcript.ImageAttachment(image)),
|
|
label: label
|
|
)
|
|
)
|
|
}
|
|
#endif // canImport(ImageIO)
|
|
|
|
/// Returns whether part data is non-empty inline JPEG data.
|
|
///
|
|
/// - Parameter data: The part data to check.
|
|
/// - Returns: `true` if `data` is inline data with the `image/jpeg` MIME type.
|
|
private static func isJPEGInlineData(_ data: Part.PartData?) -> Bool {
|
|
guard case .inlineData(let blob) = data else {
|
|
return false
|
|
}
|
|
return blob.mimeType == "image/jpeg" && blob.data?.isEmpty == false
|
|
}
|
|
}
|
|
#endif // compiler(>=6.4) && (FoundationModelsFake || canImport(FoundationModels))
|