Add workflow prewarming performance benchmark (#8206)

* Add workflow prewarming performance benchmark

* Add performance baseline recording and comparison

* Move performance baseline recording and checks into XCTest

* Declare performance tolerance in tests and exclude legacy project checks

* Simplify memory benchmark and run it on each PR

* Use published workflow for performance benchmark

* Add PR job to record and push performance baselines

* Update performance baselines

* Run baseline automation tests on PRs and use Fastlane with Xcode 27

* Update performance baselines

* Require performance benchmark in PR merge gate

---------

Co-authored-by: RCGitBot <dev+RCGitBot@revenuecat.com>
This commit is contained in:
Facundo MenzellaandRCGitBot authored and GitHub committed 2026-11-05 14:16:45 +00:00
1 parent 4e1c604b30
commit 3cc82762f7
19 files changed
+12140 -2

No files matched your search

+2 -1
View File
@@ -18,7 +18,8 @@ parameters:
update_paywall_templates,
record_paywall_screenshots,
deploy_rct_tester,
update_error_codes
update_error_codes,
record_performance_baselines
]
default: default
generate_snapshots:
+78
View File
@@ -1113,6 +1113,66 @@ jobs:
path: fastlane/test_output/xctest
destination: scan-test-output-macos
performance-automation-tests:
docker:
- image: cimg/ruby:3.3.0-node
resource_class: small
steps:
- checkout
- run:
name: Install Minitest
command: gem install minitest --version 5.27.0 --no-document
- run:
name: Test performance baseline automation
command: ruby fastlane/performance_baseline_helper_test.rb
performance-tests:
parameters:
push_baselines:
type: boolean
default: false
record_baselines:
type: boolean
default: false
executor:
name: macos-executor
xcode_version: "27.0.0"
steps:
- checkout
- tuist-generate-workspace:
query: "PerformanceTests"
include_test_dependencies: true
- run:
name: Run memory performance tests
no_output_timeout: 15m
command: |
if [ "<< parameters.record_baselines >>" = "true" ]; then
export TEST_RUNNER_RECORD_PERFORMANCE_BASELINES=1
else
export TEST_RUNNER_RECORD_PERFORMANCE_BASELINES=0
fi
xcodebuild test \
-workspace RevenueCat-Tuist.xcworkspace \
-scheme PerformanceTests \
-destination 'platform=iOS Simulator,name=iPhone 17,OS=27.0' \
-parallel-testing-enabled NO \
-resultBundlePath /tmp/performance-tests.xcresult \
ENABLE_TESTABILITY=YES CODE_SIGNING_ALLOWED=NO ONLY_ACTIVE_ARCH=YES
- store_artifacts:
path: /tmp/performance-tests.xcresult
- store_artifacts:
path: Tests/PerformanceTests/Baselines.json
- when:
condition: << parameters.push_baselines >>
steps:
- revenuecat/install-mise-tools:
tools: ruby
- install-bundle-dependencies
- revenuecat/setup-git-credentials
- run:
name: Push performance baselines to PR branch
command: bundle exec fastlane ios push_performance_baselines
run-test-ios-27:
executor:
name: macos-executor
@@ -2412,6 +2472,14 @@ jobs:
command: echo "All tasks passed"
workflows:
record-performance-baselines:
when:
equal: [record_performance_baselines, << pipeline.parameters.action >>]
jobs:
- performance-tests:
name: record-performance-baselines
record_baselines: true
generate-swiftinterface:
when: << pipeline.parameters.generate_swiftinterface >>
jobs:
@@ -2453,6 +2521,8 @@ workflows:
equal: ["release-train", << pipeline.schedule.name >>]
- not:
equal: [bump, << pipeline.parameters.action >>]
- not:
equal: [record_performance_baselines, << pipeline.parameters.action >>]
- not: << pipeline.parameters.generate_snapshots >>
- not: << pipeline.parameters.generate_revenuecatui_snapshots >>
- not: << pipeline.parameters.generate_swiftinterface >>
@@ -2472,6 +2542,8 @@ workflows:
# =============================================================
# Reduced Test Suite: Auto-run on every push
# =============================================================
- performance-tests
- performance-automation-tests
- lint:
context:
- slack-secrets
@@ -2676,6 +2748,8 @@ workflows:
- all-tasks-passed:
requires:
# Reduced Test Suite
- performance-tests
- performance-automation-tests
- lint
- check-api-changes
- run-test-ios-27
@@ -2778,6 +2852,8 @@ workflows:
equal: [scheduled_pipeline, << pipeline.trigger_source >>]
- not:
equal: [bump, << pipeline.parameters.action >>]
- not:
equal: [record_performance_baselines, << pipeline.parameters.action >>]
- equal: [true, << pipeline.parameters.version_file_changed >>]
jobs:
- prepare-next-version:
@@ -2931,6 +3007,8 @@ workflows:
equal: [scheduled_pipeline, << pipeline.trigger_source >>]
- not:
equal: [bump, << pipeline.parameters.action >>]
- not:
equal: [record_performance_baselines, << pipeline.parameters.action >>]
- not:
equal: [update_error_codes, << pipeline.parameters.action >>]
- not: << pipeline.parameters.generate_snapshots >>
+8 -1
View File
@@ -32,6 +32,9 @@ const JOBS = {
"integration-tests-all": ["slack-secrets"],
"lint": ["slack-secrets"],
"pod-lib-lint": ["slack-secrets"],
"performance-tests": [],
"performance-automation-tests": [],
"record-performance": [],
"remote-config-production-tests": ["config-endpoint-tests", "slack-secrets"],
"revenuecat-admob-tests": ["slack-secrets"],
"run-all-maestro-e2e-tests": ["e2e-tests", "slack-secrets"],
@@ -57,6 +60,10 @@ const JOBS = {
// than standalone jobs. They map to the real job name plus the parameters to pass,
// matching how they're invoked in default_config.yml's workflows.
const PARAMETERIZED_JOBS = {
"record-performance": {
job: "performance-tests",
parameters: { record_baselines: true, push_baselines: true },
},
"installation-tests-carthage-xcode-27": {
job: "installation-tests-carthage",
parameters: { xcode_version: "27.0.0" },
@@ -102,7 +109,7 @@ const workflow = requestedJobs
if (variant) {
lines.push(` name: ${job}`);
for (const [key, value] of Object.entries(variant.parameters)) {
lines.push(` ${key}: "${value}"`);
lines.push(` ${key}: ${JSON.stringify(value)}`);
}
}
if (contexts.length > 0) {
+36
View File
@@ -130,6 +130,42 @@ Tests use **XCTest**, **Nimble**, **swift-snapshot-testing**, and **OHHTTPStubs*
Before you add or change a test, read [Contributing/TESTING.md](Contributing/TESTING.md).
### Performance tests
- Add memory benchmarks in `Tests/PerformanceTests/`. Keep setup outside the measurement interval.
- Use fixed fixtures. Verify that each iteration completes the expected work.
- Set the tolerance in each test. Keep the measured baseline in `Baselines.json`.
- Verify that a known memory regression fails before you use a benchmark as evidence.
- Keep the Xcode version, simulator, and workload consistent with the baseline.
- Never record a new baseline automatically after a failure.
The `performance-tests` CI job runs on each PR branch push. It uses Xcode 27.0.0, an iPhone 17 simulator with iOS 27.0, and Release configuration. It does not need an API key.
The workflow fixture is the published revision 24 of `[performance-benchmark-1]` (`wff675a07c6c8b4540`) in [project 6627e5bc](https://app.revenuecat.com/projects/6627e5bc/paywalls). It contains seven screens and its UI configuration. The benchmark loads `Tests/PerformanceTests/Fixtures/performance-benchmark-1-revision-24.json` without duplicate workflows or screens.
Export an approved published revision with:
```sh
mafdet workflow content --project-id 6627e5bc --workflow-id wff675a07c6c8b4540 \
--output /tmp/performance-benchmark-1.json
```
Keep the export unchanged. Include the revision in its filename. Review the fixture and record a new baseline after an intentional workload change.
For a local comparison:
```sh
xcodebuild test -workspace RevenueCat-Tuist.xcworkspace -scheme PerformanceTests \
-destination 'platform=iOS Simulator,name=iPhone 17,OS=27.0' \
-parallel-testing-enabled NO ENABLE_TESTABILITY=YES CODE_SIGNING_ALLOWED=NO
```
To update the baseline on a PR, comment `@RCGitBot please test record-performance`. The job records the measurements and pushes only `Baselines.json` to the PR branch. It stops if the branch changed during the run. Review the baseline diff before you merge.
To record a candidate baseline, start a CircleCI pipeline with `action=record_performance_baselines`. Download the `Baselines.json` artifact. Review its measurements and workload hash before you update the checked-in file. The job does not commit the file.
For local record mode, prefix the command with `TEST_RUNNER_RECORD_PERFORMANCE_BASELINES=1`. Use the same test selection for record and comparison runs. CI runs the full `PerformanceTests` target. Disable record mode before you compare results.
## Development Workflow
For environment setup, see **`Contributing/CONTRIBUTING.md`**. For code style, see **`Contributing/SwiftStyleGuide.swift`**.
+1
View File
@@ -1,5 +1,6 @@
# Where to test behavior
- **PerformanceTests**: Memory regressions and changes to memory-sensitive SDK paths. Use `Tests/PerformanceTests/` and the baseline instructions in `AGENTS.md`.
- **Unit tests**: Logic, state changes, and serialization. Use `Tests/UnitTests/` for the core SDK, `Tests/StoreKitUnitTests/` for StoreKit, and `Tests/RevenueCatUITests/` for RevenueCatUI logic.
- **PaywallFixtures**: Paywall UI behavior with fixed data and no backend dependency. Examples include carousel pages across tabs, package selection, gestures, and accessibility. Add fixtures in `Tests/TestingApps/PaywallFixtures/` and XCUITests in `Tests/TestingApps/PaywallFixturesUITests/`.
- **Maestro**: Complete app flows, such as purchases, entitlement updates, and workflow navigation. Use the app and YAML flows in `Projects/AutomatedSDKTestsApp/`.
+1
View File
@@ -5,6 +5,7 @@ require 'set'
EXCLUDED_SWIFT_PATH_PREFIXES = [
'Tests/APITesters/',
'Tests/PerformanceTests/',
# Tuist-only: these targets exist solely in Projects/PaywallFixtures/Project.swift and are
# deliberately absent from RevenueCat.xcodeproj.
'Tests/TestingApps/PaywallFixtures/',
+31
View File
@@ -256,6 +256,25 @@ let project = Project(
metadata: .metadata(tags: ["RevenueCatTests"])
),
.target(
name: "PerformanceTests",
destinations: .iOS,
product: .unitTests,
bundleId: "com.revenuecat.PerformanceTests",
deploymentTargets: .iOS("15.0"),
infoPlist: .extendingDefault(with: ["PerformanceXcodeBuild": "$(XCODE_PRODUCT_BUILD_VERSION)"]),
sources: [
"../../Tests/PerformanceTests/**/*.swift"
],
resources: [
"../../Tests/PerformanceTests/Fixtures/*.json"
],
dependencies: [
.revenueCat
],
metadata: .metadata(tags: ["RevenueCatTests"])
),
// MARK: – RevenueCatAdMobTests
.target(
name: "RevenueCatAdMobTests",
@@ -328,6 +347,18 @@ let project = Project(
analyzeAction: .analyzeAction(configuration: "Debug")
),
.scheme(
name: "PerformanceTests",
shared: true,
buildAction: .buildAction(targets: ["PerformanceTests"]),
testAction: .targets(
[.testableTarget(target: "PerformanceTests", parallelization: .disabled)],
configuration: "Release",
attachDebugger: false,
options: .options(coverage: false)
)
),
.scheme(
name: "ReceiptParserTests",
shared: true,
+12
View File
@@ -0,0 +1,12 @@
{
"WorkflowPrewarmingMemoryTests/testPublishWebBundleURLsPeakMemory()" : {
"environment" : {
"architecture" : "arm64",
"model" : "iPhone18,3",
"os" : "Version 27.0 (Build 24A434)",
"xcodeBuild" : "27A266a"
},
"peakMemoryMiB" : 42.83704833984375,
"workloadSHA256" : "959a1c94069c07a67398bdbdb641bf6b48a18ab7bd956f7b48e982d4bffc6d04"
}
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,72 @@
//
// Copyright RevenueCat Inc. All Rights Reserved.
//
// Licensed under the MIT License (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://opensource.org/licenses/MIT
//
// MemoryBaseline.swift
//
import Foundation
struct MemoryBaseline: Codable {
var peakMemoryMiB: Double
var workloadSHA256: String
var environment: [String: String]
static let fileURL = URL(fileURLWithPath: #filePath)
.deletingLastPathComponent().deletingLastPathComponent().appendingPathComponent("Baselines.json")
static func checkOrRecord(
peakMemoryMiB: Double,
tolerancePercent: Double,
testName: String,
workloadSHA256: String,
environment: [String: String],
record: Bool = false,
fileURL: URL = Self.fileURL
) throws {
var baselines: [String: Self] = [:]
if FileManager.default.fileExists(atPath: fileURL.path) {
let data = try Data(contentsOf: fileURL)
baselines = try JSONDecoder().decode([String: Self].self, from: data)
}
if record {
baselines[testName] = Self(
peakMemoryMiB: peakMemoryMiB,
workloadSHA256: workloadSHA256,
environment: environment
)
let encoder = JSONEncoder()
encoder.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes]
try (encoder.encode(baselines) + Data([0x0a])).write(to: fileURL, options: .atomic)
print("Recorded \(testName): \(peakMemoryMiB) MiB in \(fileURL.path). Review the baseline diff.")
return
}
guard let baseline = baselines[testName] else {
throw Failure("No baseline for \(testName). Set RECORD_PERFORMANCE_BASELINES=1 to record it.")
}
guard baseline.environment == environment else {
throw Failure("\(testName): environment differs. Use the baseline simulator and Xcode version.")
}
guard baseline.workloadSHA256 == workloadSHA256 else {
throw Failure("\(testName): workload differs. Review the change before you record a new baseline.")
}
let limit = baseline.peakMemoryMiB * (1 + tolerancePercent / 100)
let values = String(format: "peak %.2f MiB, baseline %.2f MiB, limit %.2f MiB",
peakMemoryMiB, baseline.peakMemoryMiB, limit)
guard peakMemoryMiB <= limit else { throw Failure("\(testName): \(values)") }
print("PASS \(testName): \(values)")
}
struct Failure: Error, CustomStringConvertible {
let description: String
init(_ description: String) {
self.description = description
}
}
}
@@ -0,0 +1,98 @@
//
// Copyright RevenueCat Inc. All Rights Reserved.
//
// Licensed under the MIT License (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://opensource.org/licenses/MIT
//
// PerformanceFixture.swift
//
import CryptoKit
import Foundation
@testable import RevenueCat
import XCTest
enum PerformanceFixture {
static func makeManager(rootURL: URL) throws -> RemoteConfigManager {
let diskCache = RemoteConfigDiskCache(cache: SynchronizedLargeItemCache(
cache: FileManager.default,
basePath: "performance",
directoryType: .applicationSupport(overrideURL: rootURL)
))
let blobStore = RemoteConfigBlobStore(directoryURL: rootURL.appendingPathComponent("blobs"))
var topics: [String: RemoteConfiguration.ConfigTopic] = [:]
func store(_ data: Data, topic: RemoteConfigTopic, key: String) {
let ref = Data(SHA256.hash(data: data).prefix(24)).base64EncodedString()
.replacingOccurrences(of: "+", with: "-").replacingOccurrences(of: "/", with: "_")
XCTAssertTrue(data.withUnsafeBytes { blobStore.write(ref: ref, bytes: $0) })
topics[topic.wireName, default: [:]][key] = .init(
blobRef: ref,
prefetch: true,
content: topic == .workflows ? ["offering_identifier": .string("[performance-benchmark-1]")] : [:]
)
}
let url = try XCTUnwrap(Bundle(for: WorkflowPrewarmingMemoryTests.self)
.url(forResource: "performance-benchmark-1-revision-24", withExtension: "json"))
let data = try Data(contentsOf: url)
let workflow = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any])
let identifier = try XCTUnwrap(workflow["id"] as? String)
store(data, topic: .workflows, key: identifier)
let uiConfig = try XCTUnwrap(workflow["ui_config"] as? [String: Any])
for (key, value) in uiConfig {
store(try JSONSerialization.data(withJSONObject: value, options: [.sortedKeys]), topic: .uiConfig, key: key)
}
XCTAssertTrue(diskCache.write(.init(
manifest: "performance-fixture",
activeTopics: topics.keys.sorted(),
topics: .init(entries: topics)
)))
let backend = OfflinePerformanceBackend()
return RemoteConfigManager(
remoteConfigAPI: backend,
diskCache: diskCache,
blobStore: blobStore,
blobFetcher: RemoteConfigBlobFetcher(
blobStore: blobStore,
sourceProvider: RemoteConfigSourceProvider(topicStore: diskCache),
downloader: backend
),
currentUserProvider: backend
)
}
}
private final class OfflinePerformanceBackend: RemoteConfigAPIType,
RemoteConfigBlobDownloaderType,
CurrentUserProvider {
var currentAppUserID: String { "workflow-memory-benchmark" }
var currentUserIsAnonymous: Bool { true }
func getRemoteConfig(
request: RemoteConfigRequest,
isAppBackgrounded: Bool,
completion: @escaping RemoteConfigResponseHandler
) {
XCTFail("The performance fixture must not request remote config.")
DispatchQueue.global().async { completion(.failure(.networkError(.unexpectedResponse(nil)))) }
}
func getRemoteConfigFallback(
domain: String,
isAppBackgrounded: Bool,
completion: @escaping FallbackResponseHandler
) {
XCTFail("The performance fixture must not request fallback config.")
DispatchQueue.global().async { completion(.failure(.networkError(.unexpectedResponse(nil)))) }
}
func data(from url: URL) async throws -> Data {
XCTFail("The performance fixture must not download blobs.")
throw URLError(.notConnectedToInternet)
}
}
@@ -0,0 +1,46 @@
//
// Copyright RevenueCat Inc. All Rights Reserved.
//
// Licensed under the MIT License (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://opensource.org/licenses/MIT
//
// RecordingMemoryMetric.swift
//
import XCTest
final class RecordingMemoryMetric: XCTMemoryMetric {
final class Samples {
var peaksMiB: [Double] = []
}
let samples: Samples
init(samples: Samples = Samples()) {
self.samples = samples
super.init()
}
override func copy(with zone: NSZone? = nil) -> Any {
return RecordingMemoryMetric(samples: self.samples)
}
override func reportMeasurements(
from startTime: XCTPerformanceMeasurementTimestamp,
to endTime: XCTPerformanceMeasurementTimestamp
) throws -> [XCTPerformanceMeasurement] {
let measurements = try super.reportMeasurements(from: startTime, to: endTime)
let peak = try XCTUnwrap(measurements.first {
$0.identifier == "com.apple.dt.XCTMetric_Memory.physical_peak"
})
guard peak.unitSymbol == "byte" else {
throw MemoryBaseline.Failure("Unsupported memory unit: \(peak.unitSymbol)")
}
self.samples.peaksMiB.append(peak.doubleValue / 1_048_576)
return measurements
}
}
@@ -0,0 +1,149 @@
//
// Copyright RevenueCat Inc. All Rights Reserved.
//
// Licensed under the MIT License (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://opensource.org/licenses/MIT
//
// WorkflowPrewarmingWorkload.swift
//
import CryptoKit
import Foundation
@_spi(Internal) @testable import RevenueCat
import XCTest
final class WorkflowPrewarmingWorkload {
private let manager: RemoteConfigManager
private let rootURL: URL
private var offeringIDs: [String] = []
private var expectedScreens: [String: Set<String>] = [:]
private(set) var sha256 = ""
private init(manager: RemoteConfigManager, rootURL: URL) {
self.manager = manager
self.rootURL = rootURL
}
static func load() async throws -> WorkflowPrewarmingWorkload {
let rootURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
let manager = try PerformanceFixture.makeManager(rootURL: rootURL)
let workload = WorkflowPrewarmingWorkload(manager: manager, rootURL: rootURL)
do {
try await workload.prepare()
return workload
} catch {
workload.close()
throw error
}
}
private func prepare() async throws {
let workflows = await self.manager.awaitTopicAndPrefetchBlobsReady(.workflows)
let topic = try XCTUnwrap(workflows)
self.offeringIDs = Set(topic.values.compactMap { item in
guard case let .string(identifier)? = item.content["offering_identifier"] else { return nil }
return identifier
}).sorted()
XCTAssertFalse(self.offeringIDs.isEmpty, "The fixture needs a mapped workflow.")
let provider = WorkflowsConfigProvider(manager: self.manager)
var fingerprint = SHA256()
for offeringID in self.offeringIDs {
let mappedID = await provider.workflowId(forOfferingId: offeringID)
let workflowID = try XCTUnwrap(mappedID)
let body = await self.manager.blobData(for: .workflows, itemKey: workflowID)
let data = try XCTUnwrap(body)
let workflow = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any])
let screens = try XCTUnwrap(workflow["screens"] as? [String: Any])
self.expectedScreens[workflowID] = Set(screens.keys)
fingerprint.update(data: Data(offeringID.utf8))
fingerprint.update(data: data)
}
let uiTopic = await self.manager.awaitTopicAndPrefetchBlobsReady(.uiConfig)
for key in try XCTUnwrap(uiTopic).keys.sorted() {
let body = await self.manager.blobData(for: .uiConfig, itemKey: key)
fingerprint.update(data: try XCTUnwrap(body))
}
self.sha256 = fingerprint.finalize().map { String(format: "%02x", $0) }.joined()
}
func close() {
self.manager.close()
try? FileManager.default.removeItem(at: self.rootURL)
}
func prepareIteration() async throws -> WorkflowPrewarmingIteration {
let decodedScreens = Atomic<[String: Set<String>]>([:])
let uiConfigProvider = UiConfigProvider(manager: self.manager)
let provider = WorkflowsConfigProvider(
manager: self.manager,
uiConfigProvider: uiConfigProvider,
workflowDecoder: { data in
let workflow = try JSONDecoder.default.decode(PublishedWorkflow.self, from: data)
decodedScreens.modify { $0[workflow.id] = Set(workflow.screens.keys) }
return workflow
}
)
for identifier in self.offeringIDs {
await provider.cachePrefetchedWorkflowBodyData(includingOfferingId: identifier)
}
let config = await uiConfigProvider.getUiConfig()
_ = try XCTUnwrap(config)
return WorkflowPrewarmingIteration(
manager: WorkflowManager(
workflowsConfigProvider: provider,
paywallCache: nil,
operationDispatcher: .default,
webBundleURLBatcher: WebBundleURLBatcher(eventBus: WebBundleEventBus())
),
offerings: self.makeOfferings(),
decodedScreens: decodedScreens,
expectedScreens: self.expectedScreens
)
}
private func makeOfferings() -> Offerings {
let offerings = self.offeringIDs.map {
Offering(identifier: $0, serverDescription: $0, availablePackages: [], webCheckoutUrl: nil)
}
let response = OfferingsResponse(
currentOfferingId: self.offeringIDs.first,
offerings: [],
placements: nil,
targeting: nil,
uiConfig: nil
)
return Offerings(
offerings: Dictionary(uniqueKeysWithValues: offerings.map { ($0.identifier, $0) }),
currentOfferingID: self.offeringIDs.first,
placements: .init(
fallbackOfferingId: nil,
offeringIdsByPlacement: Dictionary(uniqueKeysWithValues: self.offeringIDs.map { ($0, $0) })
),
targeting: nil,
contents: .init(response: response, httpResponseOriginalSource: .mainServer),
loadedFromDiskCache: false
)
}
}
struct WorkflowPrewarmingIteration {
let manager: WorkflowManager
let offerings: Offerings
let decodedScreens: Atomic<[String: Set<String>]>
let expectedScreens: [String: Set<String>]
func run() async {
await self.manager.publishWebBundleURLs(offerings: self.offerings)
}
func validate() {
XCTAssertEqual(self.decodedScreens.value, self.expectedScreens,
"Every workflow and screen must decode in every iteration.")
}
}
@@ -0,0 +1,35 @@
//
// Copyright RevenueCat Inc. All Rights Reserved.
//
// Licensed under the MIT License (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://opensource.org/licenses/MIT
//
// XCTestCase+WaitForCompletion.swift
//
import Foundation
@testable import RevenueCat
import XCTest
extension XCTestCase {
func waitForCompletion<Value>(_ operation: @escaping () async throws -> Value) throws -> Value {
let completed = self.expectation(description: "Performance operation completed")
let result = Atomic<Result<Value, Error>?>(nil)
let task = Task {
do {
result.value = .success(try await operation())
} catch {
result.value = .failure(error)
}
completed.fulfill()
}
defer { task.cancel() }
self.wait(for: [completed], timeout: 120)
return try XCTUnwrap(result.value).get()
}
}
@@ -0,0 +1,83 @@
//
// Copyright RevenueCat Inc. All Rights Reserved.
//
// Licensed under the MIT License (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://opensource.org/licenses/MIT
//
// MemoryBaselineTests.swift
//
import XCTest
// swiftlint:disable:next xctestcase_superclass
final class MemoryBaselineTests: XCTestCase {
private var fileURL: URL!
override func setUpWithError() throws {
try super.setUpWithError()
self.fileURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
}
override func tearDownWithError() throws {
if FileManager.default.fileExists(atPath: self.fileURL.path) {
try FileManager.default.removeItem(at: self.fileURL)
}
try super.tearDownWithError()
}
func testComparisonUsesToleranceAndRejectsChangedInputs() throws {
try Data("""
{"Example/testMemory()": {
"peakMemoryMiB": 100,
"workloadSHA256": "workload", "environment": {"model": "simulator"}
}}
""".utf8).write(to: self.fileURL)
XCTAssertNoThrow(try self.check(peak: 110))
XCTAssertNoThrow(try self.check(peak: 80))
XCTAssertNoThrow(try self.check(peak: 115, tolerancePercent: 20))
XCTAssertThrowsError(try self.check(peak: 110, tolerancePercent: 5))
XCTAssertThrowsError(try self.check(peak: 111)) { error in
XCTAssertTrue(String(describing: error).contains("limit 110.00 MiB"))
}
XCTAssertThrowsError(try self.check(peak: 80, workload: "changed"))
XCTAssertThrowsError(try self.check(peak: 80, model: "another simulator"))
XCTAssertThrowsError(try self.check(peak: 80, test: "Missing/testMemory()"))
}
func testRecordCreatesFileAndPreservesOtherBaselines() throws {
try self.check(peak: 100, record: true)
try self.check(peak: 50, test: "Another/testMemory()", record: true)
try self.check(peak: 200, workload: "changed", record: true)
let data = try Data(contentsOf: self.fileURL)
let json = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: [String: Any]])
XCTAssertEqual(json.count, 2)
XCTAssertEqual(json["Example/testMemory()"]?["peakMemoryMiB"] as? Double, 200)
XCTAssertEqual(json["Example/testMemory()"]?["workloadSHA256"] as? String, "changed")
XCTAssertEqual(json["Another/testMemory()"]?["peakMemoryMiB"] as? Double, 50)
}
private func check(
peak: Double,
tolerancePercent: Double = 10,
workload: String = "workload",
model: String = "simulator",
test: String = "Example/testMemory()",
record: Bool = false
) throws {
try MemoryBaseline.checkOrRecord(
peakMemoryMiB: peak,
tolerancePercent: tolerancePercent,
testName: test,
workloadSHA256: workload,
environment: ["model": model],
record: record,
fileURL: self.fileURL
)
}
}
@@ -0,0 +1,89 @@
//
// Copyright RevenueCat Inc. All Rights Reserved.
//
// Licensed under the MIT License (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://opensource.org/licenses/MIT
//
// WorkflowPrewarmingMemoryTests.swift
//
import Foundation
@testable import RevenueCat
import XCTest
// swiftlint:disable:next xctestcase_superclass
final class WorkflowPrewarmingMemoryTests: XCTestCase {
private var workload: WorkflowPrewarmingWorkload!
private var previousLogLevel: LogLevel = .info
override func setUp() async throws {
try await super.setUp()
self.previousLogLevel = Purchases.logLevel
Purchases.logLevel = .error
self.workload = try await WorkflowPrewarmingWorkload.load()
}
override func tearDown() {
self.workload?.close()
self.workload = nil
Purchases.logLevel = self.previousLogLevel
super.tearDown()
}
func testPublishWebBundleURLsPeakMemory() throws {
let metric = RecordingMemoryMetric()
let failuresBeforeMeasurement = self.testRun?.failureCount
let options = XCTMeasureOptions()
options.iterationCount = 5
options.invocationOptions = [.manuallyStart, .manuallyStop]
var failure: Error?
self.measure(metrics: [metric, XCTClockMetric()], options: options) {
do {
let context = try self.waitForCompletion { try await self.workload.prepareIteration() }
self.startMeasuring()
do {
defer { self.stopMeasuring() }
try self.waitForCompletion { await context.run() }
}
context.validate()
} catch {
failure = error
}
}
if let failure { throw failure }
guard self.testRun?.failureCount == failuresBeforeMeasurement else { return }
// XCTest excludes the first iteration from its results.
let peaks = metric.samples.peaksMiB.dropFirst()
XCTAssertEqual(peaks.count, options.iterationCount)
guard peaks.count == options.iterationCount else { return }
try MemoryBaseline.checkOrRecord(
peakMemoryMiB: peaks.reduce(0, +) / Double(peaks.count),
tolerancePercent: 10,
testName: "\(type(of: self))/\(#function)",
workloadSHA256: self.workload.sha256,
environment: try self.baselineEnvironment(),
record: ProcessInfo.processInfo.environment["RECORD_PERFORMANCE_BASELINES"] == "1"
)
}
private func baselineEnvironment() throws -> [String: String] {
let environment = ProcessInfo.processInfo.environment
#if arch(arm64)
let architecture = "arm64"
#else
let architecture = "x86_64"
#endif
return [
"architecture": architecture,
"model": try XCTUnwrap(environment["SIMULATOR_MODEL_IDENTIFIER"]),
"os": ProcessInfo.processInfo.operatingSystemVersionString,
"xcodeBuild": try XCTUnwrap(Bundle(for: Self.self)
.object(forInfoDictionaryKey: "PerformanceXcodeBuild") as? String)
]
}
}
+8
View File
@@ -11,6 +11,7 @@
#
require_relative 'api_diff_helper'
require_relative 'performance_baseline_helper'
require 'rexml/document'
require 'json'
require 'open3'
@@ -1771,6 +1772,13 @@ platform :ios do
UI.important "Workflow: #{workflow_url}"
end
desc "Push updated performance baselines to the PR branch"
lane :push_performance_baselines do
Dir.chdir(File.expand_path('..', __dir__)) do
PerformanceBaselineHelper.push
end
end
desc "Creates a new PR after new snapshot files were generated"
lane :create_snapshot_pr do |options|
push_snapshot_pr(
+35
View File
@@ -0,0 +1,35 @@
require 'open3'
module PerformanceBaselineHelper
def self.push
branch = ENV.fetch('CIRCLE_BRANCH')
revision = ENV.fetch('CIRCLE_SHA1')
baseline = 'Tests/PerformanceTests/Baselines.json'
if branch == 'main' || !ENV.fetch('CIRCLE_TAG', '').empty?
raise 'Baseline updates require a PR branch.'
end
git('check-ref-format', "refs/heads/#{branch}")
git('fetch', 'origin', "refs/heads/#{branch}")
unless git('rev-parse', 'FETCH_HEAD') == revision && git('rev-parse', 'HEAD') == revision
raise 'The PR branch changed. Run record-performance again.'
end
if git('diff', '--name-only', 'HEAD', '--', baseline).empty?
puts 'No baseline changes to commit.'
return
end
puts git('commit', '--only', '-m', 'Update performance baselines', '--', baseline)
puts git('push', 'origin', "HEAD:refs/heads/#{branch}")
end
def self.git(*arguments)
output, status = Open3.capture2e('git', *arguments)
raise output unless status.success?
output.strip
end
private_class_method :git
end
@@ -0,0 +1,87 @@
require 'minitest/autorun'
require 'tmpdir'
require 'fileutils'
require 'open3'
require 'yaml'
require 'rbconfig'
class PerformanceBaselineHelperTest < Minitest::Test
ROOT = File.expand_path('..', __dir__)
BASELINE = 'Tests/PerformanceTests/Baselines.json'
def command(*args, env: {}, directory: @checkout)
output, status = Open3.capture2e(env, *args, chdir: directory)
assert status.success?, output
output.strip
end
def setup
@temporary = Dir.mktmpdir
@remote = File.join(@temporary, 'remote.git')
@checkout = File.join(@temporary, 'checkout')
command('git', 'init', '--bare', @remote, directory: @temporary)
command('git', 'clone', @remote, @checkout, directory: @temporary)
command('git', 'config', 'user.name', 'Test')
command('git', 'config', 'user.email', 'test@example.com')
command('git', 'checkout', '-b', 'test-pr')
FileUtils.mkdir_p(File.join(@checkout, File.dirname(BASELINE)))
File.write(File.join(@checkout, BASELINE), "{}\n")
File.write(File.join(@checkout, 'other.txt'), 'original')
command('git', 'add', '.')
command('git', 'commit', '-m', 'Initial')
command('git', 'push', 'origin', 'test-pr')
@head = command('git', 'rev-parse', 'HEAD')
end
def teardown
FileUtils.remove_entry(@temporary)
end
def publish
Open3.capture2e({'CIRCLE_BRANCH' => 'test-pr', 'CIRCLE_SHA1' => @head},
RbConfig.ruby, '-r', File.join(ROOT, 'fastlane/performance_baseline_helper.rb'),
'-e', 'PerformanceBaselineHelper.push', chdir: @checkout)
end
def test_pushes_only_baseline_and_skips_unchanged_result
File.write(File.join(@checkout, BASELINE), '{"peakMemoryMiB":42}')
File.write(File.join(@checkout, 'other.txt'), 'unrelated')
command('git', 'add', 'other.txt')
output, status = publish
assert status.success?, output
assert_equal BASELINE, command('git', '--git-dir', @remote, 'diff-tree', '--no-commit-id', '--name-only', '-r', 'test-pr')
assert_equal '{"peakMemoryMiB":42}', command('git', '--git-dir', @remote, 'show', "test-pr:#{BASELINE}")
@head = command('git', 'rev-parse', 'HEAD')
output, status = publish
assert status.success?, output
assert_equal @head, command('git', '--git-dir', @remote, 'rev-parse', 'test-pr')
end
def test_rejects_results_after_branch_changes
command('git', 'commit', '--allow-empty', '-m', 'New PR commit')
command('git', 'push', 'origin', 'test-pr')
remote_head = command('git', 'rev-parse', 'HEAD')
command('git', 'checkout', '--detach', @head)
File.write(File.join(@checkout, BASELINE), '{"peakMemoryMiB":42}')
output, status = publish
refute status.success?, output
assert_includes output, 'branch changed'
assert_equal remote_head, command('git', '--git-dir', @remote, 'rev-parse', 'test-pr')
end
def test_pr_merge_gate_requires_the_memory_benchmark
config = YAML.safe_load(File.read(File.join(ROOT, '.circleci/default_config.yml')), aliases: true)
jobs = config.fetch('workflows').fetch('run-all-tests').fetch('jobs')
gate = jobs.find { |job| job.is_a?(Hash) && job.key?('all-tasks-passed') }
assert_includes gate.fetch('all-tasks-passed').fetch('requires'), 'performance-tests'
end
def test_requested_job_enables_record_and_push_with_boolean_parameters
command('node', '.circleci/generate-requested-jobs-config.js', env: {'REQUESTED_JOBS' => 'record-performance'}, directory: ROOT)
config = YAML.safe_load(File.read('/tmp/requested-jobs-config.yml'), aliases: true)
job = config.fetch('workflows').fetch('on-demand-jobs').fetch('jobs').first.fetch('performance-tests')
assert_equal 'record-performance', job.fetch('name')
assert_equal true, job.fetch('record_baselines')
assert_equal true, job.fetch('push_baselines')
end
end