mirror of
https://github.com/RevenueCat/purchases-ios
synced 2026-11-09 22:35:56 +05:45
Add workflow prewarming performance benchmark (#8206)
* Add workflow prewarming performance benchmark * Add performance baseline recording and comparison * Move performance baseline recording and checks into XCTest * Declare performance tolerance in tests and exclude legacy project checks * Simplify memory benchmark and run it on each PR * Use published workflow for performance benchmark * Add PR job to record and push performance baselines * Update performance baselines * Run baseline automation tests on PRs and use Fastlane with Xcode 27 * Update performance baselines * Require performance benchmark in PR merge gate --------- Co-authored-by: RCGitBot <dev+RCGitBot@revenuecat.com>
This commit is contained in:
1 parent
4e1c604b30
commit
3cc82762f7
19 files changed
+12140
-2
No files matched your search
@@ -18,7 +18,8 @@ parameters:
|
||||
update_paywall_templates,
|
||||
record_paywall_screenshots,
|
||||
deploy_rct_tester,
|
||||
update_error_codes
|
||||
update_error_codes,
|
||||
record_performance_baselines
|
||||
]
|
||||
default: default
|
||||
generate_snapshots:
|
||||
|
||||
@@ -1113,6 +1113,66 @@ jobs:
|
||||
path: fastlane/test_output/xctest
|
||||
destination: scan-test-output-macos
|
||||
|
||||
performance-automation-tests:
|
||||
docker:
|
||||
- image: cimg/ruby:3.3.0-node
|
||||
resource_class: small
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: Install Minitest
|
||||
command: gem install minitest --version 5.27.0 --no-document
|
||||
- run:
|
||||
name: Test performance baseline automation
|
||||
command: ruby fastlane/performance_baseline_helper_test.rb
|
||||
|
||||
performance-tests:
|
||||
parameters:
|
||||
push_baselines:
|
||||
type: boolean
|
||||
default: false
|
||||
record_baselines:
|
||||
type: boolean
|
||||
default: false
|
||||
executor:
|
||||
name: macos-executor
|
||||
xcode_version: "27.0.0"
|
||||
steps:
|
||||
- checkout
|
||||
- tuist-generate-workspace:
|
||||
query: "PerformanceTests"
|
||||
include_test_dependencies: true
|
||||
- run:
|
||||
name: Run memory performance tests
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
if [ "<< parameters.record_baselines >>" = "true" ]; then
|
||||
export TEST_RUNNER_RECORD_PERFORMANCE_BASELINES=1
|
||||
else
|
||||
export TEST_RUNNER_RECORD_PERFORMANCE_BASELINES=0
|
||||
fi
|
||||
xcodebuild test \
|
||||
-workspace RevenueCat-Tuist.xcworkspace \
|
||||
-scheme PerformanceTests \
|
||||
-destination 'platform=iOS Simulator,name=iPhone 17,OS=27.0' \
|
||||
-parallel-testing-enabled NO \
|
||||
-resultBundlePath /tmp/performance-tests.xcresult \
|
||||
ENABLE_TESTABILITY=YES CODE_SIGNING_ALLOWED=NO ONLY_ACTIVE_ARCH=YES
|
||||
- store_artifacts:
|
||||
path: /tmp/performance-tests.xcresult
|
||||
- store_artifacts:
|
||||
path: Tests/PerformanceTests/Baselines.json
|
||||
- when:
|
||||
condition: << parameters.push_baselines >>
|
||||
steps:
|
||||
- revenuecat/install-mise-tools:
|
||||
tools: ruby
|
||||
- install-bundle-dependencies
|
||||
- revenuecat/setup-git-credentials
|
||||
- run:
|
||||
name: Push performance baselines to PR branch
|
||||
command: bundle exec fastlane ios push_performance_baselines
|
||||
|
||||
run-test-ios-27:
|
||||
executor:
|
||||
name: macos-executor
|
||||
@@ -2412,6 +2472,14 @@ jobs:
|
||||
command: echo "All tasks passed"
|
||||
|
||||
workflows:
|
||||
record-performance-baselines:
|
||||
when:
|
||||
equal: [record_performance_baselines, << pipeline.parameters.action >>]
|
||||
jobs:
|
||||
- performance-tests:
|
||||
name: record-performance-baselines
|
||||
record_baselines: true
|
||||
|
||||
generate-swiftinterface:
|
||||
when: << pipeline.parameters.generate_swiftinterface >>
|
||||
jobs:
|
||||
@@ -2453,6 +2521,8 @@ workflows:
|
||||
equal: ["release-train", << pipeline.schedule.name >>]
|
||||
- not:
|
||||
equal: [bump, << pipeline.parameters.action >>]
|
||||
- not:
|
||||
equal: [record_performance_baselines, << pipeline.parameters.action >>]
|
||||
- not: << pipeline.parameters.generate_snapshots >>
|
||||
- not: << pipeline.parameters.generate_revenuecatui_snapshots >>
|
||||
- not: << pipeline.parameters.generate_swiftinterface >>
|
||||
@@ -2472,6 +2542,8 @@ workflows:
|
||||
# =============================================================
|
||||
# Reduced Test Suite: Auto-run on every push
|
||||
# =============================================================
|
||||
- performance-tests
|
||||
- performance-automation-tests
|
||||
- lint:
|
||||
context:
|
||||
- slack-secrets
|
||||
@@ -2676,6 +2748,8 @@ workflows:
|
||||
- all-tasks-passed:
|
||||
requires:
|
||||
# Reduced Test Suite
|
||||
- performance-tests
|
||||
- performance-automation-tests
|
||||
- lint
|
||||
- check-api-changes
|
||||
- run-test-ios-27
|
||||
@@ -2778,6 +2852,8 @@ workflows:
|
||||
equal: [scheduled_pipeline, << pipeline.trigger_source >>]
|
||||
- not:
|
||||
equal: [bump, << pipeline.parameters.action >>]
|
||||
- not:
|
||||
equal: [record_performance_baselines, << pipeline.parameters.action >>]
|
||||
- equal: [true, << pipeline.parameters.version_file_changed >>]
|
||||
jobs:
|
||||
- prepare-next-version:
|
||||
@@ -2931,6 +3007,8 @@ workflows:
|
||||
equal: [scheduled_pipeline, << pipeline.trigger_source >>]
|
||||
- not:
|
||||
equal: [bump, << pipeline.parameters.action >>]
|
||||
- not:
|
||||
equal: [record_performance_baselines, << pipeline.parameters.action >>]
|
||||
- not:
|
||||
equal: [update_error_codes, << pipeline.parameters.action >>]
|
||||
- not: << pipeline.parameters.generate_snapshots >>
|
||||
|
||||
@@ -32,6 +32,9 @@ const JOBS = {
|
||||
"integration-tests-all": ["slack-secrets"],
|
||||
"lint": ["slack-secrets"],
|
||||
"pod-lib-lint": ["slack-secrets"],
|
||||
"performance-tests": [],
|
||||
"performance-automation-tests": [],
|
||||
"record-performance": [],
|
||||
"remote-config-production-tests": ["config-endpoint-tests", "slack-secrets"],
|
||||
"revenuecat-admob-tests": ["slack-secrets"],
|
||||
"run-all-maestro-e2e-tests": ["e2e-tests", "slack-secrets"],
|
||||
@@ -57,6 +60,10 @@ const JOBS = {
|
||||
// than standalone jobs. They map to the real job name plus the parameters to pass,
|
||||
// matching how they're invoked in default_config.yml's workflows.
|
||||
const PARAMETERIZED_JOBS = {
|
||||
"record-performance": {
|
||||
job: "performance-tests",
|
||||
parameters: { record_baselines: true, push_baselines: true },
|
||||
},
|
||||
"installation-tests-carthage-xcode-27": {
|
||||
job: "installation-tests-carthage",
|
||||
parameters: { xcode_version: "27.0.0" },
|
||||
@@ -102,7 +109,7 @@ const workflow = requestedJobs
|
||||
if (variant) {
|
||||
lines.push(` name: ${job}`);
|
||||
for (const [key, value] of Object.entries(variant.parameters)) {
|
||||
lines.push(` ${key}: "${value}"`);
|
||||
lines.push(` ${key}: ${JSON.stringify(value)}`);
|
||||
}
|
||||
}
|
||||
if (contexts.length > 0) {
|
||||
|
||||
@@ -130,6 +130,42 @@ Tests use **XCTest**, **Nimble**, **swift-snapshot-testing**, and **OHHTTPStubs*
|
||||
|
||||
Before you add or change a test, read [Contributing/TESTING.md](Contributing/TESTING.md).
|
||||
|
||||
### Performance tests
|
||||
|
||||
- Add memory benchmarks in `Tests/PerformanceTests/`. Keep setup outside the measurement interval.
|
||||
- Use fixed fixtures. Verify that each iteration completes the expected work.
|
||||
- Set the tolerance in each test. Keep the measured baseline in `Baselines.json`.
|
||||
- Verify that a known memory regression fails before you use a benchmark as evidence.
|
||||
- Keep the Xcode version, simulator, and workload consistent with the baseline.
|
||||
- Never record a new baseline automatically after a failure.
|
||||
|
||||
The `performance-tests` CI job runs on each PR branch push. It uses Xcode 27.0.0, an iPhone 17 simulator with iOS 27.0, and Release configuration. It does not need an API key.
|
||||
|
||||
The workflow fixture is the published revision 24 of `[performance-benchmark-1]` (`wff675a07c6c8b4540`) in [project 6627e5bc](https://app.revenuecat.com/projects/6627e5bc/paywalls). It contains seven screens and its UI configuration. The benchmark loads `Tests/PerformanceTests/Fixtures/performance-benchmark-1-revision-24.json` without duplicate workflows or screens.
|
||||
|
||||
Export an approved published revision with:
|
||||
|
||||
```sh
|
||||
mafdet workflow content --project-id 6627e5bc --workflow-id wff675a07c6c8b4540 \
|
||||
--output /tmp/performance-benchmark-1.json
|
||||
```
|
||||
|
||||
Keep the export unchanged. Include the revision in its filename. Review the fixture and record a new baseline after an intentional workload change.
|
||||
|
||||
For a local comparison:
|
||||
|
||||
```sh
|
||||
xcodebuild test -workspace RevenueCat-Tuist.xcworkspace -scheme PerformanceTests \
|
||||
-destination 'platform=iOS Simulator,name=iPhone 17,OS=27.0' \
|
||||
-parallel-testing-enabled NO ENABLE_TESTABILITY=YES CODE_SIGNING_ALLOWED=NO
|
||||
```
|
||||
|
||||
To update the baseline on a PR, comment `@RCGitBot please test record-performance`. The job records the measurements and pushes only `Baselines.json` to the PR branch. It stops if the branch changed during the run. Review the baseline diff before you merge.
|
||||
|
||||
To record a candidate baseline, start a CircleCI pipeline with `action=record_performance_baselines`. Download the `Baselines.json` artifact. Review its measurements and workload hash before you update the checked-in file. The job does not commit the file.
|
||||
|
||||
For local record mode, prefix the command with `TEST_RUNNER_RECORD_PERFORMANCE_BASELINES=1`. Use the same test selection for record and comparison runs. CI runs the full `PerformanceTests` target. Disable record mode before you compare results.
|
||||
|
||||
## Development Workflow
|
||||
|
||||
For environment setup, see **`Contributing/CONTRIBUTING.md`**. For code style, see **`Contributing/SwiftStyleGuide.swift`**.
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# Where to test behavior
|
||||
|
||||
- **PerformanceTests**: Memory regressions and changes to memory-sensitive SDK paths. Use `Tests/PerformanceTests/` and the baseline instructions in `AGENTS.md`.
|
||||
- **Unit tests**: Logic, state changes, and serialization. Use `Tests/UnitTests/` for the core SDK, `Tests/StoreKitUnitTests/` for StoreKit, and `Tests/RevenueCatUITests/` for RevenueCatUI logic.
|
||||
- **PaywallFixtures**: Paywall UI behavior with fixed data and no backend dependency. Examples include carousel pages across tabs, package selection, gestures, and accessibility. Add fixtures in `Tests/TestingApps/PaywallFixtures/` and XCUITests in `Tests/TestingApps/PaywallFixturesUITests/`.
|
||||
- **Maestro**: Complete app flows, such as purchases, entitlement updates, and workflow navigation. Use the app and YAML flows in `Projects/AutomatedSDKTestsApp/`.
|
||||
|
||||
@@ -5,6 +5,7 @@ require 'set'
|
||||
|
||||
EXCLUDED_SWIFT_PATH_PREFIXES = [
|
||||
'Tests/APITesters/',
|
||||
'Tests/PerformanceTests/',
|
||||
# Tuist-only: these targets exist solely in Projects/PaywallFixtures/Project.swift and are
|
||||
# deliberately absent from RevenueCat.xcodeproj.
|
||||
'Tests/TestingApps/PaywallFixtures/',
|
||||
|
||||
@@ -256,6 +256,25 @@ let project = Project(
|
||||
metadata: .metadata(tags: ["RevenueCatTests"])
|
||||
),
|
||||
|
||||
.target(
|
||||
name: "PerformanceTests",
|
||||
destinations: .iOS,
|
||||
product: .unitTests,
|
||||
bundleId: "com.revenuecat.PerformanceTests",
|
||||
deploymentTargets: .iOS("15.0"),
|
||||
infoPlist: .extendingDefault(with: ["PerformanceXcodeBuild": "$(XCODE_PRODUCT_BUILD_VERSION)"]),
|
||||
sources: [
|
||||
"../../Tests/PerformanceTests/**/*.swift"
|
||||
],
|
||||
resources: [
|
||||
"../../Tests/PerformanceTests/Fixtures/*.json"
|
||||
],
|
||||
dependencies: [
|
||||
.revenueCat
|
||||
],
|
||||
metadata: .metadata(tags: ["RevenueCatTests"])
|
||||
),
|
||||
|
||||
// MARK: – RevenueCatAdMobTests
|
||||
.target(
|
||||
name: "RevenueCatAdMobTests",
|
||||
@@ -328,6 +347,18 @@ let project = Project(
|
||||
analyzeAction: .analyzeAction(configuration: "Debug")
|
||||
),
|
||||
|
||||
.scheme(
|
||||
name: "PerformanceTests",
|
||||
shared: true,
|
||||
buildAction: .buildAction(targets: ["PerformanceTests"]),
|
||||
testAction: .targets(
|
||||
[.testableTarget(target: "PerformanceTests", parallelization: .disabled)],
|
||||
configuration: "Release",
|
||||
attachDebugger: false,
|
||||
options: .options(coverage: false)
|
||||
)
|
||||
),
|
||||
|
||||
.scheme(
|
||||
name: "ReceiptParserTests",
|
||||
shared: true,
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"WorkflowPrewarmingMemoryTests/testPublishWebBundleURLsPeakMemory()" : {
|
||||
"environment" : {
|
||||
"architecture" : "arm64",
|
||||
"model" : "iPhone18,3",
|
||||
"os" : "Version 27.0 (Build 24A434)",
|
||||
"xcodeBuild" : "27A266a"
|
||||
},
|
||||
"peakMemoryMiB" : 42.83704833984375,
|
||||
"workloadSHA256" : "959a1c94069c07a67398bdbdb641bf6b48a18ab7bd956f7b48e982d4bffc6d04"
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,72 @@
|
||||
//
|
||||
// Copyright RevenueCat Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the MIT License (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://opensource.org/licenses/MIT
|
||||
//
|
||||
// MemoryBaseline.swift
|
||||
//
|
||||
|
||||
import Foundation
|
||||
|
||||
struct MemoryBaseline: Codable {
|
||||
var peakMemoryMiB: Double
|
||||
var workloadSHA256: String
|
||||
var environment: [String: String]
|
||||
|
||||
static let fileURL = URL(fileURLWithPath: #filePath)
|
||||
.deletingLastPathComponent().deletingLastPathComponent().appendingPathComponent("Baselines.json")
|
||||
|
||||
static func checkOrRecord(
|
||||
peakMemoryMiB: Double,
|
||||
tolerancePercent: Double,
|
||||
testName: String,
|
||||
workloadSHA256: String,
|
||||
environment: [String: String],
|
||||
record: Bool = false,
|
||||
fileURL: URL = Self.fileURL
|
||||
) throws {
|
||||
var baselines: [String: Self] = [:]
|
||||
if FileManager.default.fileExists(atPath: fileURL.path) {
|
||||
let data = try Data(contentsOf: fileURL)
|
||||
baselines = try JSONDecoder().decode([String: Self].self, from: data)
|
||||
}
|
||||
if record {
|
||||
baselines[testName] = Self(
|
||||
peakMemoryMiB: peakMemoryMiB,
|
||||
workloadSHA256: workloadSHA256,
|
||||
environment: environment
|
||||
)
|
||||
let encoder = JSONEncoder()
|
||||
encoder.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes]
|
||||
try (encoder.encode(baselines) + Data([0x0a])).write(to: fileURL, options: .atomic)
|
||||
print("Recorded \(testName): \(peakMemoryMiB) MiB in \(fileURL.path). Review the baseline diff.")
|
||||
return
|
||||
}
|
||||
guard let baseline = baselines[testName] else {
|
||||
throw Failure("No baseline for \(testName). Set RECORD_PERFORMANCE_BASELINES=1 to record it.")
|
||||
}
|
||||
guard baseline.environment == environment else {
|
||||
throw Failure("\(testName): environment differs. Use the baseline simulator and Xcode version.")
|
||||
}
|
||||
guard baseline.workloadSHA256 == workloadSHA256 else {
|
||||
throw Failure("\(testName): workload differs. Review the change before you record a new baseline.")
|
||||
}
|
||||
let limit = baseline.peakMemoryMiB * (1 + tolerancePercent / 100)
|
||||
let values = String(format: "peak %.2f MiB, baseline %.2f MiB, limit %.2f MiB",
|
||||
peakMemoryMiB, baseline.peakMemoryMiB, limit)
|
||||
guard peakMemoryMiB <= limit else { throw Failure("\(testName): \(values)") }
|
||||
print("PASS \(testName): \(values)")
|
||||
}
|
||||
|
||||
struct Failure: Error, CustomStringConvertible {
|
||||
let description: String
|
||||
|
||||
init(_ description: String) {
|
||||
self.description = description
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
//
|
||||
// Copyright RevenueCat Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the MIT License (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://opensource.org/licenses/MIT
|
||||
//
|
||||
// PerformanceFixture.swift
|
||||
//
|
||||
|
||||
import CryptoKit
|
||||
import Foundation
|
||||
@testable import RevenueCat
|
||||
import XCTest
|
||||
|
||||
enum PerformanceFixture {
|
||||
|
||||
static func makeManager(rootURL: URL) throws -> RemoteConfigManager {
|
||||
let diskCache = RemoteConfigDiskCache(cache: SynchronizedLargeItemCache(
|
||||
cache: FileManager.default,
|
||||
basePath: "performance",
|
||||
directoryType: .applicationSupport(overrideURL: rootURL)
|
||||
))
|
||||
let blobStore = RemoteConfigBlobStore(directoryURL: rootURL.appendingPathComponent("blobs"))
|
||||
var topics: [String: RemoteConfiguration.ConfigTopic] = [:]
|
||||
func store(_ data: Data, topic: RemoteConfigTopic, key: String) {
|
||||
let ref = Data(SHA256.hash(data: data).prefix(24)).base64EncodedString()
|
||||
.replacingOccurrences(of: "+", with: "-").replacingOccurrences(of: "/", with: "_")
|
||||
XCTAssertTrue(data.withUnsafeBytes { blobStore.write(ref: ref, bytes: $0) })
|
||||
topics[topic.wireName, default: [:]][key] = .init(
|
||||
blobRef: ref,
|
||||
prefetch: true,
|
||||
content: topic == .workflows ? ["offering_identifier": .string("[performance-benchmark-1]")] : [:]
|
||||
)
|
||||
}
|
||||
|
||||
let url = try XCTUnwrap(Bundle(for: WorkflowPrewarmingMemoryTests.self)
|
||||
.url(forResource: "performance-benchmark-1-revision-24", withExtension: "json"))
|
||||
let data = try Data(contentsOf: url)
|
||||
let workflow = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any])
|
||||
let identifier = try XCTUnwrap(workflow["id"] as? String)
|
||||
store(data, topic: .workflows, key: identifier)
|
||||
let uiConfig = try XCTUnwrap(workflow["ui_config"] as? [String: Any])
|
||||
for (key, value) in uiConfig {
|
||||
store(try JSONSerialization.data(withJSONObject: value, options: [.sortedKeys]), topic: .uiConfig, key: key)
|
||||
}
|
||||
XCTAssertTrue(diskCache.write(.init(
|
||||
manifest: "performance-fixture",
|
||||
activeTopics: topics.keys.sorted(),
|
||||
topics: .init(entries: topics)
|
||||
)))
|
||||
let backend = OfflinePerformanceBackend()
|
||||
return RemoteConfigManager(
|
||||
remoteConfigAPI: backend,
|
||||
diskCache: diskCache,
|
||||
blobStore: blobStore,
|
||||
blobFetcher: RemoteConfigBlobFetcher(
|
||||
blobStore: blobStore,
|
||||
sourceProvider: RemoteConfigSourceProvider(topicStore: diskCache),
|
||||
downloader: backend
|
||||
),
|
||||
currentUserProvider: backend
|
||||
)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private final class OfflinePerformanceBackend: RemoteConfigAPIType,
|
||||
RemoteConfigBlobDownloaderType,
|
||||
CurrentUserProvider {
|
||||
var currentAppUserID: String { "workflow-memory-benchmark" }
|
||||
var currentUserIsAnonymous: Bool { true }
|
||||
|
||||
func getRemoteConfig(
|
||||
request: RemoteConfigRequest,
|
||||
isAppBackgrounded: Bool,
|
||||
completion: @escaping RemoteConfigResponseHandler
|
||||
) {
|
||||
XCTFail("The performance fixture must not request remote config.")
|
||||
DispatchQueue.global().async { completion(.failure(.networkError(.unexpectedResponse(nil)))) }
|
||||
}
|
||||
|
||||
func getRemoteConfigFallback(
|
||||
domain: String,
|
||||
isAppBackgrounded: Bool,
|
||||
completion: @escaping FallbackResponseHandler
|
||||
) {
|
||||
XCTFail("The performance fixture must not request fallback config.")
|
||||
DispatchQueue.global().async { completion(.failure(.networkError(.unexpectedResponse(nil)))) }
|
||||
}
|
||||
|
||||
func data(from url: URL) async throws -> Data {
|
||||
XCTFail("The performance fixture must not download blobs.")
|
||||
throw URLError(.notConnectedToInternet)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
//
|
||||
// Copyright RevenueCat Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the MIT License (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://opensource.org/licenses/MIT
|
||||
//
|
||||
// RecordingMemoryMetric.swift
|
||||
//
|
||||
|
||||
import XCTest
|
||||
|
||||
final class RecordingMemoryMetric: XCTMemoryMetric {
|
||||
|
||||
final class Samples {
|
||||
var peaksMiB: [Double] = []
|
||||
}
|
||||
|
||||
let samples: Samples
|
||||
|
||||
init(samples: Samples = Samples()) {
|
||||
self.samples = samples
|
||||
super.init()
|
||||
}
|
||||
|
||||
override func copy(with zone: NSZone? = nil) -> Any {
|
||||
return RecordingMemoryMetric(samples: self.samples)
|
||||
}
|
||||
|
||||
override func reportMeasurements(
|
||||
from startTime: XCTPerformanceMeasurementTimestamp,
|
||||
to endTime: XCTPerformanceMeasurementTimestamp
|
||||
) throws -> [XCTPerformanceMeasurement] {
|
||||
let measurements = try super.reportMeasurements(from: startTime, to: endTime)
|
||||
let peak = try XCTUnwrap(measurements.first {
|
||||
$0.identifier == "com.apple.dt.XCTMetric_Memory.physical_peak"
|
||||
})
|
||||
guard peak.unitSymbol == "byte" else {
|
||||
throw MemoryBaseline.Failure("Unsupported memory unit: \(peak.unitSymbol)")
|
||||
}
|
||||
self.samples.peaksMiB.append(peak.doubleValue / 1_048_576)
|
||||
return measurements
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
//
|
||||
// Copyright RevenueCat Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the MIT License (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://opensource.org/licenses/MIT
|
||||
//
|
||||
// WorkflowPrewarmingWorkload.swift
|
||||
//
|
||||
|
||||
import CryptoKit
|
||||
import Foundation
|
||||
@_spi(Internal) @testable import RevenueCat
|
||||
import XCTest
|
||||
|
||||
final class WorkflowPrewarmingWorkload {
|
||||
|
||||
private let manager: RemoteConfigManager
|
||||
private let rootURL: URL
|
||||
private var offeringIDs: [String] = []
|
||||
private var expectedScreens: [String: Set<String>] = [:]
|
||||
private(set) var sha256 = ""
|
||||
|
||||
private init(manager: RemoteConfigManager, rootURL: URL) {
|
||||
self.manager = manager
|
||||
self.rootURL = rootURL
|
||||
}
|
||||
|
||||
static func load() async throws -> WorkflowPrewarmingWorkload {
|
||||
let rootURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
|
||||
let manager = try PerformanceFixture.makeManager(rootURL: rootURL)
|
||||
let workload = WorkflowPrewarmingWorkload(manager: manager, rootURL: rootURL)
|
||||
do {
|
||||
try await workload.prepare()
|
||||
return workload
|
||||
} catch {
|
||||
workload.close()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
private func prepare() async throws {
|
||||
let workflows = await self.manager.awaitTopicAndPrefetchBlobsReady(.workflows)
|
||||
let topic = try XCTUnwrap(workflows)
|
||||
self.offeringIDs = Set(topic.values.compactMap { item in
|
||||
guard case let .string(identifier)? = item.content["offering_identifier"] else { return nil }
|
||||
return identifier
|
||||
}).sorted()
|
||||
XCTAssertFalse(self.offeringIDs.isEmpty, "The fixture needs a mapped workflow.")
|
||||
|
||||
let provider = WorkflowsConfigProvider(manager: self.manager)
|
||||
var fingerprint = SHA256()
|
||||
for offeringID in self.offeringIDs {
|
||||
let mappedID = await provider.workflowId(forOfferingId: offeringID)
|
||||
let workflowID = try XCTUnwrap(mappedID)
|
||||
let body = await self.manager.blobData(for: .workflows, itemKey: workflowID)
|
||||
let data = try XCTUnwrap(body)
|
||||
let workflow = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any])
|
||||
let screens = try XCTUnwrap(workflow["screens"] as? [String: Any])
|
||||
self.expectedScreens[workflowID] = Set(screens.keys)
|
||||
fingerprint.update(data: Data(offeringID.utf8))
|
||||
fingerprint.update(data: data)
|
||||
}
|
||||
let uiTopic = await self.manager.awaitTopicAndPrefetchBlobsReady(.uiConfig)
|
||||
for key in try XCTUnwrap(uiTopic).keys.sorted() {
|
||||
let body = await self.manager.blobData(for: .uiConfig, itemKey: key)
|
||||
fingerprint.update(data: try XCTUnwrap(body))
|
||||
}
|
||||
self.sha256 = fingerprint.finalize().map { String(format: "%02x", $0) }.joined()
|
||||
}
|
||||
|
||||
func close() {
|
||||
self.manager.close()
|
||||
try? FileManager.default.removeItem(at: self.rootURL)
|
||||
}
|
||||
|
||||
func prepareIteration() async throws -> WorkflowPrewarmingIteration {
|
||||
let decodedScreens = Atomic<[String: Set<String>]>([:])
|
||||
let uiConfigProvider = UiConfigProvider(manager: self.manager)
|
||||
let provider = WorkflowsConfigProvider(
|
||||
manager: self.manager,
|
||||
uiConfigProvider: uiConfigProvider,
|
||||
workflowDecoder: { data in
|
||||
let workflow = try JSONDecoder.default.decode(PublishedWorkflow.self, from: data)
|
||||
decodedScreens.modify { $0[workflow.id] = Set(workflow.screens.keys) }
|
||||
return workflow
|
||||
}
|
||||
)
|
||||
for identifier in self.offeringIDs {
|
||||
await provider.cachePrefetchedWorkflowBodyData(includingOfferingId: identifier)
|
||||
}
|
||||
let config = await uiConfigProvider.getUiConfig()
|
||||
_ = try XCTUnwrap(config)
|
||||
return WorkflowPrewarmingIteration(
|
||||
manager: WorkflowManager(
|
||||
workflowsConfigProvider: provider,
|
||||
paywallCache: nil,
|
||||
operationDispatcher: .default,
|
||||
webBundleURLBatcher: WebBundleURLBatcher(eventBus: WebBundleEventBus())
|
||||
),
|
||||
offerings: self.makeOfferings(),
|
||||
decodedScreens: decodedScreens,
|
||||
expectedScreens: self.expectedScreens
|
||||
)
|
||||
}
|
||||
|
||||
private func makeOfferings() -> Offerings {
|
||||
let offerings = self.offeringIDs.map {
|
||||
Offering(identifier: $0, serverDescription: $0, availablePackages: [], webCheckoutUrl: nil)
|
||||
}
|
||||
let response = OfferingsResponse(
|
||||
currentOfferingId: self.offeringIDs.first,
|
||||
offerings: [],
|
||||
placements: nil,
|
||||
targeting: nil,
|
||||
uiConfig: nil
|
||||
)
|
||||
return Offerings(
|
||||
offerings: Dictionary(uniqueKeysWithValues: offerings.map { ($0.identifier, $0) }),
|
||||
currentOfferingID: self.offeringIDs.first,
|
||||
placements: .init(
|
||||
fallbackOfferingId: nil,
|
||||
offeringIdsByPlacement: Dictionary(uniqueKeysWithValues: self.offeringIDs.map { ($0, $0) })
|
||||
),
|
||||
targeting: nil,
|
||||
contents: .init(response: response, httpResponseOriginalSource: .mainServer),
|
||||
loadedFromDiskCache: false
|
||||
)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
struct WorkflowPrewarmingIteration {
|
||||
let manager: WorkflowManager
|
||||
let offerings: Offerings
|
||||
let decodedScreens: Atomic<[String: Set<String>]>
|
||||
let expectedScreens: [String: Set<String>]
|
||||
|
||||
func run() async {
|
||||
await self.manager.publishWebBundleURLs(offerings: self.offerings)
|
||||
}
|
||||
|
||||
func validate() {
|
||||
XCTAssertEqual(self.decodedScreens.value, self.expectedScreens,
|
||||
"Every workflow and screen must decode in every iteration.")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
//
|
||||
// Copyright RevenueCat Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the MIT License (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://opensource.org/licenses/MIT
|
||||
//
|
||||
// XCTestCase+WaitForCompletion.swift
|
||||
//
|
||||
|
||||
import Foundation
|
||||
@testable import RevenueCat
|
||||
import XCTest
|
||||
|
||||
extension XCTestCase {
|
||||
|
||||
func waitForCompletion<Value>(_ operation: @escaping () async throws -> Value) throws -> Value {
|
||||
let completed = self.expectation(description: "Performance operation completed")
|
||||
let result = Atomic<Result<Value, Error>?>(nil)
|
||||
let task = Task {
|
||||
do {
|
||||
result.value = .success(try await operation())
|
||||
} catch {
|
||||
result.value = .failure(error)
|
||||
}
|
||||
completed.fulfill()
|
||||
}
|
||||
defer { task.cancel() }
|
||||
self.wait(for: [completed], timeout: 120)
|
||||
return try XCTUnwrap(result.value).get()
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
//
|
||||
// Copyright RevenueCat Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the MIT License (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://opensource.org/licenses/MIT
|
||||
//
|
||||
// MemoryBaselineTests.swift
|
||||
//
|
||||
|
||||
import XCTest
|
||||
|
||||
// swiftlint:disable:next xctestcase_superclass
|
||||
final class MemoryBaselineTests: XCTestCase {
|
||||
|
||||
private var fileURL: URL!
|
||||
|
||||
override func setUpWithError() throws {
|
||||
try super.setUpWithError()
|
||||
self.fileURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
|
||||
}
|
||||
|
||||
override func tearDownWithError() throws {
|
||||
if FileManager.default.fileExists(atPath: self.fileURL.path) {
|
||||
try FileManager.default.removeItem(at: self.fileURL)
|
||||
}
|
||||
try super.tearDownWithError()
|
||||
}
|
||||
|
||||
func testComparisonUsesToleranceAndRejectsChangedInputs() throws {
|
||||
try Data("""
|
||||
{"Example/testMemory()": {
|
||||
"peakMemoryMiB": 100,
|
||||
"workloadSHA256": "workload", "environment": {"model": "simulator"}
|
||||
}}
|
||||
""".utf8).write(to: self.fileURL)
|
||||
|
||||
XCTAssertNoThrow(try self.check(peak: 110))
|
||||
XCTAssertNoThrow(try self.check(peak: 80))
|
||||
XCTAssertNoThrow(try self.check(peak: 115, tolerancePercent: 20))
|
||||
XCTAssertThrowsError(try self.check(peak: 110, tolerancePercent: 5))
|
||||
XCTAssertThrowsError(try self.check(peak: 111)) { error in
|
||||
XCTAssertTrue(String(describing: error).contains("limit 110.00 MiB"))
|
||||
}
|
||||
XCTAssertThrowsError(try self.check(peak: 80, workload: "changed"))
|
||||
XCTAssertThrowsError(try self.check(peak: 80, model: "another simulator"))
|
||||
XCTAssertThrowsError(try self.check(peak: 80, test: "Missing/testMemory()"))
|
||||
}
|
||||
|
||||
func testRecordCreatesFileAndPreservesOtherBaselines() throws {
|
||||
try self.check(peak: 100, record: true)
|
||||
try self.check(peak: 50, test: "Another/testMemory()", record: true)
|
||||
try self.check(peak: 200, workload: "changed", record: true)
|
||||
|
||||
let data = try Data(contentsOf: self.fileURL)
|
||||
let json = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: [String: Any]])
|
||||
XCTAssertEqual(json.count, 2)
|
||||
XCTAssertEqual(json["Example/testMemory()"]?["peakMemoryMiB"] as? Double, 200)
|
||||
XCTAssertEqual(json["Example/testMemory()"]?["workloadSHA256"] as? String, "changed")
|
||||
XCTAssertEqual(json["Another/testMemory()"]?["peakMemoryMiB"] as? Double, 50)
|
||||
}
|
||||
|
||||
private func check(
|
||||
peak: Double,
|
||||
tolerancePercent: Double = 10,
|
||||
workload: String = "workload",
|
||||
model: String = "simulator",
|
||||
test: String = "Example/testMemory()",
|
||||
record: Bool = false
|
||||
) throws {
|
||||
try MemoryBaseline.checkOrRecord(
|
||||
peakMemoryMiB: peak,
|
||||
tolerancePercent: tolerancePercent,
|
||||
testName: test,
|
||||
workloadSHA256: workload,
|
||||
environment: ["model": model],
|
||||
record: record,
|
||||
fileURL: self.fileURL
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
//
|
||||
// Copyright RevenueCat Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the MIT License (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://opensource.org/licenses/MIT
|
||||
//
|
||||
// WorkflowPrewarmingMemoryTests.swift
|
||||
//
|
||||
|
||||
import Foundation
|
||||
@testable import RevenueCat
|
||||
import XCTest
|
||||
|
||||
// swiftlint:disable:next xctestcase_superclass
|
||||
final class WorkflowPrewarmingMemoryTests: XCTestCase {
|
||||
|
||||
private var workload: WorkflowPrewarmingWorkload!
|
||||
private var previousLogLevel: LogLevel = .info
|
||||
|
||||
override func setUp() async throws {
|
||||
try await super.setUp()
|
||||
self.previousLogLevel = Purchases.logLevel
|
||||
Purchases.logLevel = .error
|
||||
self.workload = try await WorkflowPrewarmingWorkload.load()
|
||||
}
|
||||
|
||||
override func tearDown() {
|
||||
self.workload?.close()
|
||||
self.workload = nil
|
||||
Purchases.logLevel = self.previousLogLevel
|
||||
super.tearDown()
|
||||
}
|
||||
|
||||
func testPublishWebBundleURLsPeakMemory() throws {
|
||||
let metric = RecordingMemoryMetric()
|
||||
let failuresBeforeMeasurement = self.testRun?.failureCount
|
||||
let options = XCTMeasureOptions()
|
||||
options.iterationCount = 5
|
||||
options.invocationOptions = [.manuallyStart, .manuallyStop]
|
||||
var failure: Error?
|
||||
self.measure(metrics: [metric, XCTClockMetric()], options: options) {
|
||||
do {
|
||||
let context = try self.waitForCompletion { try await self.workload.prepareIteration() }
|
||||
self.startMeasuring()
|
||||
do {
|
||||
defer { self.stopMeasuring() }
|
||||
try self.waitForCompletion { await context.run() }
|
||||
}
|
||||
context.validate()
|
||||
} catch {
|
||||
failure = error
|
||||
}
|
||||
}
|
||||
if let failure { throw failure }
|
||||
guard self.testRun?.failureCount == failuresBeforeMeasurement else { return }
|
||||
// XCTest excludes the first iteration from its results.
|
||||
let peaks = metric.samples.peaksMiB.dropFirst()
|
||||
XCTAssertEqual(peaks.count, options.iterationCount)
|
||||
guard peaks.count == options.iterationCount else { return }
|
||||
try MemoryBaseline.checkOrRecord(
|
||||
peakMemoryMiB: peaks.reduce(0, +) / Double(peaks.count),
|
||||
tolerancePercent: 10,
|
||||
testName: "\(type(of: self))/\(#function)",
|
||||
workloadSHA256: self.workload.sha256,
|
||||
environment: try self.baselineEnvironment(),
|
||||
record: ProcessInfo.processInfo.environment["RECORD_PERFORMANCE_BASELINES"] == "1"
|
||||
)
|
||||
}
|
||||
|
||||
private func baselineEnvironment() throws -> [String: String] {
|
||||
let environment = ProcessInfo.processInfo.environment
|
||||
#if arch(arm64)
|
||||
let architecture = "arm64"
|
||||
#else
|
||||
let architecture = "x86_64"
|
||||
#endif
|
||||
return [
|
||||
"architecture": architecture,
|
||||
"model": try XCTUnwrap(environment["SIMULATOR_MODEL_IDENTIFIER"]),
|
||||
"os": ProcessInfo.processInfo.operatingSystemVersionString,
|
||||
"xcodeBuild": try XCTUnwrap(Bundle(for: Self.self)
|
||||
.object(forInfoDictionaryKey: "PerformanceXcodeBuild") as? String)
|
||||
]
|
||||
}
|
||||
|
||||
}
|
||||
@@ -11,6 +11,7 @@
|
||||
#
|
||||
|
||||
require_relative 'api_diff_helper'
|
||||
require_relative 'performance_baseline_helper'
|
||||
require 'rexml/document'
|
||||
require 'json'
|
||||
require 'open3'
|
||||
@@ -1771,6 +1772,13 @@ platform :ios do
|
||||
UI.important "Workflow: #{workflow_url}"
|
||||
end
|
||||
|
||||
desc "Push updated performance baselines to the PR branch"
|
||||
lane :push_performance_baselines do
|
||||
Dir.chdir(File.expand_path('..', __dir__)) do
|
||||
PerformanceBaselineHelper.push
|
||||
end
|
||||
end
|
||||
|
||||
desc "Creates a new PR after new snapshot files were generated"
|
||||
lane :create_snapshot_pr do |options|
|
||||
push_snapshot_pr(
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
require 'open3'
|
||||
|
||||
module PerformanceBaselineHelper
|
||||
def self.push
|
||||
branch = ENV.fetch('CIRCLE_BRANCH')
|
||||
revision = ENV.fetch('CIRCLE_SHA1')
|
||||
baseline = 'Tests/PerformanceTests/Baselines.json'
|
||||
|
||||
if branch == 'main' || !ENV.fetch('CIRCLE_TAG', '').empty?
|
||||
raise 'Baseline updates require a PR branch.'
|
||||
end
|
||||
|
||||
git('check-ref-format', "refs/heads/#{branch}")
|
||||
git('fetch', 'origin', "refs/heads/#{branch}")
|
||||
unless git('rev-parse', 'FETCH_HEAD') == revision && git('rev-parse', 'HEAD') == revision
|
||||
raise 'The PR branch changed. Run record-performance again.'
|
||||
end
|
||||
|
||||
if git('diff', '--name-only', 'HEAD', '--', baseline).empty?
|
||||
puts 'No baseline changes to commit.'
|
||||
return
|
||||
end
|
||||
|
||||
puts git('commit', '--only', '-m', 'Update performance baselines', '--', baseline)
|
||||
puts git('push', 'origin', "HEAD:refs/heads/#{branch}")
|
||||
end
|
||||
|
||||
def self.git(*arguments)
|
||||
output, status = Open3.capture2e('git', *arguments)
|
||||
raise output unless status.success?
|
||||
|
||||
output.strip
|
||||
end
|
||||
private_class_method :git
|
||||
end
|
||||
@@ -0,0 +1,87 @@
|
||||
require 'minitest/autorun'
|
||||
require 'tmpdir'
|
||||
require 'fileutils'
|
||||
require 'open3'
|
||||
require 'yaml'
|
||||
require 'rbconfig'
|
||||
|
||||
class PerformanceBaselineHelperTest < Minitest::Test
|
||||
ROOT = File.expand_path('..', __dir__)
|
||||
BASELINE = 'Tests/PerformanceTests/Baselines.json'
|
||||
|
||||
def command(*args, env: {}, directory: @checkout)
|
||||
output, status = Open3.capture2e(env, *args, chdir: directory)
|
||||
assert status.success?, output
|
||||
output.strip
|
||||
end
|
||||
|
||||
def setup
|
||||
@temporary = Dir.mktmpdir
|
||||
@remote = File.join(@temporary, 'remote.git')
|
||||
@checkout = File.join(@temporary, 'checkout')
|
||||
command('git', 'init', '--bare', @remote, directory: @temporary)
|
||||
command('git', 'clone', @remote, @checkout, directory: @temporary)
|
||||
command('git', 'config', 'user.name', 'Test')
|
||||
command('git', 'config', 'user.email', 'test@example.com')
|
||||
command('git', 'checkout', '-b', 'test-pr')
|
||||
FileUtils.mkdir_p(File.join(@checkout, File.dirname(BASELINE)))
|
||||
File.write(File.join(@checkout, BASELINE), "{}\n")
|
||||
File.write(File.join(@checkout, 'other.txt'), 'original')
|
||||
command('git', 'add', '.')
|
||||
command('git', 'commit', '-m', 'Initial')
|
||||
command('git', 'push', 'origin', 'test-pr')
|
||||
@head = command('git', 'rev-parse', 'HEAD')
|
||||
end
|
||||
|
||||
def teardown
|
||||
FileUtils.remove_entry(@temporary)
|
||||
end
|
||||
|
||||
def publish
|
||||
Open3.capture2e({'CIRCLE_BRANCH' => 'test-pr', 'CIRCLE_SHA1' => @head},
|
||||
RbConfig.ruby, '-r', File.join(ROOT, 'fastlane/performance_baseline_helper.rb'),
|
||||
'-e', 'PerformanceBaselineHelper.push', chdir: @checkout)
|
||||
end
|
||||
|
||||
def test_pushes_only_baseline_and_skips_unchanged_result
|
||||
File.write(File.join(@checkout, BASELINE), '{"peakMemoryMiB":42}')
|
||||
File.write(File.join(@checkout, 'other.txt'), 'unrelated')
|
||||
command('git', 'add', 'other.txt')
|
||||
output, status = publish
|
||||
assert status.success?, output
|
||||
assert_equal BASELINE, command('git', '--git-dir', @remote, 'diff-tree', '--no-commit-id', '--name-only', '-r', 'test-pr')
|
||||
assert_equal '{"peakMemoryMiB":42}', command('git', '--git-dir', @remote, 'show', "test-pr:#{BASELINE}")
|
||||
@head = command('git', 'rev-parse', 'HEAD')
|
||||
output, status = publish
|
||||
assert status.success?, output
|
||||
assert_equal @head, command('git', '--git-dir', @remote, 'rev-parse', 'test-pr')
|
||||
end
|
||||
|
||||
def test_rejects_results_after_branch_changes
|
||||
command('git', 'commit', '--allow-empty', '-m', 'New PR commit')
|
||||
command('git', 'push', 'origin', 'test-pr')
|
||||
remote_head = command('git', 'rev-parse', 'HEAD')
|
||||
command('git', 'checkout', '--detach', @head)
|
||||
File.write(File.join(@checkout, BASELINE), '{"peakMemoryMiB":42}')
|
||||
output, status = publish
|
||||
refute status.success?, output
|
||||
assert_includes output, 'branch changed'
|
||||
assert_equal remote_head, command('git', '--git-dir', @remote, 'rev-parse', 'test-pr')
|
||||
end
|
||||
|
||||
def test_pr_merge_gate_requires_the_memory_benchmark
|
||||
config = YAML.safe_load(File.read(File.join(ROOT, '.circleci/default_config.yml')), aliases: true)
|
||||
jobs = config.fetch('workflows').fetch('run-all-tests').fetch('jobs')
|
||||
gate = jobs.find { |job| job.is_a?(Hash) && job.key?('all-tasks-passed') }
|
||||
assert_includes gate.fetch('all-tasks-passed').fetch('requires'), 'performance-tests'
|
||||
end
|
||||
|
||||
def test_requested_job_enables_record_and_push_with_boolean_parameters
|
||||
command('node', '.circleci/generate-requested-jobs-config.js', env: {'REQUESTED_JOBS' => 'record-performance'}, directory: ROOT)
|
||||
config = YAML.safe_load(File.read('/tmp/requested-jobs-config.yml'), aliases: true)
|
||||
job = config.fetch('workflows').fetch('on-demand-jobs').fetch('jobs').first.fetch('performance-tests')
|
||||
assert_equal 'record-performance', job.fetch('name')
|
||||
assert_equal true, job.fetch('record_baselines')
|
||||
assert_equal true, job.fetch('push_baselines')
|
||||
end
|
||||
end
|
||||
Reference in new issue
Block a user