mirror of
https://github.com/firebase/firebase-ios-sdk
synced 2026-11-09 22:36:01 +05:45
Konstantin Varlamov
93ad9fc551
Fuzzing triggers many assertion failures, which generates enormous log output. AFAIK, there is no way to make `NSAssertionHandler` _not_ write to log, and the assertions themselves are valid. Thus, implement a workaround: only output the last part of the logs in Travis, in order to avoid exceeding the log size limit (4 Mb).
123 lines
4.7 KiB
Bash
Executable File
123 lines
4.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
##
|
|
# Copyright 2018 Google
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# Run fuzz testing using xcodebuild. Given a total fuzzing duration, select
|
|
# a fuzzing target to run for half of the fuzzing duration. The remaining
|
|
# duration is split equally over the rest of the fuzzing targets. Must be run
|
|
# from the project root directory. The script is intended to execute on travis
|
|
# cron jobs, but can run locally (on a macOS) from the project root.
|
|
|
|
# Total time allowed for fuzzing in seconds (25 minutes for now).
|
|
readonly ALLOWED_TIME=1500
|
|
|
|
# An invalid target that is used to retrieve the list of available targets in
|
|
# the returned error message.
|
|
readonly INVALID_TARGET="INVALID_TARGET"
|
|
# The NONE target that does not perform fuzzing. It is valid but useless.
|
|
readonly NONE_TARGET="NONE"
|
|
|
|
# Script exit codes.
|
|
readonly EXIT_SUCCESS=0
|
|
readonly EXIT_FAILURE=1
|
|
|
|
# Get day of the year to use it when selecting a main target.
|
|
readonly DOY="$(date +%j)"
|
|
|
|
# How many last lines of the log to output for a failing job. This is to avoid
|
|
# exceeding Travis log size limit (4 Mb).
|
|
readonly LINES_TO_OUTPUT=500
|
|
|
|
# Run fuzz testing only if executing on xcode >= 9 because fuzzing requires
|
|
# Clang that supports -fsanitize-coverage=trace-pc-guard.
|
|
xcode_version="$(xcodebuild -version | head -n 1)"
|
|
xcode_version="${xcode_version/Xcode /}"
|
|
xcode_major="${xcode_version/.*/}"
|
|
|
|
if [[ "${xcode_major}" -lt 9 ]]; then
|
|
echo "Unsupported version of xcode."
|
|
exit ${EXIT_SUCCESS}
|
|
fi
|
|
|
|
# Helper to run fuzz tests using xcodebuild. Takes the fuzzing target as the
|
|
# first argument and the fuzzing duration as the second argument.
|
|
function run_xcode_fuzzing() {
|
|
xcodebuild \
|
|
-workspace 'Firestore/Example/Firestore.xcworkspace' \
|
|
-scheme 'Firestore_FuzzTests_iOS' \
|
|
-sdk 'iphonesimulator' \
|
|
-destination 'platform=iOS Simulator,name=iPhone 7' \
|
|
FUZZING_TARGET="$1" \
|
|
FUZZING_DURATION="$2" \
|
|
test
|
|
}
|
|
|
|
# Run fuzz testing with an INVALID_TARGET and grep the response message line
|
|
# that contains the string 'Available targets: {'.
|
|
fuzzing_targets_line="$(run_xcode_fuzzing ${INVALID_TARGET} "0" \
|
|
| grep "Available targets: {")"
|
|
|
|
# The fuzzing_targets_line string contains { TARGET1 TARGET2 ... TARGETN }.
|
|
# The following command extracts the targets and splits them into an array
|
|
# in the variable all_fuzzing_targets.
|
|
read -r -a all_fuzzing_targets <<< "$(echo ${fuzzing_targets_line} \
|
|
| cut -d "{" -f2 | cut -d "}" -f1)"
|
|
|
|
# Remove the NONE target, if it exists.
|
|
for i in "${!all_fuzzing_targets[@]}"; do
|
|
if [[ "${all_fuzzing_targets[${i}]}" == ${NONE_TARGET} ]]; then
|
|
unset all_fuzzing_targets[${i}]
|
|
fi
|
|
done
|
|
|
|
# Count all targets.
|
|
readonly fuzzing_targets_count="${#all_fuzzing_targets[@]}"
|
|
|
|
# Select a primary target to fuzz test for longer. Changes daily. The variable
|
|
# todays_primary_target contains the index of the target selected as primary.
|
|
todays_primary_target="$(( ${DOY} % ${fuzzing_targets_count} ))"
|
|
|
|
# Spend half of allowed time on the selected primary target and half on the
|
|
# remaining targets.
|
|
primary_target_time="$(( ${ALLOWED_TIME} / 2 ))"
|
|
other_targets_time="$(( ${ALLOWED_TIME} / 2 / $(( ${fuzzing_targets_count} - 1)) ))"
|
|
|
|
script_return=${EXIT_SUCCESS}
|
|
for i in "${!all_fuzzing_targets[@]}"; do
|
|
fuzzing_duration="${other_targets_time}"
|
|
if [[ "${i}" -eq "${todays_primary_target}" ]]; then
|
|
fuzzing_duration="${primary_target_time}"
|
|
fi
|
|
fuzzing_target="${all_fuzzing_targets[${i}]}"
|
|
|
|
echo "Running fuzzing target ${fuzzing_target} for ${fuzzing_duration} seconds..."
|
|
fuzzing_results="$(run_xcode_fuzzing "${fuzzing_target}" "${fuzzing_duration}")"
|
|
# Get the last line of the fuzzing results.
|
|
fuzzing_results_last_line="$(echo "${fuzzing_results}" | tail -n1)"
|
|
# If fuzzing succeeded, the last line will start with "Done"; otherwise,
|
|
# print all fuzzing results.
|
|
if [[ "${fuzzing_results_last_line}" == Done* ]]; then
|
|
echo "Fuzz testing for target ${fuzzing_target} succeeded. Ignore the ** TEST FAILED ** message."
|
|
else
|
|
echo "Error: Fuzz testing for target ${fuzzing_target} failed."
|
|
script_return=${EXIT_FAILURE}
|
|
echo "Fuzzing logs:"
|
|
echo "${fuzzing_results}" | tail -n${LINES_TO_OUTPUT}
|
|
echo "End fuzzing logs"
|
|
fi
|
|
done
|
|
|
|
exit ${script_return}
|