blob: bc6028a6de7b952e782f732aa9c1dff11a703c76 [file] [log] [blame]
#!/usr/bin/env bash
# Copyright (C) 2020 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
set -eo pipefail
API_BENCH_EXECUTABLE=$(basename "$0")
pushd "$(dirname "$0")" &> /dev/null
API_BENCH_DIR=$PWD
popd &> /dev/null
CURRENT_BENCHMARK_ID=0
BENCHMARKS=()
BUILD_DIRECTORIES=()
REFERENCES=("X")
CURRENT_API=()
UPCOMING_API=()
ARCHS=${ARCHS:-$(uname -m)}
CONFIGURATION=${CONFIGURATION:-Release}
shouldBuild=true
runReferences=true
verbose=false
iterations=5
log() {
if [ "$verbose" == true ]; then
echo "$*"
fi
}
printUsage() {
cat <<EOF
usage: $API_BENCH_EXECUTABLE [-h|--help] [--no-build] [--no-reference(s)] [-v|--verbose] [--iterations <n=$iterations>] <build_directory1> [<build_directory2> ...]
EOF
exit 0
}
collectFlags() {
while [ $# -gt 0 ]; do
case "$1" in
-h|--help)
printUsage
;;
--no-build)
shouldBuild=false
;;
--no-reference|--no-references)
runReferences=false
;;
-v|--verbose)
verbose=true
;;
--iterations)
shift
iterations=$1
;;
*)
BUILD_DIRECTORIES=("$@")
break 2
;;
esac
shift
done
if [ ${#BUILD_DIRECTORIES[@]} -lt 1 ]; then
printUsage
exit 1
fi
for i in "${!BUILD_DIRECTORIES[@]}"; do
local buildDirectory
buildDirectory=${BUILD_DIRECTORIES[$i]}
if ! [ -d "$buildDirectory" ]; then
echo "No such file or directory: $buildDirectory"
exit 1
fi
pushd "$buildDirectory" &> /dev/null
BUILD_DIRECTORIES[$i]=$PWD
popd &> /dev/null
done
}
collectBenchmarkDir() {
local benchmarkVar=$1
local benchmarkDir=$2
benchmarks=("$API_BENCH_DIR"/"$benchmarkDir"/*)
for benchmark in "${benchmarks[@]}"; do
local benchmarkID=$CURRENT_BENCHMARK_ID
CURRENT_BENCHMARK_ID=$((CURRENT_BENCHMARK_ID+1))
pushd "$benchmark" &> /dev/null
BENCHMARKS[$benchmarkID]=$PWD
eval "${benchmarkVar}[\$benchmarkID]=$benchmarkID"
popd &> /dev/null
done
}
collectBenchmarks() {
if [ "$runReferences" == true ]; then
collectBenchmarkDir "REFERENCES" References
fi
collectBenchmarkDir "CURRENT_API" CurrentAPI
collectBenchmarkDir "UPCOMING_API" UpcomingAPI
}
forEachBenchmark() {
local functor
functor="$1"
declare "$functor"
for benchmarkID in "${!BENCHMARKS[@]}"; do
local benchmark
benchmark=${BENCHMARKS[$benchmarkID]}
pushd "$benchmark" &> /dev/null
BENCHMARK_ID=$benchmarkID
BENCHMARK_NAME=$(basename "$benchmark")
"$functor"
popd &> /dev/null
done
}
buildBenchmark() {
project=$BENCHMARK_NAME.xcodeproj
if ! [ -e "$project" ]; then
return
fi
log "Building $BENCHMARK_NAME... (FRAMEWORK_SEARCH_PATHS='$BUILD_DIRECTORY')"
xcodebuild -project "$project" -target "$BENCHMARK_NAME" -configuration "$CONFIGURATION" build "FRAMEWORK_SEARCH_PATHS='$BUILD_DIRECTORY'" "ARCHS='$ARCHS'"
}
build() {
BUILD_DIRECTORY=$1
forEachBenchmark "buildBenchmark"
}
runJSBenchmark() {
"$DYLD_FRAMEWORK_PATH/jsc" "$BENCHMARK_NAME.js"
}
runNativeBenchmark() {
pushd "build/$CONFIGURATION" &> /dev/null
"./$BENCHMARK_NAME"
local exitCode=$?
if [ $exitCode -ne 0 ]; then
echo "Benchmark exited with an error: $PWD/$BENCHMARK_NAME exited with $exitCode" 1>&2
fi
popd &> /dev/null
return $exitCode
}
runBenchmarkIteration() {
if [ -e "build" ]; then
runNativeBenchmark
else
runJSBenchmark
fi
}
runBenchmark() {
log "Running $BENCHMARK_NAME..."
local result
result=$(runBenchmarkIteration)
if [ $? -ne 0 ]; then
exit 1
fi
RESULTS[$((BENCHMARK_ID * iterations + ITERATION))]=$result
log "Finished in ${result}ms"
}
computeAverage() {
local benchmarkID benchmark sum avg
benchmarkID=$1
benchmark=${BENCHMARKS[$benchmarkID]}
sum=0
for iteration in $(seq 0 1 $((iterations - 1))); do
local result
result=${RESULTS[$((benchmarkID * iterations + iteration))]}
sum=$((sum + result))
done
bc <<< "$sum.0 / $iterations.0"
}
printAverage() {
local benchmarkID benchmark name avg
benchmarkID=$1
avg=$2
benchmark=${BENCHMARKS[$benchmarkID]}
name=$(basename "$benchmark")
echo "$name: ${avg}ms"
}
printBenchmark() {
printAverage "$@" "$(computeAverage "$@")"
}
printResults() {
local buildDirectory=$1
echo "Results for $buildDirectory ($iterations iterations)"
echo "================================================================================"
if [ "$runReferences" == true ]; then
echo ""
echo "References:"
echo "----------------------------------------"
for benchmarkID in "${REFERENCES[@]}"; do
printBenchmark "$benchmarkID"
done
fi
local score count
score=0
count=0
printCategory() {
local category=$1
echo ""
echo "$category:"
echo "----------------------------------------"
for benchmarkID in $(eval 'echo ${'"$category"'[@]}'); do
avg=$(computeAverage "$benchmarkID")
printAverage "$benchmarkID" "$avg"
count=$((count + 1))
score=$(echo "$score" "$avg" | awk '{print ($1 + log($2)) }')
done
}
printCategory "CURRENT_API"
printCategory "UPCOMING_API"
score=$(echo "$score" "$count" | awk '{ print(5000.0 / exp($1 / $2)) }')
echo ""
echo "Score: $score"
}
run() {
local buildDirectory
buildDirectory=$1
export DYLD_FRAMEWORK_PATH=$buildDirectory
for iteration in $(seq 0 1 $((iterations-1))); do
ITERATION=$iteration
forEachBenchmark "runBenchmark"
done
printResults "$buildDirectory"
}
main() {
collectFlags "$@"
collectBenchmarks
for buildDirectory in "${BUILD_DIRECTORIES[@]}"; do
if [ "$shouldBuild" == true ]; then
build "$buildDirectory"
fi
run "$buildDirectory"
done
}
main "$@"