| #!/usr/bin/env bash |
| |
| # Copyright (C) 2020 Apple Inc. All rights reserved. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions |
| # are met: |
| # |
| # 1. Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # 2. Redistributions in binary form must reproduce the above copyright |
| # notice, this list of conditions and the following disclaimer in the |
| # documentation and/or other materials provided with the distribution. |
| # |
| # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY |
| # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY |
| # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
| # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| set -eo pipefail |
| |
| API_BENCH_EXECUTABLE=$(basename "$0") |
| |
| pushd "$(dirname "$0")" &> /dev/null |
| API_BENCH_DIR=$PWD |
| popd &> /dev/null |
| |
| CURRENT_BENCHMARK_ID=0 |
| BENCHMARKS=() |
| BUILD_DIRECTORIES=() |
| REFERENCES=("X") |
| CURRENT_API=() |
| UPCOMING_API=() |
| ARCHS=${ARCHS:-$(uname -m)} |
| CONFIGURATION=${CONFIGURATION:-Release} |
| |
| shouldBuild=true |
| runReferences=true |
| verbose=false |
| iterations=5 |
| |
| log() { |
| if [ "$verbose" == true ]; then |
| echo "$*" |
| fi |
| } |
| |
| printUsage() { |
| cat <<EOF |
| usage: $API_BENCH_EXECUTABLE [-h|--help] [--no-build] [--no-reference(s)] [-v|--verbose] [--iterations <n=$iterations>] <build_directory1> [<build_directory2> ...] |
| EOF |
| exit 0 |
| } |
| |
| collectFlags() { |
| while [ $# -gt 0 ]; do |
| case "$1" in |
| -h|--help) |
| printUsage |
| ;; |
| --no-build) |
| shouldBuild=false |
| ;; |
| --no-reference|--no-references) |
| runReferences=false |
| ;; |
| -v|--verbose) |
| verbose=true |
| ;; |
| --iterations) |
| shift |
| iterations=$1 |
| ;; |
| *) |
| BUILD_DIRECTORIES=("$@") |
| break 2 |
| ;; |
| esac |
| shift |
| done |
| |
| if [ ${#BUILD_DIRECTORIES[@]} -lt 1 ]; then |
| printUsage |
| exit 1 |
| fi |
| |
| for i in "${!BUILD_DIRECTORIES[@]}"; do |
| local buildDirectory |
| buildDirectory=${BUILD_DIRECTORIES[$i]} |
| |
| if ! [ -d "$buildDirectory" ]; then |
| echo "No such file or directory: $buildDirectory" |
| exit 1 |
| fi |
| |
| pushd "$buildDirectory" &> /dev/null |
| BUILD_DIRECTORIES[$i]=$PWD |
| popd &> /dev/null |
| done |
| } |
| |
| collectBenchmarkDir() { |
| local benchmarkVar=$1 |
| local benchmarkDir=$2 |
| benchmarks=("$API_BENCH_DIR"/"$benchmarkDir"/*) |
| for benchmark in "${benchmarks[@]}"; do |
| local benchmarkID=$CURRENT_BENCHMARK_ID |
| CURRENT_BENCHMARK_ID=$((CURRENT_BENCHMARK_ID+1)) |
| pushd "$benchmark" &> /dev/null |
| BENCHMARKS[$benchmarkID]=$PWD |
| eval "${benchmarkVar}[\$benchmarkID]=$benchmarkID" |
| popd &> /dev/null |
| done |
| } |
| |
| collectBenchmarks() { |
| if [ "$runReferences" == true ]; then |
| collectBenchmarkDir "REFERENCES" References |
| fi |
| collectBenchmarkDir "CURRENT_API" CurrentAPI |
| collectBenchmarkDir "UPCOMING_API" UpcomingAPI |
| } |
| |
| forEachBenchmark() { |
| local functor |
| functor="$1" |
| declare "$functor" |
| for benchmarkID in "${!BENCHMARKS[@]}"; do |
| local benchmark |
| benchmark=${BENCHMARKS[$benchmarkID]} |
| pushd "$benchmark" &> /dev/null |
| BENCHMARK_ID=$benchmarkID |
| BENCHMARK_NAME=$(basename "$benchmark") |
| "$functor" |
| popd &> /dev/null |
| done |
| } |
| |
| buildBenchmark() { |
| project=$BENCHMARK_NAME.xcodeproj |
| |
| if ! [ -e "$project" ]; then |
| return |
| fi |
| |
| log "Building $BENCHMARK_NAME... (FRAMEWORK_SEARCH_PATHS='$BUILD_DIRECTORY')" |
| xcodebuild -project "$project" -target "$BENCHMARK_NAME" -configuration "$CONFIGURATION" build "FRAMEWORK_SEARCH_PATHS='$BUILD_DIRECTORY'" "ARCHS='$ARCHS'" |
| } |
| |
| build() { |
| BUILD_DIRECTORY=$1 |
| forEachBenchmark "buildBenchmark" |
| } |
| |
| runJSBenchmark() { |
| "$DYLD_FRAMEWORK_PATH/jsc" "$BENCHMARK_NAME.js" |
| } |
| |
| runNativeBenchmark() { |
| pushd "build/$CONFIGURATION" &> /dev/null |
| "./$BENCHMARK_NAME" |
| local exitCode=$? |
| if [ $exitCode -ne 0 ]; then |
| echo "Benchmark exited with an error: $PWD/$BENCHMARK_NAME exited with $exitCode" 1>&2 |
| fi |
| popd &> /dev/null |
| return $exitCode |
| } |
| |
| runBenchmarkIteration() { |
| if [ -e "build" ]; then |
| runNativeBenchmark |
| else |
| runJSBenchmark |
| fi |
| } |
| |
| runBenchmark() { |
| log "Running $BENCHMARK_NAME..." |
| |
| local result |
| result=$(runBenchmarkIteration) |
| if [ $? -ne 0 ]; then |
| exit 1 |
| fi |
| RESULTS[$((BENCHMARK_ID * iterations + ITERATION))]=$result |
| log "Finished in ${result}ms" |
| } |
| |
| computeAverage() { |
| local benchmarkID benchmark sum avg |
| benchmarkID=$1 |
| benchmark=${BENCHMARKS[$benchmarkID]} |
| sum=0 |
| for iteration in $(seq 0 1 $((iterations - 1))); do |
| local result |
| result=${RESULTS[$((benchmarkID * iterations + iteration))]} |
| sum=$((sum + result)) |
| done |
| bc <<< "$sum.0 / $iterations.0" |
| } |
| |
| printAverage() { |
| local benchmarkID benchmark name avg |
| benchmarkID=$1 |
| avg=$2 |
| benchmark=${BENCHMARKS[$benchmarkID]} |
| name=$(basename "$benchmark") |
| echo "$name: ${avg}ms" |
| } |
| |
| printBenchmark() { |
| printAverage "$@" "$(computeAverage "$@")" |
| } |
| |
| printResults() { |
| local buildDirectory=$1 |
| |
| echo "Results for $buildDirectory ($iterations iterations)" |
| echo "================================================================================" |
| |
| if [ "$runReferences" == true ]; then |
| echo "" |
| echo "References:" |
| echo "----------------------------------------" |
| for benchmarkID in "${REFERENCES[@]}"; do |
| printBenchmark "$benchmarkID" |
| done |
| fi |
| |
| local score count |
| score=0 |
| count=0 |
| |
| printCategory() { |
| local category=$1 |
| |
| echo "" |
| echo "$category:" |
| echo "----------------------------------------" |
| |
| for benchmarkID in $(eval 'echo ${'"$category"'[@]}'); do |
| avg=$(computeAverage "$benchmarkID") |
| printAverage "$benchmarkID" "$avg" |
| count=$((count + 1)) |
| score=$(echo "$score" "$avg" | awk '{print ($1 + log($2)) }') |
| done |
| } |
| printCategory "CURRENT_API" |
| printCategory "UPCOMING_API" |
| score=$(echo "$score" "$count" | awk '{ print(5000.0 / exp($1 / $2)) }') |
| |
| echo "" |
| echo "Score: $score" |
| } |
| |
| run() { |
| local buildDirectory |
| buildDirectory=$1 |
| |
| export DYLD_FRAMEWORK_PATH=$buildDirectory |
| for iteration in $(seq 0 1 $((iterations-1))); do |
| ITERATION=$iteration |
| forEachBenchmark "runBenchmark" |
| done |
| |
| printResults "$buildDirectory" |
| } |
| |
| |
| main() { |
| collectFlags "$@" |
| collectBenchmarks |
| for buildDirectory in "${BUILD_DIRECTORIES[@]}"; do |
| if [ "$shouldBuild" == true ]; then |
| build "$buildDirectory" |
| fi |
| |
| run "$buildDirectory" |
| done |
| } |
| |
| main "$@" |