#!/usr/bin/env ruby

# Copyright (C) 2011-2015 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGE.

require 'rubygems'

require 'getoptlong'
require 'pathname'
require 'shellwords'
require 'socket'

begin
  require 'json'
rescue LoadError => e
  $stderr.puts "It does not appear that you have the 'json' package installed.  Try running 'sudo gem install json'."
  exit 1
end

SCRIPT_PATH = Pathname.new(__FILE__).realpath

raise unless SCRIPT_PATH.dirname.basename.to_s == "Scripts"
raise unless SCRIPT_PATH.dirname.dirname.basename.to_s == "Tools"

OPENSOURCE_PATH = SCRIPT_PATH.dirname.dirname.dirname

SUNSPIDER_PATH = OPENSOURCE_PATH + "PerformanceTests" + "SunSpider" + "tests" + "sunspider-1.0"
LONGSPIDER_PATH = OPENSOURCE_PATH + "PerformanceTests" + "LongSpider"
V8_PATH = OPENSOURCE_PATH + "PerformanceTests" + "SunSpider" + "tests" + "v8-v6"
JSREGRESS_PATH = OPENSOURCE_PATH + "LayoutTests" + "js" + "regress" + "script-tests"
OCTANE_WRAPPER_PATH = OPENSOURCE_PATH + "PerformanceTests" + "Octane" + "wrappers"

TEMP_PATH = OPENSOURCE_PATH + "BenchmarkTemp"

if TEMP_PATH.exist?
  raise unless TEMP_PATH.directory?
else
  Dir.mkdir(TEMP_PATH)
end

BENCH_DATA_PATH = TEMP_PATH + "benchdata"

IBR_LOOKUP=[0.00615583, 0.0975, 0.22852, 0.341628, 0.430741, 0.500526, 0.555933, 
            0.600706, 0.637513, 0.668244, 0.694254, 0.716537, 0.735827, 0.752684, 
            0.767535, 0.780716, 0.792492, 0.803074, 0.812634, 0.821313, 0.829227, 
            0.836472, 0.843129, 0.849267, 0.854943, 0.860209, 0.865107, 0.869674, 
            0.873942, 0.877941, 0.881693, 0.885223, 0.888548, 0.891686, 0.894652, 
            0.897461, 0.900124, 0.902652, 0.905056, 0.907343, 0.909524, 0.911604, 
            0.91359, 0.91549, 0.917308, 0.919049, 0.920718, 0.92232, 0.923859, 0.925338, 
            0.926761, 0.92813, 0.929449, 0.930721, 0.931948, 0.933132, 0.934275, 0.93538, 
            0.936449, 0.937483, 0.938483, 0.939452, 0.940392, 0.941302, 0.942185, 
            0.943042, 0.943874, 0.944682, 0.945467, 0.94623, 0.946972, 0.947694, 
            0.948396, 0.94908, 0.949746, 0.950395, 0.951027, 0.951643, 0.952244, 
            0.952831, 0.953403, 0.953961, 0.954506, 0.955039, 0.955559, 0.956067, 
            0.956563, 0.957049, 0.957524, 0.957988, 0.958443, 0.958887, 0.959323, 
            0.959749, 0.960166, 0.960575, 0.960975, 0.961368, 0.961752, 0.962129, 
            0.962499, 0.962861, 0.963217, 0.963566, 0.963908, 0.964244, 0.964574, 
            0.964897, 0.965215, 0.965527, 0.965834, 0.966135, 0.966431, 0.966722, 
            0.967007, 0.967288, 0.967564, 0.967836, 0.968103, 0.968366, 0.968624, 
            0.968878, 0.969128, 0.969374, 0.969617, 0.969855, 0.97009, 0.970321, 
            0.970548, 0.970772, 0.970993, 0.97121, 0.971425, 0.971636, 0.971843, 
            0.972048, 0.97225, 0.972449, 0.972645, 0.972839, 0.973029, 0.973217, 
            0.973403, 0.973586, 0.973766, 0.973944, 0.97412, 0.974293, 0.974464, 
            0.974632, 0.974799, 0.974963, 0.975125, 0.975285, 0.975443, 0.975599, 
            0.975753, 0.975905, 0.976055, 0.976204, 0.97635, 0.976495, 0.976638, 
            0.976779, 0.976918, 0.977056, 0.977193, 0.977327, 0.97746, 0.977592, 
            0.977722, 0.97785, 0.977977, 0.978103, 0.978227, 0.978349, 0.978471, 
            0.978591, 0.978709, 0.978827, 0.978943, 0.979058, 0.979171, 0.979283, 
            0.979395, 0.979504, 0.979613, 0.979721, 0.979827, 0.979933, 0.980037, 
            0.98014, 0.980242, 0.980343, 0.980443, 0.980543, 0.980641, 0.980738, 
            0.980834, 0.980929, 0.981023, 0.981116, 0.981209, 0.9813, 0.981391, 0.981481, 
            0.981569, 0.981657, 0.981745, 0.981831, 0.981916, 0.982001, 0.982085, 
            0.982168, 0.982251, 0.982332, 0.982413, 0.982493, 0.982573, 0.982651, 
            0.982729, 0.982807, 0.982883, 0.982959, 0.983034, 0.983109, 0.983183, 
            0.983256, 0.983329, 0.983401, 0.983472, 0.983543, 0.983613, 0.983683, 
            0.983752, 0.98382, 0.983888, 0.983956, 0.984022, 0.984089, 0.984154, 
            0.984219, 0.984284, 0.984348, 0.984411, 0.984474, 0.984537, 0.984599, 
            0.98466, 0.984721, 0.984782, 0.984842, 0.984902, 0.984961, 0.985019, 
            0.985077, 0.985135, 0.985193, 0.985249, 0.985306, 0.985362, 0.985417, 
            0.985472, 0.985527, 0.985582, 0.985635, 0.985689, 0.985742, 0.985795, 
            0.985847, 0.985899, 0.985951, 0.986002, 0.986053, 0.986103, 0.986153, 
            0.986203, 0.986252, 0.986301, 0.98635, 0.986398, 0.986446, 0.986494, 
            0.986541, 0.986588, 0.986635, 0.986681, 0.986727, 0.986773, 0.986818, 
            0.986863, 0.986908, 0.986953, 0.986997, 0.987041, 0.987084, 0.987128, 
            0.987171, 0.987213, 0.987256, 0.987298, 0.98734, 0.987381, 0.987423, 
            0.987464, 0.987504, 0.987545, 0.987585, 0.987625, 0.987665, 0.987704, 
            0.987744, 0.987783, 0.987821, 0.98786, 0.987898, 0.987936, 0.987974, 
            0.988011, 0.988049, 0.988086, 0.988123, 0.988159, 0.988196, 0.988232, 
            0.988268, 0.988303, 0.988339, 0.988374, 0.988409, 0.988444, 0.988479, 
            0.988513, 0.988547, 0.988582, 0.988615, 0.988649, 0.988682, 0.988716, 
            0.988749, 0.988782, 0.988814, 0.988847, 0.988879, 0.988911, 0.988943, 
            0.988975, 0.989006, 0.989038, 0.989069, 0.9891, 0.989131, 0.989161, 0.989192, 
            0.989222, 0.989252, 0.989282, 0.989312, 0.989342, 0.989371, 0.989401, 
            0.98943, 0.989459, 0.989488, 0.989516, 0.989545, 0.989573, 0.989602, 0.98963, 
            0.989658, 0.989685, 0.989713, 0.98974, 0.989768, 0.989795, 0.989822, 
            0.989849, 0.989876, 0.989902, 0.989929, 0.989955, 0.989981, 0.990007, 
            0.990033, 0.990059, 0.990085, 0.99011, 0.990136, 0.990161, 0.990186, 
            0.990211, 0.990236, 0.990261, 0.990285, 0.99031, 0.990334, 0.990358, 
            0.990383, 0.990407, 0.99043, 0.990454, 0.990478, 0.990501, 0.990525, 
            0.990548, 0.990571, 0.990594, 0.990617, 0.99064, 0.990663, 0.990686, 
            0.990708, 0.990731, 0.990753, 0.990775, 0.990797, 0.990819, 0.990841, 
            0.990863, 0.990885, 0.990906, 0.990928, 0.990949, 0.99097, 0.990991, 
            0.991013, 0.991034, 0.991054, 0.991075, 0.991096, 0.991116, 0.991137, 
            0.991157, 0.991178, 0.991198, 0.991218, 0.991238, 0.991258, 0.991278, 
            0.991298, 0.991317, 0.991337, 0.991356, 0.991376, 0.991395, 0.991414, 
            0.991433, 0.991452, 0.991471, 0.99149, 0.991509, 0.991528, 0.991547, 
            0.991565, 0.991584, 0.991602, 0.99162, 0.991639, 0.991657, 0.991675, 
            0.991693, 0.991711, 0.991729, 0.991746, 0.991764, 0.991782, 0.991799, 
            0.991817, 0.991834, 0.991851, 0.991869, 0.991886, 0.991903, 0.99192, 
            0.991937, 0.991954, 0.991971, 0.991987, 0.992004, 0.992021, 0.992037, 
            0.992054, 0.99207, 0.992086, 0.992103, 0.992119, 0.992135, 0.992151, 
            0.992167, 0.992183, 0.992199, 0.992215, 0.99223, 0.992246, 0.992262, 
            0.992277, 0.992293, 0.992308, 0.992324, 0.992339, 0.992354, 0.992369, 
            0.992384, 0.9924, 0.992415, 0.992429, 0.992444, 0.992459, 0.992474, 0.992489, 
            0.992503, 0.992518, 0.992533, 0.992547, 0.992561, 0.992576, 0.99259, 
            0.992604, 0.992619, 0.992633, 0.992647, 0.992661, 0.992675, 0.992689, 
            0.992703, 0.992717, 0.99273, 0.992744, 0.992758, 0.992771, 0.992785, 
            0.992798, 0.992812, 0.992825, 0.992839, 0.992852, 0.992865, 0.992879, 
            0.992892, 0.992905, 0.992918, 0.992931, 0.992944, 0.992957, 0.99297, 
            0.992983, 0.992995, 0.993008, 0.993021, 0.993034, 0.993046, 0.993059, 
            0.993071, 0.993084, 0.993096, 0.993109, 0.993121, 0.993133, 0.993145, 
            0.993158, 0.99317, 0.993182, 0.993194, 0.993206, 0.993218, 0.99323, 0.993242, 
            0.993254, 0.993266, 0.993277, 0.993289, 0.993301, 0.993312, 0.993324, 
            0.993336, 0.993347, 0.993359, 0.99337, 0.993382, 0.993393, 0.993404, 
            0.993416, 0.993427, 0.993438, 0.993449, 0.99346, 0.993472, 0.993483, 
            0.993494, 0.993505, 0.993516, 0.993527, 0.993538, 0.993548, 0.993559, 
            0.99357, 0.993581, 0.993591, 0.993602, 0.993613, 0.993623, 0.993634, 
            0.993644, 0.993655, 0.993665, 0.993676, 0.993686, 0.993697, 0.993707, 
            0.993717, 0.993727, 0.993738, 0.993748, 0.993758, 0.993768, 0.993778, 
            0.993788, 0.993798, 0.993808, 0.993818, 0.993828, 0.993838, 0.993848, 
            0.993858, 0.993868, 0.993877, 0.993887, 0.993897, 0.993907, 0.993916, 
            0.993926, 0.993935, 0.993945, 0.993954, 0.993964, 0.993973, 0.993983, 
            0.993992, 0.994002, 0.994011, 0.99402, 0.99403, 0.994039, 0.994048, 0.994057, 
            0.994067, 0.994076, 0.994085, 0.994094, 0.994103, 0.994112, 0.994121, 
            0.99413, 0.994139, 0.994148, 0.994157, 0.994166, 0.994175, 0.994183, 
            0.994192, 0.994201, 0.99421, 0.994218, 0.994227, 0.994236, 0.994244, 
            0.994253, 0.994262, 0.99427, 0.994279, 0.994287, 0.994296, 0.994304, 
            0.994313, 0.994321, 0.994329, 0.994338, 0.994346, 0.994354, 0.994363, 
            0.994371, 0.994379, 0.994387, 0.994395, 0.994404, 0.994412, 0.99442, 
            0.994428, 0.994436, 0.994444, 0.994452, 0.99446, 0.994468, 0.994476, 
            0.994484, 0.994492, 0.9945, 0.994508, 0.994516, 0.994523, 0.994531, 0.994539, 
            0.994547, 0.994554, 0.994562, 0.99457, 0.994577, 0.994585, 0.994593, 0.9946, 
            0.994608, 0.994615, 0.994623, 0.994631, 0.994638, 0.994645, 0.994653, 
            0.99466, 0.994668, 0.994675, 0.994683, 0.99469, 0.994697, 0.994705, 0.994712, 
            0.994719, 0.994726, 0.994734, 0.994741, 0.994748, 0.994755, 0.994762, 
            0.994769, 0.994777, 0.994784, 0.994791, 0.994798, 0.994805, 0.994812, 
            0.994819, 0.994826, 0.994833, 0.99484, 0.994847, 0.994854, 0.99486, 0.994867, 
            0.994874, 0.994881, 0.994888, 0.994895, 0.994901, 0.994908, 0.994915, 
            0.994922, 0.994928, 0.994935, 0.994942, 0.994948, 0.994955, 0.994962, 
            0.994968, 0.994975, 0.994981, 0.994988, 0.994994, 0.995001, 0.995007, 
            0.995014, 0.99502, 0.995027, 0.995033, 0.99504, 0.995046, 0.995052, 0.995059, 
            0.995065, 0.995071, 0.995078, 0.995084, 0.99509, 0.995097, 0.995103, 
            0.995109, 0.995115, 0.995121, 0.995128, 0.995134, 0.99514, 0.995146, 
            0.995152, 0.995158, 0.995164, 0.995171, 0.995177, 0.995183, 0.995189, 
            0.995195, 0.995201, 0.995207, 0.995213, 0.995219, 0.995225, 0.995231, 
            0.995236, 0.995242, 0.995248, 0.995254, 0.99526, 0.995266, 0.995272, 
            0.995277, 0.995283, 0.995289, 0.995295, 0.995301, 0.995306, 0.995312, 
            0.995318, 0.995323, 0.995329, 0.995335, 0.99534, 0.995346, 0.995352, 
            0.995357, 0.995363, 0.995369, 0.995374, 0.99538, 0.995385, 0.995391, 
            0.995396, 0.995402, 0.995407, 0.995413, 0.995418, 0.995424, 0.995429, 
            0.995435, 0.99544, 0.995445, 0.995451, 0.995456, 0.995462, 0.995467, 
            0.995472, 0.995478, 0.995483, 0.995488, 0.995493, 0.995499, 0.995504, 
            0.995509, 0.995515, 0.99552, 0.995525, 0.99553, 0.995535, 0.995541, 0.995546, 
            0.995551, 0.995556, 0.995561, 0.995566, 0.995571, 0.995577, 0.995582, 
            0.995587, 0.995592, 0.995597, 0.995602, 0.995607, 0.995612, 0.995617, 
            0.995622, 0.995627, 0.995632, 0.995637, 0.995642, 0.995647, 0.995652, 
            0.995657, 0.995661, 0.995666, 0.995671, 0.995676, 0.995681, 0.995686, 
            0.995691, 0.995695, 0.9957, 0.995705, 0.99571, 0.995715, 0.995719, 0.995724, 
            0.995729, 0.995734, 0.995738, 0.995743, 0.995748, 0.995753, 0.995757, 
            0.995762, 0.995767, 0.995771, 0.995776, 0.995781, 0.995785, 0.99579, 
            0.995794, 0.995799, 0.995804, 0.995808, 0.995813, 0.995817, 0.995822, 
            0.995826, 0.995831, 0.995835, 0.99584, 0.995844, 0.995849, 0.995853, 
            0.995858, 0.995862, 0.995867, 0.995871, 0.995876, 0.99588, 0.995885, 
            0.995889, 0.995893, 0.995898, 0.995902, 0.995906, 0.995911, 0.995915, 
            0.99592, 0.995924, 0.995928, 0.995932, 0.995937, 0.995941, 0.995945, 0.99595, 
            0.995954, 0.995958, 0.995962, 0.995967, 0.995971, 0.995975, 0.995979, 
            0.995984, 0.995988, 0.995992, 0.995996, 0.996, 0.996004, 0.996009, 0.996013, 
            0.996017, 0.996021, 0.996025, 0.996029, 0.996033, 0.996037, 0.996041, 
            0.996046, 0.99605, 0.996054, 0.996058, 0.996062, 0.996066, 0.99607, 0.996074, 
            0.996078, 0.996082, 0.996086, 0.99609, 0.996094, 0.996098, 0.996102, 
            0.996106, 0.99611, 0.996114, 0.996117, 0.996121, 0.996125, 0.996129, 
            0.996133, 0.996137, 0.996141, 0.996145, 0.996149, 0.996152, 0.996156, 
            0.99616, 0.996164]

# Run-time configuration parameters (can be set with command-line options)

$rerun=1
$inner=1
$warmup=1
$outer=4
$quantum=1000
$includeSunSpider=true
$includeLongSpider=true
$includeV8=true
$includeKraken=true
$includeJSBench=true
$includeJSRegress=true
$includeAsmBench=true
$includeDSPJS=true
$includeBrowsermarkJS=false
$includeBrowsermarkDOM=false
$includeOctane=true
$includeCompressionBench = true
$measureGC=false
$benchmarkPattern=nil
$verbosity=0
$timeMode=:preciseTime
$forceVMKind=nil
$brief=false
$silent=false
$remoteHosts=[]
$alsoLocal=false
$sshOptions=[]
$vms = []
$environment = {}
$needToCopyVMs = false
$dontCopyVMs = false
$allDRT = true
$outputName = nil
$sunSpiderWarmup = true
$configPath = Pathname.new(ENV["HOME"]) + ".run-jsc-benchmarks"

$prepare = true
$run = true
$analyze = []

# Helpful functions and classes

def smallUsage
  puts "Use the --help option to get basic usage information."
  exit 1
end

def usage
  puts "run-jsc-benchmarks [options] <vm1> [<vm2> ...]"
  puts
  puts "Runs one or more JavaScript runtimes against SunSpider, V8, and/or Kraken"
  puts "benchmarks, and reports detailed statistics.  What makes run-jsc-benchmarks"
  puts "special is that each benchmark/VM configuration is run in a single VM invocation,"
  puts "and the invocations are run in random order.  This minimizes systematics due to"
  puts "one benchmark polluting the running time of another.  The fine-grained"
  puts "interleaving of VM invocations further minimizes systematics due to changes in"
  puts "the performance or behavior of your machine."
  puts 
  puts "Run-jsc-benchmarks is highly configurable.  You can compare as many VMs as you"
  puts "like.  You can change the amount of warm-up iterations, number of iterations"
  puts "executed per VM invocation, and the number of VM invocations per benchmark."
  puts
  puts "The <vm> should be either a path to a JavaScript runtime executable (such as"
  puts "jsc), or a string of the form <name>:<path>, where the <path> is the path to"
  puts "the executable and <name> is the name that you would like to give the"
  puts "configuration for the purposeof reporting.  If no name is given, a generic name"
  puts "of the form Conf#<n> will be ascribed to the configuration automatically."
  puts
  puts "It's also possible to specify per-VM environment variables. For example, you"
  puts "might specify a VM like Foo:JSC_useJIT=false:/path/to/jsc, in which case the"
  puts "harness will set the JSC_useJIT environment variable to false just before running"
  puts "the given VM. Note that the harness will not unset the environment variable, so"
  puts "you must ensure that your other VMs will use the opposite setting"
  puts "(JSC_useJIT=true in this case)."
  puts
  puts "Options:"
  puts "--rerun <n>          Set the number of iterations of the benchmark that"
  puts "                     contribute to the measured run time.  Default is #{$rerun}."
  puts "--inner <n>          Set the number of inner (per-runtime-invocation)"
  puts "                     iterations.  Default is #{$inner}."
  puts "--outer <n>          Set the number of runtime invocations for each benchmark."
  puts "                     Default is #{$outer}."
  puts "--warmup <n>         Set the number of warm-up runs per invocation.  Default"
  puts "                     is #{$warmup}. This has a different effect on different kinds"
  puts "                     benchmarks. Some benchmarks have no notion of warm-up."
  puts "--no-ss-warmup       Disable SunSpider-based warm-up runs."
  puts "--quantum <n>        Set the duration in milliseconds for which an iteration of"
  puts "                     a throughput benchmark should be run.  Default is #{$quantum}."
  puts "--timing-mode        Set the way that time is measured.  Possible values"
  puts "                     are 'preciseTime' and 'date'.  Default is 'preciseTime'."
  puts "--force-vm-kind      Turn off auto-detection of VM kind, and assume that it is"
  puts "                     the one specified.  Valid arguments are 'jsc', "
  puts "                     'DumpRenderTree', or 'WebKitTestRunner'."
  puts "--force-vm-copy      Force VM builds to be copied to the working directory."
  puts "                     This may reduce pathologies resulting from path names."
  puts "--dont-copy-vms      Don't copy VMs even when doing a remote benchmarking run;"
  puts "                     instead assume that they are already there."
  puts "--sunspider          Only run SunSpider."
  puts "--v8-spider          Only run V8."
  puts "--kraken             Only run Kraken."
  puts "--js-bench           Only run JSBench."
  puts "--js-regress         Only run JSRegress."
  puts "--dsp                Only run DSP."
  puts "--asm-bench          Only run AsmBench."
  puts "--browsermark-js     Only run browsermark-js."
  puts "--browsermark-dom    Only run browsermark-dom."
  puts "--octane             Only run Octane."
  puts "--compression-bench  Only run compression bench"
  puts "                     The default is to run all benchmarks. The above options can"
  puts "                     be combined to run any subset (so --sunspider --dsp will run"
  puts "                     both SunSpider and DSP)."
  puts "--benchmarks         Only run benchmarks matching the given regular expression."
  puts "--measure-gc         Turn off manual calls to gc(), so that GC time is measured."
  puts "                     Works best with large values of --inner.  You can also say"
  puts "                     --measure-gc <conf>, which turns this on for one"
  puts "                     configuration only."
  puts "--verbose or -v      Print more stuff."
  puts "--brief              Print only the final result for each VM."
  puts "--silent             Don't print progress. This might slightly reduce some"
  puts "                     performance perturbation."
  puts "--remote <sshhosts>  Perform performance measurements remotely, on the given"
  puts "                     SSH host(s). Easiest way to use this is to specify the SSH"
  puts "                     user@host string. However, you can also supply a comma-"
  puts "                     separated list of SSH hosts. Alternatively, you can use this"
  puts "                     option multiple times to specify multiple hosts. This"
  puts "                     automatically copies the WebKit release builds of the VMs"
  puts "                     you specified to all of the hosts."
  puts "--ssh-options        Pass additional options to SSH."
  puts "--local              Also do a local benchmark run even when doing --remote."
  puts "--vms                Use a JSON file to specify which VMs to run, as opposed to"
  puts "                     specifying them on the command line."
  puts "--prepare-only       Only prepare the runscript (a shell script that"
  puts "                     invokes the VMs to run benchmarks) but don't run it."
  puts "--analyze            Only read the output of the runscript but don't do anything"
  puts "                     else. This requires passing the same arguments that you"
  puts "                     passed when running --prepare-only."
  puts "--output-name        Base of the filenames to put results into. Will write a file"
  puts "                     called <base>_report.txt and <base>.json.  By default this"
  puts "                     name is automatically synthesized from the machine name,"
  puts "                     date, set of benchmarks run, and set of configurations."
  puts "--environment        JSON file that specifies the environment variables that should"
  puts "                     be used for particular VMs and benchmarks."
  puts "--config <path>      Specify the path of the configuration file. Defaults to"
  puts "                     ~/.run-jsc-benchmarks"
  puts "--help or -h         Display this message."
  puts
  puts "Example:"
  puts "run-jsc-benchmarks TipOfTree:/Volumes/Data/pizlo/OpenSource/WebKitBuild/Release/jsc MyChanges:/Volumes/Data/pizlo/secondary/OpenSource/WebKitBuild/Release/jsc"
  exit 1
end

def fail(reason)
  if reason.respond_to? :backtrace
    puts "FAILED: #{reason.inspect}"
    puts "Stack trace:"
    puts reason.backtrace.join("\n")
  else
    puts "FAILED: #{reason.inspect}"
  end
  smallUsage
end

def quickFail(r1,r2)
  $stderr.puts "#{$0}: #{r1}"
  puts
  fail(r2)
end

def intArg(argName,arg,min,max)
  result=arg.to_i
  unless result.to_s == arg
    quickFail("Expected an integer value for #{argName}, but got #{arg}.",
              "Invalid argument for command-line option")
  end
  if min and result<min
    quickFail("Argument for #{argName} cannot be smaller than #{min}.",
              "Invalid argument for command-line option")
  end
  if max and result>max
    quickFail("Argument for #{argName} cannot be greater than #{max}.",
              "Invalid argument for command-line option")
  end
  result
end

def computeMean(array)
  sum=0.0
  array.each {
    | value |
    sum += value
  }
  sum/array.length
end

def computeGeometricMean(array)
  sum = 0.0
  array.each {
    | value |
    sum += Math.log(value)
  }
  Math.exp(sum * (1.0/array.length))
end

def computeHarmonicMean(array)
  1.0 / computeMean(array.collect{ | value | 1.0 / value })
end

def computeStdDev(array)
  case array.length
  when 0
    0.0/0.0
  when 1
    0.0
  else
    begin
      mean=computeMean(array)
      sum=0.0
      array.each {
        | value |
        sum += (value-mean)**2
      }
      Math.sqrt(sum/(array.length-1))
    rescue
      0.0/0.0
    end
  end
end

class Array
  def shuffle!
    size.downto(1) { |n| push delete_at(rand(n)) }
    self
  end
end

def inverseBetaRegularized(n)
  IBR_LOOKUP[n-1]
end

def numToStr(num, decimalShift)
  ("%." + (4 + decimalShift).to_s + "f") % (num.to_f)
end

class CantSay
  def initialize
  end
  
  def shortForm
    " "
  end
  
  def longForm
    ""
  end
  
  def to_s
    ""
  end
end
  
class NoChange
  attr_reader :amountFaster
  
  def initialize(amountFaster)
    @amountFaster = amountFaster
  end
  
  def shortForm
    " "
  end
  
  def longForm
    "  might be #{numToStr(@amountFaster, 0)}x faster"
  end
  
  def to_s
    if @amountFaster < 1.01
      ""
    else
      longForm
    end
  end
end

class Faster
  attr_reader :amountFaster
  
  def initialize(amountFaster)
    @amountFaster = amountFaster
  end
  
  def shortForm
    "^"
  end
  
  def longForm
    "^ definitely #{numToStr(@amountFaster, 0)}x faster"
  end
  
  def to_s
    longForm
  end
end

class Slower
  attr_reader :amountSlower
  
  def initialize(amountSlower)
    @amountSlower = amountSlower
  end
  
  def shortForm
    "!"
  end
  
  def longForm
    "! definitely #{numToStr(@amountSlower, 0)}x slower"
  end
  
  def to_s
    longForm
  end
end

class MayBeSlower
  attr_reader :amountSlower
  
  def initialize(amountSlower)
    @amountSlower = amountSlower
  end
  
  def shortForm
    "?"
  end
  
  def longForm
    "? might be #{numToStr(@amountSlower, 0)}x slower"
  end
  
  def to_s
    if @amountSlower < 1.01
      "?"
    else
      longForm
    end
  end
end

def jsonSanitize(value)
  if value.is_a? Fixnum
    value
  elsif value.is_a? Float
    if value.nan? or value.infinite?
      value.to_s
    else
      value
    end
  elsif value.is_a? Array
    value.map{|v| jsonSanitize(v)}
  elsif value.nil?
    value
  else
    raise "Unrecognized value #{value.inspect}"
  end
end

class Stats
  def initialize
    @array = []
  end
  
  def add(value)
    if not value or not @array
      @array = nil
    elsif value.is_a? Float
      if value.nan? or value.infinite?
        @array = nil
      else
        @array << value
      end
    elsif value.is_a? Stats
      add(value.array)
    elsif value.respond_to? :each
      value.each {
        | v |
        add(v)
      }
    else
      @array << value.to_f
    end
  end
  
  def status
    if @array
      :ok
    else
      :error
    end
  end
  
  def error?
    # TODO: We're probably still not handling this case correctly. 
    not @array or @array.empty?
  end
  
  def ok?
    not not @array
  end
    
  def array
    @array
  end
  
  def sum
    result=0
    @array.each {
      | value |
      result += value
    }
    result
  end
  
  def min
    @array.min
  end
  
  def max
    @array.max
  end
  
  def size
    @array.length
  end
  
  def mean
    computeMean(array)
  end
  
  def arithmeticMean
    mean
  end
  
  def stdDev
    computeStdDev(array)
  end

  def stdErr
    stdDev/Math.sqrt(size)
  end
  
  # Computes a 95% Student's t distribution confidence interval
  def confInt
    if size < 2
      0.0/0.0
    else
      raise if size > 1000
      Math.sqrt(size-1.0)*stdErr*Math.sqrt(-1.0+1.0/inverseBetaRegularized(size-1))
    end
  end
  
  def lower
    mean-confInt
  end
  
  def upper
    mean+confInt
  end
  
  def geometricMean
    computeGeometricMean(array)
  end
  
  def harmonicMean
    computeHarmonicMean(array)
  end
  
  def compareTo(other)
    return CantSay.new unless ok? and other.ok?
    
    if upper < other.lower
      Faster.new(other.mean/mean)
    elsif lower > other.upper
      Slower.new(mean/other.mean)
    elsif mean > other.mean
      MayBeSlower.new(mean/other.mean)
    else
      NoChange.new(other.mean/mean)
    end
  end
  
  def to_s
    "size = #{size}, mean = #{mean}, stdDev = #{stdDev}, stdErr = #{stdErr}, confInt = #{confInt}"
  end
  
  def jsonMap
    if ok?
      {"data"=>jsonSanitize(@array), "mean"=>jsonSanitize(mean), "confInt"=>jsonSanitize(confInt)}
    else
      "ERROR"
    end
  end
end

def doublePuts(out1,out2,msg)
  out1.puts "#{out2.path}: #{msg}" if $verbosity>=3
  out2.puts msg
end

class Benchfile < File
  @@counter = 0
  
  attr_reader :filename, :basename
  
  def initialize(name)
    @basename, @filename = Benchfile.uniqueFilename(name)
    super(@filename, "w")
  end
  
  def self.uniqueFilename(name)
    if name.is_a? Array
      basename = name[0] + @@counter.to_s + name[1]
    else
      basename = name + @@counter.to_s
    end
    filename = BENCH_DATA_PATH + basename
    @@counter += 1
    raise "Benchfile #{filename} already exists" if FileTest.exist?(filename)
    [basename, filename]
  end
  
  def self.create(name)
    file = Benchfile.new(name)
    yield file
    file.close
    file.basename
  end
end

$dataFiles={}
def ensureFile(key, filename)
  unless $dataFiles[key]
    $dataFiles[key] = Benchfile.create(key) {
      | outp |
      doublePuts($stderr,outp,IO::read(filename))
    }
  end
  $dataFiles[key]
end

# Helper for files that cannot be renamed.
$absoluteFiles={}
def ensureAbsoluteFile(filename, basedir=nil)
  return if $absoluteFiles[filename]
  filename = Pathname.new(filename)

  directory = Pathname.new('')
  if basedir and filename.dirname != basedir
    remainingPath = filename.dirname
    while remainingPath != basedir
      directory = remainingPath.basename + directory
      remainingPath = remainingPath.dirname
    end
    if not $absoluteFiles[directory]
      cmd = "mkdir -p #{Shellwords.shellescape((BENCH_DATA_PATH + directory).to_s)}"
      $stderr.puts ">> #{cmd}" if $verbosity >= 2
      raise unless system(cmd)
      intermediateDirectory = Pathname.new(directory)
      while intermediateDirectory.basename.to_s != "."
        $absoluteFiles[intermediateDirectory] = true
        intermediateDirectory = intermediateDirectory.dirname
      end
    end
  end
  
  cmd = "cp #{Shellwords.shellescape(filename.to_s)} #{Shellwords.shellescape((BENCH_DATA_PATH + directory + filename.basename).to_s)}"
  $stderr.puts ">> #{cmd}" if $verbosity >= 2
  raise unless system(cmd)
  $absoluteFiles[filename] = true
end

# Helper for large benchmarks with lots of files and directories.
def ensureBenchmarkFiles(rootdir)
    toProcess = [rootdir]
    while not toProcess.empty?
      currdir = toProcess.pop
      Dir.foreach(currdir.to_s) {
        | filename |
        path = currdir + filename
        next if filename.match(/^\./)
        toProcess.push(path) if File.directory?(path.to_s)
        ensureAbsoluteFile(path, rootdir) if File.file?(path.to_s)
      }
    end
end

class JSCommand
  attr_reader :js, :html
  def initialize(js, html)
    @js = js
    @html = html
  end
end

def loadCommandForFile(key, filename)
  file = ensureFile(key, filename)
  JSCommand.new("load(#{file.inspect});", "<script src=#{file.inspect}></script>")
end

def simpleCommand(command)
  JSCommand.new(command, "<script type=\"text/javascript\">#{command}</script>")
end

# Benchmark that consists of a single file and must be loaded in its own global object each
# time (i.e. run()).
class SingleFileTimedBenchmarkParameters
  attr_reader :benchPath
  
  def initialize(benchPath)
    @benchPath = benchPath
  end
  
  def kind
    :singleFileTimedBenchmark
  end
end

# Benchmark that consists of one or more data files that should be loaded globally, followed
# by a command to run the benchmark.
class MultiFileTimedBenchmarkParameters
  attr_reader :dataPaths, :command

  def initialize(dataPaths, command)
    @dataPaths = dataPaths
    @command = command
  end
  
  def kind
    :multiFileTimedBenchmark
  end
end

# Benchmark that consists of one or more data files that should be loaded globally, followed
# by a command to run a short tick of the benchmark. The benchmark should be run for as many
# ticks as possible, for one quantum (quantum is 1000ms by default).
class ThroughputBenchmarkParameters
  attr_reader :dataPaths, :setUpCommand, :command, :tearDownCommand, :doWarmup, :deterministic, :minimumIterations

  def initialize(dataPaths, setUpCommand, command, tearDownCommand, doWarmup, deterministic, minimumIterations)
    @dataPaths = dataPaths
    @setUpCommand = setUpCommand
    @command = command
    @tearDownCommand = tearDownCommand
    @doWarmup = doWarmup
    @deterministic = deterministic
    @minimumIterations = minimumIterations
  end
  
  def kind
    :throughputBenchmark
  end
end

# Benchmark that can only run in DumpRenderTree or WebKitTestRunner, that has its own callback for reporting
# results. Other than that it's just like SingleFileTimedBenchmark.
class SingleFileTimedCallbackBenchmarkParameters
  attr_reader :callbackDecl, :benchPath
  
  def initialize(callbackDecl, benchPath)
    @callbackDecl = callbackDecl
    @benchPath = benchPath
  end
  
  def kind
    :singleFileTimedCallbackBenchmark
  end
end

def emitTimerFunctionCode(file)
  case $timeMode
  when :preciseTime
    doublePuts($stderr,file,"function __bencher_curTimeMS() {")
    doublePuts($stderr,file,"   return preciseTime()*1000")
    doublePuts($stderr,file,"}")
  when :date
    doublePuts($stderr,file,"function __bencher_curTimeMS() {")
    doublePuts($stderr,file,"   return Date.now()")
    doublePuts($stderr,file,"}")
  else
    raise
  end
end

def emitBenchRunCodeFile(name, plan, benchParams)
  case plan.vm.vmType
  when :jsc
    Benchfile.create("bencher") {
      | file |
      emitTimerFunctionCode(file)
      
      if benchParams.kind == :multiFileTimedBenchmark
        benchParams.dataPaths.each {
          | path |
          doublePuts($stderr,file,"load(#{path.inspect});")
        }
        doublePuts($stderr,file,"gc();")
        doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{$warmup+$inner}; ++__bencher_index) {")
        doublePuts($stderr,file,"   var __before = __bencher_curTimeMS();")
        $rerun.times {
          doublePuts($stderr,file,"   #{benchParams.command.js}")
        }
        doublePuts($stderr,file,"   var __after = __bencher_curTimeMS();")
        doublePuts($stderr,file,"   if (__bencher_index >= #{$warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{$warmup}) + \": Time: \"+(__after-__before));");
        doublePuts($stderr,file,"   gc();") unless plan.vm.shouldMeasureGC
        doublePuts($stderr,file,"}")
      elsif benchParams.kind == :throughputBenchmark
        emitTimerFunctionCode(file)
        benchParams.dataPaths.each {
          | path |
          doublePuts($stderr,file,"load(#{path.inspect});")
        }
        doublePuts($stderr,file,"#{benchParams.setUpCommand.js}")
        if benchParams.doWarmup
          warmup = $warmup
        else
          warmup = 0
        end
        doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{warmup + $inner}; __bencher_index++) {")
        doublePuts($stderr,file,"    var __before = __bencher_curTimeMS();")
        doublePuts($stderr,file,"    var __after = __before;")
        doublePuts($stderr,file,"    var __runs = 0;")
        doublePuts($stderr,file,"    var __expected = #{$quantum};")
        doublePuts($stderr,file,"    while (true) {")
        $rerun.times {
          doublePuts($stderr,file,"       #{benchParams.command.js}")
        }
        doublePuts($stderr,file,"       __runs++;")
        doublePuts($stderr,file,"       __after = __bencher_curTimeMS();")
        if benchParams.deterministic
          doublePuts($stderr,file,"       if (true) {")
        else
          doublePuts($stderr,file,"       if (__after - __before >= __expected) {")
        end
        doublePuts($stderr,file,"           if (__runs >= #{benchParams.minimumIterations} || __bencher_index < #{warmup})")
        doublePuts($stderr,file,"               break;")
        doublePuts($stderr,file,"           __expected += #{$quantum}")
        doublePuts($stderr,file,"       }")
        doublePuts($stderr,file,"    }")
        doublePuts($stderr,file,"    if (__bencher_index >= #{warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{warmup}) + \": Time: \"+((__after-__before)/__runs));")
        doublePuts($stderr,file,"}")
        doublePuts($stderr,file,"#{benchParams.tearDownCommand.js}")
      else
        raise unless benchParams.kind == :singleFileTimedBenchmark
        doublePuts($stderr,file,"function __bencher_run(__bencher_what) {")
        doublePuts($stderr,file,"   var __bencher_before = __bencher_curTimeMS();")
        $rerun.times {
          doublePuts($stderr,file,"   run(__bencher_what);")
        }
        doublePuts($stderr,file,"   var __bencher_after = __bencher_curTimeMS();")
        doublePuts($stderr,file,"   return __bencher_after - __bencher_before;")
        doublePuts($stderr,file,"}")
        $warmup.times {
          doublePuts($stderr,file,"__bencher_run(#{benchParams.benchPath.inspect})")
          doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
        }
        $inner.times {
          | innerIndex |
          doublePuts($stderr,file,"print(\"#{name}: #{plan.vm}: #{plan.iteration}: #{innerIndex}: Time: \"+__bencher_run(#{benchParams.benchPath.inspect}));")
          doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
        }
      end
    }
  when :dumpRenderTree, :webkitTestRunner
    case $timeMode
    when :preciseTime
      curTime = "(testRunner.preciseTime()*1000)"
    when :date
      curTime = "(Date.now())"
    else
      raise
    end

    mainCode = Benchfile.create("bencher") {
      | file |
      doublePuts($stderr,file,"__bencher_count = 0;")
      doublePuts($stderr,file,"function __bencher_doNext(result) {")
      doublePuts($stderr,file,"    if (__bencher_count >= #{$warmup})")
      doublePuts($stderr,file,"        debug(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_count - #{$warmup}) + \": Time: \" + result);")
      doublePuts($stderr,file,"    __bencher_count++;")
      doublePuts($stderr,file,"    if (__bencher_count < #{$inner+$warmup})")
      doublePuts($stderr,file,"        __bencher_runImpl(__bencher_doNext);")
      doublePuts($stderr,file,"    else")
      doublePuts($stderr,file,"        quit();")
      doublePuts($stderr,file,"}")
      doublePuts($stderr,file,"__bencher_runImpl(__bencher_doNext);")
    }
    
    cssCode = Benchfile.create("bencher-css") {
      | file |
      doublePuts($stderr,file,".pass {\n    font-weight: bold;\n    color: green;\n}\n.fail {\n    font-weight: bold;\n    color: red;\n}\n\#console {\n    white-space: pre-wrap;\n    font-family: monospace;\n}")
    }
    
    preCode = Benchfile.create("bencher-pre") {
      | file |
      doublePuts($stderr,file,"if (window.testRunner) {")
      doublePuts($stderr,file,"    testRunner.dumpAsText(window.enablePixelTesting);")
      doublePuts($stderr,file,"    testRunner.waitUntilDone();")
      doublePuts($stderr,file,"}")
      doublePuts($stderr,file,"")
      doublePuts($stderr,file,"function debug(msg)")
      doublePuts($stderr,file,"{")
      doublePuts($stderr,file,"    var span = document.createElement(\"span\");")
      doublePuts($stderr,file,"    document.getElementById(\"console\").appendChild(span); // insert it first so XHTML knows the namespace")
      doublePuts($stderr,file,"    span.innerHTML = msg + '<br />';")
      doublePuts($stderr,file,"}")
      doublePuts($stderr,file,"")
      doublePuts($stderr,file,"function quit() {")
      doublePuts($stderr,file,"    testRunner.notifyDone();")
      doublePuts($stderr,file,"}")
      doublePuts($stderr,file,"")
      doublePuts($stderr,file,"__bencher_continuation=null;")
      doublePuts($stderr,file,"")
      doublePuts($stderr,file,"function reportResult(result) {")
      doublePuts($stderr,file,"    __bencher_continuation(result);")
      doublePuts($stderr,file,"}")
      if benchParams.kind == :singleFileTimedCallbackBenchmark
        doublePuts($stderr,file,"")
        doublePuts($stderr,file,benchParams.callbackDecl)
      end
      doublePuts($stderr,file,"")
      doublePuts($stderr,file,"function __bencher_runImpl(continuation) {")
      doublePuts($stderr,file,"    function doit() {")
      doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"\";")
      doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"<iframe id='testframe'>\";")
      doublePuts($stderr,file,"        var testFrame = document.getElementById(\"testframe\");")
      doublePuts($stderr,file,"        testFrame.contentDocument.open();")
      doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<!DOCTYPE html>\\n<head></head><body><div id=\\\"console\\\"></div>\");")
      if benchParams.kind == :throughputBenchmark or benchParams.kind == :multiFileTimedBenchmark
        benchParams.dataPaths.each {
          | path |
          doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=#{path.inspect.inspect[1..-2]}></script>\");")
        }
      end
      if benchParams.kind == :throughputBenchmark
        if benchParams.doWarmup
          warmup = $warmup
        else
          warmup = 0
        end
        doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">\");")
        doublePuts($stderr,file,"        testFrame.contentDocument.write(\"#{benchParams.setUpCommand.js}\");")
        doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_before = #{curTime};\");")
        doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_after = __bencher_before;\");")
        doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_expected = #{$quantum};\");")
        doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_runs = 0;\");")
        doublePuts($stderr,file,"        testFrame.contentDocument.write(\"while (true) {\");")
        doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    #{benchParams.command.js}\");")
        doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    __bencher_runs++;\");")
        doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    __bencher_after = #{curTime};\");")
        if benchParams.deterministic
          doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    if (true) {\");")
        else
          doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    if (__bencher_after - __bencher_before >= __bencher_expected) {\");")
        end
        doublePuts($stderr,file,"        testFrame.contentDocument.write(\"        if (__bencher_runs >= #{benchParams.minimumIterations} || window.parent.__bencher_count < #{warmup})\");")
        doublePuts($stderr,file,"        testFrame.contentDocument.write(\"            break;\");")
        doublePuts($stderr,file,"        testFrame.contentDocument.write(\"        __bencher_expected += #{$quantum}\");")
        doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    }\");")
        doublePuts($stderr,file,"        testFrame.contentDocument.write(\"}\");")
        doublePuts($stderr,file,"        testFrame.contentDocument.write(\"#{benchParams.tearDownCommand.js}\");")
        doublePuts($stderr,file,"        testFrame.contentDocument.write(\"window.parent.reportResult((__bencher_after - __bencher_before) / __bencher_runs);\");")
        doublePuts($stderr,file,"        testFrame.contentDocument.write(\"</script>\");")
      else
        doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">var __bencher_before = #{curTime};</script>\");")
        if benchParams.kind == :multiFileTimedBenchmark
          doublePuts($stderr,file,"        testFrame.contentDocument.write(#{benchParams.command.html.inspect});")
        else
          doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=#{benchParams.benchPath.inspect.inspect[1..-2]}></script>\");")
        end
        unless benchParams.kind == :singleFileTimedCallbackBenchmark
          doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">window.parent.reportResult(#{curTime} - __bencher_before);</script>\");")
        end
      end
      doublePuts($stderr,file,"        testFrame.contentDocument.write(\"</body></html>\");")
      doublePuts($stderr,file,"        testFrame.contentDocument.close();")
      doublePuts($stderr,file,"    }")
      doublePuts($stderr,file,"    __bencher_continuation = continuation;")
      doublePuts($stderr,file,"    window.setTimeout(doit, 10);")
      doublePuts($stderr,file,"}")
    }

    Benchfile.create(["bencher-htmldoc",".html"]) {
      | file |
      doublePuts($stderr,file,"<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML//EN\">\n<html><head><link rel=\"stylesheet\" href=\"#{cssCode}\"><script src=\"#{preCode}\"></script></head><body><div id=\"console\"></div><div id=\"frameparent\"></div><script src=\"#{mainCode}\"></script></body></html>")
    }
  else
    raise
  end
end

def emitBenchRunCode(name, plan, benchParams)
  plan.vm.emitRunCode(emitBenchRunCodeFile(name, plan, benchParams), plan)
end

class FileCreator
  def initialize(filename)
    @filename = filename
    @state = :empty
  end
  
  def puts(text)
    $script.print "echo #{Shellwords.shellescape(text)}"
    if @state == :empty
      $script.print " > "
      @state = :nonEmpty
    else
      $script.print " >> "
    end
    $script.puts "#{Shellwords.shellescape(@filename)}"
  end
  
  def close
    if @state == :empty
      $script.puts "rm -f #{Shellwords.shellescape(text)}"
      $script.puts "touch #{Shellwords.shellescape(text)}"
    end
  end
  
  def self.open(filename)
    outp = FileCreator.new(filename)
    yield outp
    outp.close
  end
end

def emitSelfContainedBenchRunCode(name, plan, targetFile, configFile, benchmark)
  FileCreator.open(configFile) {
    | outp |
    outp.puts "__bencher_message = \"#{name}: #{plan.vm}: #{plan.iteration}: \";"
    outp.puts "__bencher_warmup = #{$warmup};"
    outp.puts "__bencher_inner = #{$inner};"
    outp.puts "__bencher_benchmark = #{benchmark.to_json};"
    case $timeMode
    when :preciseTime
      outp.puts "__bencher_curTime = (function(){ return testRunner.preciseTime() * 1000; });"
    when :date
      outp.puts "__bencher_curTime = (function(){ return Date.now(); });"
    else
      raise
    end
  }
  
  plan.vm.emitRunCode(targetFile, plan)
end

def planForDescription(string, plans, benchFullname, vmName, iteration)
  raise "Unexpected benchmark full name: #{benchFullname.inspect}, string: #{string.inspect}" unless benchFullname =~ /\//
  suiteName = $~.pre_match
  return nil if suiteName == "WARMUP"
  benchName = $~.post_match
  result = plans.select{|v| v.suite.name == suiteName and v.benchmark.name == benchName and v.vm.name == vmName and v.iteration == iteration}
  raise "Unexpected result dimensions: #{result.inspect}, string: #{string.inspect}" unless result.size == 1
  result[0]
end

class ParsedResult
  attr_reader :plan, :innerIndex, :time, :result
  
  def initialize(plan, innerIndex, time)
    @plan = plan
    @innerIndex = innerIndex
    if time == :crashed
      @result = :error
    else
      @time = time
      @result = :success
    end
    
    raise unless @plan.is_a? BenchPlan
    raise unless @innerIndex.is_a? Integer
    raise unless @time.is_a? Numeric or @result == :error
  end
  
  def benchmark
    plan.benchmark
  end
  
  def suite
    plan.suite
  end
  
  def vm
    plan.vm
  end
  
  def outerIndex
    plan.iteration
  end
  
  def self.create(plan, innerIndex, time)
    if plan
      ParsedResult.new(plan, innerIndex, time)
    else
      nil
    end
  end
  
  def self.parse(plans, string)
    if string =~ /([a-zA-Z0-9\/_.-]+): ([a-zA-Z0-9_#. ]+): ([0-9]+): ([0-9]+): Time: /
      benchFullname = $1
      vmName = $2
      outerIndex = $3.to_i
      innerIndex = $4.to_i
      time = $~.post_match.to_f
      ParsedResult.create(planForDescription(string, plans, benchFullname, vmName, outerIndex), innerIndex, time)
    elsif string =~ /([a-zA-Z0-9\/_.-]+): ([a-zA-Z0-9_#. ]+): ([0-9]+): ([0-9]+): CRASHED/
      benchFullname = $1
      vmName = $2
      outerIndex = $3.to_i
      innerIndex = $4.to_i
      time = $~.post_match.to_f
      ParsedResult.create(planForDescription(string, plans, benchFullname, vmName, outerIndex), innerIndex, :crashed)
    else
      nil
    end
  end
end

class VM
  @@extraEnvSet = {}
  
  def initialize(origPath, name, nameKind, svnRevision)
    @origPath = origPath.to_s
    @path = origPath.to_s
    @name = name
    @nameKind = nameKind
    @extraEnv = {}
    
    if $forceVMKind
      @vmType = $forceVMKind
    else
      if @origPath =~ /DumpRenderTree$/
        @vmType = :dumpRenderTree
      elsif @origPath =~ /WebKitTestRunner$/
        @vmType = :webkitTestRunner
      else
        @vmType = :jsc
      end
    end
    
    @svnRevision = svnRevision
    
    # Try to detect information about the VM.
    if path =~ /\/WebKitBuild\/(Release|Debug)+\/([a-zA-Z]+)$/
      @checkoutPath = $~.pre_match
      # FIXME: Use some variant of this: 
      # <bdash>   def retrieve_revision
      # <bdash>     `perl -I#{@path}/Tools/Scripts -MVCSUtils -e 'print svnRevisionForDirectory("#{@path}");'`.to_i
      # <bdash>   end
      unless @svnRevision
        begin
          Dir.chdir(@checkoutPath) {
            $stderr.puts ">> cd #{@checkoutPath} && svn info" if $verbosity>=2
            IO.popen("svn info", "r") {
              | inp |
              inp.each_line {
                | line |
                if line =~ /Revision: ([0-9]+)/
                  @svnRevision = $1
                end
              }
            }
          }
          unless @svnRevision
            $stderr.puts "Warning: running svn info for #{name} silently failed."
          end
        rescue => e
          # Failed to detect svn revision.
          $stderr.puts "Warning: could not get svn revision information for #{name}: #{e}"
        end
      end
    else
      $stderr.puts "Warning: could not identify checkout location for #{name}"
    end
    
    if @path =~ /\/Release\/([a-zA-Z]+)$/
      @libPath, @relativeBinPath = [$~.pre_match+"/Release"], "./#{$1}"
    elsif @path =~ /\/Debug\/([a-zA-Z]+)$/
      @libPath, @relativeBinPath = [$~.pre_match+"/Debug"], "./#{$1}"
    elsif @path =~ /\/Contents\/Resources\/([a-zA-Z]+)$/
      @libPath = [$~.pre_match + "/Contents/Resources", $~.pre_match + "/Contents/Frameworks"]
    elsif @path =~ /\/JavaScriptCore.framework\/Resources\/([a-zA-Z]+)$/
      @libPath, @relativeBinPath = [$~.pre_match], $&[1..-1]
    elsif @path =~ /(DumpRenderTree|webkitTestRunner|jsc)$/
      @libPath, @relativeBinPath = [$~.pre_match+"/"], "./#{$1}"
    end
  end
  
  def canCopyIntoBenchPath
    if @libPath and @relativeBinPath
      true
    else
      false
    end
  end
  
  def addExtraEnv(key, val)
    @extraEnv[key] = val
    @@extraEnvSet[key] = true
  end
  
  def copyIntoBenchPath
    raise unless canCopyIntoBenchPath
    basename, filename = Benchfile.uniqueFilename("vm")
    raise unless Dir.mkdir(filename)
    @libPath.each {
      | libPathPart |
      cmd = "cp -a #{Shellwords.shellescape(libPathPart)}/* #{Shellwords.shellescape(filename.to_s)}"
      $stderr.puts ">> #{cmd}" if $verbosity>=2
      raise unless system(cmd)
    }
    @path = "#{basename}/#{@relativeBinPath}"
    @libPath = [basename]
  end
  
  def to_s
    @name
  end
  
  def name
    @name
  end
  
  def shouldMeasureGC
    $measureGC == true or ($measureGC == name)
  end
  
  def origPath
    @origPath
  end
  
  def path
    @path
  end
  
  def nameKind
    @nameKind
  end
  
  def vmType
    @vmType
  end
  
  def checkoutPath
    @checkoutPath
  end
  
  def svnRevision
    @svnRevision
  end
  
  def extraEnv
    @extraEnv
  end
  
  def printFunction
    case @vmType
    when :jsc
      "print"
    when :dumpRenderTree, :webkitTestRunner
      "debug"
    else
      raise @vmType
    end
  end
  
  def emitRunCode(fileToRun, plan)
    myLibPath = @libPath
    myLibPath = [] unless myLibPath
    @@extraEnvSet.keys.each {
      | key |
      $script.puts "unset #{Shellwords.shellescape(key)}"
    }
    $script.puts "export DYLD_LIBRARY_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
    $script.puts "export DYLD_FRAMEWORK_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
    @extraEnv.each_pair {
      | key, val |
      $script.puts "export #{Shellwords.shellescape(key)}=#{Shellwords.shellescape(val)}"
    }
    plan.environment.each_pair {
        | key, val |
        $script.puts "export #{Shellwords.shellescape(key)}=#{Shellwords.shellescape(val)}"
    }
    $script.puts "#{path} #{fileToRun} 2>&1 || {"
    $script.puts "    echo " + Shellwords.shellescape("#{name} failed to run!") + " 1>&2"
    $inner.times {
      | iteration |
      $script.puts "    echo " + Shellwords.shellescape("#{plan.prefix}: #{iteration}: CRASHED")
    }
    $script.puts "}"
    plan.environment.keys.each {
        | key |
        $script.puts "export #{Shellwords.shellescape(key)}="
    }
  end
end

class StatsAccumulator
  def initialize
    @stats = []
    ($outer*$inner).times {
      @stats << Stats.new
    }
  end
  
  def statsForIteration(outerIteration, innerIteration)
    @stats[outerIteration*$inner + innerIteration]
  end
  
  def stats
    result = Stats.new
    @stats.each {
      | stat |
      if stat.ok?
        result.add(yield stat)
      else
        result.add(nil)
      end
    }
    result
  end
  
  def geometricMeanStats
    stats {
      | stat |
      stat.geometricMean
    }
  end
  
  def arithmeticMeanStats
    stats {
      | stat |
      stat.arithmeticMean
    }
  end
end

module Benchmark
  attr_accessor :benchmarkSuite
  attr_reader :name
  
  def fullname
    benchmarkSuite.name + "/" + name
  end
  
  def to_s
    fullname
  end
  
  def weight
    1
  end
  
  def weightString
    raise unless weight.is_a? Fixnum
    raise unless weight >= 1
    if weight == 1
      ""
    else
      "x#{weight} "
    end
  end
end

class SunSpiderBenchmark
  include Benchmark
  
  def initialize(name)
    @name = name
  end
  
  def emitRunCode(plan)
    emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("SunSpider-#{@name}", "#{SUNSPIDER_PATH}/#{@name}.js")))
  end
end

class LongSpiderBenchmark
  include Benchmark
  
  def initialize(name)
    @name = name
  end
  
  def emitRunCode(plan)
    emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("LongSpider-#{@name}", "#{LONGSPIDER_PATH}/#{@name}.js")))
  end
end

class V8Benchmark
  include Benchmark
  
  def initialize(name)
    @name = name
  end
  
  def emitRunCode(plan)
    emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("V8-#{@name}", "#{V8_PATH}/v8-#{@name}.js")))
  end
end

class V8RealBenchmark
  include Benchmark
  
  attr_reader :v8SuiteName
  
  def initialize(v8SuiteName, name, weight, minimumIterations)
    @v8SuiteName = v8SuiteName
    @name = name
    @weight = weight
    @minimumIterations = minimumIterations
  end
  
  def weight
    @weight
  end
  
  def emitRunCode(plan)
    emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new(["base", @v8SuiteName, "jsc-#{@name}"].collect{|v| ensureFile("V8Real-#{v}", "#{V8_REAL_PATH}/#{v}.js")}, simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), true, false, @minimumIterations))
  end
end

class OctaneBenchmark
  include Benchmark
  
  def initialize(files, name, weight, doWarmup, deterministic, minimumIterations)
    @files = files
    @name = name
    @weight = weight
    @doWarmup = doWarmup
    @deterministic = deterministic
    @minimumIterations = minimumIterations
  end
  
  def weight
    @weight
  end
  
  def emitRunCode(plan)
    files = []
    files += (["base"] + @files).collect {
      | v |
      ensureFile("Octane-#{v}", "#{OCTANE_PATH}/#{v}.js")
    }
    files += ["jsc-#{@name}"].collect {
      | v |
      ensureFile("Octane-#{v}", "#{OCTANE_WRAPPER_PATH}/#{v}.js")
    }
    emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new(files, simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), @doWarmup, @deterministic, @minimumIterations))
  end
end

class KrakenBenchmark
  include Benchmark
  
  def initialize(name)
    @name = name
  end
  
  def emitRunCode(plan)
    emitBenchRunCode(fullname, plan, MultiFileTimedBenchmarkParameters.new([ensureFile("KrakenData-#{@name}", "#{KRAKEN_PATH}/#{@name}-data.js")], loadCommandForFile("Kraken-#{@name}", "#{KRAKEN_PATH}/#{@name}.js")))
  end
end

class JSBenchBenchmark
  include Benchmark
  
  attr_reader :jsBenchMode
  
  def initialize(name, jsBenchMode)
    @name = name
    @jsBenchMode = jsBenchMode
  end
  
  def emitRunCode(plan)
    callbackDecl  = "function JSBNG_handleResult(result) {\n"
    callbackDecl += "    if (result.error) {\n"
    callbackDecl += "        console.log(\"Did not run benchmark correctly!\");\n"
    callbackDecl += "        quit();\n"
    callbackDecl += "    }\n"
    callbackDecl += "    reportResult(result.time);\n"
    callbackDecl += "}\n"
    emitBenchRunCode(fullname, plan, SingleFileTimedCallbackBenchmarkParameters.new(callbackDecl, ensureFile("JSBench-#{@name}", "#{JSBENCH_PATH}/#{@name}/#{@jsBenchMode}.js")))
  end
end

class JSRegressBenchmark
  include Benchmark
  
  def initialize(name)
    @name = name
  end
  
  def emitRunCode(plan)
    emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("JSRegress-#{@name}", "#{JSREGRESS_PATH}/#{@name}.js")))
  end
end

class AsmBenchBenchmark
  include Benchmark
  
  def initialize(name)
    @name = name
  end
  
  def emitRunCode(plan)
    emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("AsmBench-#{@name}", "#{ASMBENCH_PATH}/#{@name}.js")))
  end
end


class CompressionBenchBenchmark
  include Benchmark
  
  def initialize(files, name, model)
    @files = files
    @name = name;
    @name = name + "-" + model if !model.empty?
    @name = @name.gsub(" ", "-").downcase
    @scriptName = name
    @weight = 1
    @doWarmup = true
    @deterministic = true
    @minimumIterations = 1
    @model = model
  end
  
  def weight
    @weight
  end
  
  def emitRunCode(plan)
    emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new((["base"] + @files + ["jsc-#{@scriptName}"]).collect{|v| ensureFile("Compression-#{v}", "#{COMPRESSIONBENCH_PATH}/#{v}.js")}, simpleCommand("jscSetUp('#{@model}');"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), @doWarmup, @deterministic, @minimumIterations))
  end
end

class DSPJSFiltrrBenchmark
  include Benchmark
  
  def initialize(name, filterKey)
    @name = name
    @filterKey = filterKey
  end
  
  def emitRunCode(plan)
    ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr.js")
    ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr_back.jpg")
    ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr-jquery.min.js")
    ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr-bencher.html")
    emitSelfContainedBenchRunCode(fullname, plan, "filtrr-bencher.html", "bencher-config.js", @filterKey)
  end
end

class DSPJSVP8Benchmark
  include Benchmark
  
  def initialize
    @name = "route9-vp8"
  end
  
  def weight
    5
  end
  
  def emitRunCode(plan)
    ensureBenchmarkFiles(DSPJS_ROUTE9_PATH)
    emitSelfContainedBenchRunCode(fullname, plan, "route9-bencher.html", "bencher-config.js", "")
  end
end

class DSPStarfieldBenchmark
  include Benchmark

  def initialize
    @name = "starfield"
  end
  
  def weight
    5
  end

  def emitRunCode(plan)
    ensureBenchmarkFiles(DSPJS_STARFIELD_PATH)
    emitSelfContainedBenchRunCode(fullname, plan, "starfield-bencher.html", "bencher-config.js", "")
  end
end

class DSPJSJSLinuxBenchmark
  include Benchmark
  def initialize
    @name = "bellard-jslinux"
  end

  def weight
    5
  end

  def emitRunCode(plan)
    ensureBenchmarkFiles(DSPJS_JSLINUX_PATH)
    emitSelfContainedBenchRunCode(fullname, plan, "jslinux-bencher.html", "bencher-config.js", "")
  end
end

class DSPJSQuake3Benchmark
  include Benchmark

  def initialize
    @name = "zynaps-quake3"
  end

  def weight
    5
  end

  def emitRunCode(plan)
    ensureBenchmarkFiles(DSPJS_QUAKE3_PATH)
    emitSelfContainedBenchRunCode(fullname, plan, "quake-bencher.html", "bencher-config.js", "")
  end
end

class DSPJSMandelbrotBenchmark
  include Benchmark

  def initialize
    @name = "zynaps-mandelbrot"
  end

  def weight
    5
  end

  def emitRunCode(plan)
    ensureBenchmarkFiles(DSPJS_MANDELBROT_PATH)
    emitSelfContainedBenchRunCode(fullname, plan, "mandelbrot-bencher.html", "bencher-config.js", "")
  end
end

class DSPJSAmmoJSASMBenchmark
  include Benchmark

  def initialize
    @name = "ammojs-asm-js"
  end

  def weight
    5
  end

  def emitRunCode(plan)
    ensureBenchmarkFiles(DSPJS_AMMOJS_ASMJS_PATH)
    emitSelfContainedBenchRunCode(fullname, plan, "ammo-asmjs-bencher.html", "bencher-config.js", "")
  end
end

class DSPJSAmmoJSRegularBenchmark
  include Benchmark

  def initialize
    @name = "ammojs-regular-js"
  end

  def weight
    5
  end

  def emitRunCode(plan)
    ensureBenchmarkFiles(DSPJS_AMMOJS_REGULAR_PATH)
    emitSelfContainedBenchRunCode(fullname, plan, "ammo-regular-bencher.html", "bencher-config.js", "")
  end
end

class BrowsermarkJSBenchmark
  include Benchmark
    
  def initialize(name)
    @name = name
  end
  
  def emitRunCode(plan)
    emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new([ensureFile(name, "#{BROWSERMARK_JS_PATH}/#{name}/test.js"), ensureFile("browsermark-bencher", "#{BROWSERMARK_JS_PATH}/browsermark-bencher.js")], simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), true, 32))
  end
end

class BrowsermarkDOMBenchmark
  include Benchmark
    
  def initialize(name)
    @name = name
  end
  
  def emitRunCode(plan)
    ensureBenchmarkFiles(BROWSERMARK_PATH)
    emitSelfContainedBenchRunCode(fullname, plan, "tests/benchmarks/dom/#{name}/index.html", "bencher-config.js", name)
  end
end

class BenchmarkSuite
  def initialize(name, preferredMean, decimalShift)
    @name = name
    @preferredMean = preferredMean
    @benchmarks = []
    @subSuites = []
    @decimalShift = decimalShift
  end
  
  def name
    @name
  end
  
  def to_s
    @name
  end
  
  def decimalShift
    @decimalShift
  end
  
  def addIgnoringPattern(benchmark)
    benchmark.benchmarkSuite = self
    @benchmarks << benchmark
  end
  
  def add(benchmark)
    if not $benchmarkPattern or "#{@name}/#{benchmark.name}" =~ $benchmarkPattern
        addIgnoringPattern(benchmark)
    end
  end
  
  def addSubSuite(subSuite)
    @subSuites << subSuite
  end
  
  def benchmarks
    @benchmarks
  end
  
  def benchmarkForName(name)
    result = @benchmarks.select{|v| v.name == name}
    raise unless result.length == 1
    result[0]
  end
  
  def hasBenchmark(benchmark)
    array = @benchmarks.select{|v| v == benchmark}
    raise unless array.length == 1 or array.length == 0
    array.length == 1
  end
  
  def subSuites
    @subSuites
  end
  
  def suites
    [self] + @subSuites
  end
  
  def suitesWithBenchmark(benchmark)
    result = [self]
    @subSuites.each {
      | subSuite |
      if subSuite.hasBenchmark(benchmark)
        result << subSuite
      end
    }
    result
  end
  
  def empty?
    @benchmarks.empty?
  end
  
  def retain_if
    @benchmarks.delete_if {
      | benchmark |
      not yield benchmark
    }
  end
  
  def preferredMean
    @preferredMean
  end
  
  def computeMean(stat)
    if stat.ok?
      (stat.send @preferredMean) * (10 ** decimalShift)
    else
      nil
    end
  end
end

class BenchRunPlan
  def initialize(benchmark, vm, iteration)
    @benchmark = benchmark
    @vm = vm
    @iteration = iteration
    @environment = {}
    if $environment.has_key?(vm.name)
      if $environment[vm.name].has_key?(benchmark.benchmarkSuite.name)
        if $environment[vm.name][benchmark.benchmarkSuite.name].has_key?(benchmark.name)
          @environment = $environment[vm.name][benchmark.benchmarkSuite.name][benchmark.name]
        end
      end
    end
  end
  
  def benchmark
    @benchmark
  end
  
  def suite
    @benchmark.benchmarkSuite
  end
  
  def vm
    @vm
  end
  
  def iteration
    @iteration
  end
 
  def environment
    @environment
  end
 
  def prefix
    "#{@benchmark.fullname}: #{vm.name}: #{iteration}"
  end
  
  def emitRunCode
    @benchmark.emitRunCode(self)
  end
  
  def to_s
    benchmark.to_s + "/" + vm.to_s
  end
end

class BenchmarkOnVM
  def initialize(benchmark, suiteOnVM, subSuitesOnVM)
    @benchmark = benchmark
    @suiteOnVM = suiteOnVM
    @subSuitesOnVM = subSuitesOnVM
    @stats = Stats.new
  end
  
  def to_s
    "#{@benchmark} on #{@suiteOnVM.vm}"
  end
  
  def benchmark
    @benchmark
  end
  
  def vm
    @suiteOnVM.vm
  end
  
  def vmStats
    @suiteOnVM.vmStats
  end
  
  def suite
    @benchmark.benchmarkSuite
  end
  
  def suiteOnVM
    @suiteOnVM
  end
  
  def subSuitesOnVM
    @subSuitesOnVM
  end
  
  def stats
    @stats
  end
  
  def parseResult(result)
    raise "VM mismatch; I've got #{vm} and they've got #{result.vm}" unless result.vm == vm
    raise unless result.benchmark == @benchmark
    @stats.add(result.time)
  end
end

class NamedStatsAccumulator < StatsAccumulator
  def initialize(name)
    super()
    @name = name
  end
  
  def reportingName
    @name
  end
end

class SuiteOnVM < StatsAccumulator
  def initialize(vm, vmStats, suite)
    super()
    @vm = vm
    @vmStats = vmStats
    @suite = suite
    
    raise unless @vm.is_a? VM
    raise unless @vmStats.is_a? StatsAccumulator
    raise unless @suite.is_a? BenchmarkSuite
  end
  
  def to_s
    "#{@suite} on #{@vm}"
  end
  
  def suite
    @suite
  end
  
  def vm
    @vm
  end

  def reportingName
    @vm.name
  end
  
  def vmStats
    raise unless @vmStats
    @vmStats
  end
end

class SubSuiteOnVM < StatsAccumulator
  def initialize(vm, suite)
    super()
    @vm = vm
    @suite = suite
    raise unless @vm.is_a? VM
    raise unless @suite.is_a? BenchmarkSuite
  end
  
  def to_s
    "#{@suite} on #{@vm}"
  end
  
  def suite
    @suite
  end
  
  def vm
    @vm
  end
  
  def reportingName
    @vm.name
  end
end

class BenchPlan
  def initialize(benchmarkOnVM, iteration)
    @benchmarkOnVM = benchmarkOnVM
    @iteration = iteration
  end
  
  def to_s
    "#{@benchmarkOnVM} \##{@iteration+1}"
  end
  
  def benchmarkOnVM
    @benchmarkOnVM
  end
  
  def benchmark
    @benchmarkOnVM.benchmark
  end
  
  def suite
    @benchmarkOnVM.suite
  end
  
  def vm
    @benchmarkOnVM.vm
  end
  
  def iteration
    @iteration
  end
  
  def parseResult(result)
    raise unless result.plan == self
    @benchmarkOnVM.parseResult(result)
    benchmark.weight.times {
      @benchmarkOnVM.vmStats.statsForIteration(@iteration, result.innerIndex).add(result.time)
      @benchmarkOnVM.suiteOnVM.statsForIteration(@iteration, result.innerIndex).add(result.time)
      @benchmarkOnVM.subSuitesOnVM.each {
        | subSuite |
        subSuite.statsForIteration(@iteration, result.innerIndex).add(result.time)
      }
    }
  end
end

def lpad(str,chars)
  if str.length>chars
    str
  else
    "%#{chars}s"%(str)
  end
end

def rpad(str,chars)
  while str.length < chars
    str+=" "
  end
  str
end

def center(str,chars)
  while str.length<chars
    str+=" "
    if str.length<chars
      str=" "+str
    end
  end
  str
end

def statsToStr(stats, decimalShift)
  if stats.error?
    lpad(center("ERROR", 10+10+2), 12+10+2)
  elsif $inner*$outer == 1
    string = numToStr(stats.mean, decimalShift)
    raise unless string =~ /\./
    left = $~.pre_match
    right = $~.post_match
    lpad(left, 13 - decimalShift) + "." + rpad(right, 10 + decimalShift)
  else
    lpad(numToStr(stats.mean, decimalShift), 12) + "+-" + rpad(numToStr(stats.confInt, decimalShift), 10)
  end
end

def plural(num)
  if num == 1
    ""
  else
    "s"
  end
end

def wrap(str, columns)
  array = str.split
  result = ""
  curLine = array.shift
  array.each {
    | curStr |
    if (curLine + " " + curStr).size > columns
      result += curLine + "\n"
      curLine = curStr
    else
      curLine += " " + curStr
    end
  }
  result + curLine + "\n"
end
  
def runAndGetResults
  results = nil
  Dir.chdir(BENCH_DATA_PATH) {
    $stderr.puts ">> sh ./runscript" if $verbosity >= 2
    raise "Script did not complete correctly: #{$?}" unless system("sh ./runscript > runlog")
    results = IO::read("runlog")
  }
  raise unless results
  results
end

def parseAndDisplayResults(results)
  vmStatses = []
  $vms.each {
    | vm |
    vmStatses << NamedStatsAccumulator.new(vm.name)
  }
  
  suitesOnVMs = []
  suitesOnVMsForSuite = {}
  subSuitesOnVMsForSubSuite = {}
  $suites.each {
    | suite |
    suitesOnVMsForSuite[suite] = []
    suite.subSuites.each {
      | subSuite |
      subSuitesOnVMsForSubSuite[subSuite] = []
    }
  }
  suitesOnVMsForVM = {}
  $vms.each {
    | vm |
    suitesOnVMsForVM[vm] = []
  }
  
  benchmarksOnVMs = []
  benchmarksOnVMsForBenchmark = {}
  $benchmarks.each {
    | benchmark |
    benchmarksOnVMsForBenchmark[benchmark] = []
  }
  
  $vms.each_with_index {
    | vm, vmIndex |
    vmStats = vmStatses[vmIndex]
    $suites.each {
      | suite |
      suiteOnVM = SuiteOnVM.new(vm, vmStats, suite)
      subSuitesOnVM = suite.subSuites.map {
        | subSuite |
        result = SubSuiteOnVM.new(vm, subSuite)
        subSuitesOnVMsForSubSuite[subSuite] << result
        result
      }
      suitesOnVMs << suiteOnVM
      suitesOnVMsForSuite[suite] << suiteOnVM
      suitesOnVMsForVM[vm] << suiteOnVM
      suite.benchmarks.each {
        | benchmark |
        subSuitesOnVMForThisBenchmark = []
        subSuitesOnVM.each {
          | subSuiteOnVM |
          if subSuiteOnVM.suite.hasBenchmark(benchmark)
            subSuitesOnVMForThisBenchmark << subSuiteOnVM
          end
        }
        benchmarkOnVM = BenchmarkOnVM.new(benchmark, suiteOnVM, subSuitesOnVMForThisBenchmark)
        benchmarksOnVMs << benchmarkOnVM
        benchmarksOnVMsForBenchmark[benchmark] << benchmarkOnVM
      }
    }
  }
  
  plans = []
  benchmarksOnVMs.each {
    | benchmarkOnVM |
    $outer.times {
      | iteration |
      plans << BenchPlan.new(benchmarkOnVM, iteration)
    }
  }

  hostname = nil
  hwmodel = nil
  results.each_line {
    | line |
    line.chomp!
    if line =~ /HOSTNAME:([^.]+)/
      hostname = $1
    elsif line =~ /HARDWARE:hw\.model: /
      hwmodel = $~.post_match.chomp
    else
      result = ParsedResult.parse(plans, line)
      if result
        result.plan.parseResult(result)
      end
    end
  }
  
  # Compute the geomean of the preferred means of results on a SuiteOnVM
  overallResults = []
  $vms.each {
    | vm |
    result = Stats.new
    $outer.times {
      | outerIndex |
      $inner.times {
        | innerIndex |
        curResult = Stats.new
        suitesOnVMsForVM[vm].each {
          | suiteOnVM |
          # For a given iteration, suite, and VM, compute the suite's preferred mean
          # over the data collected for all benchmarks in that suite. We'll have one
          # sample per benchmark. For example on V8 this will be the geomean of 1
          # sample for crypto, 1 sample for deltablue, and so on, and 1 sample for
          # splay.
          curResult.add(suiteOnVM.suite.computeMean(suiteOnVM.statsForIteration(outerIndex, innerIndex)))
        }
        
        # curResult now holds 1 sample for each of the means computed in the above
        # loop. Compute the geomean over this, and store it.
        if curResult.ok?
          result.add(curResult.geometricMean)
        else
          result.add(nil)
        end
      }
    }

    # $overallResults will have a Stats for each VM. That Stats object will hold
    # $inner*$outer geomeans, allowing us to compute the arithmetic mean and
    # confidence interval of the geomeans of preferred means. Convoluted, but
    # useful and probably sound.
    overallResults << result
  }
  
  if $verbosity >= 2
    benchmarksOnVMs.each {
      | benchmarkOnVM |
      $stderr.puts "#{benchmarkOnVM}: #{benchmarkOnVM.stats}"
    }
    
    $vms.each_with_index {
      | vm, vmIndex |
      vmStats = vmStatses[vmIndex]
      $stderr.puts "#{vm} (arithmeticMean): #{vmStats.arithmeticMeanStats}"
      $stderr.puts "#{vm} (geometricMean): #{vmStats.geometricMeanStats}"
    }
  end

  if $outputName
    reportName = $outputName
  else
    reportName =
      (if ($vms.collect {
             | vm |
             vm.nameKind
           }.index :auto)
         ""
       else
         text = $vms.collect {
           | vm |
           vm.to_s
         }.join("_") + "_"
         if text.size >= 40
           ""
         else
           text
         end
       end) +
      ($suites.collect {
         | suite |
         suite.to_s
       }.join("")) + "_" +
      (if hostname
         hostname + "_"
       else
         ""
       end)+
      (begin
         time = Time.now
         "%04d%02d%02d_%02d%02d" %
           [ time.year, time.month, time.day,
             time.hour, time.min ]
       end)
  end

  unless $brief
    puts "Generating benchmark report at #{Dir.pwd}/#{reportName}_report.txt"
    puts "And raw data at #{Dir.pwd}/#{reportName}.json"
  end
  
  outp = $stdout
  json = {}
  begin
    outp = File.open(reportName + "_report.txt","w")
  rescue => e
    $stderr.puts "Error: could not save report to #{reportName}_report.txt: #{e}"
    $stderr.puts
  end
  
  def createVMsString
    result = ""
    result += "   " if $allSuites.size > 1
    result += rpad("", $benchpad + $weightpad)
    result += " "
    $vms.size.times {
      | index |
      if index != 0
        result += " "+NoChange.new(0).shortForm
      end
      result += lpad(center($vms[index].name, 10+10+2), 12+10+2)
    }
    result += "    "
    if $vms.size >= 3
      result += center("#{$vms[-1].name} v. #{$vms[0].name}",26)
    elsif $vms.size >= 2
      result += " "*26
    end
    result
  end
  
  def andJoin(list)
    if list.size == 1
      list[0].to_s
    elsif list.size == 2
      "#{list[0]} and #{list[1]}"
    else
      "#{list[0..-2].join(', ')}, and #{list[-1]}"
    end
  end
  
  json["vms"] = $vms.collect{|v| v.name}
  json["suites"] = {}
  json["runlog"] = results
  
  columns = [createVMsString.size, 78].max
  
  outp.print "Benchmark report for "
  outp.print andJoin($suites)
  if hostname
    outp.print " on #{hostname}"
  end
  if hwmodel
    outp.print " (#{hwmodel})"
  end
  outp.puts "."
  outp.puts
  
  outp.puts "VMs tested:"
  $vms.each {
    | vm |
    outp.print "\"#{vm.name}\" at #{vm.origPath}"
    if vm.svnRevision
      outp.print " (r#{vm.svnRevision})"
    end
    outp.puts
    vm.extraEnv.each_pair {
      | key, val |
      outp.puts "    export #{key}=#{val}"
    }
  }
  
  outp.puts
  
  outp.puts wrap("Collected #{$outer*$inner} sample#{plural($outer*$inner)} per benchmark/VM, "+
                 "with #{$outer} VM invocation#{plural($outer)} per benchmark."+
                 (if $rerun > 1 then (" Ran #{$rerun} benchmark iterations, and measured the "+
                                      "total time of those iterations, for each sample.")
                  else "" end)+
                 (if $measureGC == true then (" No manual garbage collection invocations were "+
                                              "emitted.")
                  elsif $measureGC then (" Emitted a call to gc() between sample measurements for "+
                                         "all VMs except #{$measureGC}.")
                  else (" Emitted a call to gc() between sample measurements.") end)+
                 (if $warmup == 0 then (" Did not include any warm-up iterations; measurements "+
                                        "began with the very first iteration.")
                  else (" Used #{$warmup*$rerun} benchmark iteration#{plural($warmup*$rerun)} per VM "+
                        "invocation for warm-up.") end)+
                 (case $timeMode
                  when :preciseTime then (" Used the jsc-specific preciseTime() function to get "+
                                          "microsecond-level timing.")
                  when :date then (" Used the portable Date.now() method to get millisecond-"+
                                   "level timing.")
                  else raise end)+
                 " Reporting benchmark execution times with 95% confidence "+
                 "intervals in milliseconds.",
                 columns)
  
  outp.puts
  
  def printVMs(outp)
    outp.puts createVMsString
  end
  
  def summaryStats(outp, json, accumulators, name, decimalShift, &proc)
    resultingJson = {}
    outp.print "   " if $allSuites.size > 1
    outp.print rpad(name, $benchpad + $weightpad)
    outp.print " "
    accumulators.size.times {
      | index |
      if index != 0
        outp.print " "+accumulators[index].stats(&proc).compareTo(accumulators[index-1].stats(&proc)).shortForm
      end
      outp.print statsToStr(accumulators[index].stats(&proc), decimalShift)
      resultingJson[accumulators[index].reportingName] = accumulators[index].stats(&proc).jsonMap
    }
    if accumulators.size>=2
      outp.print("    "+accumulators[-1].stats(&proc).compareTo(accumulators[0].stats(&proc)).longForm)
    end
    outp.puts
    json[name] = resultingJson
  end

  def allSummaryStats(outp, json, accumulators, preferredMean, decimalShift)
    meanLabel = '<' + preferredMean.to_s.sub(/Mean$/, '') + '>'
    summaryStats(outp, json, accumulators, meanLabel, decimalShift) {
      | stat |
      stat.send(preferredMean)
    }
  end
  
  $suites.each {
    | suite |
    suiteJson = {}
    subSuiteJsons = {}
    suite.subSuites.each {
      | subSuite |
      subSuiteJsons[subSuite] = {}
    }
    
    printVMs(outp)
    if $allSuites.size > 1
      outp.puts(andJoin(suite.suites.map{|v| v.name}) + ":")
    else
      outp.puts
    end
    suite.benchmarks.each {
      | benchmark |
      benchmarkJson = {}
      outp.print "   " if $allSuites.size > 1
      outp.print rpad(benchmark.name, $benchpad) + rpad(benchmark.weightString, $weightpad)
      if benchmark.name.size > $benchNameClip
        outp.puts
        outp.print "   " if $allSuites.size > 1
        outp.print((" " * $benchpad) + (" " * $weightpad))
      end
      outp.print " "
      myConfigs = benchmarksOnVMsForBenchmark[benchmark]
      myConfigs.size.times {
        | index |
        if index != 0
          outp.print " "+myConfigs[index].stats.compareTo(myConfigs[index-1].stats).shortForm
        end
        outp.print statsToStr(myConfigs[index].stats, suite.decimalShift)
        benchmarkJson[myConfigs[index].vm.name] = myConfigs[index].stats.jsonMap
      }
      if $vms.size>=2
        outp.print("    "+myConfigs[-1].stats.compareTo(myConfigs[0].stats).to_s)
      end
      outp.puts
      suiteJson[benchmark.name] = benchmarkJson
      suite.subSuites.each {
        | subSuite |
        if subSuite.hasBenchmark(benchmark)
          subSuiteJsons[subSuite][benchmark.name] = benchmarkJson
        end
      }
    }
    outp.puts
    unless suite.subSuites.empty?
      suite.subSuites.each {
        | subSuite |
        outp.puts "#{subSuite.name}:"
        allSummaryStats(outp, subSuiteJsons[subSuite], subSuitesOnVMsForSubSuite[subSuite], subSuite.preferredMean, subSuite.decimalShift)
        outp.puts
      }
      outp.puts "#{suite.name} including #{andJoin(suite.subSuites.map{|v| v.name})}:"
    end
    allSummaryStats(outp, suiteJson, suitesOnVMsForSuite[suite], suite.preferredMean, suite.decimalShift)
    outp.puts if $allSuites.size > 1
    
    json["suites"][suite.name] = suiteJson
    suite.subSuites.each {
      | subSuite |
      json["suites"][subSuite.name] = subSuiteJsons[subSuite]
    }
  }
  
  if $suites.size > 1
    scaledResultJson = {}
    printVMs(outp)
    outp.puts "Geomean of preferred means:"
    outp.print "   "
    outp.print rpad("<scaled-result>", $benchpad + $weightpad)
    outp.print " "
    $vms.size.times {
      | index |
      if index != 0
        outp.print " "+overallResults[index].compareTo(overallResults[index-1]).shortForm
      end
      outp.print statsToStr(overallResults[index], 0)
      scaledResultJson[$vms[index].name] = overallResults[index].jsonMap
    }
    if overallResults.size>=2
      outp.print("    "+overallResults[-1].compareTo(overallResults[0]).longForm)
    end
    outp.puts
    
    json["<scaled-result>"] = scaledResultJson
  end
  outp.puts
  
  if outp != $stdout
    outp.close
  end
  
  if outp != $stdout and not $brief
    puts
    File.open(reportName + "_report.txt") {
      | inp |
      puts inp.read
    }
  end
  
  if $brief
    puts(overallResults.collect{|stats| stats.mean}.join("\t"))
    puts(overallResults.collect{|stats| stats.confInt}.join("\t"))
  end
  
  File.open(reportName + ".json", "w") {
    | outp |
    outp.puts json.to_json
  }
end

begin
  $sawBenchOptions = false
  
  def resetBenchOptionsIfNecessary
    unless $sawBenchOptions
      $includeSunSpider = false
      $includeLongSpider = false
      $includeV8 = false
      $includeKraken = false
      $includeJSBench = false
      $includeJSRegress = false
      $includeAsmBench = false
      $includeDSPJS = false
      $includeBrowsermarkJS = false
      $includeBrowsermarkDOM = false
      $includeOctane = false
      $includeCompressionBench = false
      $sawBenchOptions = true
    end
  end
  
  GetoptLong.new(['--rerun', GetoptLong::REQUIRED_ARGUMENT],
                 ['--inner', GetoptLong::REQUIRED_ARGUMENT],
                 ['--outer', GetoptLong::REQUIRED_ARGUMENT],
                 ['--warmup', GetoptLong::REQUIRED_ARGUMENT],
                 ['--no-ss-warmup', GetoptLong::NO_ARGUMENT],
                 ['--quantum', GetoptLong::REQUIRED_ARGUMENT],
                 ['--minimum', GetoptLong::REQUIRED_ARGUMENT],
                 ['--timing-mode', GetoptLong::REQUIRED_ARGUMENT],
                 ['--sunspider', GetoptLong::NO_ARGUMENT],
                 ['--longspider', GetoptLong::NO_ARGUMENT],
                 ['--v8-spider', GetoptLong::NO_ARGUMENT],
                 ['--kraken', GetoptLong::NO_ARGUMENT],
                 ['--js-bench', GetoptLong::NO_ARGUMENT],
                 ['--js-regress', GetoptLong::NO_ARGUMENT],
                 ['--asm-bench', GetoptLong::NO_ARGUMENT],
                 ['--dsp', GetoptLong::NO_ARGUMENT],
                 ['--browsermark-js', GetoptLong::NO_ARGUMENT],
                 ['--browsermark-dom', GetoptLong::NO_ARGUMENT],
                 ['--octane', GetoptLong::NO_ARGUMENT],
                 ['--compression-bench', GetoptLong::NO_ARGUMENT],
                 ['--benchmarks', GetoptLong::REQUIRED_ARGUMENT],
                 ['--measure-gc', GetoptLong::OPTIONAL_ARGUMENT],
                 ['--force-vm-kind', GetoptLong::REQUIRED_ARGUMENT],
                 ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
                 ['--dont-copy-vms', GetoptLong::NO_ARGUMENT],
                 ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
                 ['--brief', GetoptLong::NO_ARGUMENT],
                 ['--silent', GetoptLong::NO_ARGUMENT],
                 ['--remote', GetoptLong::REQUIRED_ARGUMENT],
                 ['--local', GetoptLong::NO_ARGUMENT],
                 ['--ssh-options', GetoptLong::REQUIRED_ARGUMENT],
                 ['--slave', GetoptLong::NO_ARGUMENT],
                 ['--prepare-only', GetoptLong::NO_ARGUMENT],
                 ['--analyze', GetoptLong::REQUIRED_ARGUMENT],
                 ['--vms', GetoptLong::REQUIRED_ARGUMENT],
                 ['--output-name', GetoptLong::REQUIRED_ARGUMENT],
                 ['--environment', GetoptLong::REQUIRED_ARGUMENT],
                 ['--config', GetoptLong::REQUIRED_ARGUMENT],
                 ['--help', '-h', GetoptLong::NO_ARGUMENT]).each {
    | opt, arg |
    case opt
    when '--rerun'
      $rerun = intArg(opt,arg,1,nil)
    when '--inner'
      $inner = intArg(opt,arg,1,nil)
    when '--outer'
      $outer = intArg(opt,arg,1,nil)
    when '--warmup'
      $warmup = intArg(opt,arg,0,nil)
    when '--no-ss-warmup'
      $sunSpiderWarmup = false
    when '--quantum'
      $quantum = intArg(opt,arg,1,nil)
    when '--minimum'
      $minimum = intArg(opt,arg,1,nil)
    when '--timing-mode'
      if arg.upcase == "PRECISETIME"
        $timeMode = :preciseTime
      elsif arg.upcase == "DATE"
        $timeMode = :date
      elsif arg.upcase == "AUTO"
        $timeMode = :auto
      else
        quickFail("Expected either 'preciseTime', 'date', or 'auto' for --time-mode, but got '#{arg}'.",
                  "Invalid argument for command-line option")
      end
    when '--force-vm-kind'
      if arg.upcase == "JSC"
        $forceVMKind = :jsc
      elsif arg.upcase == "DUMPRENDERTREE"
        $forceVMKind = :dumpRenderTree
      elsif arg.upcase == "WEBKITTESTRUNNER"
        $forceVMKind = :webkitTestRunner
      elsif arg.upcase == "AUTO"
        $forceVMKind = nil
      else
        quickFail("Expected 'jsc', 'DumpRenderTree', or 'WebKitTestRunner' for --force-vm-kind, but got '#{arg}'.",
                  "Invalid argument for command-line option")
      end
    when '--force-vm-copy'
      $needToCopyVMs = true
    when '--dont-copy-vms'
      $dontCopyVMs = true
    when '--sunspider'
      resetBenchOptionsIfNecessary
      $includeSunSpider = true
    when '--longspider'
      resetBenchOptionsIfNecessary
      $includeLongSpider = true
    when '--v8-spider'
      resetBenchOptionsIfNecessary
      $includeV8 = true
    when '--kraken'
      resetBenchOptionsIfNecessary
      $includeKraken = true
    when '--js-bench'
      resetBenchOptionsIfNecessary
      $includeJSBench = true
    when '--js-regress'
      resetBenchOptionsIfNecessary
      $includeJSRegress = true
    when '--asm-bench'
      resetBenchOptionsIfNecessary
      $includeAsmBench = true
    when '--dsp'
      resetBenchOptionsIfNecessary
      $includeDSPJS = true
    when '--browsermark-js'
      resetBenchOptionsIfNecessary
      $includeBrowsermarkJS = true
    when '--browsermark-dom'
      resetBenchOptionsIfNecessary
      $includeBrowsermarkDOM = true
    when '--octane'
      resetBenchOptionsIfNecessary
      $includeOctane = true
    when '--compression-bench'
      resetBenchOptionsIfNecessary
      $includeCompressionBench = true
    when '--benchmarks'
      $benchmarkPattern = Regexp.new(arg)
    when '--measure-gc'
      if arg == ''
        $measureGC = true
      else
        $measureGC = arg
      end
    when '--verbose'
      $verbosity += 1
    when '--brief'
      $brief = true
    when '--silent'
      $silent = true
    when '--remote'
      $remoteHosts += arg.split(',')
      $needToCopyVMs = true
    when '--ssh-options'
      $sshOptions << arg
    when '--local'
      $alsoLocal = true
    when '--prepare-only'
      $run = false
    when '--analyze'
      $prepare = false
      $run = false
      $analyze << arg
    when '--output-name'
      $outputName = arg
    when '--vms'
      JSON::parse(IO::read(arg)).each {
        | vmDescription |
        path = Pathname.new(vmDescription["path"]).realpath
        if vmDescription["name"]
          name = vmDescription["name"]
          nameKind = :given
        else
          name = "Conf\##{$vms.length+1}"
          nameKind = :auto
        end
        vm = VM.new(path, name, nameKind, nil)
        if vmDescription["env"]
          vmDescription["env"].each_pair {
            | key, val |
            vm.addExtraEnv(key, val)
          }
        end
        $vms << vm
      }
    when '--environment'
      $environment = JSON::parse(IO::read(arg))
    when '--config'
      $configPath = Pathname.new(arg)
    when '--help'
      usage
    else
      raise "bad option: #{opt}"
    end
  }
  
  # Figure out the configuration
  if $configPath.file?
    config = JSON::parse(IO::read($configPath.to_s))
  else
    config = {}
  end

  def pathname_if_exist config
    if config
      Pathname.new config
    else
      config
    end
  end

  OCTANE_PATH = pathname_if_exist(config["OctanePath"])
  BROWSERMARK_PATH = pathname_if_exist(config["BrowserMarkPath"])
  BROWSERMARK_JS_PATH = pathname_if_exist(config["BrowserMarkJSPath"])
  BROWSERMARK_DOM_PATH = pathname_if_exist(config["BrowserMarkDOMPath"])
  ASMBENCH_PATH = pathname_if_exist(config["AsmBenchPath"])
  COMPRESSIONBENCH_PATH = pathname_if_exist(config["CompressionBenchPath"])
  DSPJS_FILTRR_PATH = pathname_if_exist(config["DSPJSFiltrrPath"])
  DSPJS_ROUTE9_PATH = pathname_if_exist(config["DSPJSRoute9Path"])
  DSPJS_STARFIELD_PATH = pathname_if_exist(config["DSPJSStarfieldPath"])
  DSPJS_QUAKE3_PATH = pathname_if_exist(config["DSPJSQuake3Path"])
  DSPJS_MANDELBROT_PATH = pathname_if_exist(config["DSPJSMandelbrotPath"])
  DSPJS_JSLINUX_PATH = pathname_if_exist(config["DSPJSLinuxPath"])
  DSPJS_AMMOJS_ASMJS_PATH = pathname_if_exist(config["DSPJSAmmoJSAsmJSPath"])
  DSPJS_AMMOJS_REGULAR_PATH = pathname_if_exist(config["DSPJSAmmoJSRegularPath"])
  JSBENCH_PATH = pathname_if_exist(config["JSBenchPath"])
  KRAKEN_PATH = pathname_if_exist(config["KrakenPath"])
  
  # If the --dont-copy-vms option was passed, it overrides the --force-vm-copy option.
  if $dontCopyVMs
    $needToCopyVMs = false
  end
  
  ARGV.each {
    | vm |
    if vm =~ /([a-zA-Z0-9_ .]+):/
      name = $1
      nameKind = :given
      vm = $~.post_match
    else
      name = "Conf\##{$vms.length+1}"
      nameKind = :auto
    end
    envs = []
    while vm =~ /([a-zA-Z0-9_]+)=([a-zA-Z0-9_:.]+):/
      envs << [$1, $2]
      vm = $~.post_match
    end
    $stderr.puts "#{name}: #{vm}" if $verbosity >= 1
    vm = VM.new(Pathname.new(vm).realpath, name, nameKind, nil)
    envs.each {
      | pair |
      vm.addExtraEnv(pair[0], pair[1])
    }
    $vms << vm
  }
  
  if $vms.empty?
    quickFail("Please specify at least on configuraiton on the command line.",
              "Insufficient arguments")
  end
  
  $vms.each {
    | vm |
    if vm.vmType == :jsc
      $allDRT = false
    end
  }
  
  SUNSPIDER = BenchmarkSuite.new("SunSpider", :arithmeticMean, 0)
  WARMUP = BenchmarkSuite.new("WARMUP", :arithmeticMean, 0)
  ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
   "access-fannkuch", "access-nbody", "access-nsieve",
   "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-bitwise-and",
   "bitops-nsieve-bits", "controlflow-recursive", "crypto-aes",
   "crypto-md5", "crypto-sha1", "date-format-tofte", "date-format-xparb",
   "math-cordic", "math-partial-sums", "math-spectral-norm", "regexp-dna",
   "string-base64", "string-fasta", "string-tagcloud",
   "string-unpack-code", "string-validate-input"].each {
    | name |
    SUNSPIDER.add SunSpiderBenchmark.new(name)
    WARMUP.addIgnoringPattern SunSpiderBenchmark.new(name)
  }

  LONGSPIDER = BenchmarkSuite.new("LongSpider", :geometricMean, 0)
  ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
   "access-fannkuch", "access-nbody", "access-nsieve",
   "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-nsieve-bits",
   "controlflow-recursive", "crypto-aes", "crypto-md5", "crypto-sha1",
   "date-format-tofte", "date-format-xparb", "hash-map", "math-cordic",
   "math-partial-sums", "math-spectral-norm", "string-base64",
   "string-fasta", "string-tagcloud"].each {
    | name |
    LONGSPIDER.add LongSpiderBenchmark.new(name)
  }

  V8 = BenchmarkSuite.new("V8Spider", :geometricMean, 0)
  ["crypto", "deltablue", "earley-boyer", "raytrace",
   "regexp", "richards", "splay"].each {
    | name |
    V8.add V8Benchmark.new(name)
  }

  OCTANE = BenchmarkSuite.new("Octane", :geometricMean, 1)
  [[["crypto"], "encrypt", 1, true, false, 32],
   [["crypto"], "decrypt", 1, true, false, 32],
   [["deltablue"], "deltablue", 2, true, false, 32],
   [["earley-boyer"], "earley", 1, true, false, 32],
   [["earley-boyer"], "boyer", 1, true, false, 32],
   [["navier-stokes"], "navier-stokes", 2, true, false, 16],
   [["raytrace"], "raytrace", 2, true, false, 32],
   [["richards"], "richards", 2, true, false, 32],
   [["splay"], "splay", 2, true, false, 32],
   [["regexp"], "regexp", 2, true, false, 16],
   [["pdfjs"], "pdfjs", 2, false, false, 4],
   [["mandreel"], "mandreel", 2, false, false, 4],
   [["gbemu-part1", "gbemu-part2"], "gbemu", 2, false, false, 4],
   [["code-load"], "closure", 1, false, false, 16],
   [["code-load"], "jquery", 1, false, false, 16],
   [["box2d"], "box2d", 2, false, false, 8],
   [["zlib", "zlib-data"], "zlib", 2, false, true, 3],
   [["typescript", "typescript-input", "typescript-compiler"], "typescript", 2, false, true, 1]].each {
    | args |
    OCTANE.add OctaneBenchmark.new(*args)
  }

  KRAKEN = BenchmarkSuite.new("Kraken", :arithmeticMean, -1)
  ["ai-astar", "audio-beat-detection", "audio-dft", "audio-fft",
   "audio-oscillator", "imaging-darkroom", "imaging-desaturate",
   "imaging-gaussian-blur", "json-parse-financial",
   "json-stringify-tinderbox", "stanford-crypto-aes",
   "stanford-crypto-ccm", "stanford-crypto-pbkdf2",
   "stanford-crypto-sha256-iterative"].each {
    | name |
    KRAKEN.add KrakenBenchmark.new(name)
  }
  
  JSBENCH = BenchmarkSuite.new("JSBench", :arithmeticMean, 0)
  [["amazon", "urm"], ["facebook", "urem"], ["google", "urem"], ["twitter", "urem"],
   ["yahoo", "urem"]].each {
    | nameAndMode |
    JSBENCH.add JSBenchBenchmark.new(*nameAndMode)
  }
  
  JSREGRESS = BenchmarkSuite.new("JSRegress", :geometricMean, 0)
  Dir.foreach(JSREGRESS_PATH) {
    | filename |
    if filename =~ /\.js$/
      name = $~.pre_match
      JSREGRESS.add JSRegressBenchmark.new(name)
    end
  }
  
  ASMBENCH = BenchmarkSuite.new("AsmBench", :geometricMean, 0)
  if JSBENCH_PATH
    Dir.foreach(ASMBENCH_PATH) {
      | filename |
      if filename =~ /\.js$/
        name = $~.pre_match
        ASMBENCH.add AsmBenchBenchmark.new(name)
      end
    }
  end

  COMPRESSIONBENCH = BenchmarkSuite.new("CompressionBench", :geometricMean, 0)
  [[["huffman", "compression-data"], "huffman", ""],
   [["arithmetic", "compression-data"], "arithmetic", "Simple"],
   [["arithmetic", "compression-data"], "arithmetic", "Precise"],
   [["arithmetic", "compression-data"], "arithmetic", "Complex Precise"],
   [["arithmetic", "compression-data"], "arithmetic", "Precise Order 0"],
   [["arithmetic", "compression-data"], "arithmetic", "Precise Order 1"],
   [["arithmetic", "compression-data"], "arithmetic", "Precise Order 2"],
   [["arithmetic", "compression-data"], "arithmetic", "Simple Order 1"],
   [["arithmetic", "compression-data"], "arithmetic", "Simple Order 2"],
   [["lz-string", "compression-data"], "lz-string", ""]
  ].each {
    | args |
    COMPRESSIONBENCH.add CompressionBenchBenchmark.new(*args)
  }

  DSPJS = BenchmarkSuite.new("DSP", :geometricMean, 0)
  DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-posterize-tint", "e2")
  DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-tint-contrast-sat-bright", "e5")
  DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-tint-sat-adj-contr-mult", "e7")
  DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-blur-overlay-sat-contr", "e8")
  DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-sat-blur-mult-sharpen-contr", "e9")
  DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-sepia-bias", "e10")
  DSPJS.add DSPJSVP8Benchmark.new
  DSPJS.add DSPStarfieldBenchmark.new
  DSPJS.add DSPJSJSLinuxBenchmark.new
  DSPJS.add DSPJSQuake3Benchmark.new
  DSPJS.add DSPJSMandelbrotBenchmark.new
  DSPJS.add DSPJSAmmoJSASMBenchmark.new
  DSPJS.add DSPJSAmmoJSRegularBenchmark.new

  BROWSERMARK_JS = BenchmarkSuite.new("BrowsermarkJS", :geometricMean, 1)
  ["array_blur", "array_weighted", "string_chat", "string_filter", "string_weighted"].each {
    | name |
      BROWSERMARK_JS.add BrowsermarkJSBenchmark.new(name)
  }
  
  BROWSERMARK_DOM = BenchmarkSuite.new("BrowsermarkDOM", :geometricMean, 1)
  ["advanced_search", "create_source", "dynamic_create", "search"].each {
    | name |
      BROWSERMARK_DOM.add BrowsermarkDOMBenchmark.new(name)
  }

  $suites = []
  
  if $includeSunSpider and not SUNSPIDER.empty?
    $suites << SUNSPIDER
  end
  
  if $includeLongSpider and not LONGSPIDER.empty?
    $suites << LONGSPIDER
  end
  
  if $includeV8 and not V8.empty?
    $suites << V8
  end
  
  if $includeOctane and not OCTANE.empty?
    if OCTANE_PATH
      $suites << OCTANE
    else
      $stderr.puts "Warning: refusing to run Octane because \"OctanePath\" isn't set in #{$configPath}."
    end
  end
  
  if $includeKraken and not KRAKEN.empty?
    if KRAKEN_PATH
      $suites << KRAKEN
    else
      $stderr.puts "Warning: refusing to run Kraken because \"KrakenPath\" isn't set in #{$configPath}."
    end
  end
  
  if $includeJSBench and not JSBENCH.empty?
    if $allDRT
      if JSBENCH_PATH
        $suites << JSBENCH
      else
        $stderr.puts "Warning: refusing to run JSBench because \"JSBenchPath\" isn't set in #{$configPath}"
      end
    else
      $stderr.puts "Warning: refusing to run JSBench because not all VMs are DumpRenderTree or WebKitTestRunner."
    end
  end
  
  if $includeJSRegress and not JSREGRESS.empty?
    $suites << JSREGRESS
  end
  
  if $includeAsmBench and not ASMBENCH.empty?
    if ASMBENCH_PATH
      $suites << ASMBENCH
    else
      $stderr.puts "Warning: refusing to run AsmBench because \"AsmBenchPath\" isn't set in #{$configPath}."
    end
  end
  
  if $includeDSPJS and not DSPJS.empty?
    if $allDRT
      if DSPJS_FILTRR_PATH and DSPJS_ROUTE9_PATH and DSPJS_STARFIELD_PATH and DSPJS_QUAKE3_PATH and DSPJS_MANDELBROT_PATH and DSPJS_JSLINUX_PATH and DSPJS_AMMOJS_ASMJS_PATH and DSPJS_AMMOJS_REGULAR_PATH
        $suites << DSPJS
      else
        $stderr.puts "Warning: refusing to run DSPJS because one of the following isn't set in #{$configPath}: \"DSPJSFiltrrPath\", \"DSPJSRoute9Path\", \"DSPJSStarfieldPath\", \"DSPJSQuake3Path\", \"DSPJSMandelbrotPath\", \"DSPJSLinuxPath\", \"DSPJSAmmoJSAsmJSPath\", \"DSPJSAmmoJSRegularPath\"."
      end
    else
      $stderr.puts "Warning: refusing to run DSPJS because not all VMs are DumpRenderTree or WebKitTestRunner."
    end
  end
  
  if $includeBrowsermarkJS and not BROWSERMARK_JS.empty?
    if BROWSERMARK_PATH and BROWSERMARK_JS_PATH
      $suites << BROWSERMARK_JS
    else
      $stderr.puts "Warning: refusing to run Browsermark-JS because one of the following isn't set in #{$configPath}: \"BrowserMarkPath\" or \"BrowserMarkJSPath\"."
    end
  end

  if $includeBrowsermarkDOM and not BROWSERMARK_DOM.empty?
    if $allDRT
      if BROWSERMARK_PATH and BROWSERMARK_JS_PATH and BROWSERMARK_DOM_PATH
        $suites << BROWSERMARK_DOM
      else
        $stderr.puts "Warning: refusing to run Browsermark-DOM because one of the following isn't set in #{$configPath}: \"BrowserMarkPath\", \"BrowserMarkJSPath\", or \"BrowserMarkDOMPath\"."
      end
    else
      $stderr.puts "Warning: refusing to run Browsermark-DOM because not all VMs are DumpRenderTree or WebKitTestRunner."
    end
  end

  if $includeCompressionBench and not COMPRESSIONBENCH.empty?
    if COMPRESSIONBENCH_PATH
      $suites << COMPRESSIONBENCH
    else
      $stderr.puts "Warning: refusing to run CompressionBench because \"CompressionBenchPath\" isn't set in #{$configPath}"
    end
  end

  $allSuites = $suites.map{|v| v.suites}.flatten(1)
  
  $benchmarks = []
  $suites.each {
    | suite |
    $benchmarks += suite.benchmarks
  }
  
  if $suites.empty? or $benchmarks.empty?
    $stderr.puts "No benchmarks found.  Bailing out."
    exit 1
  end
  
  if $outer*$inner == 1
    $stderr.puts "Warning: will only collect one sample per benchmark/VM.  Confidence interval calculation will fail."
  end
  
  $stderr.puts "Using timeMode = #{$timeMode}." if $verbosity >= 1
  
  $runPlans = []
  $vms.each {
    | vm |
    $benchmarks.each {
      | benchmark |
      $outer.times {
        | iteration |
        $runPlans << BenchRunPlan.new(benchmark, vm, iteration)
      }
    }
  }
  
  $runPlans.shuffle!
  
  if $sunSpiderWarmup
    warmupPlans = []
    $vms.each {
      | vm |
      WARMUP.benchmarks.each {
        | benchmark |
        warmupPlans << BenchRunPlan.new(benchmark, vm, 0)
      }
    }
    
    $runPlans = warmupPlans.shuffle + $runPlans
  end
  
  $suitepad = $suites.collect {
    | suite |
    suite.to_s.size
  }.max + 1
  
  $planpad = $runPlans.collect {
    | plan |
    plan.to_s.size
  }.max + 1
  
  maxBenchNameLength =
    ($benchmarks + ["<arithmetic> *", "<geometric> *", "<harmonic> *"]).collect {
    | benchmark |
    if benchmark.respond_to? :name
      benchmark.name.size
    else
      benchmark.size
    end
  }.max
  $benchNameClip = 40
  $benchpad = [maxBenchNameLength, $benchNameClip].min + 1
  
  $weightpad = $benchmarks.collect {
    | benchmark |
    benchmark.weightString.size
  }.max

  $vmpad = $vms.collect {
    | vm |
    vm.to_s.size
  }.max + 1
  
  $analyze.each_with_index {
    | filename, index |
    if index >= 1
      puts
    end
    parseAndDisplayResults(IO::read(filename))
  }
  
  if not $prepare and not $run
    exit 0
  end
  
  if FileTest.exist? BENCH_DATA_PATH
    cmd = "rm -rf #{BENCH_DATA_PATH}"
    $stderr.puts ">> #{cmd}" if $verbosity >= 2
    raise unless system cmd
  end
  
  Dir.mkdir BENCH_DATA_PATH
  
  if $needToCopyVMs
    canCopyIntoBenchPath = true
    $vms.each {
      | vm |
      canCopyIntoBenchPath = false unless vm.canCopyIntoBenchPath
    }
    
    if canCopyIntoBenchPath
      $vms.each {
        | vm |
        $stderr.puts "Copying #{vm} into #{BENCH_DATA_PATH}..."
        vm.copyIntoBenchPath
      }
      $stderr.puts "All VMs are in place."
    else
      $stderr.puts "Warning: don't know how to copy some VMs into #{BENCH_DATA_PATH}, so I won't do it."
    end
  end
  
  if $measureGC and $measureGC != true
    found = false
    $vms.each {
      | vm |
      if vm.name == $measureGC
        found = true
      end
    }
    unless found
      $stderr.puts "Warning: --measure-gc option ignored because no VM is named #{$measureGC}"
    end
  end
  
  if $prepare
    File.open("#{BENCH_DATA_PATH}/runscript", "w") {
      | file |
      file.puts "echo \"HOSTNAME:\\c\""
      file.puts "hostname"
      file.puts "echo"
      file.puts "echo \"HARDWARE:\\c\""
      file.puts "/usr/sbin/sysctl hw.model"
      file.puts "echo"
      file.puts "set -e"
      $script = file
      $runPlans.each_with_index {
        | plan, idx |
        if $verbosity == 0 and not $silent
          text1 = lpad(idx.to_s,$runPlans.size.to_s.size)+"/"+$runPlans.size.to_s
          text2 = plan.to_s
          file.puts("echo " + Shellwords.shellescape("\r#{text1} #{rpad(text2,$planpad)}") + "\"\\c\" 1>&2")
          file.puts("echo " + Shellwords.shellescape("\r#{text1} #{text2}") + "\"\\c\" 1>&2")
        end
        plan.emitRunCode
      }
      if $verbosity == 0 and not $silent
        file.puts("echo " + Shellwords.shellescape("\r#{$runPlans.size}/#{$runPlans.size} #{' '*($suitepad+1+$benchpad+1+$vmpad)}") + "\"\\c\" 1>&2")
        file.puts("echo " + Shellwords.shellescape("\r#{$runPlans.size}/#{$runPlans.size}") + " 1>&2")
      end
    }
  end
  
  if $run
    unless $remoteHosts.empty?
      $stderr.puts "Packaging benchmarking directory for remote hosts..." if $verbosity==0
      Dir.chdir(TEMP_PATH) {
        cmd = "tar -czf payload.tar.gz benchdata"
        $stderr.puts ">> #{cmd}" if $verbosity>=2
        raise unless system(cmd)
      }
      
      def grokHost(host)
        if host =~ /:([0-9]+)$/
          "-p " + $1 + " " + Shellwords.shellescape($~.pre_match)
        else
          Shellwords.shellescape(host)
        end
      end
      
      def sshRead(host, command)
        cmd = "ssh #{$sshOptions.collect{|x| Shellwords.shellescape(x)}.join(' ')} #{grokHost(host)} #{Shellwords.shellescape(command)}"
        $stderr.puts ">> #{cmd}" if $verbosity>=2
        result = ""
        IO.popen(cmd, "r") {
          | inp |
          inp.each_line {
            | line |
            $stderr.puts "#{host}: #{line}" if $verbosity>=2
            result += line
          }
        }
        raise "#{$?}" unless $?.success?
        result
      end
      
      def sshWrite(host, command, data)
        cmd = "ssh #{$sshOptions.collect{|x| Shellwords.shellescape(x)}.join(' ')} #{grokHost(host)} #{Shellwords.shellescape(command)}"
        $stderr.puts ">> #{cmd}" if $verbosity>=2
        IO.popen(cmd, "w") {
          | outp |
          outp.write(data)
        }
        raise "#{$?}" unless $?.success?
      end
      
      $remoteHosts.each {
        | host |
        $stderr.puts "Sending benchmark payload to #{host}..." if $verbosity==0
        
        remoteTempPath = JSON::parse(sshRead(host, "cat ~/.bencher"))["tempPath"]
        raise unless remoteTempPath
        
        sshWrite(host, "cd #{Shellwords.shellescape(remoteTempPath)} && rm -rf benchdata && tar -xz", IO::read("#{TEMP_PATH}/payload.tar.gz"))
        
        $stderr.puts "Running on #{host}..." if $verbosity==0
        
        parseAndDisplayResults(sshRead(host, "cd #{Shellwords.shellescape(remoteTempPath + '/benchdata')} && sh runscript"))
      }
    end
    
    if not $remoteHosts.empty? and $alsoLocal
      $stderr.puts "Running locally..."
    end
    
    if $remoteHosts.empty? or $alsoLocal
      parseAndDisplayResults(runAndGetResults)
    end
  end
  
  if $prepare and not $run and $analyze.empty?
    puts wrap("Benchmarking script and data are in #{BENCH_DATA_PATH}. You can run "+
              "the benchmarks and get the results by doing:", 78)
    puts
    puts "cd #{BENCH_DATA_PATH}"
    puts "sh runscript > results.txt"
    puts
    puts wrap("Then you can analyze the results by running bencher with the same arguments "+
              "as now, but replacing --prepare-only with --analyze results.txt.", 78)
  end
rescue => e
  fail(e)
end