| #!/usr/bin/env ruby |
| # coding: utf-8 |
| |
| require 'getoptlong' |
| require 'pathname' |
| |
| $benchmarks_all = [ |
| # Single-threaded benchmarks. |
| "churn", |
| "list_allocate", |
| "tree_allocate", |
| "tree_churn", |
| "fragment", |
| "fragment_iterate", |
| "medium", |
| "big", |
| |
| # Benchmarks based on browser recordings. |
| "facebook", |
| "reddit", |
| "flickr", |
| "theverge", |
| "nimlang", |
| |
| # Multi-threaded benchmark variants. |
| "message_one", |
| "message_many", |
| "churn --parallel", |
| "list_allocate --parallel", |
| "tree_allocate --parallel", |
| "tree_churn --parallel", |
| "fragment --parallel", |
| "fragment_iterate --parallel", |
| |
| # These tests often crash TCMalloc: <rdar://problem/13657137>. |
| "medium --parallel", |
| "big --parallel", |
| |
| # Enable these tests to test memory footprint. The way they run is not |
| # really compatible with throughput testing. |
| # "reddit_memory_warning --runs 0", |
| # "flickr_memory_warning --runs 0", |
| # "theverge_memory_warning --runs 0", |
| |
| # Enable this test to test shrinking back down from a large heap while a process remains active. |
| # The way it runs is not really compatible with throughput testing. |
| # "balloon" |
| "facebook --parallel", |
| "reddit --parallel", |
| "flickr --parallel", |
| "theverge --parallel", |
| # "nimlang --use-thread-id", |
| ] |
| |
| $benchmarks_memory = [ |
| "facebook", |
| "reddit", |
| "flickr", |
| "theverge", |
| "nimlang" |
| ] |
| |
| $benchmarks_memory_warning = [ |
| "reddit_memory_warning --runs 0", |
| "flickr_memory_warning --runs 0", |
| "theverge_memory_warning --runs 0", |
| ] |
| |
| $benchmarks = $benchmarks_all |
| $heap = 0 |
| |
| def usage |
| puts "run-malloc-benchmarks [options] /path/to/MallocBench Name:/path/to/libmbmalloc.dylib [ Name:/path/to/libmbmalloc.dylib ]" |
| puts |
| puts " Runs a suite of memory allocation and access benchmarks." |
| puts |
| puts " <Name:/path/to/libmbmalloc.dylib> is a symbolic name followed by a path to libmbmalloc.dylib." |
| puts |
| puts " Specify \"SystemMalloc\" to test the built-in libc malloc." |
| puts " Specify \"NanoMalloc\" to test the built-in libc malloc using the NanoMalloc zone." |
| puts |
| puts " Example usage:" |
| puts |
| puts " run-malloc-benchmarks /BUILD/MallocBench SystemMalloc:/BUILD/libmbmalloc.dylib NanoMalloc:/BUILD/libmbmalloc.dylib" |
| puts " run-malloc-benchmarks /BUILD/MallocBench FastMalloc:/BUILD/FastMalloc/libmbmalloc.dylib" |
| puts " run-malloc-benchmarks --benchmark churn SystemMalloc:/BUILD/libmbmalloc.dylib FastMalloc:/BUILD/FastMalloc/libmbmalloc.dylib" |
| puts |
| puts "Options:" |
| puts |
| puts " --benchmark <benchmark> Select a single benchmark to run instead of the full suite." |
| puts " --heap <heap> Set a baseline heap size." |
| puts |
| end |
| |
| class Dylib |
| attr_reader :name |
| attr_reader :path |
| |
| def initialize(name, path) |
| @name = name |
| @path = path |
| end |
| end |
| |
| class Results |
| attr_reader :executionTime |
| attr_reader :peakMemory |
| attr_reader :memoryAtEnd |
| |
| def initialize(executionTime, peakMemory, memoryAtEnd) |
| @executionTime = executionTime |
| @peakMemory = peakMemory |
| @memoryAtEnd = memoryAtEnd |
| end |
| end |
| |
| class Stat |
| attr_reader :benchmark |
| attr_reader :result |
| |
| def initialize(benchmark, result) |
| @benchmark = benchmark |
| @result = result[/\d+/].to_i |
| end |
| end |
| |
| class TimeStat < Stat |
| def to_s |
| @result + "ms" |
| end |
| end |
| |
| class MemoryStat < Stat |
| def to_s |
| @result + "kB" |
| end |
| end |
| |
| class PeakMemoryStat < Stat |
| def to_s |
| @result + "kB" |
| end |
| end |
| |
| def lpad(str, chars) |
| if str.length > chars |
| str |
| else |
| "%#{chars}s"%(str) |
| end |
| end |
| |
| def rpad(str, chars) |
| while str.length < chars |
| str += " " |
| end |
| str |
| end |
| |
| def computeArithmeticMean(array) |
| sum = 0.0 |
| array.each { |
| | value | |
| sum += value |
| } |
| (sum / array.length) |
| end |
| |
| def computeGeometricMean(array) |
| mult = 1.0 |
| array.each { |
| | value | |
| mult *= value ? value : 1.0 |
| } |
| (mult ** (1.0 / array.length)) |
| end |
| |
| def computeHarmonicMean(array) |
| 1.0 / computeArithmeticMean(array.collect{ | value | 1.0 / value }) |
| end |
| |
| def lowerIsBetter(a, b, better, worse) |
| if b < a |
| return "^ " + (a.to_f / b.to_f).round(2).to_s + "x " + better |
| end |
| |
| if b == a |
| return "" |
| end |
| |
| "! " + (b.to_f / a.to_f).round(2).to_s + "x " + worse |
| end |
| |
| |
| def lowerIsFaster(a, b) |
| lowerIsBetter(a, b, "faster", "slower") |
| end |
| |
| def lowerIsSmaller(a, b) |
| lowerIsBetter(a, b, "smaller", "bigger") |
| end |
| |
| def numberWithDelimiter(number) |
| number.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse |
| end |
| |
| def prettify(number, suffix) |
| numberWithDelimiter(number) + suffix |
| end |
| |
| def parseOptions |
| GetoptLong.new( |
| ['--benchmark', GetoptLong::REQUIRED_ARGUMENT], |
| ['--memory', GetoptLong::NO_ARGUMENT], |
| ['--memory_warning', GetoptLong::NO_ARGUMENT], |
| ['--heap', GetoptLong::REQUIRED_ARGUMENT], |
| ['--help', GetoptLong::NO_ARGUMENT], |
| ).each { |
| | opt, arg | |
| case opt |
| when '--benchmark' |
| $benchmarks = [ arg ] |
| when '--memory' |
| $benchmarks = $benchmarks_memory |
| when '--memory_warning' |
| $benchmarks = $benchmarks_memory_warning |
| when '--heap' |
| $heap = arg |
| when '--help' |
| usage |
| exit 1 |
| else |
| raise "bad option: #{opt}" |
| end |
| } |
| |
| if ARGV.length < 1 |
| puts "Error: No MallocBench specified." |
| exit 1 |
| end |
| |
| if ARGV.length < 2 |
| puts "Error: No dylib specified." |
| exit 1 |
| end |
| |
| $mallocBench = File.absolute_path(ARGV.shift) |
| if !File.exists?($mallocBench) |
| puts "File not found: #{$mallocBench}." |
| exit 1 |
| end |
| |
| $buildDir = Pathname.new($mallocBench).dirname |
| |
| dylibs = [] |
| ARGV.each { |
| | arg | |
| name, path = arg.split(":") |
| if !name || name.length < 1 || |
| !path || path.length < 1 |
| puts "Invalid <Name:/path/to/dylib>: '#{arg}'." |
| exit 1 |
| end |
| |
| dylib = Dylib.new(name, File.expand_path(path)) |
| |
| if !File.exists?(dylib.path) |
| puts "File not found: #{dylib.path}." |
| exit 1 |
| end |
| |
| dylibs.push(dylib) |
| } |
| dylibs |
| end |
| |
| def runBenchmarks(dylibs) |
| executionTime = [] |
| peakMemory = [] |
| memoryAtEnd = [] |
| |
| $benchmarks.each { |
| | benchmark | |
| |
| executionTime.push([]) |
| peakMemory.push([]) |
| memoryAtEnd.push([]) |
| |
| dylibs.each { |
| | dylib | |
| |
| $stderr.print "\rRUNNING #{dylib.name}: #{benchmark}... " |
| env = "DYLD_LIBRARY_PATH='#{Pathname.new(dylib.path).dirname}' " |
| env += "LD_LIBRARY_PATH='#{Pathname.new(dylib.path).dirname}' " |
| if dylib.name == "NanoMalloc" |
| env += "MallocNanoZone=1 " |
| elsif dylib.name == "SystemMalloc" |
| env += "MallocNanoZone=0 " |
| end |
| input = "cd '#{$buildDir}'; #{env} '#{$mallocBench}' --benchmark #{benchmark} --heap #{$heap}}" |
| output =`#{input}` |
| splitOutput = output.split("\n") |
| |
| executionTime[-1].push(TimeStat.new(benchmark, splitOutput[1])) |
| peakMemory[-1].push(PeakMemoryStat.new(benchmark, splitOutput.length > 3 ? splitOutput[2] : "0")) |
| memoryAtEnd[-1].push(MemoryStat.new(benchmark, splitOutput.length > 2 ? splitOutput[3] : "0")) |
| } |
| } |
| $stderr.print "\r \n" |
| |
| Results.new(executionTime, peakMemory, memoryAtEnd) |
| end |
| |
| def printResults(dylibs, results) |
| def printHeader(dylibs, fieldSize) |
| print |
| print lpad("", fieldSize) |
| print lpad(dylibs[0].name, fieldSize) |
| if dylibs.length > 1 |
| print lpad(dylibs[1].name, fieldSize) |
| print lpad("Δ", fieldSize) |
| end |
| print "\n" |
| end |
| |
| def printMetric(name, results, compareFunction, suffix, fieldSize) |
| def printMean(name, results, meanFunction, compareFunction, suffix, fieldSize) |
| means = [] |
| |
| means.push(meanFunction.call(results.collect { | stats | stats[0].result })) |
| print rpad(" " + name, fieldSize) |
| print lpad("#{prettify(means[0].round, suffix)}", fieldSize) |
| |
| if results[0][1] |
| means.push(meanFunction.call(results.collect { | stats | stats[1].result })) |
| print lpad("#{prettify(means[1].round, suffix)}", fieldSize) |
| print lpad(compareFunction.call(means[0], means[1]), fieldSize) |
| end |
| |
| print "\n" |
| end |
| |
| if results[0][0].result == 0 |
| return |
| end |
| |
| print name + ":\n" |
| results.each { |
| | stats | |
| |
| print rpad(" " + stats[0].benchmark, fieldSize) |
| print lpad("#{prettify(stats[0].result, suffix)}", fieldSize) |
| |
| if stats[1] |
| print lpad("#{prettify(stats[1].result, suffix)}", fieldSize) |
| print lpad(compareFunction.call(stats[0].result, stats[1].result), fieldSize) |
| end |
| |
| print "\n" |
| } |
| |
| print "\n" |
| |
| printMean("<geometric mean>", results, method(:computeGeometricMean), compareFunction, suffix, fieldSize) |
| printMean("<arithmetic mean>", results, method(:computeArithmeticMean), compareFunction, suffix, fieldSize) |
| printMean("<harmonic mean>", results, method(:computeHarmonicMean), compareFunction, suffix, fieldSize) |
| |
| print "\n" |
| end |
| |
| fieldSize = ($benchmarks + ["<arithmetic mean>"]).collect { |
| | benchmark | |
| benchmark.size |
| }.max + 4 |
| |
| printHeader(dylibs, fieldSize) |
| printMetric("Execution Time", results.executionTime, method(:lowerIsFaster), "ms", fieldSize) |
| printMetric("Peak Memory", results.peakMemory, method(:lowerIsSmaller), "kB", fieldSize) |
| printMetric("Memory at End", results.memoryAtEnd, method(:lowerIsSmaller), "kB", fieldSize) |
| end |
| |
| def main |
| begin |
| dylibs = parseOptions() |
| results = runBenchmarks(dylibs) |
| printResults(dylibs, results) |
| rescue => exception |
| puts |
| puts |
| puts exception |
| puts exception.backtrace |
| puts |
| end |
| end |
| |
| main() |
