WebKitTools/Scripts/run-webkit-tests - WebKit - Git at Google

 #!/usr/bin/perl -w

 # Copyright (C) 2005 Apple Computer, Inc.  All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 #
 # 1.  Redistributions of source code must retain the above copyright
 #     notice, this list of conditions and the following disclaimer.
 # 2.  Redistributions in binary form must reproduce the above copyright
 #     notice, this list of conditions and the following disclaimer in the
 #     documentation and/or other materials provided with the distribution.
 # 3.  Neither the name of Apple Computer, Inc. ("Apple") nor the names of
 #     its contributors may be used to endorse or promote products derived
 #     from this software without specific prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 # Script to run the Web Kit Open Source Project layout tests.

 use strict;
 use IPC::Open2;
 use Getopt::Long;
 use FindBin;
 use Cwd;
 use lib $FindBin::Bin;
 use webkitdirs;

 # Run all the tests passed in on the command line.
 # If no tests are passed, find all the .html, .xml, .xhtml (and svg) files in the test directory.

 # Run each text.
 # Compare against the existing file xxx-expected.txt.
 # If there is a mismatch, generate xxx-actual.txt and xxx-diffs.txt.

 # At the end, report:
 #   the number of tests that got the expected results
 #   the number of tests that ran, but did not get the expected results
 #   the number of tests that failed to run
 #   the number of tests that were run but had no expected results to compare against

 setConfiguration();
 my $productDir = productDir();

 chdirWebKit();

 # Argument handling
 my $testSVGs = '';
 my $pixelTests = '';
 my $maxWidth = '';
 my $maxHeight = '';
 my $verbose = '';
 my $quiet = '';

 GetOptions('svg' => \$testSVGs,
     'pixel-tests' => \$pixelTests,
     'max-width' => \$maxWidth,
     'max-height' => \$maxHeight,
     'verbose' => \$verbose,
     'quiet' => \$quiet);

 my $toolname = "DumpRenderTree";

 my $result = system "WebKitTools/Scripts/build-dumprendertree", @ARGV;
 exit $result if $result;
 if ($testSVGs) {
     my $result = system "WebKitTools/Scripts/build-dumpkcanvastree", @ARGV;
     exit $result if $result;
     $toolname = "DumpKCanvasTree";
 }

 my $tool = "$productDir/$toolname";
 my $imageDiffTool = "$productDir/ImageDiff";
 die "can't find executable $toolname tool (looked in $productDir)\n" if !-x $tool;
 die "can't find executable $imageDiffTool tool (looked in $productDir)\n" if $pixelTests && !-x $imageDiffTool;

 if ($testSVGs) {
     checkSVGFrameworks();
 } else {
     checkFrameworks();
 }

 my $layoutTestsName = "layout-tests";
 if ($testSVGs) {
     $layoutTestsName = "svg-tests";
 }

 my $workingDir = getcwd();
 my $WebCoreDirectory = "$workingDir/WebCore";
 my $testDirectory = "$WebCoreDirectory/$layoutTestsName";
 my $testResultsDirectory = "/tmp/layout-test-results";
 my $testResults = "$testResultsDirectory/results.html";

 $ENV{DYLD_FRAMEWORK_PATH} = $productDir;

 my @tests;

 my $findArguments = "\\( -name resources \\! -prune \\) -or -name '*.html' -or -name '*.xml' -or -name '*.xhtml'";
 if ($testSVGs) {
     $findArguments = "\\( -name resources \\! -prune \\) -or -name '*.svg'";
 }
 my $foundTestName = 0;
 for my $test (@ARGV) {
     next if $test =~ /^-/;
     $foundTestName = 1;
     $test =~ s/^$testDirectory\///;
     if ($test =~ /^\//) {
         print "can't run test outside $testDirectory\n";
     } elsif (-f "$testDirectory/$test") {
         if ($test !~ /\.(html|xml|xhtml|svg)$/) {
             print "test $test does not have a supported extension\n";
         } else {
             push @tests, $test;
         }
     } elsif (-d "$testDirectory/$test") {
         push @tests, map { chomp; s-^$testDirectory/--; $_; } `find -Ls $testDirectory/$test $findArguments`;
     } else {
         print "test $test not found\n";
     }
 }
 if (!$foundTestName) {
     @tests = map { chomp; s-^$testDirectory/--; $_; } `find -Ls $testDirectory $findArguments`;
 }

 die "no tests to run\n" if !@tests;

 my %counts;
 my %tests;
 my %imagesPresent;
 my $count;
 my @toolArgs;

 if ($pixelTests) {
     push @toolArgs, "--pixel-tests";
     if ($maxWidth) {
         push @toolArgs, ("--width", $maxWidth);
     }

     if ($maxWidth) {
         push @toolArgs, ("--height", $maxWidth);
     }
 }

 push  @toolArgs, "-";

 $| = 1;

 open2(\*IN, \*OUT, $tool, @toolArgs) or die;

 if ($pixelTests) {
     open2(\*DIFFIN, \*DIFFOUT, $imageDiffTool, "") or die "unable to open $imageDiffTool\n";
 }

 my $column = 0;
 my $lastDirectory = "";

 for my $test (@tests) {
     next if $test eq 'results.html';

     my $base = $test;
     $base =~ s/\.(html|xml|xhtml|svg)$//;

     if ($verbose) {
         print "running $base test";
     } elsif (!$quiet) {
         my $dir = $base;
         $dir =~ s|/[^/]+$||;
         if ($dir ne $lastDirectory) {
             print "\n" unless $column == 0;
             print $dir . " ";
             $lastDirectory = $dir;
             $column = 0;
         }

         print ".";
         $column++;
     }

     my $result;

     print OUT "$testDirectory/$test\n";

     my $actual = "";
     while (<IN>) {
         last if /#EOF/;
         $actual .= $_;
     }

     my $expected;
     if (open EXPECTED, "<", "$testDirectory/$base-expected.txt") {
         $expected = "";
         while (<EXPECTED>) {
             $expected .= $_;
         }
         close EXPECTED;
     }

     my $textDumpMatches = $expected && ($actual eq $expected);
     my $actualHash = "";
     my $expectedHash = "";
     my $hashMatches = "";
     my $actualPNG = "";
     my $actualPNGSize = 0;
     my $expectedPNG = "";
     my $expectedPNGSize = 0;
     my $diffPNG = "";
     my $diffPercentage = "";
     my $diffResult = "passed";

     if ($pixelTests) {
         while (<IN>) {
             last if /#EOF/;
             if (/ActualHash: ([a-f0-9]{32})/) {
                 $actualHash = $1;
             } elsif (/BaselineHash: ([a-f0-9]{32})/) {
                 $expectedHash = $1;
             } elsif (/Content-length: (\d+)\s*/) {
                 $actualPNGSize = $1;
                 read(IN, $actualPNG, $actualPNGSize);
             }
         }

         if ($hashMatches = ($expectedHash eq $actualHash)) {
             $diffResult = "passed";
         }

         if (!$hashMatches && -f "$testDirectory/$base-expected.png" && $textDumpMatches) {
             $expectedPNGSize = getFilesize("$testDirectory/$base-expected.png");
             open EXPECTEDPNG, "$testDirectory/$base-expected.png";
             read(EXPECTEDPNG, $expectedPNG, $expectedPNGSize);

             print DIFFOUT "Content-length: $actualPNGSize\n";
             print DIFFOUT $actualPNG;

             print DIFFOUT "Content-length: $expectedPNGSize\n";
             print DIFFOUT $expectedPNG;

             while (<DIFFIN>) {
 		last if /^error/ || /^diff:/;
                 if (/Content-length: (\d+)\s*/) {
                     read(DIFFIN, $diffPNG, $1);
                 }
             }

             if (/^diff: (.+)% (passed|failed)/) {
                 $diffPercentage = $1;
                 $diffResult = $2;
             }
         }
     }

     if ($pixelTests) {
         if ($actualPNGSize != 0 && ! -f "$testDirectory/$base-expected.png") {
             open EXPECTED, ">", "$testDirectory/$base-expected.png" or die "could not create $testDirectory/$base-expected.png\n";
             print EXPECTED $actualPNG;
             close EXPECTED;
         }

         # update the expected hash if the image diff said that there was no difference
         if ($actualHash ne "" && ! -f "$testDirectory/$base-expected.checksum") {
             open EXPECTED, ">", "$testDirectory/$base-expected.checksum" or die "could not create $testDirectory/$base-expected.checksum\n";
             print EXPECTED $actualHash;
             close EXPECTED;
         }
     }

     if (!defined $expected) {
         if ($verbose) {
             print " -> new test\n";
         }
         $result = "new";
         open EXPECTED, ">", "$testDirectory/$base-expected.txt" or die "could not create $testDirectory/$base-expected.txt\n";
         print EXPECTED $actual;
         close EXPECTED;

         unlink "$testResultsDirectory/$base-actual.txt";
         unlink "$testResultsDirectory/$base-diffs.txt";
     } elsif ($textDumpMatches && (!$pixelTests || ($pixelTests && $diffResult eq "passed"))) {
         if ($verbose) {
             print " -> succeeded\n";
         }
         $result = "match";
         unlink "$testResultsDirectory/$base-actual.txt";
         unlink "$testResultsDirectory/$base-diffs.txt";
     } elsif (!$textDumpMatches || ($pixelTests && $diffResult ne "passed")) {
         print "\n" unless $column == 0;
         print "$test -> failed\n";
         $column = 0;

         $result = "mismatch";

         my $dir = "$testResultsDirectory/$base";
         $dir =~ s|/[^/]+$||;
         system "mkdir", "-p", $dir;

         open ACTUAL, ">", "$testResultsDirectory/$base-actual.txt" or die;
         print ACTUAL $actual;
         close ACTUAL;

         system "diff -u \"$testDirectory/$base-expected.txt\" \"$testResultsDirectory/$base-actual.txt\" > \"$testResultsDirectory/$base-diffs.txt\"";

         if ($pixelTests && $diffPNG && $diffPNG ne "") {
             $imagesPresent{$base} = 1;

             open ACTUAL, ">", "$testResultsDirectory/$base-actual.png" or die;
             print ACTUAL $actualPNG;
             close ACTUAL;

             open DIFF, ">", "$testResultsDirectory/$base-diffs.png" or die;
             print DIFF $diffPNG;
             close DIFF;

             open DIFFHTML, ">$testResultsDirectory/$base-diffs.html" or die;
             print DIFFHTML "<html>\n";
             print DIFFHTML "<head>\n";
             print DIFFHTML "<title>$base Image Compare</title>\n";
             print DIFFHTML "<script language=\"Javascript\" type=\"text/javascript\">\n";
             print DIFFHTML "var actualImageVisible = true;\n";
             print DIFFHTML "function animateImage() {\n";
             print DIFFHTML "    var image = document.getElementById(\"animatedImage\");\n";
             print DIFFHTML "    var imageText = document.getElementById(\"imageText\");\n";
             print DIFFHTML "    if (actualImageVisible) {\n";
             print DIFFHTML "        image.src=\"$testDirectory/$base-expected.png\";\n";
             print DIFFHTML "        imageText.innerHTML = \"Expected Image\";\n";
             print DIFFHTML "        actualImageVisible = false;\n";
             print DIFFHTML "    } else {\n";
             print DIFFHTML "        image.src=\"$testResultsDirectory/$base-actual.png\";\n";
             print DIFFHTML "        imageText.innerHTML = \"Actual Image\";\n";
             print DIFFHTML "        actualImageVisible = true;\n";
             print DIFFHTML "    }\n";
             print DIFFHTML "    setTimeout('animateImage()',2000);\n";
             print DIFFHTML "}\n";
             print DIFFHTML "</script>\n";
             print DIFFHTML "</head>\n";
             print DIFFHTML "<body onLoad=\"animateImage();\">\n";
             print DIFFHTML "<table>\n";
             if ($diffPercentage) {
                 print DIFFHTML "<tr>\n";
                 print DIFFHTML "<td>Difference between images: <a href=\"$testResultsDirectory/$base-diffs.png\">$diffPercentage%</a></td>\n";
                 print DIFFHTML "</tr>\n";
             }
             print DIFFHTML "<tr>\n";
             print DIFFHTML "<td id=\"imageText\" style=\"text-weight: bold;\">Actual Image</td>\n";
             print DIFFHTML "</tr>\n";
             print DIFFHTML "<tr>\n";
             print DIFFHTML "<td><img src=\"$testResultsDirectory/$base-actual.png\" id=\"animatedImage\"></td>\n";
             print DIFFHTML "</tr>\n";
             print DIFFHTML "</table>\n";
             print DIFFHTML "</body>\n";
             print DIFFHTML "</html>\n";
         }
     } else {
         $result = "fail";
         print "\n" unless $column == 0;
         print "$test -> crashed\n";
         $column = 0;

         close IN;
         close OUT;
         open2(\*IN, \*OUT, $tool, @toolArgs) or die;
     }

     $count += 1;
     $counts{$result} += 1;
     push @{$tests{$result}}, $test;
 }

 close IN;
 close OUT;

 my %text = (
     match => "succeeded",
     mismatch => "had incorrect layout",
     new => "were new",
     fail => "failed (tool did not execute successfully)",
 );

 print "\n";

 if ($counts{match} && $counts{match} == $count) {
     print "all $count test cases succeeded\n";
     unlink $testResults;
 } else {
     for my $type ("match", "mismatch", "new", "fail") {
         my $c = $counts{$type};
         if ($c) {
             my $t = $text{$type};
             my $message;
             if ($c == 1) {
                 $t =~ s/were/was/;
                 $message = sprintf "1 test case (%d%%) %s\n", 1 * 100 / $count, $t;
             } else {
                 $message = sprintf "%d test cases (%d%%) %s\n", $c, $c * 100 / $count, $t;
             }
             $message =~ s-\(0%\)-(<1%)-;
             print $message;
         }
     }

     system "mkdir", "-p", $testResultsDirectory;

     open HTML, ">", $testResults or die;
     print HTML "<html>\n";
     print HTML "<head>\n";
     print HTML "<title>Layout Test Results</title>\n";
     print HTML "</head>\n";
     print HTML "<body>\n";

     if ($counts{mismatch}) {
         print HTML "<p>Tests where results did not match expected results:</p>\n";
         print HTML "<table>\n";
         for my $test (@{$tests{mismatch}}) {
             my $base = $test;
             $base =~ s/\.(html|xml|xhtml|svg)$//;
             print HTML "<tr>\n";
             print HTML "<td><a href=\"$testDirectory/$test\">$base</a></td>\n";
             print HTML "<td><a href=\"$testDirectory/$base-expected.txt\">expected</a></td>\n";
             print HTML "<td><a href=\"$base-actual.txt\">actual</a></td>\n";
             print HTML "<td><a href=\"$base-diffs.txt\">diffs</a></td>\n";
             if ($pixelTests) {
                 print HTML "<td><a href=\"$testDirectory/$base-expected.png\">expected image</a></td>\n";
                 if ($imagesPresent{$base}) {
                     print HTML "<td><a href=\"$base-diffs.html\">image diffs</a></td>\n";
                 }
             }
             print HTML "</tr>\n";
         }
         print HTML "</table>\n";
     }

     if ($counts{fail}) {
         print HTML "<p>Tests that caused the DumpRenderTree tool to fail:</p>\n";
         print HTML "<table>\n";
         for my $test (@{$tests{fail}}) {
             my $base = $test;
             $base =~ s/\.(html|xml|xhtml|svg)$//;
             print HTML "<tr>\n";
             print HTML "<td><a href=\"$testDirectory/$test\">$base</a></td>\n";
             print HTML "</tr>\n";
         }
         print HTML "</table>\n";
     }

     if ($counts{new}) {
         print HTML "<p>Tests that had no expected results (probably new):</p>\n";
         print HTML "<table>\n";
         for my $test (@{$tests{new}}) {
             my $base = $test;
             $base =~ s/\.(html|xml|xhtml|svg)$//;
             print HTML "<tr>\n";
             print HTML "<td><a href=\"$testDirectory/$test\">$base</a></td>\n";
             print HTML "<td><a href=\"$testDirectory/$base-expected.txt\">results</a></td>\n";
             if ($pixelTests) {
                 print HTML "<td><a href=\"$testDirectory/$base-expected.png\">image</a></td>\n";
             }
             print HTML "</tr>\n";
         }
         print HTML "</table>\n";
     }

     print HTML "</body>\n";
     print HTML "</html>\n";
     close HTML;

     system "open", $testResults;
 }

 sub getFilesize
 {
     my $filename = shift;
     my @fileStats;

     @fileStats = stat($filename);

     $fileStats[7];
 }
	#!/usr/bin/perl -w

	# Copyright (C) 2005 Apple Computer, Inc. All rights reserved.
	#
	# Redistribution and use in source and binary forms, with or without
	# modification, are permitted provided that the following conditions
	# are met:
	#
	# 1. Redistributions of source code must retain the above copyright
	# notice, this list of conditions and the following disclaimer.
	# 2. Redistributions in binary form must reproduce the above copyright
	# notice, this list of conditions and the following disclaimer in the
	# documentation and/or other materials provided with the distribution.
	# 3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of
	# its contributors may be used to endorse or promote products derived
	# from this software without specific prior written permission.
	#
	# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
	# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
	# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
	# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
	# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
	# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
	# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

	# Script to run the Web Kit Open Source Project layout tests.

	use strict;
	use IPC::Open2;
	use Getopt::Long;
	use FindBin;
	use Cwd;
	use lib $FindBin::Bin;
	use webkitdirs;

	# Run all the tests passed in on the command line.
	# If no tests are passed, find all the .html, .xml, .xhtml (and svg) files in the test directory.

	# Run each text.
	# Compare against the existing file xxx-expected.txt.
	# If there is a mismatch, generate xxx-actual.txt and xxx-diffs.txt.

	# At the end, report:
	# the number of tests that got the expected results
	# the number of tests that ran, but did not get the expected results
	# the number of tests that failed to run
	# the number of tests that were run but had no expected results to compare against

	setConfiguration();
	my $productDir = productDir();

	chdirWebKit();

	# Argument handling
	my $testSVGs = '';
	my $pixelTests = '';
	my $maxWidth = '';
	my $maxHeight = '';
	my $verbose = '';
	my $quiet = '';

	GetOptions('svg' => \$testSVGs,
	'pixel-tests' => \$pixelTests,
	'max-width' => \$maxWidth,
	'max-height' => \$maxHeight,
	'verbose' => \$verbose,
	'quiet' => \$quiet);

	my $toolname = "DumpRenderTree";

	my $result = system "WebKitTools/Scripts/build-dumprendertree", @ARGV;
	exit $result if $result;
	if ($testSVGs) {
	my $result = system "WebKitTools/Scripts/build-dumpkcanvastree", @ARGV;
	exit $result if $result;
	$toolname = "DumpKCanvasTree";
	}

	my $tool = "$productDir/$toolname";
	my $imageDiffTool = "$productDir/ImageDiff";
	die "can't find executable $toolname tool (looked in $productDir)\n" if !-x $tool;
	die "can't find executable $imageDiffTool tool (looked in $productDir)\n" if $pixelTests && !-x $imageDiffTool;

	if ($testSVGs) {
	checkSVGFrameworks();
	} else {
	checkFrameworks();
	}

	my $layoutTestsName = "layout-tests";
	if ($testSVGs) {
	$layoutTestsName = "svg-tests";
	}

	my $workingDir = getcwd();
	my $WebCoreDirectory = "$workingDir/WebCore";
	my $testDirectory = "$WebCoreDirectory/$layoutTestsName";
	my $testResultsDirectory = "/tmp/layout-test-results";
	my $testResults = "$testResultsDirectory/results.html";

	$ENV{DYLD_FRAMEWORK_PATH} = $productDir;

	my @tests;

	my $findArguments = "\\( -name resources \\! -prune \\) -or -name '.html' -or -name '.xml' -or -name '*.xhtml'";
	if ($testSVGs) {
	$findArguments = "\\( -name resources \\! -prune \\) -or -name '*.svg'";
	}
	my $foundTestName = 0;
	for my $test (@ARGV) {
	next if $test =~ /^-/;
	$foundTestName = 1;
	$test =~ s/^$testDirectory\///;
	if ($test =~ /^\//) {
	print "can't run test outside $testDirectory\n";
	} elsif (-f "$testDirectory/$test") {
	if ($test !~ /\.(html\|xml\|xhtml\|svg)$/) {
	print "test $test does not have a supported extension\n";
	} else {
	push @tests, $test;
	}
	} elsif (-d "$testDirectory/$test") {
	push @tests, map { chomp; s-^$testDirectory/--; $_; } `find -Ls $testDirectory/$test $findArguments`;
	} else {
	print "test $test not found\n";
	}
	}
	if (!$foundTestName) {
	@tests = map { chomp; s-^$testDirectory/--; $_; } `find -Ls $testDirectory $findArguments`;
	}

	die "no tests to run\n" if !@tests;

	my %counts;
	my %tests;
	my %imagesPresent;
	my $count;
	my @toolArgs;

	if ($pixelTests) {
	push @toolArgs, "--pixel-tests";
	if ($maxWidth) {
	push @toolArgs, ("--width", $maxWidth);
	}

	if ($maxWidth) {
	push @toolArgs, ("--height", $maxWidth);
	}
	}

	push @toolArgs, "-";

	$\| = 1;

	open2(\IN, \OUT, $tool, @toolArgs) or die;

	if ($pixelTests) {
	open2(\DIFFIN, \DIFFOUT, $imageDiffTool, "") or die "unable to open $imageDiffTool\n";
	}

	my $column = 0;
	my $lastDirectory = "";

	for my $test (@tests) {
	next if $test eq 'results.html';

	my $base = $test;
	$base =~ s/\.(html\|xml\|xhtml\|svg)$//;

	if ($verbose) {
	print "running $base test";
	} elsif (!$quiet) {
	my $dir = $base;
	$dir =~ s\|/[^/]+$\|\|;
	if ($dir ne $lastDirectory) {
	print "\n" unless $column == 0;
	print $dir . " ";
	$lastDirectory = $dir;
	$column = 0;
	}

	print ".";
	$column++;
	}

	my $result;

	print OUT "$testDirectory/$test\n";

	my $actual = "";
	while (<IN>) {
	last if /#EOF/;
	$actual .= $_;
	}

	my $expected;
	if (open EXPECTED, "<", "$testDirectory/$base-expected.txt") {
	$expected = "";
	while (<EXPECTED>) {
	$expected .= $_;
	}
	close EXPECTED;
	}

	my $textDumpMatches = $expected && ($actual eq $expected);
	my $actualHash = "";
	my $expectedHash = "";
	my $hashMatches = "";
	my $actualPNG = "";
	my $actualPNGSize = 0;
	my $expectedPNG = "";
	my $expectedPNGSize = 0;
	my $diffPNG = "";
	my $diffPercentage = "";
	my $diffResult = "passed";

	if ($pixelTests) {
	while (<IN>) {
	last if /#EOF/;
	if (/ActualHash: ([a-f0-9]{32})/) {
	$actualHash = $1;
	} elsif (/BaselineHash: ([a-f0-9]{32})/) {
	$expectedHash = $1;
	} elsif (/Content-length: (\d+)\s*/) {
	$actualPNGSize = $1;
	read(IN, $actualPNG, $actualPNGSize);
	}
	}

	if ($hashMatches = ($expectedHash eq $actualHash)) {
	$diffResult = "passed";
	}

	if (!$hashMatches && -f "$testDirectory/$base-expected.png" && $textDumpMatches) {
	$expectedPNGSize = getFilesize("$testDirectory/$base-expected.png");
	open EXPECTEDPNG, "$testDirectory/$base-expected.png";
	read(EXPECTEDPNG, $expectedPNG, $expectedPNGSize);

	print DIFFOUT "Content-length: $actualPNGSize\n";
	print DIFFOUT $actualPNG;

	print DIFFOUT "Content-length: $expectedPNGSize\n";
	print DIFFOUT $expectedPNG;

	while (<DIFFIN>) {
	last if /^error/ \|\| /^diff:/;
	if (/Content-length: (\d+)\s*/) {
	read(DIFFIN, $diffPNG, $1);
	}
	}

	if (/^diff: (.+)% (passed\|failed)/) {
	$diffPercentage = $1;
	$diffResult = $2;
	}
	}
	}

	if ($pixelTests) {
	if ($actualPNGSize != 0 && ! -f "$testDirectory/$base-expected.png") {
	open EXPECTED, ">", "$testDirectory/$base-expected.png" or die "could not create $testDirectory/$base-expected.png\n";
	print EXPECTED $actualPNG;
	close EXPECTED;
	}

	# update the expected hash if the image diff said that there was no difference
	if ($actualHash ne "" && ! -f "$testDirectory/$base-expected.checksum") {
	open EXPECTED, ">", "$testDirectory/$base-expected.checksum" or die "could not create $testDirectory/$base-expected.checksum\n";
	print EXPECTED $actualHash;
	close EXPECTED;
	}
	}

	if (!defined $expected) {
	if ($verbose) {
	print " -> new test\n";
	}
	$result = "new";
	open EXPECTED, ">", "$testDirectory/$base-expected.txt" or die "could not create $testDirectory/$base-expected.txt\n";
	print EXPECTED $actual;
	close EXPECTED;

	unlink "$testResultsDirectory/$base-actual.txt";
	unlink "$testResultsDirectory/$base-diffs.txt";
	} elsif ($textDumpMatches && (!$pixelTests \|\| ($pixelTests && $diffResult eq "passed"))) {
	if ($verbose) {
	print " -> succeeded\n";
	}
	$result = "match";
	unlink "$testResultsDirectory/$base-actual.txt";
	unlink "$testResultsDirectory/$base-diffs.txt";
	} elsif (!$textDumpMatches \|\| ($pixelTests && $diffResult ne "passed")) {
	print "\n" unless $column == 0;
	print "$test -> failed\n";
	$column = 0;

	$result = "mismatch";

	my $dir = "$testResultsDirectory/$base";
	$dir =~ s\|/[^/]+$\|\|;
	system "mkdir", "-p", $dir;

	open ACTUAL, ">", "$testResultsDirectory/$base-actual.txt" or die;
	print ACTUAL $actual;
	close ACTUAL;

	system "diff -u \"$testDirectory/$base-expected.txt\" \"$testResultsDirectory/$base-actual.txt\" > \"$testResultsDirectory/$base-diffs.txt\"";

	if ($pixelTests && $diffPNG && $diffPNG ne "") {
	$imagesPresent{$base} = 1;

	open ACTUAL, ">", "$testResultsDirectory/$base-actual.png" or die;
	print ACTUAL $actualPNG;
	close ACTUAL;

	open DIFF, ">", "$testResultsDirectory/$base-diffs.png" or die;
	print DIFF $diffPNG;
	close DIFF;

	open DIFFHTML, ">$testResultsDirectory/$base-diffs.html" or die;
	print DIFFHTML "<html>\n";
	print DIFFHTML "<head>\n";
	print DIFFHTML "<title>$base Image Compare</title>\n";
	print DIFFHTML "<script language=\"Javascript\" type=\"text/javascript\">\n";
	print DIFFHTML "var actualImageVisible = true;\n";
	print DIFFHTML "function animateImage() {\n";
	print DIFFHTML " var image = document.getElementById(\"animatedImage\");\n";
	print DIFFHTML " var imageText = document.getElementById(\"imageText\");\n";
	print DIFFHTML " if (actualImageVisible) {\n";
	print DIFFHTML " image.src=\"$testDirectory/$base-expected.png\";\n";
	print DIFFHTML " imageText.innerHTML = \"Expected Image\";\n";
	print DIFFHTML " actualImageVisible = false;\n";
	print DIFFHTML " } else {\n";
	print DIFFHTML " image.src=\"$testResultsDirectory/$base-actual.png\";\n";
	print DIFFHTML " imageText.innerHTML = \"Actual Image\";\n";
	print DIFFHTML " actualImageVisible = true;\n";
	print DIFFHTML " }\n";
	print DIFFHTML " setTimeout('animateImage()',2000);\n";
	print DIFFHTML "}\n";
	print DIFFHTML "</script>\n";
	print DIFFHTML "</head>\n";
	print DIFFHTML "<body onLoad=\"animateImage();\">\n";
	print DIFFHTML "<table>\n";
	if ($diffPercentage) {
	print DIFFHTML "<tr>\n";
	print DIFFHTML "<td>Difference between images: <a href=\"$testResultsDirectory/$base-diffs.png\">$diffPercentage%</a></td>\n";
	print DIFFHTML "</tr>\n";
	}
	print DIFFHTML "<tr>\n";
	print DIFFHTML "<td id=\"imageText\" style=\"text-weight: bold;\">Actual Image</td>\n";
	print DIFFHTML "</tr>\n";
	print DIFFHTML "<tr>\n";
	print DIFFHTML "<td><img src=\"$testResultsDirectory/$base-actual.png\" id=\"animatedImage\"></td>\n";
	print DIFFHTML "</tr>\n";
	print DIFFHTML "</table>\n";
	print DIFFHTML "</body>\n";
	print DIFFHTML "</html>\n";
	}
	} else {
	$result = "fail";
	print "\n" unless $column == 0;
	print "$test -> crashed\n";
	$column = 0;

	close IN;
	close OUT;
	open2(\IN, \OUT, $tool, @toolArgs) or die;
	}

	$count += 1;
	$counts{$result} += 1;
	push @{$tests{$result}}, $test;
	}

	close IN;
	close OUT;

	my %text = (
	match => "succeeded",
	mismatch => "had incorrect layout",
	new => "were new",
	fail => "failed (tool did not execute successfully)",
	);

	print "\n";

	if ($counts{match} && $counts{match} == $count) {
	print "all $count test cases succeeded\n";
	unlink $testResults;
	} else {
	for my $type ("match", "mismatch", "new", "fail") {
	my $c = $counts{$type};
	if ($c) {
	my $t = $text{$type};
	my $message;
	if ($c == 1) {
	$t =~ s/were/was/;
	$message = sprintf "1 test case (%d%%) %s\n", 1 * 100 / $count, $t;
	} else {
	$message = sprintf "%d test cases (%d%%) %s\n", $c, $c * 100 / $count, $t;
	}
	$message =~ s-\(0%\)-(<1%)-;
	print $message;
	}
	}

	system "mkdir", "-p", $testResultsDirectory;

	open HTML, ">", $testResults or die;
	print HTML "<html>\n";
	print HTML "<head>\n";
	print HTML "<title>Layout Test Results</title>\n";
	print HTML "</head>\n";
	print HTML "<body>\n";

	if ($counts{mismatch}) {
	print HTML "<p>Tests where results did not match expected results:</p>\n";
	print HTML "<table>\n";
	for my $test (@{$tests{mismatch}}) {
	my $base = $test;
	$base =~ s/\.(html\|xml\|xhtml\|svg)$//;
	print HTML "<tr>\n";
	print HTML "<td><a href=\"$testDirectory/$test\">$base</a></td>\n";
	print HTML "<td><a href=\"$testDirectory/$base-expected.txt\">expected</a></td>\n";
	print HTML "<td><a href=\"$base-actual.txt\">actual</a></td>\n";
	print HTML "<td><a href=\"$base-diffs.txt\">diffs</a></td>\n";
	if ($pixelTests) {
	print HTML "<td><a href=\"$testDirectory/$base-expected.png\">expected image</a></td>\n";
	if ($imagesPresent{$base}) {
	print HTML "<td><a href=\"$base-diffs.html\">image diffs</a></td>\n";
	}
	}
	print HTML "</tr>\n";
	}
	print HTML "</table>\n";
	}

	if ($counts{fail}) {
	print HTML "<p>Tests that caused the DumpRenderTree tool to fail:</p>\n";
	print HTML "<table>\n";
	for my $test (@{$tests{fail}}) {
	my $base = $test;
	$base =~ s/\.(html\|xml\|xhtml\|svg)$//;
	print HTML "<tr>\n";
	print HTML "<td><a href=\"$testDirectory/$test\">$base</a></td>\n";
	print HTML "</tr>\n";
	}
	print HTML "</table>\n";
	}

	if ($counts{new}) {
	print HTML "<p>Tests that had no expected results (probably new):</p>\n";
	print HTML "<table>\n";
	for my $test (@{$tests{new}}) {
	my $base = $test;
	$base =~ s/\.(html\|xml\|xhtml\|svg)$//;
	print HTML "<tr>\n";
	print HTML "<td><a href=\"$testDirectory/$test\">$base</a></td>\n";
	print HTML "<td><a href=\"$testDirectory/$base-expected.txt\">results</a></td>\n";
	if ($pixelTests) {
	print HTML "<td><a href=\"$testDirectory/$base-expected.png\">image</a></td>\n";
	}
	print HTML "</tr>\n";
	}
	print HTML "</table>\n";
	}

	print HTML "</body>\n";
	print HTML "</html>\n";
	close HTML;

	system "open", $testResults;
	}

	sub getFilesize
	{
	my $filename = shift;
	my @fileStats;

	@fileStats = stat($filename);

	$fileStats[7];
	}