Tools/Scripts/svn-apply

#!/usr/bin/env perl

# Copyright (C) 2005, 2006, 2007 Apple Inc.  All rights reserved.
# Copyright (C) 2009 Cameron McCormack <cam@mcc.id.au>
# Copyright (C) 2010 Chris Jerdonek (chris.jerdonek@gmail.com)
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1.  Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution. 
# 3.  Neither the name of Apple Inc. ("Apple") nor the names of
#     its contributors may be used to endorse or promote products derived
#     from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# "patch" script for WebKit Open Source Project, used to apply patches.

# Differences from invoking "patch -p0":
#
#   Handles added files (does a svn add with logic to handle local changes).
#   Handles added directories (does a svn add).
#   Handles removed files (does a svn rm with logic to handle local changes).
#   Handles removed directories--those with no more files or directories left in them
#       (does a svn rm).
#   Has mode where it will roll back to svn version numbers in the patch file so svn
#       can do a 3-way merge.
#   Paths from Index: lines are used rather than the paths on the patch lines, which
#       makes patches generated by "cvs diff" work (increasingly unimportant since we
#       use Subversion now).
#   ChangeLog patches use --fuzz=3 to prevent rejects.
#   Handles binary files (requires patches made by svn-create-patch).
#   Handles copied and moved files (requires patches made by svn-create-patch).
#   Handles git-diff patches (without binary changes) created at the top-level directory
#
# Missing features:
#
#   Handle property changes.
#   Handle copied and moved directories (would require patches made by svn-create-patch).
#   When doing a removal, check that old file matches what's being removed.
#   Notice a patch that's being applied at the "wrong level" and make it work anyway.
#   Do a dry run on the whole patch and don't do anything if part of the patch is
#       going to fail (probably too strict unless we exclude ChangeLog).
#   Handle git-diff patches with binary delta

use strict;
use warnings;

use Digest::MD5;
use File::Basename;
use File::Copy qw(copy);
use File::Spec;
use Getopt::Long;
use MIME::Base64;
use POSIX qw(strftime);

use FindBin;
use lib $FindBin::Bin;
use VCSUtils;

sub addDirectoriesIfNeeded($);
sub applyPatch($$;$);
sub checksum($);
sub handleBinaryChange($$);
sub handleGitBinaryChange($$);
sub isDirectoryEmptyForRemoval($);
sub patch($);
sub removeDirectoriesIfNeeded();

# These should be replaced by an scm class/module:
sub scmKnowsOfFile($);
sub scmCopy($$);
sub scmAdd($);
sub scmAddQueued($);
sub scmCommitQueueToggledExecutableBit();
sub scmCommitQueueAdded();
sub scmRemove($);
sub scmToggleExecutableBitQueued($$);

my $merge = 0;
my $showHelp = 0;
my $reviewer;
my $force = 0;
my $skipChangeLogs = 0;
my @scmQueuedFilesToAdd = ();
my %scmQueuedExecutableBits;

my $optionParseSuccess = GetOptions(
    "merge!" => \$merge,
    "help!" => \$showHelp,
    "reviewer=s" => \$reviewer,
    "force!" => \$force,
    "skip-changelogs" => \$skipChangeLogs
);

if (!$optionParseSuccess || $showHelp) {
    print STDERR basename($0) . " [-h|--help] [--force] [-m|--merge] [-r|--reviewer name] [--skip-changelogs] patch1 [patch2 ...]\n";
    exit 1;
}

my %removeDirectoryIgnoreList = (
    '.' => 1,
    '..' => 1,
    '.git' => 1,
    '.svn' => 1,
    '_svn' => 1,
);

my $epochTime = time(); # This is used to set the date in ChangeLog files.
my $globalExitStatus = 0;

my $repositoryRootPath = determineVCSRoot();

my %checkedDirectories;

# Need to use a typeglob to pass the file handle as a parameter,
# otherwise get a bareword error.
my @diffHashRefs = parsePatch(*ARGV);

print "Parsed " . @diffHashRefs . " diffs from patch file(s).\n";
die "No diff found." unless @diffHashRefs;

my $preparedPatchHash = prepareParsedPatch($force, @diffHashRefs);

my @copyDiffHashRefs = @{$preparedPatchHash->{copyDiffHashRefs}};
my @nonCopyDiffHashRefs = @{$preparedPatchHash->{nonCopyDiffHashRefs}};
my %sourceRevisions = %{$preparedPatchHash->{sourceRevisionHash}};

if ($merge) {
    die "--merge is currently only supported for SVN" unless isSVN();
    # How do we handle Git patches applied to an SVN checkout here?
    for my $file (sort keys %sourceRevisions) {
        my $version = $sourceRevisions{$file};
        print "Getting version $version of $file\n";
        my $escapedFile = escapeSubversionPath($file);
        system("svn", "update", "-r", $version, $escapedFile) == 0 or die "Failed to run svn update -r $version $escapedFile.";
    }
}

# Handle copied and moved files first since moved files may have their
# source deleted before the move.
for my $copyDiffHashRef (@copyDiffHashRefs) {
    my $indexPath = $copyDiffHashRef->{indexPath};
    my $copiedFromPath = $copyDiffHashRef->{copiedFromPath};

    addDirectoriesIfNeeded(dirname($indexPath));
    scmCopy($copiedFromPath, $indexPath);
}

for my $diffHashRef (@nonCopyDiffHashRefs) {
    patch($diffHashRef);
}

# For git we need to toggle the executable bit before adding the files
# For SVN is the other way around.
if (isGit()) {
    scmCommitQueueToggledExecutableBit();
    scmCommitQueueAdded();
} elsif (isSVN()) {
    scmCommitQueueAdded();
    scmCommitQueueToggledExecutableBit();
}

removeDirectoriesIfNeeded();

exit $globalExitStatus;

sub addDirectoriesIfNeeded($)
{
    my ($path) = @_;
    my @dirs = File::Spec->splitdir($path);
    my $dir = ".";
    while (scalar @dirs) {
        $dir = File::Spec->catdir($dir, shift @dirs);
        die "'$dir' exists, but is not a directory" if ( -e $dir && ! -d $dir);
        if (isGit()) {
            # Git removes a directory once the last file in it is removed. We need
            # explicitly check for the existence of each directory along the path
            # (and create it if it doesn't) so as to support patches that move all files in
            # directory A to A/B. That is, we cannot depend on %checkedDirectories.
            mkdir $dir if (! -e $dir);
        } elsif (isSVN()) {
            next if exists $checkedDirectories{$dir};
            if (! -e $dir) {
                mkdir $dir or die "Failed to create required directory '$dir' for path '$path'\n";
                scmAdd($dir)
            }
            elsif (-d $dir) {
                # SVN prints "svn: warning: 'directory' is already under version control"
                # if you try and add a directory which is already in the repository
                # So we check first to see if the directory is under version control first.
                if (!scmKnowsOfFile($dir)) {
                    scmAdd($dir);
                }
            }
        }
        $checkedDirectories{$dir} = 1;
    }
}

# Args:
#   $patch: a patch string.
#   $pathRelativeToRoot: the path of the file to be patched, relative to the
#                        repository root. This should normally be the path
#                        found in the patch's "Index:" line.
#   $options: a reference to an array of options to pass to the patch command.
sub applyPatch($$;$)
{
    my ($patch, $pathRelativeToRoot, $options) = @_;

    my $optionalArgs = {options => $options, ensureForce => $force};

    my $exitStatus = runPatchCommand($patch, $repositoryRootPath, $pathRelativeToRoot, $optionalArgs);

    if ($exitStatus) {
        $globalExitStatus = $exitStatus;
    }
}

sub checksum($)
{
    my $file = shift;
    open(FILE, $file) or die "Can't open '$file': $!";
    binmode(FILE);
    my $checksum = Digest::MD5->new->addfile(*FILE)->hexdigest();
    close(FILE);
    return $checksum;
}

sub handleBinaryChange($$)
{
    my ($fullPath, $contents) = @_;
    # [A-Za-z0-9+/] is the class of allowed base64 characters.
    # One or more lines, at most 76 characters in length.
    # The last line is allowed to have up to two '=' characters at the end (to signify padding).
    if ($contents =~ m#((\n[A-Za-z0-9+/]{76})*\n[A-Za-z0-9+/]{2,74}?[A-Za-z0-9+/=]{2}\n)#) {
        # Addition or Modification
        open FILE, ">", $fullPath or die "Failed to open $fullPath.";
        print FILE decode_base64($1);
        close FILE;
        if (!scmKnowsOfFile($fullPath)) {
            # Addition
            scmAddQueued($fullPath);
        }
    } else {
        # Deletion
        scmRemove($fullPath);
    }
}

sub handleGitBinaryChange($$)
{
    my ($fullPath, $diffHashRef) = @_;

    my $contents = $diffHashRef->{svnConvertedText};

    my ($binaryChunkType, $binaryChunk, $reverseBinaryChunkType, $reverseBinaryChunk) = decodeGitBinaryPatch($contents, $fullPath);
    if (!$binaryChunkType) {
        die "$fullPath: unknown git binary patch format";
    }

    my $isFileAddition = $diffHashRef->{isNew};
    my $isFileDeletion = $diffHashRef->{isDeletion};

    my $originalContents = "";
    if (open FILE, $fullPath) {
        die "$fullPath already exists" if $isFileAddition;

        binmode(FILE);
        $originalContents = join("", <FILE>);
        close FILE;
    }

    if ($reverseBinaryChunkType eq "literal") {
        die "Original content of $fullPath mismatches" if $originalContents ne $reverseBinaryChunk;
    }

    if ($isFileDeletion) {
        scmRemove($fullPath);
    } else {
        # Addition or Modification
        my $out = "";
        if ($binaryChunkType eq "delta") {
            $out = applyGitBinaryPatchDelta($binaryChunk, $originalContents);
        } else {
            $out = $binaryChunk;
        }
        if ($reverseBinaryChunkType eq "delta") {
            die "Original content of $fullPath mismatches" if $originalContents ne applyGitBinaryPatchDelta($reverseBinaryChunk, $out);
        }
        open FILE, ">", $fullPath or die "Failed to open $fullPath.";
        binmode(FILE);
        print FILE $out;
        close FILE;
        if ($isFileAddition) {
            scmAddQueued($fullPath);
        }
    }
}

sub isDirectoryEmptyForRemoval($)
{
    my ($dir) = @_;
    return 1 unless -d $dir;
    my $directoryIsEmpty = 1;
    opendir DIR, $dir or die "Could not open '$dir' to list files: $?";
    for (my $item = readdir DIR; $item && $directoryIsEmpty; $item = readdir DIR) {
        next if exists $removeDirectoryIgnoreList{$item};
        if (-d File::Spec->catdir($dir, $item)) {
            $directoryIsEmpty = 0;
        } else {
            next if (scmWillDeleteFile(File::Spec->catdir($dir, $item)));
            $directoryIsEmpty = 0;
        }
    }
    closedir DIR;
    return $directoryIsEmpty;
}

# Args:
#   $diffHashRef: a diff hash reference of the type returned by parsePatch().
sub patch($)
{
    my ($diffHashRef) = @_;

    # Make sure $patch is initialized to some value.  A deletion can have no
    # svnConvertedText property in the case of a deletion resulting from a
    # Git rename.
    my $patch = $diffHashRef->{svnConvertedText} || "";

    my $fullPath = $diffHashRef->{indexPath};
    my $isBinary = $diffHashRef->{isBinary};
    my $isGit = $diffHashRef->{isGit};
    my $hasTextChunks = $patch && $diffHashRef->{numTextChunks};

    my $deletion = 0;
    my $addition = 0;

    $addition = 1 if ($diffHashRef->{isNew} || $patch =~ /\n@@ -0,0 .* @@/);
    $deletion = 1 if ($diffHashRef->{isDeletion} || $patch =~ /\n@@ .* \+0,0 @@/);

    if (basename($fullPath) eq "ChangeLog" && $skipChangeLogs) {
        print "Skipping '$fullPath' since --skip-changelogs was passed on the command line.";
        return;
    }

    if (!$addition && !$deletion && !$isBinary && $hasTextChunks) {
        # Standard patch, patch tool can handle this.
        if (basename($fullPath) eq "ChangeLog") {
            my $changeLogDotOrigExisted = -f "${fullPath}.orig";
            my $changeLogHash = fixChangeLogPatch($patch);
            my $newPatch = setChangeLogDateAndReviewer($changeLogHash->{patch}, $reviewer, $epochTime);
            applyPatch($newPatch, $fullPath, ["--fuzz=3"]);
            unlink("${fullPath}.orig") if (! $changeLogDotOrigExisted);
        } else {
            applyPatch($patch, $fullPath);
        }
    } else {
        # Either a deletion, an addition or a binary change.

        addDirectoriesIfNeeded(dirname($fullPath));

        if ($isBinary) {
            if ($isGit) {
                handleGitBinaryChange($fullPath, $diffHashRef);
            } else {
                handleBinaryChange($fullPath, $patch) if $patch;
            }
        } elsif ($deletion) {
            applyPatch($patch, $fullPath, ["--force"]) if ($patch && $hasTextChunks);
            scmRemove($fullPath);
        } elsif ($addition && $hasTextChunks) {
            # Addition
            rename($fullPath, "$fullPath.orig") if -e $fullPath;
            applyPatch($patch, $fullPath);
            unlink("$fullPath.orig") if -e "$fullPath.orig" && checksum($fullPath) eq checksum("$fullPath.orig");
            scmAddQueued($fullPath);
        } elsif ($addition && !$hasTextChunks) {
            # Add empty file.
            die "\"$fullPath\" already exists" if -e $fullPath;
            open(my $FH, ">>", $fullPath) or die "Could not open \"$fullPath\" for writing: $!";
            close($FH);
            scmAddQueued($fullPath);
        } elsif (!defined($diffHashRef->{executableBitDelta})) {
            die "Can't handle patch for \"$fullPath\".";
        }
    }
    scmToggleExecutableBitQueued($fullPath, $diffHashRef->{executableBitDelta}) if defined($diffHashRef->{executableBitDelta});
}

sub removeDirectoriesIfNeeded()
{
    foreach my $dir (reverse sort keys %checkedDirectories) {
        if (isDirectoryEmptyForRemoval($dir)) {
            scmRemove($dir);
        }
    }
}

# This could be made into a more general "status" call, except svn and git
# have different ideas about "moving" files which might get confusing.
sub scmWillDeleteFile($)
{
    my ($path) = @_;
    if (isSVN()) {
        my $svnOutput = svnStatus($path);
        return 1 if $svnOutput && substr($svnOutput, 0, 1) eq "D";
    } elsif (isGit()) {
        my $command = runCommand("git", "diff-index", "--name-status", "HEAD", "--", $path);
        return 1 if $command->{stdout} && substr($command->{stdout}, 0, 1) eq "D";
    }
    return 0;
}

# Return whether the file at the given path is known to Git.
#
# This method outputs a message like the following to STDERR when
# returning false:
#
# "error: pathspec 'test.png' did not match any file(s) known to git.
#  Did you forget to 'git add'?"
sub gitKnowsOfFile($)
{
    my $path = shift;

    `git ls-files --error-unmatch -- $path`;
    my $exitStatus = exitStatus($?);
    return $exitStatus == 0;
}

sub scmKnowsOfFile($)
{
    my ($path) = @_;
    if (isSVN()) {
        my $svnOutput = svnStatus($path);
        # This will match more than intended.  ? might not be the first field in the status
        if ($svnOutput && $svnOutput =~ m#\?\s+$path\n#) {
            return 0;
        }
        # This does not handle errors well.
        return 1;
    } elsif (isGit()) {
        my @result = callSilently(\&gitKnowsOfFile, $path);
        return $result[0];
    }
}

sub scmCopy($$)
{
    my ($source, $destination) = @_;
    if (isSVN()) {
        my $escapedSource = escapeSubversionPath($source);
        my $escapedDestination = escapeSubversionPath($destination);
        system("svn", "copy", $escapedSource, $escapedDestination) == 0 or die "Failed to svn copy $escapedSource $escapedDestination.";
    } elsif (isGit()) {
        copy($source, $destination) or die "Failed to copy $source $destination.";
        system("git", "add", "-v", $destination) == 0 or die "Failed to git add $destination.";
    }
}

sub scmAdd($)
{
    my ($path) = @_;
    if (isSVN()) {
        my $escapedPath = escapeSubversionPath($path);
        system("svn", "add", $escapedPath) == 0 or die "Failed to svn add $escapedPath.";
    } elsif (isGit()) {
        system("git", "add", "-v", $path) == 0 or die "Failed to git add $path.";
    }
}

sub scmRemove($)
{
    my ($path) = @_;
    if (isSVN()) {
        # SVN is very verbose when removing directories.  Squelch all output except the last line.
        my $svnOutput;
        my $escapedPath = escapeSubversionPath($path);
        open SVN, "svn rm --force '$escapedPath' |" or die "svn rm --force '$escapedPath' failed!";
        # Only print the last line.  Subversion outputs all changed statuses below $dir
        while (<SVN>) {
            $svnOutput = $_;
        }
        close SVN;
        print $svnOutput if $svnOutput;
    } elsif (isGit()) {
        # Git removes a directory if it becomes empty when the last file it contains is
        # removed by `git rm`. In svn-apply this can happen when a directory is being
        # removed in a patch, and all of the files inside of the directory are removed
        # before attemping to remove the directory itself. In this case, Git will have 
        # already deleted the directory and `git rm` would exit with an error claiming
        # there was no file. The --ignore-unmatch switch gracefully handles this case.
        system("git", "rm", "--force", "--ignore-unmatch", $path) == 0 or die "Failed to git rm --force --ignore-unmatch $path.";
    }
}

# Calling "git add" / "svn add" per each file its very slow on big patches
# We queue the filenames to execute them at the end in a few commands.
sub scmAddQueued($)
{
    my ($path) = @_;
    if (isSVN()) {
        push(@scmQueuedFilesToAdd, escapeSubversionPath($path));
    } elsif (isGit()) {
        push(@scmQueuedFilesToAdd, $path);
    }
}

sub scmCommitQueueAdded()
{
    my @cmdBase;
    if (isSVN()) {
        @cmdBase = ("svn", "add")
    } elsif (isGit()) {
        @cmdBase = ("git", "add", "-v")
    }

    # When we are handling a very large patch (more than 1000 files modified)
    # Instead of executing only one add command, we execute several commands
    # with 1000 files each one. We do that to avoid running into E2BIG errors
    # on the execve() syscall (argument list is too long).
    while (@scmQueuedFilesToAdd) {
        my @cmdGrouped = splice(@scmQueuedFilesToAdd, 0, 1000);
        unshift (@cmdGrouped, @cmdBase);
        system (@cmdGrouped);
    }
}

sub scmToggleExecutableBitQueued($$)
{
    my ($path, $executableBitDelta) = @_;
    $scmQueuedExecutableBits{$path} = $executableBitDelta;
}

sub scmCommitQueueToggledExecutableBit()
{
    foreach my $path (keys %scmQueuedExecutableBits) {
        scmToggleExecutableBit($path, $scmQueuedExecutableBits{$path});
    }
}