Tools/Scripts/update-webkit-dependency - WebKit - Git at Google

 #!/usr/bin/env perl

 # Copyright (C) 2005, 2006, 2007 Apple Inc.  All rights reserved.
 # Copyright (C) 2011 Carl Lobo.  All rights reserved.
 # Copyright (C) 2016 Jeremy Zerfas.  All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 #
 # 1.  Redistributions of source code must retain the above copyright
 #     notice, this list of conditions and the following disclaimer.
 # 2.  Redistributions in binary form must reproduce the above copyright
 #     notice, this list of conditions and the following disclaimer in the
 #     documentation and/or other materials provided with the distribution.
 # 3.  Neither the name of Apple Inc. ("Apple") nor the names of
 #     its contributors may be used to endorse or promote products derived
 #     from this software without specific prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 # Updates a dependency for the development environment.

 use strict;
 use warnings;

 use Archive::Zip qw( :ERROR_CODES );
 use File::Copy;
 use File::Find;
 use File::Spec;
 use File::Temp ();
 use FindBin;
 use HTTP::Date qw(str2time time2str);
 use HTTP::Request;
 use LWP::Simple;
 use LWP::UserAgent;
 use POSIX;
 use lib $FindBin::Bin;
 use webkitdirs;

 if ($#ARGV != 1) {
     die <<EOF;
 Usage:
         update-webkit-dependancy <URL with the dependancy zip file> <*prefix dir inside zip without filename>

         * If filename is requirements.zip and the contents of the zip file are "requirements/x" then prefix = "."
         * If filename is xyz.zip and the contents of the zip file are xyz/abc/x" then prefix = "abc"
         * x is lib or include or bin.
 EOF
 }

 sub getLibraryName($);

 # Time in seconds that the new zip file must be newer than the old for us to
 # consider them to be different. If the difference in modification time is less
 # than this threshold, we assume that the files are the same. We need this
 # because the zip file is served from a set of mirrors with slightly different
 # Last-Modified times.
 my $newnessThreshold = 30;

 my $libsURL = shift;
 my $prefixInZip = shift;
 my $sourceDir = sourceDir();
 my $file = getLibraryName($libsURL);
 my $zipFile = "$file.zip";
 my $webkitLibrariesDir = $ENV{'WEBKIT_LIBRARIES'} || File::Spec->catdir($sourceDir, "WebKitLibraries", "win");
 my $tmpRelativeDir = File::Temp::tempdir("webkitlibsXXXXXXX", TMPDIR => 1, CLEANUP => 1);
 my $tmpAbsDir = File::Spec->rel2abs($tmpRelativeDir);
 my $ua = LWP::UserAgent->new();
 $ua->env_proxy;

 print "Checking for newer version of $zipFile...\n";

 # Ideally we could just use a previous modification time and/or ETag to do a single conditional request for the file,
 # however there are some problems with this approach. Some servers may not support conditional requests (Dropbox and
 # GitHub don't currently support conditional requests using If-Modified-Since headers plus they also don't emit
 # Last-Modified headers either). If mirrors are being used, some servers may have different ETags for the same file.
 # Some servers may not implement conditional requests properly. Etc...
 #
 # Instead we will assume that we need to download a new version of the file unless we can be reasonably certain that we
 # already previously got an up to date copy of the file. We can do this by making a quick, small request to check for
 # the presence of Last-Modified and/or ETag headers (or alternatively a separate *.headers file that contains a
 # Last-Modified header) and see if they match values that we may have gotten previously.

 # It would be nice to be able to just use a HEAD request for this initial check but developer.apple.com returns 401
 # errors for those requests. Instead we can just set a size limit for a GET request (which we can also set the range for
 # to request just the first byte) so that we don't have to retrieve the entire file.
 $ua->max_size(0);
 my $response = $ua->get($libsURL, Accept => '*/*', Range => "bytes=0-0");
 $ua->max_size(undef);

 if (! $response->is_success) {
     print STDERR "Could not access $libsURL:\n";
     print STDERR $response->message, "\n";
     print STDERR $response->headers_as_string, "\n";
     print STDERR "Please ensure that $libsURL is reachable";
     if ($libsURL =~ /^https/i) {
         print STDERR " and that Perl can use LWP::Simple to connect to HTTPS URLs.\n";
         print STDERR "You may have to run \"\$ cpan LWP::Protocol::https\"";
     }
     print STDERR ".\n";

     if (-f File::Spec->catfile($webkitLibrariesDir, "$file.headers")) {
         print STDERR "Falling back to existing version of $file.\n";
         exit 0;
     } else {
         print STDERR "No existing version of $file to fall back to.\n";
         exit 1;
     }
 }

 my $contentType = $response->header('Content-Type');
 # Try to determine the file size based on the Content-Range or Content-Length headers. Normally we will need to use the
 # Content-Range header since we are just requesting a range but if the server doesn't support ranges then we will try
 # using the Content-Length header instead. Also note that neither header is affected by using max_size() to limit the
 # response size and the Content-Length header also shouldn't be affected from the use of HTTP compression since LWP
 # doesn't use it by default.
 my $contentLength = $response->header('Content-Length');
 # If the Content-Range header is defined and has a number after the slash, then that's the file size we want.
 if (defined $response->header('Content-Range') && $response->header('Content-Range') =~ /^.+\/(\d+)/) {
     $contentLength=$1;
 }
 my $lastModified = $response->header('Last-Modified');
 my $etag = $response->header('ETag');

 print "Located a file";
 print " of type $contentType" if defined $contentType;
 print " of size $contentLength" if defined $contentLength;
 print ".\n";

 # Get any old headers that are available.
 my $oldLastModified;
 my $oldETag;
 if (open OLDHEADERSFILE, "<", File::Spec->catfile($webkitLibrariesDir, "$file.headers")) {
     local $/ = undef;
     my $oldHeaders = <OLDHEADERSFILE>;
     close OLDHEADERSFILE;

     ($oldLastModified) = $oldHeaders =~ /^Last-Modified: (.+)$/mi;
     ($oldETag) = $oldHeaders =~ /^ETag: (.+)$/mi;
 }

 # If we have matching old and new ETags, then that is a very good indication that we already had gotten an up to date
 # copy of the file previously. On the other hand if we have ETags that don't match, that doesn't necessarily indicate
 # that the files are different since different servers may compute different ETags for the same file. If they don't
 # match we will continue to see if we can skip downloading the file via other tests.
 if (defined $etag && defined $oldETag && $etag eq $oldETag) {
     print "Current $file is up to date.\n";
     exit 0;
 }

 # If on the initial request we didn't get a Last-Modified header, then we will try getting the Last-Modified header from
 # a separate *.headers file that may have been put on the server.
 if (! defined $lastModified) {
     my $headerURL = $libsURL;
     $headerURL =~ s/\.zip$/\.headers/;

     $response = $ua->get($headerURL);

     if (! $response->is_success) {
         print STDERR "Could not access $headerURL:\n";
         print STDERR $response->message, "\n";
         print STDERR $response->headers_as_string, "\n";
         print STDERR "Please ensure that $headerURL is reachable";
         if ($headerURL =~ /^https/i) {
             print STDERR " and that Perl can use LWP::Simple to connect to HTTPS URLs.\n";
             print STDERR "You may have to run \"\$ cpan LWP::Protocol::https\"";
         }
         print STDERR ".\n";
     }

     ($lastModified) = $response->content =~ /^Last-Modified: (.+)$/mi;
 }

 my $lastModifiedTime = str2time($lastModified);
 my $oldLastModifiedTime = str2time($oldLastModified);
 if (defined $lastModifiedTime && defined $oldLastModifiedTime
     && abs($lastModifiedTime-$oldLastModifiedTime) < $newnessThreshold) {
     print "Current $file is up to date.\n";
     exit 0;
 }

 print "Downloading $zipFile...\n\n";
 print "$libsURL\n";
 my $request = HTTP::Request->new(GET => $libsURL);
 $request->header(Accept => "*/*");
 $response = $ua->request($request, File::Spec->catfile($tmpAbsDir, $zipFile));
 die "Couldn't download $zipFile!" if is_error($response->code);

 my $zip = Archive::Zip->new(File::Spec->catfile($tmpAbsDir, $zipFile));

 # Make sure the zip file contains a directory with the same name as the zip file (minus the extension) and a prefix
 # directory if applicable.
 my $prefixDirectoryPathInZipFile = "$file/".(($prefixInZip eq ".") ? "" : "$prefixInZip/");
 if (! $zip->memberNamed($prefixDirectoryPathInZipFile)) {
     print STDERR "Couldn't find $prefixDirectoryPathInZipFile directory in zip file.\n";

     if (-f File::Spec->catfile($webkitLibrariesDir, "$file.headers")) {
         print STDERR "Falling back to existing version of $file.\n";
         exit 0;
     } else {
         print STDERR "No existing version of $file to fall back to.\n";
         exit 1;
     }
 }

 my $result = $zip->extractTree("", $tmpAbsDir);
 die "Couldn't unzip $zipFile." if $result != AZ_OK;

 print "\nInstalling $file...\n";

 sub wanted
 {
     my $relativeName = File::Spec->abs2rel($File::Find::name, File::Spec->catdir($tmpAbsDir, $file, $prefixInZip));
     my $destination = File::Spec->catfile($webkitLibrariesDir, $relativeName);

     if (-d $_) {
         mkdir $destination;
         return;
     }

     copy($_, $destination);
 }

 File::Find::find(\&wanted, File::Spec->catfile($tmpAbsDir, $file));

 if (open HEADERSFILE, ">", File::Spec->catfile($webkitLibrariesDir, "$file.headers")) {
     print HEADERSFILE "Last-Modified: $lastModified\n" if defined $lastModified;
     print HEADERSFILE "ETag: $etag\n" if defined $etag;
     close HEADERSFILE;
 } else {
     print STDERR "Couldn't write $file.headers to $webkitLibrariesDir.\n"
 }

 print "The $file has been sucessfully installed in\n $webkitLibrariesDir\n";
 exit;

 sub getLibraryName($)
 {
     my $url = shift;
     $url =~ m#/([^/]+)\.zip$#;
     return $1;
 }
	#!/usr/bin/env perl

	# Copyright (C) 2005, 2006, 2007 Apple Inc. All rights reserved.
	# Copyright (C) 2011 Carl Lobo. All rights reserved.
	# Copyright (C) 2016 Jeremy Zerfas. All rights reserved.
	#
	# Redistribution and use in source and binary forms, with or without
	# modification, are permitted provided that the following conditions
	# are met:
	#
	# 1. Redistributions of source code must retain the above copyright
	# notice, this list of conditions and the following disclaimer.
	# 2. Redistributions in binary form must reproduce the above copyright
	# notice, this list of conditions and the following disclaimer in the
	# documentation and/or other materials provided with the distribution.
	# 3. Neither the name of Apple Inc. ("Apple") nor the names of
	# its contributors may be used to endorse or promote products derived
	# from this software without specific prior written permission.
	#
	# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
	# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
	# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
	# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
	# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
	# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
	# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

	# Updates a dependency for the development environment.

	use strict;
	use warnings;

	use Archive::Zip qw( :ERROR_CODES );
	use File::Copy;
	use File::Find;
	use File::Spec;
	use File::Temp ();
	use FindBin;
	use HTTP::Date qw(str2time time2str);
	use HTTP::Request;
	use LWP::Simple;
	use LWP::UserAgent;
	use POSIX;
	use lib $FindBin::Bin;
	use webkitdirs;

	if ($#ARGV != 1) {
	die <<EOF;
	Usage:
	update-webkit-dependancy <URL with the dependancy zip file> <*prefix dir inside zip without filename>

	* If filename is requirements.zip and the contents of the zip file are "requirements/x" then prefix = "."
	* If filename is xyz.zip and the contents of the zip file are xyz/abc/x" then prefix = "abc"
	* x is lib or include or bin.
	EOF
	}

	sub getLibraryName($);

	# Time in seconds that the new zip file must be newer than the old for us to
	# consider them to be different. If the difference in modification time is less
	# than this threshold, we assume that the files are the same. We need this
	# because the zip file is served from a set of mirrors with slightly different
	# Last-Modified times.
	my $newnessThreshold = 30;

	my $libsURL = shift;
	my $prefixInZip = shift;
	my $sourceDir = sourceDir();
	my $file = getLibraryName($libsURL);
	my $zipFile = "$file.zip";
	my $webkitLibrariesDir = $ENV{'WEBKIT_LIBRARIES'} \|\| File::Spec->catdir($sourceDir, "WebKitLibraries", "win");
	my $tmpRelativeDir = File::Temp::tempdir("webkitlibsXXXXXXX", TMPDIR => 1, CLEANUP => 1);
	my $tmpAbsDir = File::Spec->rel2abs($tmpRelativeDir);
	my $ua = LWP::UserAgent->new();
	$ua->env_proxy;

	print "Checking for newer version of $zipFile...\n";

	# Ideally we could just use a previous modification time and/or ETag to do a single conditional request for the file,
	# however there are some problems with this approach. Some servers may not support conditional requests (Dropbox and
	# GitHub don't currently support conditional requests using If-Modified-Since headers plus they also don't emit
	# Last-Modified headers either). If mirrors are being used, some servers may have different ETags for the same file.
	# Some servers may not implement conditional requests properly. Etc...
	#
	# Instead we will assume that we need to download a new version of the file unless we can be reasonably certain that we
	# already previously got an up to date copy of the file. We can do this by making a quick, small request to check for
	# the presence of Last-Modified and/or ETag headers (or alternatively a separate *.headers file that contains a
	# Last-Modified header) and see if they match values that we may have gotten previously.

	# It would be nice to be able to just use a HEAD request for this initial check but developer.apple.com returns 401
	# errors for those requests. Instead we can just set a size limit for a GET request (which we can also set the range for
	# to request just the first byte) so that we don't have to retrieve the entire file.
	$ua->max_size(0);
	my $response = $ua->get($libsURL, Accept => '/', Range => "bytes=0-0");
	$ua->max_size(undef);

	if (! $response->is_success) {
	print STDERR "Could not access $libsURL:\n";
	print STDERR $response->message, "\n";
	print STDERR $response->headers_as_string, "\n";
	print STDERR "Please ensure that $libsURL is reachable";
	if ($libsURL =~ /^https/i) {
	print STDERR " and that Perl can use LWP::Simple to connect to HTTPS URLs.\n";
	print STDERR "You may have to run \"\$ cpan LWP::Protocol::https\"";
	}
	print STDERR ".\n";

	if (-f File::Spec->catfile($webkitLibrariesDir, "$file.headers")) {
	print STDERR "Falling back to existing version of $file.\n";
	exit 0;
	} else {
	print STDERR "No existing version of $file to fall back to.\n";
	exit 1;
	}
	}

	my $contentType = $response->header('Content-Type');
	# Try to determine the file size based on the Content-Range or Content-Length headers. Normally we will need to use the
	# Content-Range header since we are just requesting a range but if the server doesn't support ranges then we will try
	# using the Content-Length header instead. Also note that neither header is affected by using max_size() to limit the
	# response size and the Content-Length header also shouldn't be affected from the use of HTTP compression since LWP
	# doesn't use it by default.
	my $contentLength = $response->header('Content-Length');
	# If the Content-Range header is defined and has a number after the slash, then that's the file size we want.
	if (defined $response->header('Content-Range') && $response->header('Content-Range') =~ /^.+\/(\d+)/) {
	$contentLength=$1;
	}
	my $lastModified = $response->header('Last-Modified');
	my $etag = $response->header('ETag');

	print "Located a file";
	print " of type $contentType" if defined $contentType;
	print " of size $contentLength" if defined $contentLength;
	print ".\n";

	# Get any old headers that are available.
	my $oldLastModified;
	my $oldETag;
	if (open OLDHEADERSFILE, "<", File::Spec->catfile($webkitLibrariesDir, "$file.headers")) {
	local $/ = undef;
	my $oldHeaders = <OLDHEADERSFILE>;
	close OLDHEADERSFILE;

	($oldLastModified) = $oldHeaders =~ /^Last-Modified: (.+)$/mi;
	($oldETag) = $oldHeaders =~ /^ETag: (.+)$/mi;
	}

	# If we have matching old and new ETags, then that is a very good indication that we already had gotten an up to date
	# copy of the file previously. On the other hand if we have ETags that don't match, that doesn't necessarily indicate
	# that the files are different since different servers may compute different ETags for the same file. If they don't
	# match we will continue to see if we can skip downloading the file via other tests.
	if (defined $etag && defined $oldETag && $etag eq $oldETag) {
	print "Current $file is up to date.\n";
	exit 0;
	}

	# If on the initial request we didn't get a Last-Modified header, then we will try getting the Last-Modified header from
	# a separate *.headers file that may have been put on the server.
	if (! defined $lastModified) {
	my $headerURL = $libsURL;
	$headerURL =~ s/\.zip$/\.headers/;

	$response = $ua->get($headerURL);

	if (! $response->is_success) {
	print STDERR "Could not access $headerURL:\n";
	print STDERR $response->message, "\n";
	print STDERR $response->headers_as_string, "\n";
	print STDERR "Please ensure that $headerURL is reachable";
	if ($headerURL =~ /^https/i) {
	print STDERR " and that Perl can use LWP::Simple to connect to HTTPS URLs.\n";
	print STDERR "You may have to run \"\$ cpan LWP::Protocol::https\"";
	}
	print STDERR ".\n";
	}

	($lastModified) = $response->content =~ /^Last-Modified: (.+)$/mi;
	}

	my $lastModifiedTime = str2time($lastModified);
	my $oldLastModifiedTime = str2time($oldLastModified);
	if (defined $lastModifiedTime && defined $oldLastModifiedTime
	&& abs($lastModifiedTime-$oldLastModifiedTime) < $newnessThreshold) {
	print "Current $file is up to date.\n";
	exit 0;
	}

	print "Downloading $zipFile...\n\n";
	print "$libsURL\n";
	my $request = HTTP::Request->new(GET => $libsURL);
	$request->header(Accept => "/");
	$response = $ua->request($request, File::Spec->catfile($tmpAbsDir, $zipFile));
	die "Couldn't download $zipFile!" if is_error($response->code);

	my $zip = Archive::Zip->new(File::Spec->catfile($tmpAbsDir, $zipFile));

	# Make sure the zip file contains a directory with the same name as the zip file (minus the extension) and a prefix
	# directory if applicable.
	my $prefixDirectoryPathInZipFile = "$file/".(($prefixInZip eq ".") ? "" : "$prefixInZip/");
	if (! $zip->memberNamed($prefixDirectoryPathInZipFile)) {
	print STDERR "Couldn't find $prefixDirectoryPathInZipFile directory in zip file.\n";

	if (-f File::Spec->catfile($webkitLibrariesDir, "$file.headers")) {
	print STDERR "Falling back to existing version of $file.\n";
	exit 0;
	} else {
	print STDERR "No existing version of $file to fall back to.\n";
	exit 1;
	}
	}

	my $result = $zip->extractTree("", $tmpAbsDir);
	die "Couldn't unzip $zipFile." if $result != AZ_OK;

	print "\nInstalling $file...\n";

	sub wanted
	{
	my $relativeName = File::Spec->abs2rel($File::Find::name, File::Spec->catdir($tmpAbsDir, $file, $prefixInZip));
	my $destination = File::Spec->catfile($webkitLibrariesDir, $relativeName);

	if (-d $_) {
	mkdir $destination;
	return;
	}

	copy($_, $destination);
	}

	File::Find::find(\&wanted, File::Spec->catfile($tmpAbsDir, $file));

	if (open HEADERSFILE, ">", File::Spec->catfile($webkitLibrariesDir, "$file.headers")) {
	print HEADERSFILE "Last-Modified: $lastModified\n" if defined $lastModified;
	print HEADERSFILE "ETag: $etag\n" if defined $etag;
	close HEADERSFILE;
	} else {
	print STDERR "Couldn't write $file.headers to $webkitLibrariesDir.\n"
	}

	print "The $file has been sucessfully installed in\n $webkitLibrariesDir\n";
	exit;

	sub getLibraryName($)
	{
	my $url = shift;
	$url =~ m#/([^/]+)\.zip$#;
	return $1;
	}