| #!/usr/bin/env python |
| # |
| # Copyright (C) 2018 Igalia S.L. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions are met: |
| # |
| # 1. Redistributions of source code must retain the above copyright notice, this |
| # list of conditions and the following disclaimer. |
| # 2. Redistributions in binary form must reproduce the above copyright notice, |
| # this list of conditions and the following disclaimer in the documentation |
| # and/or other materials provided with the distribution. |
| # |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
| # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR |
| # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| import base64 |
| import datetime |
| import hashlib |
| import json |
| import optparse |
| import os |
| import shutil |
| import subprocess |
| import sys |
| import tempfile |
| import zipfile |
| |
| top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..')) |
| sys.path.insert(0, os.path.join(top_level_directory, 'Tools', 'flatpak')) |
| sys.path.insert(0, os.path.join(top_level_directory, 'Tools', 'jhbuild')) |
| import jhbuildutils |
| import flatpakutils |
| |
| |
| # Ideally we should use something like lddtree or create our own version of that |
| # But in practice for jsc bundles there isn't recursive library entries, so we |
| # use standard ldd here just to avoid having to require lddtree. |
| def ldd_get_libs_and_interpreter(binary): |
| lddCommand = ['ldd', binary] |
| lddProcess = subprocess.Popen(lddCommand, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
| stdout, stderr = lddProcess.communicate() |
| if lddProcess.returncode != 0: |
| raise RuntimeError('The ldd command returned non-zero status') |
| libs = [] |
| for line in stdout.splitlines(): |
| line = line.strip() |
| if '=>' in line: |
| line = line.split('=>')[1].strip() |
| if 'not found' in line: |
| raise RuntimeError('Some dependencies can not be found with ldd.') |
| line = line.split(' ')[0].strip() |
| if os.path.isfile(line): |
| libs.append(line) |
| else: |
| line = line.split(' ')[0].strip() |
| if os.path.isfile(line): |
| interpreter = line |
| return libs, interpreter |
| |
| |
| def generate_readme(bundleTmpDir, builderName, configuration, platform, revision): |
| print('Generate README.txt file') |
| readmeFile = os.path.join(bundleTmpDir, 'README.txt') |
| with open(readmeFile, 'w') as readmeHandle: |
| readmeHandle.write('JSC bundle details\n') |
| readmeHandle.write(' Builder name: %s\n' % builderName) |
| readmeHandle.write(' Builder date: %s\n' % datetime.datetime.now().isoformat()) |
| readmeHandle.write(' Configuration: %s\n' % configuration) |
| readmeHandle.write(' WebKit Platform: %s\n' % platform) |
| readmeHandle.write(' WebKit Revision: %s\n' % revision) |
| readmeHandle.write('\nInstructions: Execute the run-jsc wrapper script.\n') |
| return True |
| |
| |
| def generate_wrapper_script(bundleTmpDir, interpreter): |
| print('Generate wrapper script run-jsc') |
| scriptFile = os.path.join(bundleTmpDir, 'run-jsc') |
| with open(scriptFile, 'w') as scriptHandle: |
| scriptHandle.write('#!/bin/sh\n') |
| scriptHandle.write('MYDIR="$(dirname $(readlink -f $0))"\n') |
| scriptHandle.write('export LD_LIBRARY_PATH="${MYDIR}/lib"\n') |
| scriptHandle.write('exec "${MYDIR}/lib/%s" "${MYDIR}/bin/jsc" "$@"\n' % os.path.basename(interpreter)) |
| os.chmod(scriptFile, 0755) |
| |
| |
| def copy_and_remove_rpath(origFile, destinationDir, type='bin'): |
| if not os.path.isfile(origFile): |
| raise ValueError('Can not find file %s' % origFile) |
| print('Copy to bundle [%s]: %s' % (type, origFile)) |
| shutil.copy(origFile, destinationDir) |
| try: |
| patchElfCommand = ['patchelf', '--remove-rpath', os.path.join(destinationDir, os.path.basename(origFile))] |
| if subprocess.call(patchElfCommand) != 0: |
| print('WARNING: The patchelf command returned non-zero status') |
| except OSError as e: |
| if e.errno == os.errno.ENOENT: |
| print('WARNING: patchelf not found. Not modifying rpath') |
| else: |
| raise |
| |
| |
| def createJSCBundle(configuration, revision=None, builderName=None, platform=None): |
| buildDir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'WebKitBuild')) |
| binDir = os.path.join(buildDir, configuration.capitalize(), 'bin') |
| libDir = os.path.join(buildDir, configuration.capitalize(), 'lib') |
| jscBinary = os.path.join(binDir, 'jsc') |
| if not os.path.isfile(jscBinary) or not os.access(jscBinary, os.X_OK): |
| raise ValueError('Cannot find jsc at %s' % jscBinary) |
| |
| # Define names and paths for the generation of the bundle. |
| bundleTmpDir = os.path.join(buildDir, 'jsc_tmp') |
| bundleTmpLibDir = os.path.join(bundleTmpDir, 'lib') |
| bundleTmpBinDir = os.path.join(bundleTmpDir, 'bin') |
| bundleFileName = 'jsc_' + configuration |
| bundleFileCompressed = os.path.join(buildDir, bundleFileName + '.zip') |
| |
| # Clean everything from previous runs |
| if os.path.isdir(bundleTmpDir): |
| shutil.rmtree(bundleTmpDir) |
| if os.path.isfile(bundleFileCompressed): |
| os.remove(bundleFileCompressed) |
| |
| # Create bundleTmpDir and put there everything needed. |
| os.makedirs(bundleTmpDir) |
| os.makedirs(bundleTmpLibDir) |
| os.makedirs(bundleTmpBinDir) |
| copy_and_remove_rpath(jscBinary, bundleTmpBinDir, type='bin') |
| libraries, interpreter = ldd_get_libs_and_interpreter(jscBinary) |
| copy_and_remove_rpath(interpreter, bundleTmpLibDir, type='interpreter') |
| for library in libraries: |
| copy_and_remove_rpath(library, bundleTmpLibDir, type='lib') |
| generate_readme(bundleTmpDir, builderName, configuration, platform, revision) |
| generate_wrapper_script(bundleTmpDir, interpreter) |
| |
| # jsvu project prefers .zip rather than .tar.xz |
| with zipfile.ZipFile(bundleFileCompressed, 'w', compression=zipfile.ZIP_DEFLATED) as zipHandle: |
| for dirname, subdirs, files in os.walk(bundleTmpDir): |
| for filename in files: |
| systemFilePath = os.path.join(dirname, filename) |
| zipFilePath = systemFilePath.replace(bundleTmpDir, '', 1).lstrip('/') |
| zipHandle.write(systemFilePath, zipFilePath) |
| |
| if not os.path.isfile(bundleFileCompressed): |
| raise RuntimeError('Unable to create the file %s' % bundleFileCompressed) |
| return bundleFileCompressed |
| |
| |
| def sha256sum(bundleFilePath): |
| hash = hashlib.sha256() |
| with open(bundleFilePath, 'rb') as f: |
| for chunk in iter(lambda: f.read(4096), b''): |
| hash.update(chunk) |
| return hash.hexdigest() |
| |
| |
| # The expected format for --remote-config-file is something like: |
| # { |
| # "servername": "webkitgtk.org", |
| # "serveraddress": "webkitgtk.intranet-address.local", |
| # "serverport": "23", |
| # "username": "upload-bot-64", |
| # "baseurl": "https://webkitgtk.org/jsc-built-products/x86_64", |
| # "sshkey": "output of the priv key in base64. E.g. cat ~/.ssh/id_rsa|base64 -w0" |
| # } |
| def uploadJSCBundle(bundleFilePath, remoteConfigFile, configuration, revision): |
| remoteData = json.load(open(remoteConfigFile)) |
| remoteFileName = str(revision) + '.zip' |
| remoteFileBundlePathName = os.path.join(configuration, remoteFileName) |
| remoteFileHashPathName = os.path.join(configuration, str(revision) + '.sha256sum') |
| with tempfile.NamedTemporaryFile() as sshkeyfile: |
| # In theory NamedTemporaryFile() is already created 0600. But it don't hurts ensuring this again here. |
| os.chmod(sshkeyfile.name, 0600) |
| sshkeyfile.write(base64.b64decode(remoteData['sshkey'])) |
| sshkeyfile.flush() |
| # Generate and upload also a sha256 hash |
| with tempfile.NamedTemporaryFile() as hashcheckfile: |
| hashforbundle = sha256sum(bundleFilePath) |
| os.chmod(hashcheckfile.name, 0644) |
| hashcheckfile.write('%s %s\n' % (hashforbundle, remoteFileName)) |
| hashcheckfile.flush() |
| with tempfile.NamedTemporaryFile() as uploadinstructionsfile: |
| uploadinstructionsfile.write('progress\n') |
| uploadinstructionsfile.write('put %s %s\n' % (bundleFilePath, remoteFileBundlePathName)) |
| uploadinstructionsfile.write('put %s %s\n' % (hashcheckfile.name, remoteFileHashPathName)) |
| uploadinstructionsfile.write('quit\n') |
| uploadinstructionsfile.flush() |
| # The idea of this is to ensure scp doesn't ask any question (not even on the first run). |
| # This should be secure enough according to https://www.gremwell.com/ssh-mitm-public-key-authentication |
| sftpCommand = ['sftp', |
| '-o', 'StrictHostKeyChecking=no', |
| '-o', 'UserKnownHostsFile=/dev/null', |
| '-o', 'LogLevel=ERROR', |
| '-P', remoteData['serverport'], |
| '-i', sshkeyfile.name, |
| '-b', uploadinstructionsfile.name, |
| '%s@%s' % (remoteData['username'], remoteData['serveraddress'])] |
| print('Uploading bundle to %s as %s with sha256 hash %s' % (remoteData['servername'], remoteFileBundlePathName, hashforbundle)) |
| if subprocess.call(sftpCommand) != 0: |
| raise RuntimeError('The sftp command returned non-zero status') |
| |
| print('Done: archive sucesfully uploaded to %s/%s' % (remoteData['baseurl'], remoteFileBundlePathName)) |
| return 0 |
| |
| |
| def main(): |
| parser = optparse.OptionParser('usage: %prog [options]') |
| parser.add_option('--platform', dest='platform') |
| parser.add_option('--debug', action='store_const', const='debug', dest='configuration') |
| parser.add_option('--release', action='store_const', const='release', dest='configuration') |
| parser.add_option('--revision', action='store', type='string', dest='revision') |
| parser.add_option('--builder-name', action='store', type='string', dest='buildername') |
| parser.add_option('--remote-config-file', action='store', type='string', dest='remoteConfigFile') |
| options, args = parser.parse_args() |
| |
| if not options.platform: |
| parser.error('Platform is required') |
| return 1 |
| if not options.configuration: |
| parser.error('Configuration is required') |
| return 1 |
| |
| platform = options.platform.lower() |
| configuration = options.configuration.lower() |
| if platform == 'gtk': |
| flatpakutils.run_in_sandbox_if_available(sys.argv) |
| if not flatpakutils.is_sandboxed(): |
| jhbuildutils.enter_jhbuild_environment_if_available("gtk") |
| else: |
| raise NotImplementedError('Unsupported platform') |
| |
| bundleFilePath = createJSCBundle(configuration, options.revision, options.buildername, platform) |
| print('Bundle file created at: %s' % bundleFilePath) |
| if options.remoteConfigFile is not None and os.path.isfile(options.remoteConfigFile): |
| return uploadJSCBundle(bundleFilePath, options.remoteConfigFile, options.configuration, options.revision) |
| return 0 |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |