| #!/usr/bin/env python |
| # |
| # Copyright (C) 2017 Igalia S.L. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions are |
| # met: |
| # |
| # * Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # |
| # * Redistributions in binary form must reproduce the above |
| # copyright notice, this list of conditions and the following disclaimer |
| # in the documentation and/or other materials provided with the |
| # distribution. |
| # |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| # |
| import sys |
| import signal |
| import os |
| import argparse |
| import subprocess |
| import tempfile |
| import shutil |
| import socket |
| import json |
| import traceback |
| import multiprocessing |
| from time import sleep |
| |
| test_buildbot_master_tac = """ |
| import os |
| from twisted.application import service |
| try: |
| from buildbot.master.bot import BuildMaster |
| except: |
| from buildbot.master import BuildMaster |
| |
| basedir = os.path.dirname(os.path.realpath(__file__)) |
| configfile = r'master.cfg' |
| |
| application = service.Application('buildmaster') |
| BuildMaster(basedir, configfile).setServiceParent(application) |
| """ |
| |
| worker_buildbot_master_tac = """ |
| import os |
| from twisted.application import service |
| from buildslave.bot import BuildSlave |
| |
| basedir = os.path.dirname(os.path.realpath(__file__)) |
| buildmaster_host = 'localhost' |
| port = 17000 |
| slavename = '%(worker)s' |
| passwd = '1234' |
| keepalive = 600 |
| usepty = 1 |
| |
| application = service.Application('buildslave') |
| BuildSlave(buildmaster_host, port, slavename, passwd, basedir, keepalive, usepty).setServiceParent(application) |
| """ |
| |
| |
| def check_tcp_port_open(address, port): |
| s = socket.socket() |
| try: |
| s.connect((address, port)) |
| return True |
| except: |
| return False |
| |
| |
| def upgrade_db_needed(log): |
| try: |
| with open(log) as f: |
| for l in f: |
| if 'upgrade the database' in l: |
| return True |
| except: |
| return False |
| return False |
| |
| |
| def create_tempdir(tmpdir=None): |
| if tmpdir is not None: |
| if not os.path.isdir(tmpdir): |
| raise ValueError('%s is not a directory' % tmpdir) |
| return tempfile.mkdtemp(prefix=os.path.join(os.path.abspath(tmpdir), 'tmp')) |
| return tempfile.mkdtemp() |
| |
| |
| def print_if_error_stdout_stderr(cmd, retcode, stdout=None, stderr=None, extramsg=None): |
| if retcode != 0: |
| if type(cmd) == type([]): |
| cmd = ' '.join(cmd) |
| print('WARNING: "%s" returned %s status code' % (cmd, retcode)) |
| if stdout is not None: |
| print(stdout) |
| if stderr is not None: |
| print(stderr) |
| if extramsg is not None: |
| print(extramsg) |
| |
| |
| def setup_master_workdir(configdir, base_workdir): |
| master_workdir = os.path.join(base_workdir, 'master') |
| print('Copying files from %s to %s ...' % (configdir, master_workdir)) |
| shutil.copytree(configdir, master_workdir) |
| print('Generating buildbot files at %s ...' % master_workdir) |
| with open(os.path.join(master_workdir, 'buildbot.tac'), 'w') as f: |
| f.write(test_buildbot_master_tac) |
| mkpwd_cmd = ['./make_passwords_json.py'] |
| mkpwd_process = subprocess.Popen(mkpwd_cmd, cwd=master_workdir, |
| stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
| stdout, stderr = mkpwd_process.communicate() |
| print_if_error_stdout_stderr(mkpwd_cmd, mkpwd_process.returncode, stdout, stderr) |
| return master_workdir |
| |
| |
| def wait_for_master_ready(master_workdir): |
| master_ready_check_counter = 0 |
| while True: |
| if os.path.isfile(os.path.join(master_workdir, '.master-is-ready')): |
| return |
| if master_ready_check_counter > 60: |
| raise RuntimeError('ERROR: Aborting after waiting 60 seconds for the master to start.') |
| sleep(1) |
| master_ready_check_counter += 1 |
| |
| |
| def start_master(master_workdir): |
| # This is started via multiprocessing. We set a new process group here |
| # to be able to reliably kill this subprocess and all of its child on clean. |
| os.setsid() |
| buildmasterlog = os.path.join(master_workdir, 'buildmaster.log') |
| dbupgraded = False |
| retry = True |
| if check_tcp_port_open('localhost', 8710): |
| print('ERROR: There is some process already listening in port 8170') |
| return 1 |
| while retry: |
| retry = False |
| print('Starting the twistd process ...') |
| twistd_cmd = ['twistd', '-l', buildmasterlog, '-noy', 'buildbot.tac'] |
| twistd_process = subprocess.Popen(twistd_cmd, cwd=master_workdir, |
| stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
| while twistd_process.poll() is None: |
| if check_tcp_port_open('localhost', 8710): |
| print('Test buildmaster ready!.\n\n' |
| + ' - See buildmaster log:\n' |
| + ' tail -f %s\n' % buildmasterlog |
| + ' - Open a browser to:\n' |
| + ' http://localhost:8710\n' |
| + ' - Credentials for triggering manual builds:\n' |
| + ' login: committer@webkit.org\n' |
| + ' password: committerpassword\n') |
| with open(os.path.join(master_workdir, '.master-is-ready'), 'w') as f: |
| f.write('ready') |
| twistd_process.wait() |
| return 0 |
| sleep(1) |
| stdout, stderr = twistd_process.communicate() |
| if twistd_process.returncode == 0 and upgrade_db_needed(buildmasterlog) and not dbupgraded: |
| retry = True |
| dbupgraded = True |
| print('Upgrading the database ...') |
| upgrade_cmd = ['buildbot', 'upgrade-master', master_workdir] |
| upgrade_process = subprocess.Popen(upgrade_cmd, cwd=master_workdir, |
| stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
| stdout, stderr = upgrade_process.communicate() |
| print_if_error_stdout_stderr(upgrade_cmd, upgrade_process.returncode, stdout, stderr) |
| else: |
| print_if_error_stdout_stderr(twistd_cmd, twistd_process.returncode, stdout, stderr, |
| 'Check the log at %s' % buildmasterlog) |
| return 0 |
| |
| |
| def get_list_workers(master_workdir): |
| password_list = os.path.join(master_workdir, 'passwords.json') |
| with open(password_list) as f: |
| passwords = json.load(f) |
| list_workers = [] |
| for worker in passwords.keys(): |
| list_workers.append(str(worker)) |
| return list_workers |
| |
| |
| def start_worker(base_workdir, worker): |
| # This is started via multiprocessing. We set a new process group here |
| # to be able to reliably kill this subprocess and all of its child on clean. |
| os.setsid() |
| worker_workdir = os.path.join(base_workdir, worker) |
| os.mkdir(worker_workdir) |
| with open(os.path.join(worker_workdir, 'buildbot.tac'), 'w') as f: |
| f.write(worker_buildbot_master_tac % {'worker': worker}) |
| twistd_cmd = ['twistd', '-l', 'worker.log', '-noy', 'buildbot.tac'] |
| twistd_worker_process = subprocess.Popen(twistd_cmd, cwd=worker_workdir, |
| stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
| try: |
| stdout, stderr = twistd_worker_process.communicate() |
| except: |
| twistd_worker_process.kill() |
| return |
| print_if_error_stdout_stderr(twistd_cmd, twistd_worker_process.returncode, stdout, stderr, |
| 'Check the log at %s' % os.path.join(worker_workdir, 'worker.log')) |
| |
| |
| def clean(temp_dir): |
| if os.path.isdir(temp_dir): |
| print('\n\nCleaning %s ... \n' % (temp_dir)) |
| # shutil.rmtree can fail if we hold an open file descriptor on temp_dir |
| # (which is very likely when cleaning) or if temp_dir is a NFS mount. |
| # Use rm instead that always works. |
| rm = subprocess.Popen(['rm', '-fr', temp_dir]) |
| rm.wait() |
| |
| |
| def cmd_exists(cmd): |
| return any(os.access(os.path.join(path, cmd), os.X_OK) |
| for path in os.environ['PATH'].split(os.pathsep)) |
| |
| |
| def check_buildbot_installed(): |
| if cmd_exists('twistd') and cmd_exists('buildbot'): |
| return |
| raise RuntimeError('Buildbot is not installed.') |
| |
| |
| def setup_virtualenv(base_workdir_temp): |
| if cmd_exists('virtualenv'): |
| print('Setting up virtualenv at %s ... ' % base_workdir_temp) |
| virtualenv_cmd = ['virtualenv', '-p', 'python2', 'venv'] |
| virtualenv_process = subprocess.Popen(virtualenv_cmd, cwd=base_workdir_temp, |
| stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
| stdout, stderr = virtualenv_process.communicate() |
| print_if_error_stdout_stderr(virtualenv_cmd, virtualenv_process.returncode, stdout, stderr) |
| virtualenv_bindir = os.path.join(base_workdir_temp, 'venv', 'bin') |
| virtualenv_pip = os.path.join(virtualenv_bindir, 'pip') |
| if not os.access(virtualenv_pip, os.X_OK): |
| print('Something went wrong setting up virtualenv' |
| 'Trying to continue using the system version of buildbot') |
| return |
| print('Setting up buildbot dependencies on the virtualenv ... ') |
| # The idea is to install the very same version of buildbot and its |
| # dependencies than the ones used for running https://build.webkit.org/about |
| pip_cmd = [virtualenv_pip, 'install', |
| 'buildbot==0.8.6p1', |
| 'buildbot-slave==0.8.6p1', |
| 'twisted==12.1.0', |
| 'jinja2==2.6', |
| 'sqlalchemy==0.7.8', |
| 'sqlalchemy-migrate==0.12.0'] |
| pip_process = subprocess.Popen(pip_cmd, cwd=base_workdir_temp, |
| stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
| stdout, stderr = pip_process.communicate() |
| print_if_error_stdout_stderr(pip_cmd, pip_process.returncode, stdout, stderr) |
| os.environ['PATH'] = virtualenv_bindir + ':' + os.environ['PATH'] |
| return |
| print('WARNING: virtualenv not installed. ' |
| 'Trying to continue using the system version of buildbot') |
| |
| |
| def configdir_is_valid(configdir): |
| return(os.path.isdir(configdir) and |
| os.path.isfile(os.path.join(configdir, 'config.json')) and |
| os.path.isfile(os.path.join(configdir, 'master.cfg')) and |
| os.access(os.path.join(configdir, 'make_passwords_json.py'), os.X_OK)) |
| |
| |
| def main(configdir, basetempdir=None, no_clean=False, no_workers=False, use_system_version=False): |
| configdir = os.path.abspath(os.path.realpath(configdir)) |
| if not configdir_is_valid(configdir): |
| raise ValueError('The configdir %s dont contains the buildmaster files expected by this script' % configdir) |
| base_workdir_temp = os.path.abspath(os.path.realpath(create_tempdir(basetempdir))) |
| if base_workdir_temp.startswith(configdir): |
| raise ValueError('The temporal working directory %s cant be located inside configdir %s' % (base_workdir_temp, configdir)) |
| try: |
| if not use_system_version: |
| setup_virtualenv(base_workdir_temp) |
| check_buildbot_installed() |
| master_workdir = setup_master_workdir(configdir, base_workdir_temp) |
| master_runner = multiprocessing.Process(target=start_master, args=(master_workdir,)) |
| master_runner.start() |
| wait_for_master_ready(master_workdir) |
| if no_workers: |
| print(' - To manually attach a build worker use this info:\n' |
| + ' TCP port for the worker-to-master connection: 17000\n' |
| + ' worker-id: the one defined at %s\n' % os.path.join(master_workdir, 'passwords.json') |
| + ' password: 1234\n') |
| else: |
| worker_runners = [] |
| for worker in get_list_workers(master_workdir): |
| worker_runner = multiprocessing.Process(target=start_worker, args=(base_workdir_temp, worker,)) |
| worker_runner.start() |
| worker_runners.append(worker_runner) |
| print(' - Workers started!.\n' |
| + ' Check the log for each one at %s/${worker-name-id}/worker.log\n' % base_workdir_temp |
| + ' tail -f %s/*/worker.log\n' % base_workdir_temp) |
| for worker_runner in worker_runners: |
| worker_runner.join() |
| master_runner.join() |
| except: |
| traceback.print_exc() |
| finally: |
| try: |
| # The children may exit between the check and the kill call. |
| # Ignore any exception raised here. |
| for c in multiprocessing.active_children(): |
| # Send the signal to the whole process group. |
| # Otherwise some twistd sub-childs can remain alive. |
| os.killpg(os.getpgid(c.pid), signal.SIGKILL) |
| except: |
| pass |
| if not no_clean: |
| clean(base_workdir_temp) |
| sys.exit(0) |
| |
| |
| if __name__ == '__main__': |
| parser = argparse.ArgumentParser() |
| parser.add_argument('--config-dir', help='Path to the directory of the build master config files. ' |
| 'Defauls to the directory where this script is located.', |
| dest='configdir', type=str, |
| default=os.path.dirname(__file__)) |
| parser.add_argument('--base-temp-dir', help='Path where the temporal working directory will be created. ' |
| 'Note: To trigger test builds with the test workers you need enough free space on that path.', |
| dest='basetempdir', default=None, type=str) |
| parser.add_argument('--no-clean', help='Do not clean the temporal working dir on exit.', |
| dest='no_clean', action='store_true') |
| parser.add_argument('--no-workers', help='Do not start the test workers.', |
| dest='no_workers', action='store_true') |
| parser.add_argument('--use-system-version', help='Instead of setting up a virtualenv with the buildbot version ' |
| 'used by build.webkit.org, use the buildbot version installed on this system.', |
| dest='use_system_version', action='store_true') |
| args = parser.parse_args() |
| main(args.configdir, args.basetempdir, args.no_clean, args.no_workers, args.use_system_version) |
