| #!/usr/bin/env python3 |
| # |
| # Copyright (C) 2017, 2021 Igalia S.L. |
| # Copyright (C) 2020 Apple Inc. All rights reserved. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions are |
| # met: |
| # |
| # * Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # |
| # * Redistributions in binary form must reproduce the above |
| # copyright notice, this list of conditions and the following disclaimer |
| # in the documentation and/or other materials provided with the |
| # distribution. |
| # |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| # |
| import sys |
| import signal |
| import os |
| import argparse |
| import subprocess |
| import tempfile |
| import shutil |
| import socket |
| import json |
| import traceback |
| import multiprocessing |
| from time import sleep |
| |
| buildbot_server_tac = """ |
| import os |
| from twisted.application import service |
| from buildbot.master import BuildMaster |
| |
| basedir = os.path.dirname(os.path.realpath(__file__)) |
| configfile = r'%(config_file)s' |
| |
| application = service.Application('buildmaster') |
| BuildMaster(basedir, configfile).setServiceParent(application) |
| """ |
| |
| buildbot_worker_tac = """ |
| import os |
| from buildbot_worker.bot import Worker |
| from twisted.application import service |
| |
| basedir = os.path.dirname(os.path.realpath(__file__)) |
| application = service.Application('buildbot-worker') |
| |
| buildmaster_host = 'localhost' |
| port = %(server_pb_port)s |
| workername = '%(worker_name)s' |
| passwd = 'password' |
| keepalive = 600 |
| |
| s = Worker(buildmaster_host, port, workername, passwd, basedir, keepalive) |
| s.setServiceParent(application) |
| """ |
| |
| |
| def check_tcp_port_open(address, port): |
| s = socket.socket() |
| try: |
| s.connect((address, port)) |
| return True |
| except ConnectionRefusedError: |
| return False |
| |
| |
| def create_tempdir(tmpdir=None): |
| if tmpdir is not None: |
| if not os.path.isdir(tmpdir): |
| raise ValueError('{} is not a directory'.format(tmpdir)) |
| return tempfile.mkdtemp(prefix=os.path.join(os.path.abspath(tmpdir), 'tmp')) |
| return tempfile.mkdtemp() |
| |
| |
| def print_if_error_stdout_stderr(cmd, retcode, stdout=None, stderr=None, extramsg=None): |
| if retcode != 0: |
| if type(cmd) is list: |
| cmd = ' '.join(cmd) |
| print('WARNING: "{cmd}" returned {retcode} status code'.format(cmd=cmd, retcode=retcode)) |
| if stdout is not None: |
| print('STDOUT:\n' + stdout.decode('utf-8')) |
| if stderr is not None: |
| print('STDERR:\n' + stderr.decode('utf-8')) |
| if extramsg is not None: |
| print(extramsg) |
| |
| |
| def cmd_exists(cmd): |
| return any(os.access(os.path.join(path, cmd), os.X_OK) |
| for path in os.environ['PATH'].split(os.pathsep)) |
| |
| |
| class BuildbotTestRunner(object): |
| |
| def __init__(self, configdir): |
| self._configdir = os.path.abspath(os.path.realpath(configdir)) |
| if not os.path.isdir(self._configdir): |
| raise RuntimeError('The configdir {} is not a directory'.format(self._configdir)) |
| if not os.path.isfile(os.path.join(self._configdir, 'config.json')): |
| raise RuntimeError('The configdir {} does not contain a config.json file'.format(self._configdir)) |
| |
| number_config_files = 0 |
| for file in os.listdir(self._configdir): |
| if file.endswith('.cfg'): |
| self._server_config_file_name = file |
| number_config_files += 1 |
| if number_config_files == 0: |
| raise RuntimeError('The configdir {} does not contain a .cfg file'.format(self._configdir)) |
| if number_config_files != 1: |
| raise RuntimeError('The configdir {} has more than one .cfg file'.format(self._configdir)) |
| |
| self._server_http_port, self._server_pb_port = self._get_config_tcp_ports() |
| |
| def _get_config_tcp_ports(self): |
| pb_port = 17000 |
| http_port = 8010 |
| try: |
| with open(os.path.join(self._configdir, self._server_config_file_name)) as f: |
| for line in f: |
| if '=' in line: |
| if 'protocols' in line and 'pb' in line and 'port' in line: |
| pb = eval(line.split('=', 1)[1]) |
| pb_port = int((pb['pb']['port'])) |
| if 'www' in line and 'port' in line: |
| http = eval(line.split('=', 1)[1]) |
| http_port = int((http['port'])) |
| except: |
| print("Unable to detect http and pb ports from config. Using defaults") |
| return http_port, pb_port |
| |
| def start(self, basetempdir=None, no_clean=False, number_workers=1, use_system_version=False): |
| try: |
| self._base_workdir_temp = os.path.abspath(os.path.realpath(create_tempdir(basetempdir))) |
| if self._base_workdir_temp.startswith(self._configdir): |
| raise ValueError('The temporal working directory {} cant be located inside configdir {}'.format(self._base_workdir_temp, self._configdir)) |
| if not use_system_version: |
| self._setup_virtualenv() |
| if not (cmd_exists('twistd') and cmd_exists('buildbot')): |
| raise RuntimeError('Buildbot is not installed.') |
| self._setup_server_workdir() |
| server_runner = multiprocessing.Process(target=self._start_server) |
| server_runner.start() |
| self._wait_for_server_ready() |
| if number_workers == 0: |
| print(' - To manually attach a build worker use this info:\n' |
| + ' TCP port for the worker-to-server connection: {}\n'.format(self._server_pb_port) |
| + ' worker-id: the one defined at {}\n'.format(os.path.join(self._server_wordir, 'passwords.json')) |
| + ' password: password\n') |
| elif number_workers == 1: |
| worker = 'local-worker' |
| worker_runner = multiprocessing.Process(target=self._start_worker, args=(worker,)) |
| worker_runner.start() |
| print(' - Worker started!.\n' |
| + ' Check the log for at {}/local-worker/worker.log\n'.format(self._base_workdir_temp) |
| + ' tail -f {}/local-worker/worker.log\n'.format(self._base_workdir_temp)) |
| worker_runner.join() |
| else: |
| worker_runners = [] |
| for worker in self._get_list_workers(): |
| worker_runner = multiprocessing.Process(target=self._start_worker, args=(worker,)) |
| worker_runner.start() |
| worker_runners.append(worker_runner) |
| print(' - Workers started!.\n' |
| + ' Check the log for each one at {}/WORKERNAMEID/worker.log\n'.format(self._base_workdir_temp) |
| + ' tail -f {}/*/worker.log\n'.format(self._base_workdir_temp)) |
| for worker_runner in worker_runners: |
| worker_runner.join() |
| server_runner.join() |
| except KeyboardInterrupt: |
| pass # no print the exception |
| except: |
| traceback.print_exc() |
| finally: |
| try: |
| # The children may exit between the check and the kill call. |
| # Ignore any exception raised here. |
| for c in multiprocessing.active_children(): |
| # Send the signal to the whole process group. |
| # Otherwise some twistd sub-childs can remain alive. |
| os.killpg(os.getpgid(c.pid), signal.SIGKILL) |
| except: |
| pass |
| if not no_clean: |
| self._clean() |
| sys.exit(0) |
| |
| def _wait_for_server_ready(self): |
| server_ready_check_counter = 0 |
| while True: |
| if os.path.isfile(self._server_ready_fd): |
| return |
| if server_ready_check_counter > 60: |
| print('ERROR: buildbot server has not started after waiting 60 seconds for it.') |
| |
| if os.path.isfile(self._server_log): |
| print('The server.log file contains the following:') |
| with open(self._server_log, 'r') as f: |
| print(f.read()) |
| else: |
| print('There is no server.log on the directory. That means a general failure starting buildbot.') |
| |
| raise RuntimeError('buildbot server has not started after waiting 60 seconds for it.') |
| sleep(1) |
| server_ready_check_counter += 1 |
| |
| def _create_mock_worker_passwords_dict(self): |
| with open(os.path.join(self._server_wordir, 'config.json'), 'r') as config_json: |
| config_dict = json.load(config_json) |
| result = dict([(worker['name'], 'password') for worker in config_dict['workers']]) |
| return result |
| |
| def _setup_server_workdir(self): |
| self._server_wordir = os.path.join(self._base_workdir_temp, 'buildbot-server') |
| assert(not os.path.exists(self._server_wordir)) |
| self._server_log = os.path.join(self._server_wordir, 'server.log') |
| self._server_ready_fd = os.path.join(self._server_wordir, '.server-is-ready') |
| print('Copying files from {} to {} ...'.format(self._configdir, self._server_wordir)) |
| shutil.copytree(self._configdir, self._server_wordir) |
| print('Generating buildbot files at {} ...'.format(self._server_wordir)) |
| with open(os.path.join(self._server_wordir, 'buildbot.tac'), 'w') as f: |
| f.write(buildbot_server_tac % {'config_file': self._server_config_file_name}) |
| with open(os.path.join(self._server_wordir, 'passwords.json'), 'w') as passwords_file: |
| passwords_file.write(json.dumps(self._create_mock_worker_passwords_dict(), indent=4, sort_keys=True)) |
| |
| def _setup_virtualenv(self): |
| if cmd_exists('virtualenv'): |
| print('Setting up virtualenv at {} ... '.format(self._base_workdir_temp)) |
| virtualenv_cmd = ['virtualenv', '-p', 'python3', 'venv'] |
| virtualenv_process = subprocess.Popen(virtualenv_cmd, cwd=self._base_workdir_temp, |
| stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
| stdout, stderr = virtualenv_process.communicate() |
| print_if_error_stdout_stderr(virtualenv_cmd, virtualenv_process.returncode, stdout, stderr) |
| virtualenv_bindir = os.path.join(self._base_workdir_temp, 'venv', 'bin') |
| virtualenv_pip = os.path.join(virtualenv_bindir, 'pip') |
| if not os.access(virtualenv_pip, os.X_OK): |
| print('Something went wrong setting up virtualenv' |
| 'Trying to continue using the system version of buildbot') |
| return |
| print('Setting up buildbot dependencies on the virtualenv ... ') |
| # The idea is to install the very same version of buildbot and its |
| # dependencies than the ones used for running https://build.webkit.org/about |
| buildbot_version = '2.10.5' |
| pip_cmd = [virtualenv_pip, 'install', |
| 'buildbot=={}'.format(buildbot_version), |
| 'buildbot-console-view=={}'.format(buildbot_version), |
| 'buildbot-grid-view=={}'.format(buildbot_version), |
| 'buildbot-waterfall-view=={}'.format(buildbot_version), |
| 'buildbot-worker=={}'.format(buildbot_version), |
| 'buildbot-www=={}'.format(buildbot_version), |
| 'twisted==21.2.0', |
| 'requests==2.21.0', |
| 'lz4==1.1.0'] |
| pip_process = subprocess.Popen(pip_cmd, cwd=self._base_workdir_temp, |
| stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
| stdout, stderr = pip_process.communicate() |
| print_if_error_stdout_stderr(pip_cmd, pip_process.returncode, stdout, stderr) |
| os.environ['PATH'] = virtualenv_bindir + ':' + os.environ['PATH'] |
| return |
| print('WARNING: virtualenv not installed. ' |
| 'Trying to continue using the system version of buildbot') |
| |
| def _upgrade_db_needed(self): |
| with open(self._server_log) as f: |
| for l in f: |
| if 'upgrade the database' in l: |
| return True |
| return False |
| |
| def _start_server(self): |
| # This is started via multiprocessing. We set a new process group here |
| # to be able to reliably kill this subprocess and all of its child on clean. |
| os.setsid() |
| dbupgraded = False |
| retry = True |
| if check_tcp_port_open('localhost', self._server_pb_port): |
| print('ERROR: There is some process already listening in port {}'.format(self._server_pb_port)) |
| return 1 |
| while retry: |
| retry = False |
| print('Starting the buildbot server process ...') |
| twistd_cmd = ['twistd', '-l', self._server_log, '-noy', 'buildbot.tac'] |
| twistd_server_process = subprocess.Popen(twistd_cmd, cwd=self._server_wordir, |
| stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
| while twistd_server_process.poll() is None: |
| if check_tcp_port_open('localhost', self._server_pb_port): |
| print('Test buildbot server ready!\n' |
| + 'Press CTRL-C to stop\n\n' |
| + ' - See buildbot server log:\n' |
| + ' tail -f {}\n\n'.format(self._server_log) |
| + ' - Open a browser to:\n' |
| + ' http://localhost:{}\n'.format(self._server_http_port)) |
| with open(self._server_ready_fd, 'w') as f: |
| f.write('ready') |
| twistd_server_process.wait() |
| return 0 |
| sleep(1) |
| stdout, stderr = twistd_server_process.communicate() |
| if twistd_server_process.returncode == 0 and self._upgrade_db_needed() and not dbupgraded: |
| retry = True |
| dbupgraded = True |
| print('Upgrading the database ...') |
| upgrade_cmd = ['buildbot', 'upgrade-master', self._server_wordir] |
| upgrade_process = subprocess.Popen(upgrade_cmd, cwd=self._server_wordir, |
| stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
| stdout, stderr = upgrade_process.communicate() |
| print_if_error_stdout_stderr(upgrade_cmd, upgrade_process.returncode, stdout, stderr) |
| else: |
| print_if_error_stdout_stderr(twistd_cmd, twistd_server_process.returncode, stdout, stderr, |
| 'Check the log at {}'.format(self._server_log)) |
| return twistd_server_process.returncode |
| |
| def _get_list_workers(self): |
| password_list = os.path.join(self._server_wordir, 'passwords.json') |
| with open(password_list) as f: |
| passwords = json.load(f) |
| list_workers = [] |
| for worker in passwords: |
| list_workers.append(str(worker)) |
| return list_workers |
| |
| def _start_worker(self, worker): |
| # This is started via multiprocessing. We set a new process group here |
| # to be able to reliably kill this subprocess and all of its child on clean. |
| os.setsid() |
| worker_workdir = os.path.join(self._base_workdir_temp, worker) |
| os.mkdir(worker_workdir) |
| with open(os.path.join(worker_workdir, 'buildbot.tac'), 'w') as f: |
| f.write(buildbot_worker_tac % {'worker_name': worker, 'server_pb_port': self._server_pb_port}) |
| twistd_cmd = ['twistd', '-l', 'worker.log', '-noy', 'buildbot.tac'] |
| twistd_worker_process = subprocess.Popen(twistd_cmd, cwd=worker_workdir, |
| stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
| stdout, stderr = twistd_worker_process.communicate() |
| print_if_error_stdout_stderr(twistd_cmd, twistd_worker_process.returncode, stdout, stderr, |
| 'Check the log at {}'.format(os.path.join(worker_workdir, 'worker.log'))) |
| return twistd_worker_process.returncode |
| |
| def _clean(self): |
| if os.path.isdir(self._base_workdir_temp): |
| print('\n\nCleaning {} ... \n'.format(self._base_workdir_temp)) |
| # shutil.rmtree can fail if we hold an open file descriptor on temp_dir |
| # (which is very likely when cleaning) or if temp_dir is a NFS mount. |
| # Use rm instead that always works. |
| rm = subprocess.Popen(['rm', '-fr', self._base_workdir_temp]) |
| rm.wait() |
| |
| |
| if __name__ == '__main__': |
| if sys.version_info < (3, 5): |
| print('ERROR: Please use Python 3. This script is not compatible with Python 2.') |
| sys.exit(1) |
| |
| parser = argparse.ArgumentParser() |
| configuration = parser.add_mutually_exclusive_group(required=True) |
| configuration.add_argument('--ews', action='store_const', const='ews', dest='configuration', |
| help='Simulate the EWS buildbot server (ews-build.webkit.org)') |
| configuration.add_argument('--post-commit', action='store_const', const='post_commit', dest='configuration', |
| help='Simulate the post-commit buildbot server (build.webkit.org)') |
| configuration.add_argument('--config-dir', default=None, dest='configdir', type=str, |
| help='Specify the directory that contains the buildbot config files') |
| parser.add_argument('--base-temp-dir', help='Path where the temporal working directory will be created. ' |
| 'Note: To trigger test builds with the test workers you need enough free space on that path.', |
| dest='basetempdir', default=None, type=str) |
| parser.add_argument('--no-clean', help='Do not clean the temporal working dir on exit.', |
| dest='no_clean', action='store_true') |
| workers = parser.add_mutually_exclusive_group(required=False) |
| workers.add_argument('--no-workers', help='Do not start any workers.', |
| dest='number_workers', action='store_const', const=0) |
| workers.add_argument('--all-workers', help='Instead of starting only one worker that round-robins between all the queues, ' |
| 'start multiple parallel workers as defined on the server config.', |
| dest='number_workers', action='store_const', const=float('inf')) |
| parser.add_argument('--use-system-version', help='Instead of setting up a virtualenv with the buildbot version ' |
| 'used by build.webkit.org, use the buildbot version installed on this system.', |
| dest='use_system_version', action='store_true') |
| args = parser.parse_args() |
| |
| if args.configuration == "ews": |
| configdir = os.path.join(os.path.dirname(__file__), 'ews-build') |
| elif args.configuration == "post_commit": |
| configdir = os.path.join(os.path.dirname(__file__), 'build-webkit-org') |
| else: |
| configdir = args.configdir |
| |
| if args.number_workers is None: |
| args.number_workers = 1 |
| |
| buildbot_test_runner = BuildbotTestRunner(configdir) |
| buildbot_test_runner.start(args.basetempdir, args.no_clean, args.number_workers, args.use_system_version) |
