blob: ad68c95d33f0e63a121679b15ea8c499d200f45b [file] [log] [blame]
# Copyright (C) 2019, 2022 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import calendar
import datetime
import json
import os
import time
import twisted
from base64 import b64encode
from buildbot.process.results import SUCCESS, FAILURE, CANCELLED, WARNINGS, SKIPPED, EXCEPTION, RETRY
from buildbot.util import httpclientservice, service
from buildbot.www.hooks.github import GitHubEventHandler
from steps import GitHub
from twisted.internet import defer
from twisted.internet import reactor
from twisted.internet.defer import succeed
from twisted.python import log
from twisted.web.client import Agent
from twisted.web.http_headers import Headers
from twisted.web.iweb import IBodyProducer
from zope.interface import implementer
custom_suffix = '-uat' if os.getenv('BUILDBOT_UAT') else ''
@implementer(IBodyProducer)
class JSONProducer(object):
"""
Perform JSON asynchronously as to not lock the buildbot main event loop
"""
def __init__(self, data):
try:
self.body = json.dumps(data, default=self.json_serialize_datetime).encode('utf-8')
except TypeError:
self.body = ''
self.length = len(self.body)
def startProducing(self, consumer):
if self.body:
consumer.write(self.body)
return succeed(None)
def pauseProducing(self):
pass
def stopProducing(self):
pass
def json_serialize_datetime(self, obj):
"""
Serializing buildbot dates into UNIX epoch timestamps.
"""
if isinstance(obj, datetime.datetime):
return int(calendar.timegm(obj.timetuple()))
raise TypeError("Type %s not serializable" % type(obj))
class Events(service.BuildbotService):
EVENT_SERVER_ENDPOINT = 'https://ews.webkit{}.org/results/'.format(custom_suffix).encode()
MAX_GITHUB_DESCRIPTION = 140
SHORT_STEPS = (
'configure-build',
'validate-change',
'configuration',
'clean-up-git-repo',
'fetch-branch-references',
'show-identifier',
'update-working-directory',
'apply-patch',
'kill-old-processes',
'set-build-summary',
)
def __init__(self, master_hostname, type_prefix='', name='Events'):
"""
Initialize the Events Plugin. Sends data to event server on specific buildbot events.
:param type_prefix: [optional] prefix we want to add to the 'type' field on the json we send
to event server. (i.e. ews-build, where 'ews-' is the prefix.
:return: Events Object
"""
service.BuildbotService.__init__(self, name=name)
if type_prefix and not type_prefix.endswith("-"):
type_prefix += "-"
self.type_prefix = type_prefix
self.master_hostname = master_hostname
def sendDataToEWS(self, data):
if os.getenv('EWS_API_KEY', None):
data['EWS_API_KEY'] = os.getenv('EWS_API_KEY')
agent = Agent(reactor)
body = JSONProducer(data)
agent.request(b'POST', self.EVENT_SERVER_ENDPOINT, Headers({'Content-Type': ['application/json']}), body)
def sendDataToGitHub(self, repository, sha, data, user=None):
username, access_token = GitHub.credentials(user=user)
data['description'] = data.get('description', '')
if len(data['description']) > self.MAX_GITHUB_DESCRIPTION:
data['description'] = '{}...'.format(data['description'][:self.MAX_GITHUB_DESCRIPTION - 3])
auth_header = b64encode('{}:{}'.format(username, access_token).encode('utf-8')).decode('utf-8')
agent = Agent(reactor)
body = JSONProducer(data)
d = agent.request(b'POST', GitHub.commit_status_url(sha, repository).encode('utf-8'), Headers({
'Authorization': ['Basic {}'.format(auth_header)],
'User-Agent': ['python-twisted/{}'.format(twisted.__version__)],
'Accept': ['application/vnd.github.v3+json'],
'Content-Type': ['application/json'],
}), body)
def getBuilderName(self, build):
if not (build and 'properties' in build):
return ''
return build.get('properties').get('buildername')[0]
def extractProperty(self, build, property_name):
if not (build and 'properties' in build and property_name in build['properties']):
return None
return build.get('properties').get(property_name)[0]
@defer.inlineCallbacks
def buildStarted(self, key, build):
if not build.get('properties'):
build['properties'] = yield self.master.db.builds.getBuildProperties(build.get('buildid'))
builder = yield self.master.db.builders.getBuilder(build.get('builderid'))
builder_display_name = builder.get('description')
data = {
"type": self.type_prefix + "build",
"status": "started",
"hostname": self.master_hostname,
"patch_id": self.extractProperty(build, 'patch_id'),
"build_id": build.get('buildid'),
"builder_id": build.get('builderid'),
"number": build.get('number'),
"result": build.get('results'),
"started_at": build.get('started_at'),
"complete_at": build.get('complete_at'),
"state_string": build.get('state_string'),
"builder_name": self.getBuilderName(build),
"builder_display_name": builder_display_name,
}
self.sendDataToEWS(data)
def buildFinishedGitHub(self, build):
sha = self.extractProperty(build, 'github.head.sha')
repository = self.extractProperty(build, 'repository')
if not sha or not repository:
print('Pull request number defined, but sha is {} and repository {}, which are invalid'.format(sha, repository))
print('Not reporting build result to GitHub')
return
data_to_send = dict(
owner=(self.extractProperty(build, 'owners') or [None])[0],
repo=(self.extractProperty(build, 'github.head.repo.full_name') or '').split('/')[-1],
sha=sha,
target_url='{}#/builders/{}/builds/{}'.format(self.master.config.buildbotURL, build.get('builderid'), build.get('number')),
state={
SUCCESS: 'success',
WARNINGS: 'success',
SKIPPED: 'success',
RETRY: 'pending',
FAILURE: 'failure'
}.get(build.get('results'), 'error'),
description=build.get('state_string'),
context=build['description'] + custom_suffix,
)
self.sendDataToGitHub(repository, sha, data_to_send, user=GitHub.user_for_queue(self.extractProperty(build, 'buildername')))
@defer.inlineCallbacks
def buildFinished(self, key, build):
if not build.get('properties'):
build['properties'] = yield self.master.db.builds.getBuildProperties(build.get('buildid'))
if not build.get('steps'):
build['steps'] = yield self.master.db.steps.getSteps(build.get('buildid'))
builder = yield self.master.db.builders.getBuilder(build.get('builderid'))
build['description'] = builder.get('description', '?')
if self.extractProperty(build, 'github.number'):
return self.buildFinishedGitHub(build)
patch_id = self.extractProperty(build, 'patch_id')
if not patch_id:
return
data = {
"type": self.type_prefix + "build",
"status": "finished",
"hostname": self.master_hostname,
"patch_id": self.extractProperty(build, 'patch_id'),
"build_id": build.get('buildid'),
"builder_id": build.get('builderid'),
"number": build.get('number'),
"result": build.get('results'),
"started_at": build.get('started_at'),
"complete_at": build.get('complete_at'),
"state_string": build.get('state_string'),
"builder_name": self.getBuilderName(build),
"builder_display_name": builder.get('description'),
"steps": build.get('steps'),
}
self.sendDataToEWS(data)
@defer.inlineCallbacks
def stepStartedGitHub(self, build, state_string):
sha = self.extractProperty(build, 'github.head.sha')
repository = self.extractProperty(build, 'repository')
if not sha or not repository:
print('Pull request number defined, but sha is {} and repository {}, which are invalid'.format(sha, repository))
print('Not reporting step started to GitHub')
return
builder = yield self.master.db.builders.getBuilder(build.get('builderid'))
data_to_send = dict(
owner=(self.extractProperty(build, 'owners') or [None])[0],
repo=(self.extractProperty(build, 'github.head.repo.full_name') or '').split('/')[-1],
sha=sha,
target_url='{}#/builders/{}/builds/{}'.format(self.master.config.buildbotURL, build.get('builderid'), build.get('number')),
state={
SUCCESS: 'pending',
WARNINGS: 'pending',
FAILURE: 'failure',
EXCEPTION: 'error',
}.get(build.get('results'), 'pending'),
description=state_string,
context=builder.get('description', '?') + custom_suffix,
)
self.sendDataToGitHub(repository, sha, data_to_send, user=GitHub.user_for_queue(self.extractProperty(build, 'buildername')))
@defer.inlineCallbacks
def stepStarted(self, key, step):
state_string = step.get('state_string')
if state_string == 'pending':
state_string = 'Running {}'.format(step.get('name'))
build = yield self.master.db.builds.getBuild(step.get('buildid'))
if not build.get('properties'):
build['properties'] = yield self.master.db.builds.getBuildProperties(step.get('buildid'))
# We need to force the defered properties to resolve
if build['properties'].get('github.number') and build.get('step') not in self.SHORT_STEPS:
self.stepStartedGitHub(build, state_string)
data = {
"type": self.type_prefix + "step",
"status": "started",
"hostname": self.master_hostname,
"step_id": step.get('stepid'),
"build_id": step.get('buildid'),
"result": step.get('results'),
"state_string": state_string,
"started_at": step.get('started_at'),
"complete_at": step.get('complete_at'),
}
self.sendDataToEWS(data)
def stepFinished(self, key, step):
data = {
"type": self.type_prefix + "step",
"status": "finished",
"hostname": self.master_hostname,
"step_id": step.get('stepid'),
"build_id": step.get('buildid'),
"result": step.get('results'),
"state_string": step.get('state_string'),
"started_at": step.get('started_at'),
"complete_at": step.get('complete_at'),
}
self.sendDataToEWS(data)
@defer.inlineCallbacks
def startService(self):
yield service.BuildbotService.startService(self)
startConsuming = self.master.mq.startConsuming
self._buildStartedConsumer = yield startConsuming(self.buildStarted, ('builds', None, 'new'))
self._buildCompleteConsumer = yield startConsuming(self.buildFinished, ('builds', None, 'finished'))
self._stepStartedConsumer = yield startConsuming(self.stepStarted, ('steps', None, 'started'))
self._stepFinishedConsumer = yield startConsuming(self.stepFinished, ('steps', None, 'finished'))
def stopService(self):
self._buildStartedConsumer.stopConsuming()
self._buildCompleteConsumer.stopConsuming()
self._stepStartedConsumer.stopConsuming()
self._stepFinishedConsumer.stopConsuming()
class GitHubEventHandlerNoEdits(GitHubEventHandler):
OPEN_STATES = ('open',)
PUBLIC_REPOS = ('WebKit/WebKit',)
SENSATIVE_FIELDS = ('github.title',)
UNSAFE_MERGE_QUEUE_LABEL = 'unsafe-merge-queue'
MERGE_QUEUE_LABEL = 'merge-queue'
LABEL_PROCESS_DELAY = 10
@classmethod
def file_with_status_sign(cls, info):
if info.get('status') in ('removed', 'renamed'):
return '--- {}'.format(info['filename'])
return '+++ {}'.format(info['filename'])
def _get_commit_msg(self, repo, sha):
return ''
@defer.inlineCallbacks
def _get_pr_files(self, repo, number):
# Copied from https://github.com/buildbot/buildbot/blob/v2.10.5/master/buildbot/www/hooks/github.py to include added/modified/deleted
headers = {"User-Agent": "Buildbot"}
if self._token:
headers["Authorization"] = "token " + self._token
url = "/repos/{}/pulls/{}/files".format(repo, number)
http = yield httpclientservice.HTTPClientService.getService(
self.master,
self.github_api_endpoint,
headers=headers,
debug=self.debug,
verify=self.verify,
)
res = yield http.get(url)
if 200 <= res.code < 300:
data = yield res.json()
return [self.file_with_status_sign(f) for f in data]
log.msg('Failed fetching PR files: response code {}'.format(res.code))
return []
def extractProperties(self, payload):
result = super(GitHubEventHandlerNoEdits, self).extractProperties(payload)
if payload.get('base', {}).get('repo', {}).get('full_name') not in self.PUBLIC_REPOS:
for field in self.SENSATIVE_FIELDS:
if field in result:
del result[field]
return result
def handle_pull_request(self, payload, event):
pr_number = payload['number']
action = payload.get('action')
state = payload.get('pull_request', {}).get('state')
labels = [label.get('name') for label in payload.get('pull_request', {}).get('labels', [])]
if state not in self.OPEN_STATES:
log.msg("PR #{} is '{}', which triggers nothing".format(pr_number, state))
return ([], 'git')
if action == 'labeled' and self.UNSAFE_MERGE_QUEUE_LABEL in labels:
log.msg("PR #{} was labeled for unsafe-merge-queue".format(pr_number))
# 'labeled' is usually an ignored action, override it to force build
payload['action'] = 'synchronize'
time.sleep(self.LABEL_PROCESS_DELAY)
return super(GitHubEventHandlerNoEdits, self).handle_pull_request(payload, 'unsafe_merge_queue')
if action == 'labeled' and self.MERGE_QUEUE_LABEL in labels:
log.msg("PR #{} was labeled for merge-queue".format(pr_number))
# 'labeled' is usually an ignored action, override it to force build
payload['action'] = 'synchronize'
time.sleep(self.LABEL_PROCESS_DELAY)
return super(GitHubEventHandlerNoEdits, self).handle_pull_request(payload, 'merge_queue')
return super(GitHubEventHandlerNoEdits, self).handle_pull_request(payload, event)