Tools/resultsdbpy/resultsdbpy/model/suite_context.py

# Copyright (C) 2019 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1.  Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import calendar
import json
import time

from cassandra.cqlengine import columns
from datetime import datetime
from resultsdbpy.controller.commit import Commit
from resultsdbpy.model.commit_context import CommitContext
from resultsdbpy.model.configuration_context import ClusteredByConfiguration
from resultsdbpy.model.test_context import Expectations
from resultsdbpy.model.upload_context import UploadCallbackContext


class SuiteContext(UploadCallbackContext):
    DEFAULT_LIMIT = 100

    class SuiteResultsBase(ClusteredByConfiguration):
        suite = columns.Text(partition_key=True, required=True)
        branch = columns.Text(partition_key=True, required=True)
        details = columns.Text(required=False)
        stats = columns.Text(required=False)

        def unpack(self):
            return dict(
                uuid=self.uuid,
                start_time=calendar.timegm(self.start_time.timetuple()),
                details=json.loads(self.details) if self.details else {},
                stats=json.loads(self.stats) if self.stats else {},
            )

    class SuiteResultsByCommit(SuiteResultsBase):
        __table_name__ = 'suite_results_by_commit'
        uuid = columns.BigInt(primary_key=True, required=True, clustering_order='DESC')
        sdk = columns.Text(primary_key=True, required=True)
        start_time = columns.DateTime(primary_key=True, required=True)

    class SuiteResultsByStartTime(SuiteResultsBase):
        __table_name__ = 'suite_results_by_start_time'
        start_time = columns.DateTime(primary_key=True, required=True, clustering_order='DESC')
        sdk = columns.Text(primary_key=True, required=True)
        uuid = columns.BigInt(primary_key=True, required=True)

    def __init__(self, *args, **kwargs):
        super(SuiteContext, self).__init__('suite-results', *args, **kwargs)

        with self:
            self.cassandra.create_table(self.SuiteResultsByCommit)
            self.cassandra.create_table(self.SuiteResultsByStartTime)

    def register(self, configuration, commits, suite, test_results, timestamp=None):
        timestamp = timestamp or time.time()
        if not isinstance(timestamp, datetime):
            timestamp = datetime.utcfromtimestamp(int(timestamp))

        try:
            if not isinstance(suite, str):
                raise TypeError(f'Expected type {str}, got {type(suite)}')

            if isinstance(timestamp, datetime):
                timestamp = calendar.timegm(timestamp.timetuple())

            run_stats = test_results.get('run_stats', {})
            failure_trigger_points = dict(
                failed=Expectations.STRING_TO_STATE_ID[Expectations.ERROR],
                timedout=Expectations.STRING_TO_STATE_ID[Expectations.TIMEOUT],
                crashed=Expectations.STRING_TO_STATE_ID[Expectations.CRASH],
            )
            run_stats['tests_run'] = 0
            for key in failure_trigger_points.keys():
                run_stats[f'tests_{key}'] = 0
                run_stats[f'tests_unexpected_{key}'] = 0

            def callback(test, result):
                run_stats['tests_run'] += 1
                actual_results = Expectations.string_to_state_ids(result.get('actual', ''))
                expected_results = set(Expectations.string_to_state_ids(result.get('expected', '')))

                # FAIL is a special case, we want to treat tests with TEXT, AUDIO and IMAGE diffs as failures
                if Expectations.STRING_TO_STATE_ID[Expectations.FAIL] in expected_results:
                    expected_results.add(Expectations.STRING_TO_STATE_ID[Expectations.TEXT])
                    expected_results.add(Expectations.STRING_TO_STATE_ID[Expectations.AUDIO])
                    expected_results.add(Expectations.STRING_TO_STATE_ID[Expectations.IMAGE])

                worst_result = min(actual_results)
                unexpected_results = set(actual_results) - expected_results
                worst_unexpected_result = min(unexpected_results or [Expectations.STRING_TO_STATE_ID[Expectations.PASS]])

                for key, point in failure_trigger_points.items():
                    if worst_result <= point:
                        run_stats[f'tests_{key}'] += 1
                    if worst_unexpected_result <= point:
                        run_stats[f'tests_unexpected_{key}'] += 1

            Expectations.iterate_through_nested_results(test_results.get('results'), callback)

            uuid = self.commit_context.uuid_for_commits(commits)
            ttl = int((uuid // Commit.TIMESTAMP_TO_UUID_MULTIPLIER) + self.ttl_seconds - time.time()) if self.ttl_seconds else None

            with self, self.cassandra.batch_query_context():
                for branch in self.commit_context.branch_keys_for_commits(commits):
                    for table in [self.SuiteResultsByCommit, self.SuiteResultsByStartTime]:
                        self.configuration_context.insert_row_with_configuration(
                            table.__table_name__, configuration=configuration, suite=suite,
                            branch=branch, uuid=uuid, ttl=ttl,
                            sdk=configuration.sdk or '?', start_time=timestamp,
                            details=json.dumps(test_results.get('details', {})),
                            stats=json.dumps(run_stats),
                        )
        except Exception as e:
            return self.partial_status(e)
        return self.partial_status()

    def _find_results(
            self, table, configurations, suite, recent=True,
            branch=None, begin=None, end=None,
            begin_query_time=None, end_query_time=None,
            limit=DEFAULT_LIMIT,
    ):
        if not isinstance(suite, str):
            raise TypeError(f'Expected type {str}, got {type(suite)}')

        def get_time(time):
            if isinstance(time, datetime):
                return time
            elif time:
                return datetime.utcfromtimestamp(int(time))
            return None

        with self:
            result = {}
            for configuration in configurations:
                result.update({config: [value.unpack() for value in values] for config, values in self.configuration_context.select_from_table_with_configurations(
                    table.__table_name__, configurations=[configuration], recent=recent,
                    suite=suite, sdk=configuration.sdk, branch=branch or self.commit_context.DEFAULT_BRANCH_KEY,
                    uuid__gte=CommitContext.convert_to_uuid(begin),
                    uuid__lte=CommitContext.convert_to_uuid(end, CommitContext.timestamp_to_uuid()),
                    start_time__gte=get_time(begin_query_time), start_time__lte=get_time(end_query_time),
                    limit=limit,
                ).items()})
            return result

    def find_by_commit(self, *args, **kwargs):
        return self._find_results(self.SuiteResultsByCommit, *args, **kwargs)

    def find_by_start_time(self, *args, **kwargs):
        return self._find_results(self.SuiteResultsByStartTime, *args, **kwargs)