Tools/Scripts/run-bindings-tests

#!/usr/bin/python
# Copyright (C) 2010 Google Inc.  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE COMPUTER, INC. OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# This script generates h and cpp file for TestObj.idl using the V8 code
# generator. Please execute the script whenever changes are made to
# CodeGeneratorV8.pm, and submit the changes in V8TestObj.h/cpp in the same
# patch. This makes it easier to track and review changes in generated code.
# To execute, invoke: 'python run_tests.py'

import os
import os.path
import shutil
import subprocess
import sys
import tempfile
from webkitpy.common.checkout.scm.detection import detect_scm_system


def generate_from_idl(generator, idl_file, output_directory):
    cmd = ['perl', '-w',
           '-IWebCore/bindings/scripts',
           'WebCore/bindings/scripts/generate-bindings.pl',
           # idl include directories (path relative to generate-bindings.pl)
           '--include', '.',
           '--defines', 'TESTING_%s' % generator,
           '--generator', generator,
           '--outputDir', output_directory,
           idl_file]

    process = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
    exit_code = process.wait()
    output = process.communicate()[0]
    if output:
        print output
    return exit_code


def detect_changes(generator, work_directory, reference_directory):
    changes_found = False
    for output_file in os.listdir(work_directory):
        cmd = ['diff',
               '-u',
               '-N',
               os.path.join(reference_directory, output_file),
               os.path.join(work_directory, output_file)]

        process = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
        process.wait()
        output = process.communicate()[0]
        if output:
            print 'FAIL: (%s) %s' % (generator, output_file)
            print output
            changes_found = True
        else:
            print 'PASS: (%s) %s' % (generator, output_file)

    return changes_found


def run_tests(generator, input_directory, reference_directory, reset_results):
    work_directory = reference_directory

    passed = True
    for input_file in os.listdir(input_directory):
        (name, extension) = os.path.splitext(input_file)
        if extension != '.idl':
            continue
        # Generate output into the work directory (either the given one or a
        # temp one if not reset_results is performed)
        if not reset_results:
            work_directory = tempfile.mkdtemp()

        if generate_from_idl(generator,
                             os.path.join(input_directory, input_file),
                             work_directory):
            passed = False

        if reset_results:
            print "Reset results: (%s) %s" % (generator, input_file)
            continue

        # Detect changes
        if detect_changes(generator, work_directory, reference_directory):
            passed = False
        shutil.rmtree(work_directory)

    return passed


def main(argv):
    """Runs WebCore bindings code generators on test IDL files and compares
    the results with reference files.

    Options:
       --reset-results: Overwrites the reference files with the generated results.

    """
    reset_results = "--reset-results" in argv

    current_scm = detect_scm_system(os.curdir)
    os.chdir(os.path.join(current_scm.checkout_root, 'Source'))

    all_tests_passed = True

    generators = [
        'JS',
        'V8',
        'ObjC',
        'GObject',
        'CPP'
    ]

    for generator in generators:
        input_directory = os.path.join('WebCore', 'bindings', 'scripts', 'test')
        reference_directory = os.path.join('WebCore', 'bindings', 'scripts', 'test', generator)
        if not run_tests(generator, input_directory, reference_directory, reset_results):
            all_tests_passed = False

    print ''
    if all_tests_passed:
        print 'All tests PASS!'
        return 0
    else:
        print 'Some tests FAIL! (To update the reference files, execute "run-bindings-tests --reset-results")'
        return -1


if __name__ == '__main__':
    sys.exit(main(sys.argv))