# Copyright (c) 2017, Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1.  Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE INC. AND ITS CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""
 This script uploads changes made to W3C web-platform-tests tests.
"""

import argparse
import logging
import os
import time
import json
from urllib2 import HTTPError

from webkitpy.common.checkout.scm.git import Git
from webkitpy.common.host import Host
from webkitpy.common.net.bugzilla import Bugzilla
from webkitpy.common.webkit_finder import WebKitFinder
from webkitpy.w3c.wpt_github import WPTGitHub
from webkitpy.w3c.wpt_linter import WPTLinter
from webkitpy.w3c.common import WPT_GH_ORG, WPT_GH_REPO_NAME, WPT_GH_URL, WPTPaths
from webkitpy.common.memoized import memoized

_log = logging.getLogger(__name__)

WEBKIT_WPT_DIR = 'LayoutTests/imported/w3c/web-platform-tests'
WPT_PR_URL = "%s/pull/" % WPT_GH_URL
WEBKIT_EXPORT_PR_LABEL = 'webkit-export'

EXCLUDED_FILE_SUFFIXES = ['-expected.txt', '.worker.html', '.any.html', '.any.worker.html', 'w3c-import.log']


class WebPlatformTestExporter(object):
    def __init__(self, host, options, gitClass=Git, bugzillaClass=Bugzilla, WPTGitHubClass=WPTGitHub, WPTLinterClass=WPTLinter):
        self._host = host
        self._filesystem = host.filesystem
        self._options = options

        self._host.initialize_scm()

        self._WPTGitHubClass = WPTGitHubClass
        self._gitClass = gitClass
        self._bugzilla = bugzillaClass()
        self._bug_id = options.bug_id
        if not self._bug_id:
            if options.attachment_id:
                self._bug_id = self._bugzilla.bug_id_for_attachment_id(options.attachment_id)
            elif options.git_commit:
                self._bug_id = self._host.checkout().bug_id_for_this_commit(options.git_commit)

        if not self._options.repository_directory:
            webkit_finder = WebKitFinder(self._filesystem)
            self._options.repository_directory = WPTPaths.wpt_checkout_path(webkit_finder)

        self._linter = WPTLinterClass(self._options.repository_directory, host.filesystem)

        self._bugzilla_url = "https://bugs.webkit.org/show_bug.cgi?id=" + str(self._bug_id)
        self._commit_message = options.message
        if not self._commit_message:
            self._commit_message = 'WebKit export of ' + self._bugzilla_url if self._bug_id else 'Export made from a WebKit repository'

    @property
    def username(self):
        if hasattr(self, '_username'):
            return self._username

        self._ensure_username_and_token(self._options)
        return self._username

    @property
    def token(self):
        if hasattr(self, '_token'):
            return self._token

        self._ensure_username_and_token(self._options)
        return self._token

    @property
    @memoized
    def _github(self):
        return self._WPTGitHubClass(self._host, self.username, self.token) if self.username and self.token else None

    @property
    @memoized
    def _wpt_fork_branch_github_url(self):
        return "https://github.com/%s/%s/tree/%s" % (self.username, WPT_GH_REPO_NAME, self._public_branch_name)

    @property
    @memoized
    def _wpt_fork_remote(self):
        wpt_fork_remote = self._options.repository_remote
        if not wpt_fork_remote:
            wpt_fork_remote = self.username

        return wpt_fork_remote

    @property
    @memoized
    def _wpt_fork_push_url(self):
        wpt_fork_push_url = self._options.repository_remote_url
        if not wpt_fork_push_url:
            wpt_fork_push_url = "https://%s@github.com/%s/%s.git" % (self.username, self.username, WPT_GH_REPO_NAME)

        return wpt_fork_push_url

    @property
    @memoized
    def _git(self):
        return self._ensure_wpt_repository("%s.git" % WPT_GH_URL, self._options.repository_directory, self._gitClass)

    @property
    @memoized
    def _branch_name(self):
        return self._ensure_new_branch_name()

    @property
    @memoized
    def _public_branch_name(self):
        options = self._options
        return options.public_branch_name if options.public_branch_name else self._branch_name

    @property
    @memoized
    def _wpt_patch(self):
        patch_data = self._host.scm().create_patch(self._options.git_commit, [WEBKIT_WPT_DIR]) or ''
        patch_data = self._strip_ignored_files_from_diff(patch_data)
        if not 'diff' in patch_data:
            return ''
        return patch_data

    def has_wpt_changes(self):
        return bool(self._wpt_patch)

    def _find_filename(self, line):
        return line.split(' ')[-1]

    def _is_ignored_file(self, filename):
        for suffix in EXCLUDED_FILE_SUFFIXES:
            if filename.endswith(suffix):
                return True
        return False

    def _strip_ignored_files_from_diff(self, diff):
        lines = diff.split('\n')
        include_file = True
        new_lines = []
        for line in lines:
            if line.startswith('diff'):
                include_file = True
                filename = self._find_filename(line)
                if self._is_ignored_file(filename):
                    include_file = False
            if include_file:
                new_lines.append(line)

        return '\n'.join(new_lines)

    def write_git_patch_file(self):
        _, patch_file = self._filesystem.open_binary_tempfile('wpt_export_patch')
        patch_data = self._wpt_patch
        if not 'diff' in patch_data:
            _log.info('No changes to upstream, patch data is: "%s"' % (patch_data))
            return ''
        # FIXME: We can probably try to use --relative git parameter to not do that replacement.
        patch_data = patch_data.replace(WEBKIT_WPT_DIR + '/', '')

        # FIXME: Support stripping of <!-- webkit-test-runner --> comments.
        self.has_webkit_test_runner_specific_changes = 'webkit-test-runner' in patch_data
        if self.has_webkit_test_runner_specific_changes:
            _log.warning("Patch contains webkit-test-runner specific changes, please remove them before creating a PR")
            return ''

        self._filesystem.write_text_file(patch_file, patch_data)
        return patch_file

    def _prompt_for_token(self, options):
        if options.non_interactive:
            return None
        return self._host.user.prompt_password('Enter GitHub OAuth token (or empty string to skip creating a pull request): ')

    def _prompt_for_username(self, options):
        if options.non_interactive:
            return None
        return self._host.user.prompt('Enter your GitHub username: ')

    def _ensure_username_and_token(self, options):
        self._username = options.username
        if not self._username:
            # FIXME: Use the keychain to store username and oauth token instead of .git/config
            self._username = self._git.local_config('github.username').rstrip()
            if not self._username:
                self._username = os.environ.get('GITHUB_USERNAME')
            if not self._username:
                self._username = self._prompt_for_username(options)
            if not self._username:
                raise ValueError("Missing GitHub username, please provide it as a command argument (see help for the command).")

        self._token = options.token
        if not self._token:
            self._token = self._git.local_config('github.token').rstrip()
            if not self._token:
                self._token = os.environ.get('GITHUB_TOKEN')
            if not self._token:
                self._token = self._prompt_for_token(options)
            if not self._token:
                _log.info("Missing GitHub token, the script will not be able to create a pull request to %s's %s repository." % (WPT_GH_ORG, WPT_GH_REPO_NAME))

        if self._token:
            self._validate_and_save_token(self._username, self._token)

    def _validate_and_save_token(self, username, token):
        url = 'https://api.github.com/user?access_token=%s' % (token,)
        try:
            response = self._host.web.request(method='GET', url=url, data=None)
        except HTTPError as e:
            raise Exception("OAuth token is not valid")
        data = json.load(response)
        login = data.get('login', None)
        if login != username:
            raise Exception("OAuth token does not match the provided username. Provided user: %s, github login: %s" % (username, login))
        else:
            # Username and token are valid. Save them in the git config so we
            # do not need to ask for them again
            if not self._git.local_config('github.token'):
                self._git.set_local_config('github.token', token)
            if not self._git.local_config('github.username'):
                self._git.set_local_config('github.username', username)

    def _ensure_wpt_repository(self, url, wpt_repository_directory, gitClass):
        git = None
        if not self._filesystem.exists(wpt_repository_directory):
            _log.info('Cloning %s into %s...' % (url, wpt_repository_directory))
            gitClass.clone(url, wpt_repository_directory, self._host.executive)
        git = gitClass(wpt_repository_directory, None, executive=self._host.executive, filesystem=self._filesystem)
        return git

    def _fetch_wpt_repository(self):
        _log.info('Fetching web-platform-tests repository')
        self._git.fetch()

    def _ensure_new_branch_name(self):
        branch_name_prefix = "wpt-export-for-webkit-" + (str(self._bug_id) if self._bug_id else "0")
        branch_name = branch_name_prefix
        counter = 0
        while self._git.branch_ref_exists(branch_name):
            branch_name = ("%s-%s") % (branch_name_prefix, str(counter))
            counter = counter + 1
        return branch_name

    def download_and_commit_patch(self):
        if self._options.git_commit:
            return True

        patch_options = ["--no-update", "--no-clean", "--local-commit"]
        if self._options.attachment_id:
            patch_options.insert("apply-attachment")
            patch_options.append(self._options.attachment_id)
        elif self._options.bug_id:
            patch_options.insert("apply-from-bug")
            patch_options.append(self._options.bug_id)
        else:
            _log.info("Exporting local changes")
            return
        raise TypeError("Retrieval of patch from bugzilla is not yet implemented")

    def clean(self):
        _log.info('Cleaning web-platform-tests master branch')
        self._git.checkout('master')
        self._git.reset_hard('origin/master')

    def create_branch_with_patch(self, patch):
        _log.info('Applying patch to web-platform-tests branch ' + self._branch_name)
        try:
            self._git.checkout_new_branch(self._branch_name)
        except Exception as e:
            _log.warning(e)
            _log.info("Retrying to create the branch")
            self._git.delete_branch(self._branch_name)
            self._git.checkout_new_branch(self._branch_name)
        try:
            self._git.apply_mail_patch([patch])
        except Exception as e:
            _log.warning(e)
            return False
        self._git.commit(['-a', '-m', self._commit_message])
        return True

    def push_to_wpt_fork(self):
        self.create_upload_remote_if_needed()
        _log.info('Pushing branch ' + self._branch_name + " to " + self._git.remote(["get-url", self._wpt_fork_remote]).rstrip())
        _log.info('This may take some time')
        self._git.push([self._wpt_fork_remote, self._branch_name + ":" + self._public_branch_name, '-f'])
        _log.info('Branch available at ' + self._wpt_fork_branch_github_url)
        return True

    def make_pull_request(self):
        if self.has_webkit_test_runner_specific_changes:
            _log.error('Cannot create a WPT PR since it contains webkit test runner specific changes')
            return

        if not self._github:
            _log.info('Skipping pull request because OAuth token was not provided. You can open the pull request manually using the branch ' + self._wpt_fork_branch_github_url)
            return

        _log.info('Making pull request')
        title = self._bugzilla.fetch_bug_dictionary(self._bug_id)["title"].replace("[", "\\[").replace("]", "\\]")
        # NOTE: this should contain the exact string "WebKit export" to match the condition in
        # https://github.com/web-platform-tests/wpt-pr-bot/blob/f53e625c4871010277dc68336b340b5cd86e2a10/lib/metadata/index.js#L87
        description = "WebKit export from bug: [%s](https://bugs.webkit.org/show_bug.cgi?id=%s)" % (title, self._bug_id)
        pr_number = self.create_wpt_pull_request(self._wpt_fork_remote + ':' + self._public_branch_name, self._commit_message, description)
        if pr_number:
            try:
                self._github.add_label(pr_number, WEBKIT_EXPORT_PR_LABEL)
            except Exception as e:
                _log.warning(e)
                _log.info('Could not add label "%s" to pr #%s. User "%s" may not have permission to update labels in the %s/%s repo.' % (WEBKIT_EXPORT_PR_LABEL, pr_number, self.username, WPT_GH_ORG, WPT_GH_REPO_NAME))
        if self._bug_id and pr_number:
            pr_url = WPT_PR_URL + str(pr_number)
            self._bugzilla.post_comment_to_bug(self._bug_id, "Submitted web-platform-tests pull request: " + pr_url, see_also=[pr_url])

    def create_wpt_pull_request(self, remote_branch_name, title, body):
        pr_number = None
        try:
            pr_number = self._github.create_pr(remote_branch_name, title, body)
        except Exception as e:
            if e.code == 422:
                _log.info('Unable to create a new pull request for branch "%s" because a pull request already exists. The branch has been updated and there is no further action needed.' % (remote_branch_name))
            else:
                _log.warning(e)
                _log.info('Error creating a pull request on github. Please ensure that the provided github token has the "public_repo" scope.')
        return pr_number

    def delete_local_branch(self):
        _log.info('Removing local branch ' + self._branch_name)
        self._git.checkout('master')
        self._git.delete_branch(self._branch_name)

    def create_upload_remote_if_needed(self):
        if not self._wpt_fork_remote in self._git.remote([]):
            self._git.remote(["add", self._wpt_fork_remote, self._wpt_fork_push_url])

    def do_export(self):
        git_patch_file = self.write_git_patch_file()

        if not git_patch_file:
            _log.error("Unable to create a patch to apply to web-platform-tests repository")
            return

        self._fetch_wpt_repository()
        self.clean()

        if not self.create_branch_with_patch(git_patch_file):
            _log.error("Cannot create web-platform-tests local branch from the patch")
            self.delete_local_branch()
            return

        if git_patch_file:
            self._filesystem.remove(git_patch_file)

        lint_errors = self._linter.lint()
        if lint_errors:
            _log.error("The wpt linter detected %s linting error(s). Please address the above errors before attempting to export changes to the web-platform-test repository." % (lint_errors,))
            self.delete_local_branch()
            self.clean()
            return

        try:
            if self.push_to_wpt_fork():
                if self._options.create_pull_request:
                    self.make_pull_request()
        finally:
            self.delete_local_branch()
            _log.info("Finished")
            self.clean()


def parse_args(args):
    description = """Script to generate a pull request to W3C web-platform-tests repository
    'Tools/Scripts/export-w3c-test-changes -c -g HEAD -b XYZ' will do the following:
    - Clone web-platform-tests repository if not done already and set it up for pushing branches.
    - Gather WebKit bug id XYZ bug and changes to apply to web-platform-tests repository based on the HEAD commit
    - Create a remote branch named webkit-XYZ on https://github.com/USERNAME/%s.git repository based on the locally applied patch.
    -    USERNAME may be set using the environment variable GITHUB_USERNAME or as a command line option. It is then stored in git config as github.username.
    -    Github credential may be set using the environment variable GITHUB_TOKEN or as a command line option. (Please provide a valid GitHub 'Personal access token' with 'repo' as scope). It is then stored in git config as github.token.
    - Make the related pull request on %s.git repository.
    - Clean the local Git repository
    Notes:
    - It is safer to provide a bug id using -b option (bug id from a git commit is not always working).
    - As a dry run, one can start by running the script without -c. This will only create the branch on the user public GitHub repository.
    - By default, the script will create an https remote URL that will require a password-based authentication to GitHub. If you are using an SSH key, please use the --remote-url option.
    FIXME:
    - The script is not yet able to update an existing pull request
    - Need a way to monitor the progress of the pul request so that status of all pending pull requests can be done at import time.
    """ % (WPT_GH_REPO_NAME, WPT_GH_URL)
    parser = argparse.ArgumentParser(prog='export-w3c-test-changes ...', description=description, formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument('-g', '--git-commit', dest='git_commit', default=None, help='Git commit to apply')
    parser.add_argument('-b', '--bug', dest='bug_id', default=None, help='Bug ID to search for patch')
    parser.add_argument('-a', '--attachment', dest='attachment_id', default=None, help='Attachment ID to search for patch')
    parser.add_argument('-n', '--name', dest='username', default=None, help='github user name if GITHUB_USERNAME is not defined or github.username in the WPT repo config is not defined')
    parser.add_argument('-t', '--token', dest='token', default=None, help='github token, needed for creating pull requests only if GITHUB_TOKEN env variable is not defined or github.token in the WPT repo config is not defined')
    parser.add_argument('-bn', '--branch-name', dest='public_branch_name', default=None, help='Branch name to push to')
    parser.add_argument('-m', '--message', dest='message', default=None, help='Commit message')
    parser.add_argument('-r', '--remote', dest='repository_remote', default=None, help='repository origin to use to push')
    parser.add_argument('-u', '--remote-url', dest='repository_remote_url', default=None, help='repository url to use to push')
    parser.add_argument('-d', '--repository', dest='repository_directory', default=None, help='repository directory')
    parser.add_argument('-c', '--create-pr', dest='create_pull_request', action='store_true', default=False, help='create pull request to w3c web-platform-tests')
    parser.add_argument('--non-interactive', action='store_true', dest='non_interactive', default=False, help='Never prompt the user, fail as fast as possible.')

    options, args = parser.parse_known_args(args)

    return options


def configure_logging():
    class LogHandler(logging.StreamHandler):

        def format(self, record):
            if record.levelno > logging.INFO:
                return "%s: %s" % (record.levelname, record.getMessage())
            return record.getMessage()

    logger = logging.getLogger('webkitpy.w3c.test_exporter')
    logger.propagate = False
    logger.setLevel(logging.INFO)
    handler = LogHandler()
    handler.setLevel(logging.INFO)
    logger.addHandler(handler)
    return handler


def main(_argv, _stdout, _stderr):
    options = parse_args(_argv)

    configure_logging()
    test_exporter = WebPlatformTestExporter(Host(), options)

    if not test_exporter.has_wpt_changes():
        _log.info('No changes to upstream. Exiting...')
        return

    test_exporter.do_export()