# This file is part of Buildbot.  Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
"""
BuildSteps that are specific to the Twisted source tree
"""

import re
from typing import List

from twisted.internet import defer
from twisted.python import log

from buildbot import util
from buildbot.process import buildstep
from buildbot.process import logobserver
from buildbot.process.results import FAILURE
from buildbot.process.results import SKIPPED
from buildbot.process.results import SUCCESS
from buildbot.process.results import WARNINGS
from buildbot.steps import shell


class HLint(buildstep.ShellMixin, buildstep.BuildStep):
    """I run a 'lint' checker over a set of .xhtml files. Any deviations
    from recommended style is flagged and put in the output log.

    This step looks at .changes in the parent Build to extract a list of
    Lore XHTML files to check."""

    name = "hlint"
    description = "running hlint"
    descriptionDone = "hlint"
    warnOnWarnings = True
    warnOnFailure = True
    # TODO: track time, but not output
    warnings = 0

    def __init__(self, python=None, **kwargs):
        kwargs = self.setupShellMixin(kwargs, prohibitArgs=['command'])
        super().__init__(**kwargs)
        self.python = python
        self.warningLines = []
        self.addLogObserver('stdio', logobserver.LineConsumerLogObserver(self.logConsumer))

    @defer.inlineCallbacks
    def run(self):
        # create the command
        html_files = set()
        for f in self.build.allFiles():
            if f.endswith(".xhtml") and not f.startswith("sandbox/"):
                html_files.add(f)
        # remove duplicates
        hlintTargets = sorted(list(html_files))
        if not hlintTargets:
            return SKIPPED
        self.hlintFiles = hlintTargets

        command = []
        if self.python:
            command.append(self.python)
        command += ["bin/lore", "-p", "--output", "lint", *self.hlintFiles]

        cmd = yield self.makeRemoteShellCommand(command=command)
        yield self.runCommand(cmd)

        stdio_log = yield self.getLog('stdio')
        yield stdio_log.finish()

        yield self.addCompleteLog('warnings', '\n'.join(self.warningLines))
        yield self.addCompleteLog("files", "\n".join(self.hlintFiles) + "\n")

        # warnings are in stdout, rc is always 0, unless the tools break
        if cmd.didFail():
            return FAILURE

        self.descriptionDone = f"{self.warnings} hlin{self.warnings == 1 and 't' or 'ts'}"

        if self.warnings:
            return WARNINGS
        return SUCCESS

    def logConsumer(self):
        while True:
            _, line = yield
            if ':' in line:
                self.warnings += 1
                self.warningLines.append(line)


class TrialTestCaseCounter(logobserver.LogLineObserver):
    _line_re = re.compile(r'^(?:Doctest: )?([\w\.]+) \.\.\. \[([^\]]+)\]$')

    def __init__(self):
        super().__init__()
        self.numTests = 0
        self.finished = False
        self.counts = {
            'total': None,
            'failures': 0,
            'errors': 0,
            'skips': 0,
            'expectedFailures': 0,
            'unexpectedSuccesses': 0,
        }

    def outLineReceived(self, line):
        # different versions of Twisted emit different per-test lines with
        # the bwverbose reporter.
        #  2.0.0: testSlave (buildbot.test.test_runner.Create) ... [OK]
        #  2.1.0: buildbot.test.test_runner.Create.testSlave ... [OK]
        #  2.4.0: buildbot.test.test_runner.Create.testSlave ... [OK]
        # Let's just handle the most recent version, since it's the easiest.
        # Note that doctests create lines line this:
        #  Doctest: viff.field.GF ... [OK]

        if line.startswith("=" * 40):
            self.finished = True
        if not self.finished:
            m = self._line_re.search(line.strip())
            if m:
                m.groups()
                self.numTests += 1
                self.step.setProgress('tests', self.numTests)

        out = re.search(r'Ran (\d+) tests', line)
        if out:
            self.counts['total'] = int(out.group(1))
        if line.startswith("OK") or line.startswith("FAILED ") or line.startswith("PASSED"):
            # the extra space on FAILED_ is to distinguish the overall
            # status from an individual test which failed. The lack of a
            # space on the OK is because it may be printed without any
            # additional text (if there are no skips,etc)
            out = re.search(r'failures=(\d+)', line)
            if out:
                self.counts['failures'] = int(out.group(1))
            out = re.search(r'errors=(\d+)', line)
            if out:
                self.counts['errors'] = int(out.group(1))
            out = re.search(r'skips=(\d+)', line)
            if out:
                self.counts['skips'] = int(out.group(1))
            out = re.search(r'expectedFailures=(\d+)', line)
            if out:
                self.counts['expectedFailures'] = int(out.group(1))
            out = re.search(r'unexpectedSuccesses=(\d+)', line)
            if out:
                self.counts['unexpectedSuccesses'] = int(out.group(1))
            # successes= is a Twisted-2.0 addition, and is not currently used
            out = re.search(r'successes=(\d+)', line)
            if out:
                self.counts['successes'] = int(out.group(1))


UNSPECIFIED = ()  # since None is a valid choice


class Trial(buildstep.ShellMixin, buildstep.BuildStep):
    """
    There are some class attributes which may be usefully overridden
    by subclasses. 'trialMode' and 'trialArgs' can influence the trial
    command line.
    """

    name = "trial"
    progressMetrics = ('output', 'tests', 'test.log')
    # note: the slash only works on unix workers, of course, but we have
    # no way to know what the worker uses as a separator.
    # TODO: figure out something clever.
    logfiles = {"test.log": "_trial_temp/test.log"}
    # we use test.log to track Progress at the end of __init__()

    renderables = ['tests', 'jobs']
    flunkOnFailure = True
    python = None
    trial = "trial"
    trialMode = ["--reporter=bwverbose"]  # requires Twisted-2.1.0 or newer
    # for Twisted-2.0.0 or 1.3.0, use ["-o"] instead
    trialArgs: List[str] = []
    jobs = None
    testpath = UNSPECIFIED  # required (but can be None)
    testChanges = False  # TODO: needs better name
    recurse = False
    reactor = None
    randomly = False
    tests = None  # required

    description = 'testing'
    descriptionDone = 'tests'

    def __init__(
        self,
        reactor=UNSPECIFIED,
        python=None,
        trial=None,
        testpath=UNSPECIFIED,
        tests=None,
        testChanges=None,
        recurse=None,
        randomly=None,
        trialMode=None,
        trialArgs=None,
        jobs=None,
        **kwargs,
    ):
        kwargs = self.setupShellMixin(kwargs, prohibitArgs=['command'])
        super().__init__(**kwargs)

        if python:
            self.python = python
        if self.python is not None:
            if isinstance(self.python, str):
                self.python = [self.python]
            for s in self.python:
                if " " in s:
                    # this is not strictly an error, but I suspect more
                    # people will accidentally try to use python="python2.3
                    # -Wall" than will use embedded spaces in a python flag
                    log.msg("python= component '%s' has spaces")
                    log.msg("To add -Wall, use python=['python', '-Wall']")
                    why = "python= value has spaces, probably an error"
                    raise ValueError(why)

        if trial:
            self.trial = trial
        if " " in self.trial:
            raise ValueError("trial= value has spaces")
        if trialMode is not None:
            self.trialMode = trialMode
        if trialArgs is not None:
            self.trialArgs = trialArgs
        if jobs is not None:
            self.jobs = jobs

        if testpath is not UNSPECIFIED:
            self.testpath = testpath
        if self.testpath is UNSPECIFIED:
            raise ValueError("You must specify testpath= (it can be None)")
        assert isinstance(self.testpath, str) or self.testpath is None

        if reactor is not UNSPECIFIED:
            self.reactor = reactor

        if tests is not None:
            self.tests = tests
        if isinstance(self.tests, str):
            self.tests = [self.tests]
        if testChanges is not None:
            self.testChanges = testChanges
            # self.recurse = True  # not sure this is necessary

        if not self.testChanges and self.tests is None:
            raise ValueError("Must either set testChanges= or provide tests=")

        if recurse is not None:
            self.recurse = recurse
        if randomly is not None:
            self.randomly = randomly

        if self.reactor:
            self.description = f"testing ({self.reactor})"

        # this counter will feed Progress along the 'test cases' metric
        self.observer = TrialTestCaseCounter()
        self.addLogObserver('stdio', self.observer)

        # this observer consumes multiple lines in a go, so it can't be easily
        # handled in TrialTestCaseCounter.
        self.addLogObserver('stdio', logobserver.LineConsumerLogObserver(self.logConsumer))
        self.problems = []
        self.warnings = {}

        # text used before commandComplete runs
        self.text = 'running'

    def setup_python_path(self):
        if self.testpath is None:
            return

        # this bit produces a list, which can be used by buildbot_worker.runprocess.RunProcess
        ppath = self.env.get('PYTHONPATH', self.testpath)
        if isinstance(ppath, str):
            ppath = [ppath]
        if self.testpath not in ppath:
            ppath.insert(0, self.testpath)
        self.env['PYTHONPATH'] = ppath

    @defer.inlineCallbacks
    def run(self):
        # choose progressMetrics and logfiles based on whether trial is being
        # run with multiple workers or not.
        output_observer = logobserver.OutputProgressObserver('test.log')

        # build up most of the command, then stash it until start()
        command = []
        if self.python:
            command.extend(self.python)
        command.append(self.trial)
        command.extend(self.trialMode)
        if self.recurse:
            command.append("--recurse")
        if self.reactor:
            command.append(f"--reactor={self.reactor}")
        if self.randomly:
            command.append("--random=0")
        command.extend(self.trialArgs)

        if self.jobs is not None:
            self.jobs = int(self.jobs)
            command.append(f"--jobs={self.jobs}")

            # using -j/--jobs flag produces more than one test log.
            self.logfiles = {}
            for i in range(self.jobs):
                self.logfiles[f'test.{i}.log'] = f'_trial_temp/{i}/test.log'
                self.logfiles[f'err.{i}.log'] = f'_trial_temp/{i}/err.log'
                self.logfiles[f'out.{i}.log'] = f'_trial_temp/{i}/out.log'
                self.addLogObserver(f'test.{i}.log', output_observer)
        else:
            # this one just measures bytes of output in _trial_temp/test.log
            self.addLogObserver('test.log', output_observer)

        # now that self.build.allFiles() is nailed down, finish building the
        # command
        if self.testChanges:
            for f in self.build.allFiles():
                if f.endswith(".py"):
                    command.append(f"--testmodule={f}")
        else:
            command.extend(self.tests)

        self.setup_python_path()

        cmd = yield self.makeRemoteShellCommand(command=command)
        yield self.runCommand(cmd)

        stdio_log = yield self.getLog('stdio')
        yield stdio_log.finish()

        # figure out all status, then let the various hook functions return
        # different pieces of it

        problems = '\n'.join(self.problems)
        warnings = self.warnings

        if problems:
            yield self.addCompleteLog("problems", problems)

        if warnings:
            lines = sorted(warnings.keys())
            yield self.addCompleteLog("warnings", "".join(lines))

        return self.build_results(cmd)

    def build_results(self, cmd):
        counts = self.observer.counts
        total = counts['total']
        failures = counts['failures']
        errors = counts['errors']
        parsed = total is not None

        desc_parts = []

        if not cmd.didFail():
            if parsed:
                results = SUCCESS
                if total:
                    desc_parts += [str(total), total == 1 and "test" or "tests", "passed"]
                else:
                    desc_parts += ["no tests", "run"]
            else:
                results = FAILURE
                desc_parts += ["testlog", "unparseable"]
        else:
            # something failed
            results = FAILURE
            if parsed:
                desc_parts += ["tests"]
                if failures:
                    desc_parts += [str(failures), failures == 1 and "failure" or "failures"]
                if errors:
                    desc_parts += [str(errors), errors == 1 and "error" or "errors"]
            else:
                desc_parts += ["tests", "failed"]

        if counts['skips']:
            desc_parts += [str(counts['skips']), counts['skips'] == 1 and "skip" or "skips"]
        if counts['expectedFailures']:
            desc_parts += [
                str(counts['expectedFailures']),
                "todo" if counts['expectedFailures'] == 1 else "todos",
            ]

        if self.reactor:
            desc_parts.append(self.rtext('({})'))

        self.descriptionDone = util.join_list(desc_parts)
        return results

    def rtext(self, fmt='{}'):
        if self.reactor:
            rtext = fmt.format(self.reactor)
            return rtext.replace("reactor", "")
        return ""

    def logConsumer(self):
        while True:
            _, line = yield
            if line.find(" exceptions.DeprecationWarning: ") != -1:
                # no source
                warning = line  # TODO: consider stripping basedir prefix here
                self.warnings[warning] = self.warnings.get(warning, 0) + 1
            elif line.find(" DeprecationWarning: ") != -1 or line.find(" UserWarning: ") != -1:
                # next line is the source
                warning = line + "\n" + (yield)[1] + "\n"
                self.warnings[warning] = self.warnings.get(warning, 0) + 1
            elif line.find("Warning: ") != -1:
                warning = line
                self.warnings[warning] = self.warnings.get(warning, 0) + 1

            if line.find("=" * 60) == 0 or line.find("-" * 60) == 0:
                # read to EOF
                while True:
                    self.problems.append(line)
                    _, line = yield


class RemovePYCs(shell.ShellCommand):
    name = "remove_pyc"
    command = ['find', '.', '-name', "'*.pyc'", '-exec', 'rm', '{}', ';']
    description = "removing .pyc files"
    descriptionDone = "remove .pycs"
