CAP/TPfutures/MiniC-futures/test_expect_pragma.py

import collections
import re
import os
import subprocess
import sys
import pytest

testinfo = collections.namedtuple(
    'testinfo',
    ['exitcode', 'execcode', 'output', 'linkargs', 'skip_test_expected'])
default_testinfo = testinfo(
    exitcode=0, execcode=0, output='', linkargs=[], skip_test_expected=False)


def cat(filename):
    with open(filename, "rb") as f:
        for line in f:
            sys.stdout.buffer.write(line)


def env_bool_variable(name, globals):
    if name not in globals:
        globals[name] = False
    if name in os.environ:
        globals[name] = True


def env_str_variable(name, globals):
    if name in os.environ:
        globals[name] = os.environ[name]


class TestExpectPragmas(object):
    """Base class for tests that read the expected result as annotations
    in test files.

    get_expect(file) will parse the file, looking EXPECT and EXITCODE
    pragmas.

    run_command(command) is a wrapper around subprocess.check_output()
    that extracts the output and exit code.

    """

    def get_expect(self, filename):
        """Parse "filename" looking for EXPECT and EXITCODE annotations.

        Look for a line "EXPECTED" (possibly with whitespaces and
        comments). Text after this "EXPECTED" line is the expected
        output.

        The file may also contain a line like "EXITCODE <n>" where <n>
        is an integer, and is the expected exitcode of the command.

        The result is cached to avoid re-parsing the file multiple
        times.
        """
        if filename not in self.__expect:
            self.__expect[filename] = self._extract_expect(filename)
        return self.__expect[filename]

    def remove(self, file):
        """Like os.remove(), but ignore errors, e.g. don't complain if the
        file doesn't exist.
        """
        try:
            os.remove(file)
        except OSError:
            pass

    def run_command(self, cmd, scope="compile"):
        """Run the command cmd (given as [command, arg1, arg2, ...]), and
        return testinfo(exitcode=..., output=...) containing the
        exit code of the command it its standard output + standard error.

        If scope="compile" (resp. "runtime"), then the exitcode (resp.
        execcode) is set with the exit status of the command, and the
        execcode (resp. exitcode) is set to 0.
        """
        try:
            output = subprocess.check_output(cmd, timeout=60,
                                             stderr=subprocess.STDOUT)
            status = 0
        except subprocess.CalledProcessError as e:
            output = e.output
            status = e.returncode
        if scope == "runtime":
            return default_testinfo._replace(execcode=status,
                                             output=output.decode())
        else:
            return default_testinfo._replace(exitcode=status,
                                             output=output.decode())

    def skip_if_partial_match(self, actual, expect, ignore_error_message):
        if not ignore_error_message:
            return False

        # TODO: Deal with undefined behavior here?

        if expect.exitcode != actual.exitcode:
            # Not the same exit code => something's wrong anyway
            return False

        if actual.exitcode == 3:
            # There's a syntax error in both expected and actual,
            # but the actual error may slightly differ if we don't
            # have the exact same .g4.
            return True

        # Let the test pass with 'return True' if appropriate.
        # Otherwise, continue to the full assertion for a
        # complete diagnostic.
        if actual.exitcode != 0 and expect.exitcode == actual.exitcode:
            if expect.output == '':
                # No output expected, but we know there must be an
                # error. If there was a particular error message
                # expected, we'd have written it in the output,
                # hence just ignore the actual message.
                return True
            # Ignore difference in error message except in the
            # line number (ignore the column too, it may
            # slightly vary, eg. in "foo" / 4, the error may
            # be considered on "foo" or on /):
            if re.match(r'^In function [^ :]*: Line [0-9]* col [0-9]*:',
                        actual.output):
                out_loc = re.sub(r' col [0-9]*:.*$', '', actual.output)
                exp_loc = re.sub(r' col [0-9]*:.*$', '', expect.output)
                if out_loc == exp_loc:
                    return True
            if any(x.output and
                   (x.output.endswith('has no value yet!' + os.linesep)
                    or x.output.endswith(' by 0' + os.linesep))
                   for x in (actual, expect)):
                # Ignore the error message when our compiler
                # raises this error (either in actual or expect,
                # depending on what we're testing).
                return True

        return False

    __expect = {}

    def _extract_expect(self, file):
        exitcode = 0
        execcode = 0
        linkargs = []
        inside_expected = False
        skip_test_expected = False
        expected_lines = []
        expected_present = False
        with open(file, encoding="utf-8") as f:
            for line in f.readlines():
                # Ignore non-comments
                if not re.match(r'\s*//', line):
                    continue
                # Cleanup comment start and whitespaces
                line = re.sub(r'\s*//\s*', '', line)
                line = re.sub(r'\s*$', '', line)

                if line == 'END EXPECTED':
                    inside_expected = False
                elif line.startswith('EXITCODE'):
                    words = line.split(' ')
                    assert len(words) == 2
                    exitcode = int(words[1])
                elif line.startswith('EXECCODE'):
                    words = line.split(' ')
                    assert len(words) == 2
                    execcode = int(words[1])
                elif line.startswith('LINKARGS'):
                    words = line.split(' ')
                    assert len(words) >= 2
                    linkargs += [w.replace("$dir", os.path.dirname(file))
                                 for w in words[1:]]
                elif line == 'EXPECTED':
                    inside_expected = True
                    expected_present = True
                elif line == 'SKIP TEST EXPECTED':
                    skip_test_expected = True
                elif inside_expected:
                    expected_lines.append(line)

        if not expected_present:
            pytest.fail("Missing EXPECTED directive in test file")

        if expected_lines == []:
            output = ''
        else:
            output = os.linesep.join(expected_lines) + os.linesep

        return testinfo(exitcode=exitcode, execcode=execcode,
                        output=output, linkargs=linkargs,
                        skip_test_expected=skip_test_expected)