Commit 0ff718e1 authored by Michael J. Sullivan's avatar Michael J. Sullivan Committed by Łukasz Langa
Browse files

Blacken .py files in blib2to3 (#1011)

* Blacken .py files in blib2to3

This is in preparation for adding type annotations to blib2to3 in
order to compiling it with mypyc (#1009, which I can rebase on top of
this).

To enforce that it stays blackened, I just cargo-culted the existing
test code used for validating formatting. It feels pretty clunky now,
though, so I can abstract the common logic out into a helper if that
seems better. (But error messages might be less clear then?)

* Tidy up the tests
parent 3bfb6697
Loading
Loading
Loading
Loading
+3 −6
Changes for blib2to3/pgen2/conv.py: 3 added lines, 6 removed lines.
Original line number Diff line number Diff line
@@ -70,8 +70,7 @@ def parse_graminit_h(self, filename):
            lineno += 1
            mo = re.match(r"^#define\s+(\w+)\s+(\d+)$", line)
            if not mo and line.strip():
                print("%s(%s): can't parse %s" % (filename, lineno,
                                                  line.strip()))
                print("%s(%s): can't parse %s" % (filename, lineno, line.strip()))
            else:
                symbol, number = mo.groups()
                number = int(number)
@@ -129,8 +128,7 @@ def parse_graminit_c(self, filename):
        states = []
        while line.startswith("static arc "):
            while line.startswith("static arc "):
                mo = re.match(r"static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$",
                              line)
                mo = re.match(r"static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$", line)
                assert mo, (lineno, line)
                n, m, k = list(map(int, mo.groups()))
                arcs = []
@@ -170,8 +168,7 @@ def parse_graminit_c(self, filename):
        ndfas = int(mo.group(1))
        for i in range(ndfas):
            lineno, line = lineno + 1, next(f)
            mo = re.match(r'\s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$',
                          line)
            mo = re.match(r'\s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$', line)
            assert mo, (lineno, line)
            symbol = mo.group(2)
            number, x, y, z = list(map(int, mo.group(1, 3, 4, 5)))
+14 −22
Changes for blib2to3/pgen2/driver.py: 14 added lines, 22 removed lines.
Original line number Diff line number Diff line
@@ -28,13 +28,7 @@


class Driver(object):

    def __init__(
        self,
        grammar,
        convert=None,
        logger=None,
    ):
    def __init__(self, grammar, convert=None, logger=None):
        self.grammar = grammar
        if logger is None:
            logger = logging.getLogger(__name__)
@@ -73,8 +67,9 @@ def parse_tokens(self, tokens, debug=False):
            if type == token.OP:
                type = grammar.opmap[value]
            if debug:
                self.logger.debug("%s %r (prefix=%r)",
                                  token.tok_name[type], value, prefix)
                self.logger.debug(
                    "%s %r (prefix=%r)", token.tok_name[type], value, prefix
                )
            if type == token.INDENT:
                indent_columns.append(len(value))
                _prefix = prefix + value
@@ -96,8 +91,7 @@ def parse_tokens(self, tokens, debug=False):
                column = 0
        else:
            # We never broke out -- EOF is too soon (how can this happen???)
            raise parse.ParseError("incomplete input",
                                   type, value, (prefix, start))
            raise parse.ParseError("incomplete input", type, value, (prefix, start))
        return p.rootnode

    def parse_stream_raw(self, stream, debug=False):
@@ -117,8 +111,7 @@ def parse_file(self, filename, encoding=None, debug=False):
    def parse_string(self, text, debug=False):
        """Parse a string and return the syntax tree."""
        tokens = tokenize.generate_tokens(
            io.StringIO(text).readline,
            grammar=self.grammar
            io.StringIO(text).readline, grammar=self.grammar
        )
        return self.parse_tokens(tokens, debug)

@@ -130,24 +123,24 @@ def _partially_consume_prefix(self, prefix, column):
        for char in prefix:
            current_line += char
            if wait_for_nl:
                if char == '\n':
                if char == "\n":
                    if current_line.strip() and current_column < column:
                        res = ''.join(lines)
                        res = "".join(lines)
                        return res, prefix[len(res) :]

                    lines.append(current_line)
                    current_line = ""
                    current_column = 0
                    wait_for_nl = False
            elif char in ' \t':
            elif char in " \t":
                current_column += 1
            elif char == '\n':
            elif char == "\n":
                # unexpected empty line
                current_column = 0
            else:
                # indent is finished
                wait_for_nl = True
        return ''.join(lines), current_line
        return "".join(lines), current_line


def _generate_pickle_name(gt, cache_dir=None):
@@ -161,8 +154,7 @@ def _generate_pickle_name(gt, cache_dir=None):
        return name


def load_grammar(gt="Grammar.txt", gp=None,
                 save=True, force=False, logger=None):
def load_grammar(gt="Grammar.txt", gp=None, save=True, force=False, logger=None):
    """Load the grammar (maybe from a pickle)."""
    if logger is None:
        logger = logging.getLogger(__name__)
@@ -219,11 +211,11 @@ def main(*args):
    """
    if not args:
        args = sys.argv[1:]
    logging.basicConfig(level=logging.INFO, stream=sys.stdout,
                        format='%(message)s')
    logging.basicConfig(level=logging.INFO, stream=sys.stdout, format="%(message)s")
    for gt in args:
        load_grammar(gt, save=True, force=True)
    return True


if __name__ == "__main__":
    sys.exit(int(not main()))
+12 −3
Changes for blib2to3/pgen2/grammar.py: 12 added lines, 3 removed lines.
Original line number Diff line number Diff line
@@ -90,7 +90,9 @@ def __init__(self):

    def dump(self, filename):
        """Dump the grammar tables to a pickle file."""
        with tempfile.NamedTemporaryFile(dir=os.path.dirname(filename), delete=False) as f:
        with tempfile.NamedTemporaryFile(
            dir=os.path.dirname(filename), delete=False
        ) as f:
            pickle.dump(self.__dict__, f, pickle.HIGHEST_PROTOCOL)
        os.replace(f.name, filename)

@@ -109,8 +111,14 @@ def copy(self):
        Copy the grammar.
        """
        new = self.__class__()
        for dict_attr in ("symbol2number", "number2symbol", "dfas", "keywords",
                          "tokens", "symbol2label"):
        for dict_attr in (
            "symbol2number",
            "number2symbol",
            "dfas",
            "keywords",
            "tokens",
            "symbol2label",
        ):
            setattr(new, dict_attr, getattr(self, dict_attr).copy())
        new.labels = self.labels[:]
        new.states = self.states[:]
@@ -121,6 +129,7 @@ def copy(self):
    def report(self):
        """Dump the grammar tables to standard output, for debugging."""
        from pprint import pprint

        print("s2n")
        pprint(self.symbol2number)
        print("n2s")
+7 −2
Changes for blib2to3/pgen2/literals.py: 7 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -5,7 +5,8 @@

import regex as re

simple_escapes = {"a": "\a",
simple_escapes = {
    "a": "\a",
    "b": "\b",
    "f": "\f",
    "n": "\n",
@@ -14,7 +15,9 @@
    "v": "\v",
    "'": "'",
    '"': '"',
                  "\\": "\\"}
    "\\": "\\",
}


def escape(m):
    all, tail = m.group(0, 1)
@@ -37,6 +40,7 @@ def escape(m):
            raise ValueError("invalid octal string escape ('\\%s')" % tail) from None
    return chr(i)


def evalString(s):
    assert s.startswith("'") or s.startswith('"'), repr(s[:1])
    q = s[0]
@@ -47,6 +51,7 @@ def evalString(s):
    s = s[len(q) : -len(q)]
    return re.sub(r"\\(\'|\"|\\|[abfnrtv]|x.{0,2}|[0-7]{1,3})", escape, s)


def test():
    for i in range(256):
        c = chr(i)
+6 −4
Changes for blib2to3/pgen2/parse.py: 6 added lines, 4 removed lines.
Original line number Diff line number Diff line
@@ -13,17 +13,20 @@
# Local imports
from . import token


class ParseError(Exception):
    """Exception to signal the parser is stuck."""

    def __init__(self, msg, type, value, context):
        Exception.__init__(self, "%s: type=%r, value=%r, context=%r" %
                           (msg, type, value, context))
        Exception.__init__(
            self, "%s: type=%r, value=%r, context=%r" % (msg, type, value, context)
        )
        self.msg = msg
        self.type = type
        self.value = value
        self.context = context


class Parser(object):
    """Parser engine.

@@ -152,8 +155,7 @@ def addtoken(self, type, value, context):
                    self.pop()
                    if not self.stack:
                        # Done parsing, but another token is input
                        raise ParseError("too much input",
                                         type, value, context)
                        raise ParseError("too much input", type, value, context)
                else:
                    # No success finding a transition
                    raise ParseError("bad input", type, value, context)
Loading