Commit 8c565d86 authored by Łukasz Langa's avatar Łukasz Langa
Browse files

blib2to3: Never put prefixes on INDENT leaves either

parent 7e1c5b2b
Loading
Loading
Loading
Loading
+2 −1
Changes for blib2to3/README: 2 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -7,5 +7,6 @@ Reasons for forking:
  *args and **kwargs
- backport of GH-6143 that restores the ability to reformat legacy usage of
  `async`
- better ability to debug (better reprs for starters)
- better ability to debug (better reprs)
- INDENT and DEDENT don't hold whitespace and comment prefixes
- ability to Cythonize
+2 −2
Changes for blib2to3/pgen2/driver.py: 2 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -69,7 +69,7 @@ def parse_tokens(self, tokens, debug=False):
            if debug:
                self.logger.debug("%s %r (prefix=%r)",
                                  token.tok_name[type], value, prefix)
            if type == token.DEDENT:
            if type in {token.INDENT, token.DEDENT}:
                _prefix = prefix
                prefix = ""
            if p.addtoken(type, value, (prefix, start)):
@@ -77,7 +77,7 @@ def parse_tokens(self, tokens, debug=False):
                    self.logger.debug("Stop.")
                break
            prefix = ""
            if type == token.DEDENT:
            if type in {token.INDENT, token.DEDENT}:
                prefix = _prefix
            lineno, column = end
            if value.endswith("\n"):
+5 −4
Changes for blib2to3/pgen2/tokenize.py: 5 added lines, 4 removed lines.
Original line number Diff line number Diff line
@@ -430,6 +430,10 @@ def generate_tokens(readline):
                yield stashed
                stashed = None

            if column > indents[-1]:           # count indents
                indents.append(column)
                yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)

            if line[pos] in '#\r\n':           # skip comments or blank lines
                if line[pos] == '#':
                    comment_token = line[pos:].rstrip('\r\n')
@@ -443,10 +447,7 @@ def generate_tokens(readline):
                           (lnum, pos), (lnum, len(line)), line)
                continue

            if column > indents[-1]:           # count indents or dedents
                indents.append(column)
                yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
            while column < indents[-1]:
            while column < indents[-1]:         # count dedents
                if column not in indents:
                    raise IndentationError(
                        "unindent does not match any outer indentation level",
+1 −0
Changes for tests/empty_lines.py: 1 added line, 0 removed lines.
Original line number Diff line number Diff line
@@ -105,6 +105,7 @@ def f():
    if not prev:
        prevp = preceding_leaf(p)
        if not prevp or prevp.type in OPENING_BRACKETS:

            return NO

        if prevp.type == token.EQUAL:
+1 −1

File changed.

Contains only whitespace changes.