Commit 9138a75b authored by Łukasz Langa's avatar Łukasz Langa
Browse files

Fix parsing of unaligned standalone comments

Fixes #99
Fixes #112
parent 1909d8cc
Loading
Loading
Loading
Loading
+4 −0
Changes for README.md: 4 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -491,6 +491,10 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md).

## Change Log

### 18.4a2 (unreleased)

* fixed parsing of unaligned standalone comments (#99, #112)

### 18.4a1

* added `--quiet` (#78)
+6 −0
Changes for blib2to3/pgen2/driver.py: 6 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -77,6 +77,12 @@ def parse_tokens(self, tokens, debug=False):
                    self.logger.debug("Stop.")
                break
            prefix = ""
            if type == token.INDENT:
                if _prefix.startswith(value):
                    # Don't double-indent.  Since we're delaying the prefix that
                    # would normally belong to INDENT, we need to put the value
                    # at the end versus at the beginning.
                    _prefix = _prefix[len(value):] + value
            if type in {token.INDENT, token.DEDENT}:
                prefix = _prefix
            lineno, column = end
+4 −4
Changes for blib2to3/pgen2/tokenize.py: 4 added lines, 4 removed lines.
Original line number Diff line number Diff line
@@ -412,10 +412,6 @@ def generate_tokens(readline):
                yield (NL, line[pos:], (lnum, pos), (lnum, len(line)), line)
                continue

            if column > indents[-1]:           # count indents
                indents.append(column)
                yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)

            if line[pos] == '#':               # skip comments
                comment_token = line[pos:].rstrip('\r\n')
                nl_pos = pos + len(comment_token)
@@ -425,6 +421,10 @@ def generate_tokens(readline):
                        (lnum, nl_pos), (lnum, len(line)), line)
                continue

            if column > indents[-1]:           # count indents
                indents.append(column)
                yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)

            while column < indents[-1]:        # count dedents
                if column not in indents:
                    raise IndentationError(
+16 −0
Changes for tests/comments2.py: 16 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -23,6 +23,14 @@
    'Generator',
]

if 'PYTHON' in os.environ:
    add_compiler(compiler_from_env())
else:
    # for compiler in compilers.values():
         # add_compiler(compiler)
    add_compiler(compilers[(7.0, 32)])
    # add_compiler(compilers[(7.1, 64)])

# Comment before function.
def inline_comments_in_brackets_ruin_everything():
    if typedargslist:
@@ -144,6 +152,14 @@ def inline_comments_in_brackets_ruin_everything():
    "Generator",
]

if "PYTHON" in os.environ:
    add_compiler(compiler_from_env())
else:
    # for compiler in compilers.values():
    # add_compiler(compiler)
    add_compiler(compilers[(7.0, 32)])
# add_compiler(compilers[(7.1, 64)])

# Comment before function.