Commit fc869039 authored by Łukasz Langa's avatar Łukasz Langa
Browse files

Don't crash and burn on empty lines with trailing whitespace

Fixes #80
parent 611737f9
Loading
Loading
Loading
Loading
+6 −0
Changes for README.md: 6 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -333,6 +333,12 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md).

## Change Log

### 18.3a5 (unreleased)

* fixed 18.3a4 regression: don't crash and burn on empty lines with
  trailing whitespace (#80)


### 18.3a4

* `# fmt: off` and `# fmt: on` are implemented (#5)
+5 −5
Changes for blib2to3/pgen2/tokenize.py: 5 added lines, 5 removed lines.
Original line number Diff line number Diff line
@@ -430,21 +430,21 @@ def generate_tokens(readline):
                yield stashed
                stashed = None

            if line[pos] in '\r\n':            # skip blank lines
                yield (NL, line[pos:], (lnum, pos), (lnum, len(line)), line)
                continue

            if column > indents[-1]:           # count indents
                indents.append(column)
                yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)

            if line[pos] in '#\r\n':           # skip comments or blank lines
                if line[pos] == '#':
            if line[pos] == '#':               # skip comments
                comment_token = line[pos:].rstrip('\r\n')
                nl_pos = pos + len(comment_token)
                yield (COMMENT, comment_token,
                        (lnum, pos), (lnum, pos + len(comment_token)), line)
                yield (NL, line[nl_pos:],
                        (lnum, nl_pos), (lnum, len(line)), line)
                else:
                    yield ((NL, COMMENT)[line[pos] == '#'], line[pos:],
                           (lnum, pos), (lnum, len(line)), line)
                continue

            while column < indents[-1]:        # count dedents
+1 −0
Changes for tests/function.py: 1 added line, 0 removed lines.
Original line number Diff line number Diff line
@@ -34,6 +34,7 @@ def spaces(a=1, b=(), c=[], d={}, e=True, f=-1, g=1 if False else 2, h="", i=r''
def spaces_types(a: int = 1, b: tuple = (), c: list = [], d: dict = {}, e: bool = True, f: int = -1, g: int = 1 if False else 2, h: str = "", i: str = r''): ...
def spaces2(result= _core.Value(None)):
 ...
    # EMPTY LINE WITH WHITESPACE (this comment will be removed)
def example(session):
    result = session.query(models.Customer.id).filter(
        models.Customer.account_id == account_id,
+2 −0
Changes for tests/test_black.py: 2 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@
fs = partial(black.format_str, line_length=ll)
THIS_FILE = Path(__file__)
THIS_DIR = THIS_FILE.parent
EMPTY_LINE = '# EMPTY LINE WITH WHITESPACE' + ' (this comment will be removed)'


def dump_to_stderr(*output: str) -> str:
@@ -33,6 +34,7 @@ def read_data(name: str) -> Tuple[str, str]:
        lines = test.readlines()
    result = _input
    for line in lines:
        line = line.replace(EMPTY_LINE, '')
        if line.rstrip() == '# output':
            result = _output
            continue