Commit 788268bc authored by Łukasz Langa's avatar Łukasz Langa
Browse files

Re-implement magic trailing comma handling:

- when a trailing comma is specified in any bracket pair, that signals to Black
  that this bracket pair needs to be always exploded, e.g. presented as "one
  item per line";

- this causes some changes to previously formatted code that erroneously left
  trailing commas embedded into single-line expressions;

- internally, Black needs to be able to identify trailing commas that it put
  itself compared to pre-existing trailing commas. We do this by using/abusing
  lib2to3's `was_checked` attribute.  It's True for internally generated
  trailing commas and False for pre-existing ones (in fact, for all
  pre-existing leaves and nodes).

Fixes #1288
parent 4938cc9e
Loading
Loading
Loading
Loading
+3 −0
Changes for CHANGES.md: 3 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -4,6 +4,9 @@

#### _Black_

- re-implemented support for explicit trailing commas: now it works consistently within
  any bracket pair, including nested structures (#1288 and duplicates)

- reindent docstrings when reindenting code around it (#1053)

- show colored diffs (#1266)
+4 −1
Changes for gallery/gallery.py: 4 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -127,7 +127,10 @@ def get_package(


def download_and_extract_top_packages(
    directory: Path, days: Days = 365, workers: int = 8, limit: slice = DEFAULT_SLICE,
    directory: Path,
    days: Days = 365,
    workers: int = 8,
    limit: slice = DEFAULT_SLICE,
) -> Generator[Path, None, None]:
    with ThreadPoolExecutor(max_workers=workers) as executor:
        bound_downloader = partial(get_package, version=None, directory=directory)
+66 −60
Changes for src/black/__init__.py: 66 added lines, 60 removed lines.
Original line number Diff line number Diff line
@@ -1442,7 +1442,8 @@ def append(self, leaf: Leaf, preformatted: bool = False) -> None:
            )
        if self.inside_brackets or not preformatted:
            self.bracket_tracker.mark(leaf)
            self.maybe_remove_trailing_comma(leaf)
            if self.maybe_should_explode(leaf):
                self.should_explode = True
        if not self.append_comment(leaf):
            self.leaves.append(leaf)

@@ -1618,59 +1619,26 @@ def contains_unsplittable_type_ignore(self) -> bool:
    def contains_multiline_strings(self) -> bool:
        return any(is_multiline_string(leaf) for leaf in self.leaves)

    def maybe_remove_trailing_comma(self, closing: Leaf) -> bool:
        """Remove trailing comma if there is one and it's safe."""
    def maybe_should_explode(self, closing: Leaf) -> bool:
        """Return True if this line should explode (always be split), that is when:
        - there's a pre-existing trailing comma here; and
        - it's not a one-tuple.
        """
        if not (
            self.leaves
            closing.type in CLOSING_BRACKETS
            and self.leaves
            and self.leaves[-1].type == token.COMMA
            and closing.type in CLOSING_BRACKETS
            and not self.leaves[-1].was_checked  # pre-existing
        ):
            return False

        if closing.type == token.RBRACE:
            self.remove_trailing_comma()
            return True

        if closing.type == token.RSQB:
            comma = self.leaves[-1]
            if comma.parent and comma.parent.type == syms.listmaker:
                self.remove_trailing_comma()
        if closing.type in {token.RBRACE, token.RSQB}:
            return True

        # For parens let's check if it's safe to remove the comma.
        # Imports are always safe.
        if self.is_import:
            self.remove_trailing_comma()
            return True

        # Otherwise, if the trailing one is the only one, we might mistakenly
        # change a tuple into a different type by removing the comma.
        depth = closing.bracket_depth + 1
        commas = 0
        opening = closing.opening_bracket
        for _opening_index, leaf in enumerate(self.leaves):
            if leaf is opening:
                break

        else:
            return False

        for leaf in self.leaves[_opening_index + 1 :]:
            if leaf is closing:
                break

            bracket_depth = leaf.bracket_depth
            if bracket_depth == depth and leaf.type == token.COMMA:
                commas += 1
                if leaf.parent and leaf.parent.type in {
                    syms.arglist,
                    syms.typedargslist,
                }:
                    commas += 1
                    break

        if commas > 1:
            self.remove_trailing_comma()
        if not is_one_tuple_between(closing.opening_bracket, closing, self.leaves):
            return True

        return False
@@ -2647,7 +2615,7 @@ def init_st(ST: Type[StringTransformer]) -> StringTransformer:
            is_line_short_enough(line, line_length=mode.line_length, line_str=line_str)
            or line.contains_unsplittable_type_ignore()
        )
        and not (line.contains_standalone_comments() and line.inside_brackets)
        and not (line.inside_brackets and line.contains_standalone_comments())
    ):
        # Only apply basic string preprocessing, since lines shouldn't be split here.
        if mode.experimental_string_processing:
@@ -4772,10 +4740,8 @@ def right_hand_split(
    tail = bracket_split_build_line(tail_leaves, line, opening_bracket)
    bracket_split_succeeded_or_raise(head, body, tail)
    if (
        # the body shouldn't be exploded
        not body.should_explode
        # the opening bracket is an optional paren
        and opening_bracket.type == token.LPAR
        opening_bracket.type == token.LPAR
        and not opening_bracket.value
        # the closing bracket is an optional paren
        and closing_bracket.type == token.RPAR
@@ -4872,7 +4838,9 @@ def bracket_split_build_line(
                        continue

                    if leaves[i].type != token.COMMA:
                        leaves.insert(i + 1, Leaf(token.COMMA, ","))
                        new_comma = Leaf(token.COMMA, ",")
                        new_comma.was_checked = True
                        leaves.insert(i + 1, new_comma)
                    break

    # Populate the line
@@ -4880,8 +4848,8 @@ def bracket_split_build_line(
        result.append(leaf, preformatted=True)
        for comment_after in original.comments_after(leaf):
            result.append(comment_after, preformatted=True)
    if is_body:
        result.should_explode = should_explode(result, opening_bracket)
    if is_body and should_split_body_explode(result, opening_bracket):
        result.should_explode = True
    return result


@@ -4966,7 +4934,9 @@ def append_to_line(leaf: Leaf) -> Iterator[Line]:
            and current_line.leaves[-1].type != token.COMMA
            and current_line.leaves[-1].type != STANDALONE_COMMENT
        ):
            current_line.append(Leaf(token.COMMA, ","))
            new_comma = Leaf(token.COMMA, ",")
            new_comma.was_checked = True
            current_line.append(new_comma)
        yield current_line


@@ -5588,24 +5558,60 @@ def ensure_visible(leaf: Leaf) -> None:
        leaf.value = ")"


def should_explode(line: Line, opening_bracket: Leaf) -> bool:
def should_split_body_explode(line: Line, opening_bracket: Leaf) -> bool:
    """Should `line` immediately be split with `delimiter_split()` after RHS?"""

    if not (
        opening_bracket.parent
        and opening_bracket.parent.type in {syms.atom, syms.import_from}
        and opening_bracket.value in "[{("
    ):
    if not (opening_bracket.parent and opening_bracket.value in "[{("):
        return False

    # We're essentially checking if the body is delimited by commas and there's more
    # than one of them (we're excluding the trailing comma and if the delimiter priority
    # is still commas, that means there's more).
    exclude = set()
    pre_existing_trailing_comma = False
    try:
        last_leaf = line.leaves[-1]
        exclude = {id(last_leaf)} if last_leaf.type == token.COMMA else set()
        if last_leaf.type == token.COMMA:
            pre_existing_trailing_comma = not last_leaf.was_checked
            exclude.add(id(last_leaf))
        max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
    except (IndexError, ValueError):
        return False

    return max_priority == COMMA_PRIORITY
    return max_priority == COMMA_PRIORITY and (
        # always explode imports
        opening_bracket.parent.type in {syms.atom, syms.import_from}
        or pre_existing_trailing_comma
    )


def is_one_tuple_between(opening: Leaf, closing: Leaf, leaves: List[Leaf]) -> bool:
    """Return True if content between `opening` and `closing` looks like a one-tuple."""
    depth = closing.bracket_depth + 1
    for _opening_index, leaf in enumerate(leaves):
        if leaf is opening:
            break

    else:
        raise LookupError("Opening paren not found in `leaves`")

    commas = 0
    _opening_index += 1
    for leaf in leaves[_opening_index:]:
        if leaf is closing:
            break

        bracket_depth = leaf.bracket_depth
        if bracket_depth == depth and leaf.type == token.COMMA:
            commas += 1
            if leaf.parent and leaf.parent.type in {
                syms.arglist,
                syms.typedargslist,
            }:
                commas += 1
                break

    return commas < 2


def get_features_used(node: Node) -> Set[Feature]:
+1 −1
Changes for src/blib2to3/pgen2/driver.py: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -128,7 +128,7 @@ def parse_stream(self, stream: IO[Text], debug: bool = False) -> NL:
        return self.parse_stream_raw(stream, debug)

    def parse_file(
        self, filename: Path, encoding: Optional[Text] = None, debug: bool = False,
        self, filename: Path, encoding: Optional[Text] = None, debug: bool = False
    ) -> NL:
        """Parse a file and return the syntax tree."""
        with io.open(filename, "r", encoding=encoding) as stream:
+24 −11
Changes for tests/data/collections.py: 24 added lines, 11 removed lines.
Original line number Diff line number Diff line
@@ -2,18 +2,18 @@

from . import A, B, C

# unwraps
# keeps existing trailing comma
from foo import (
    bar,
)

# stays wrapped
# also keeps existing structure
from foo import (
    baz,
    qux,
)

# as doesn't get confusing when unwrapped
# `as` works as well
from foo import (
    xyzzy as magic,
)
@@ -77,17 +77,21 @@

from . import A, B, C

# unwraps
from foo import bar
# keeps existing trailing comma
from foo import (
    bar,
)

# stays wrapped
# also keeps existing structure
from foo import (
    baz,
    qux,
)

# as doesn't get confusing when unwrapped
from foo import xyzzy as magic
# `as` works as well
from foo import (
    xyzzy as magic,
)

a = {
    1,
@@ -151,11 +155,20 @@

if True:
    ec2client.get_waiter("instance_stopped").wait(
        InstanceIds=[instance.id], WaiterConfig={"Delay": 5,}
        InstanceIds=[instance.id],
        WaiterConfig={
            "Delay": 5,
        },
    )
    ec2client.get_waiter("instance_stopped").wait(
        InstanceIds=[instance.id], WaiterConfig={"Delay": 5,},
        InstanceIds=[instance.id],
        WaiterConfig={
            "Delay": 5,
        },
    )
    ec2client.get_waiter("instance_stopped").wait(
        InstanceIds=[instance.id], WaiterConfig={"Delay": 5,},
        InstanceIds=[instance.id],
        WaiterConfig={
            "Delay": 5,
        },
    )
Loading