Commit d73166c4 authored by Łukasz Langa's avatar Łukasz Langa
Browse files

Move delimiter token skipping to BracketTracker

Also, added lambda argument delimiter skipping.

Fixes #133
parent b5b658da
Loading
Loading
Loading
Loading
+3 −1
Changes for README.md: 3 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -516,9 +516,11 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md).
* generalized star expression handling, including double stars; this
  fixes multiplication making expressions "unsafe" for trailing commas (#132)

* fix parsing of complex expressions after star and double stars in
* fixed parsing of complex expressions after star and double stars in
  function parameters (#2)

* fixed invalid splitting on comma in lambda arguments (#133)

### 18.4a2

* fixed parsing of unaligned standalone comments (#99, #112)
+50 −27
Changes for black.py: 50 added lines, 27 removed lines.
Original line number Diff line number Diff line
@@ -609,6 +609,8 @@ class BracketTracker:
    bracket_match: Dict[Tuple[Depth, NodeType], Leaf] = Factory(dict)
    delimiters: Dict[LeafID, Priority] = Factory(dict)
    previous: Optional[Leaf] = None
    _for_loop_variable: bool = False
    _lambda_arguments: bool = False

    def mark(self, leaf: Leaf) -> None:
        """Mark `leaf` with bracket-related metadata. Keep track of delimiters.
@@ -628,6 +630,8 @@ def mark(self, leaf: Leaf) -> None:
        if leaf.type == token.COMMENT:
            return

        self.maybe_decrement_after_for_loop_variable(leaf)
        self.maybe_decrement_after_lambda_arguments(leaf)
        if leaf.type in CLOSING_BRACKETS:
            self.depth -= 1
            opening_bracket = self.bracket_match.pop((self.depth, leaf.type))
@@ -645,6 +649,8 @@ def mark(self, leaf: Leaf) -> None:
            self.bracket_match[self.depth, BRACKET[leaf.type]] = leaf
            self.depth += 1
        self.previous = leaf
        self.maybe_increment_lambda_arguments(leaf)
        self.maybe_increment_for_loop_variable(leaf)

    def any_open_brackets(self) -> bool:
        """Return True if there is an yet unmatched open bracket on the line."""
@@ -658,6 +664,50 @@ def max_delimiter_priority(self, exclude: Iterable[LeafID] = ()) -> int:
        """
        return max(v for k, v in self.delimiters.items() if k not in exclude)

    def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool:
        """In a for loop, or comprehension, the variables are often unpacks.

        To avoid splitting on the comma in this situation, increase the depth of
        tokens between `for` and `in`.
        """
        if leaf.type == token.NAME and leaf.value == "for":
            self.depth += 1
            self._for_loop_variable = True
            return True

        return False

    def maybe_decrement_after_for_loop_variable(self, leaf: Leaf) -> bool:
        """See `maybe_increment_for_loop_variable` above for explanation."""
        if self._for_loop_variable and leaf.type == token.NAME and leaf.value == "in":
            self.depth -= 1
            self._for_loop_variable = False
            return True

        return False

    def maybe_increment_lambda_arguments(self, leaf: Leaf) -> bool:
        """In a lambda expression, there might be more than one argument.

        To avoid splitting on the comma in this situation, increase the depth of
        tokens between `lambda` and `:`.
        """
        if leaf.type == token.NAME and leaf.value == "lambda":
            self.depth += 1
            self._lambda_arguments = True
            return True

        return False

    def maybe_decrement_after_lambda_arguments(self, leaf: Leaf) -> bool:
        """See `maybe_increment_lambda_arguments` above for explanation."""
        if self._lambda_arguments and leaf.type == token.COLON:
            self.depth -= 1
            self._lambda_arguments = False
            return True

        return False


@dataclass
class Line:
@@ -668,8 +718,6 @@ class Line:
    comments: List[Tuple[Index, Leaf]] = Factory(list)
    bracket_tracker: BracketTracker = Factory(BracketTracker)
    inside_brackets: bool = False
    has_for: bool = False
    _for_loop_variable: bool = False

    def append(self, leaf: Leaf, preformatted: bool = False) -> None:
        """Add a new `leaf` to the end of the line.
@@ -690,10 +738,8 @@ def append(self, leaf: Leaf, preformatted: bool = False) -> None:
            # imports, for which we only preserve newlines.
            leaf.prefix += whitespace(leaf)
        if self.inside_brackets or not preformatted:
            self.maybe_decrement_after_for_loop_variable(leaf)
            self.bracket_tracker.mark(leaf)
            self.maybe_remove_trailing_comma(leaf)
            self.maybe_increment_for_loop_variable(leaf)

        if not self.append_comment(leaf):
            self.leaves.append(leaf)
@@ -840,29 +886,6 @@ def maybe_remove_trailing_comma(self, closing: Leaf) -> bool:

        return False

    def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool:
        """In a for loop, or comprehension, the variables are often unpacks.

        To avoid splitting on the comma in this situation, increase the depth of
        tokens between `for` and `in`.
        """
        if leaf.type == token.NAME and leaf.value == "for":
            self.has_for = True
            self.bracket_tracker.depth += 1
            self._for_loop_variable = True
            return True

        return False

    def maybe_decrement_after_for_loop_variable(self, leaf: Leaf) -> bool:
        """See `maybe_increment_for_loop_variable` above for explanation."""
        if self._for_loop_variable and leaf.type == token.NAME and leaf.value == "in":
            self.bracket_tracker.depth -= 1
            self._for_loop_variable = False
            return True

        return False

    def append_comment(self, comment: Leaf) -> bool:
        """Add an inline or standalone comment to the line."""
        if (
+9 −3
Changes for tests/expression.diff: 9 added lines, 3 removed lines.
Original line number Diff line number Diff line
@@ -11,7 +11,7 @@
 True
 False
 1
@@ -29,59 +29,73 @@
@@ -29,60 +29,78 @@
 ~great
 +value
 -1
@@ -24,8 +24,14 @@
 lambda a, b, c=True: a
-lambda a, b, c=True, *, d=(1 << v2), e='str': a
-lambda a, b, c=True, *vararg, d=(v1 << 2), e='str', **kwargs: a + b
-foo = (lambda port_id, ignore_missing: {"port1": port1_resource, "port2": port2_resource}[port_id])
+lambda a, b, c=True, *, d=(1 << v2), e="str": a
+lambda a, b, c=True, *vararg, d=(v1 << 2), e="str", **kwargs: a + b
+foo = (
+    lambda port_id, ignore_missing: {"port1": port1_resource, "port2": port2_resource}[
+        port_id
+    ]
+)
 1 if True else 2
 str or None if True else str or bytes or None
 (str or None) if True else (str or bytes or None)
@@ -104,7 +110,7 @@
 call(**self.screen_kwargs)
 call(b, **self.screen_kwargs)
 lukasz.langa.pl
@@ -90,11 +104,11 @@
@@ -91,11 +109,11 @@
 1.0 .real
 ....__class__
 list[str]
@@ -117,7 +123,7 @@
 ]
 slice[0]
 slice[0:1]
@@ -121,88 +135,122 @@
@@ -122,88 +140,122 @@
 numpy[-(c + 1):, d]
 numpy[:, l[-2]]
 numpy[:, ::-1]
+6 −0
Changes for tests/expression.py: 6 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -37,6 +37,7 @@
lambda a, b, c=True: a
lambda a, b, c=True, *, d=(1 << v2), e='str': a
lambda a, b, c=True, *vararg, d=(v1 << 2), e='str', **kwargs: a + b
foo = (lambda port_id, ignore_missing: {"port1": port1_resource, "port2": port2_resource}[port_id])
1 if True else 2
str or None if True else str or bytes or None
(str or None) if True else (str or bytes or None)
@@ -249,6 +250,11 @@ async def f():
lambda a, b, c=True: a
lambda a, b, c=True, *, d=(1 << v2), e="str": a
lambda a, b, c=True, *vararg, d=(v1 << 2), e="str", **kwargs: a + b
foo = (
    lambda port_id, ignore_missing: {"port1": port1_resource, "port2": port2_resource}[
        port_id
    ]
)
1 if True else 2
str or None if True else str or bytes or None
(str or None) if True else (str or bytes or None)