Commit 866be066 authored by Jason Fried's avatar Jason Fried Committed by Carol Willing
Browse files

Make --safe work for Python2.7 syntax, by using typed_ast for safe validation (#840)

parent 6fdbdb4e
Loading
Loading
Loading
Loading
+1 −0
Changes for Pipfile: 1 added line, 0 removed lines.
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ appdirs = "*"
toml = ">=0.9.4"
black = {path = ".",extras = ["d"],editable = true}
aiohttp-cors = "*"
typed-ast = ">=1.3.1"

[dev-packages]
pre-commit = "*"
+23 −8
Changes for black.py: 23 added lines, 8 removed lines.
Original line number Diff line number Diff line
@@ -40,6 +40,7 @@
from attr import dataclass, evolve, Factory
import click
import toml
from typed_ast import ast3, ast27

# lib2to3 fork
from blib2to3.pytree import Node, Leaf, type_repr
@@ -3380,17 +3381,31 @@ def __str__(self) -> str:
        return ", ".join(report) + "."


def parse_ast(src: str) -> Union[ast3.AST, ast27.AST]:
    try:
        return ast3.parse(src)
    except SyntaxError:
        return ast27.parse(src)


def assert_equivalent(src: str, dst: str) -> None:
    """Raise AssertionError if `src` and `dst` aren't equivalent."""

    import ast
    import traceback

    def _v(node: ast.AST, depth: int = 0) -> Iterator[str]:
    def _v(node: Union[ast3.AST, ast27.AST], depth: int = 0) -> Iterator[str]:
        """Simple visitor generating strings to compare ASTs by content."""
        yield f"{'  ' * depth}{node.__class__.__name__}("

        for field in sorted(node._fields):
            # TypeIgnore has only one field 'lineno' which breaks this comparison
            if isinstance(node, (ast3.TypeIgnore, ast27.TypeIgnore)):
                break

            # Ignore str kind which is case sensitive / and ignores unicode_literals
            if isinstance(node, (ast3.Str, ast27.Str, ast3.Bytes)) and field == "kind":
                continue

            try:
                value = getattr(node, field)
            except AttributeError:
@@ -3404,15 +3419,15 @@ def _v(node: ast.AST, depth: int = 0) -> Iterator[str]:
                    # parentheses and they change the AST.
                    if (
                        field == "targets"
                        and isinstance(node, ast.Delete)
                        and isinstance(item, ast.Tuple)
                        and isinstance(node, (ast3.Delete, ast27.Delete))
                        and isinstance(item, (ast3.Tuple, ast27.Tuple))
                    ):
                        for item in item.elts:
                            yield from _v(item, depth + 2)
                    elif isinstance(item, ast.AST):
                    elif isinstance(item, (ast3.AST, ast27.AST)):
                        yield from _v(item, depth + 2)

            elif isinstance(value, ast.AST):
            elif isinstance(value, (ast3.AST, ast27.AST)):
                yield from _v(value, depth + 2)

            else:
@@ -3421,7 +3436,7 @@ def _v(node: ast.AST, depth: int = 0) -> Iterator[str]:
        yield f"{'  ' * depth})  # /{node.__class__.__name__}"

    try:
        src_ast = ast.parse(src)
        src_ast = parse_ast(src)
    except Exception as exc:
        major, minor = sys.version_info[:2]
        raise AssertionError(
@@ -3431,7 +3446,7 @@ def _v(node: ast.AST, depth: int = 0) -> Iterator[str]:
        )

    try:
        dst_ast = ast.parse(dst)
        dst_ast = parse_ast(dst)
    except Exception as exc:
        log = dump_to_file("".join(traceback.format_tb(exc.__traceback__)), dst)
        raise AssertionError(
+7 −1
Changes for setup.py: 7 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -41,7 +41,13 @@ def get_version() -> str:
    package_data={"blib2to3": ["*.txt"]},
    python_requires=">=3.6",
    zip_safe=False,
    install_requires=["click>=6.5", "attrs>=18.1.0", "appdirs", "toml>=0.9.4"],
    install_requires=[
        "click>=6.5",
        "attrs>=18.1.0",
        "appdirs",
        "toml>=0.9.4",
        "typed-ast>=1.3.1",
    ],
    extras_require={"d": ["aiohttp>=3.3.2", "aiohttp-cors"]},
    test_suite="tests.test_black",
    classifiers=[
+2 −2
Changes for tests/data/comments6.py: 2 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -55,8 +55,8 @@ def f(
    an_element_with_a_long_value = calls() or more_calls() and more()  # type: bool

    tup = (
        another_element,  # type: int
        another_really_really_long_element_with_a_unnecessarily_long_name_to_describe_what_it_does_enterprise_style,  # type: int
        another_element,
        another_really_really_long_element_with_a_unnecessarily_long_name_to_describe_what_it_does_enterprise_style,
    )  # type: Tuple[int, int]

    a = (
+4 −16
Changes for tests/test_black.py: 4 added lines, 16 removed lines.
Original line number Diff line number Diff line
@@ -2,7 +2,7 @@
import asyncio
import logging
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager, redirect_stderr
from contextlib import contextmanager
from functools import partial, wraps
from io import BytesIO, TextIOWrapper
import os
@@ -474,7 +474,7 @@ def test_python2(self) -> None:
        source, expected = read_data("python2")
        actual = fs(source)
        self.assertFormatEqual(expected, actual)
        # black.assert_equivalent(source, actual)
        black.assert_equivalent(source, actual)
        black.assert_stable(source, actual, black.FileMode())

    @patch("black.dump_to_file", dump_to_stderr)
@@ -483,6 +483,7 @@ def test_python2_print_function(self) -> None:
        mode = black.FileMode(target_versions={TargetVersion.PY27})
        actual = fs(source, mode=mode)
        self.assertFormatEqual(expected, actual)
        black.assert_equivalent(source, actual)
        black.assert_stable(source, actual, mode)

    @patch("black.dump_to_file", dump_to_stderr)
@@ -490,6 +491,7 @@ def test_python2_unicode_literals(self) -> None:
        source, expected = read_data("python2_unicode_literals")
        actual = fs(source)
        self.assertFormatEqual(expected, actual)
        black.assert_equivalent(source, actual)
        black.assert_stable(source, actual, black.FileMode())

    @patch("black.dump_to_file", dump_to_stderr)
@@ -1562,20 +1564,6 @@ async def check(header_value: str, expected_status: int) -> None:
            await check("3.4", 204)
            await check("py3.4", 204)

    @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed")
    @async_test
    async def test_blackd_fast(self) -> None:
        with open(os.devnull, "w") as dn, redirect_stderr(dn):
            app = blackd.make_app()
            async with TestClient(TestServer(app)) as client:
                response = await client.post("/", data=b"ur'hello'")
                self.assertEqual(response.status, 500)
                self.assertIn("failed to parse source file", await response.text())
                response = await client.post(
                    "/", data=b"ur'hello'", headers={blackd.FAST_OR_SAFE_HEADER: "fast"}
                )
                self.assertEqual(response.status, 200)

    @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed")
    @async_test
    async def test_blackd_line_length(self) -> None: