Commit e9d4e7b6 authored by jgirardet's avatar jgirardet Committed by Łukasz Langa
Browse files

add gitignore support using pathspec (#878)

parent a6d86699
Loading
Loading
Loading
Loading
+1 −0
Changes for Pipfile: 1 added line, 0 removed lines.
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ black = {path = ".",extras = ["d"],editable = true}
aiohttp-cors = "*"
typed-ast = ">=1.3.1"
regex = ">=2019.8"
pathspec = ">=0.6"

[dev-packages]
pre-commit = "*"
+25 −2
Changes for black.py: 25 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -43,6 +43,7 @@
import click
import toml
from typed_ast import ast3, ast27
from pathspec import PathSpec

# lib2to3 fork
from blib2to3.pytree import Node, Leaf, type_repr
@@ -436,7 +437,9 @@ def main(
        p = Path(s)
        if p.is_dir():
            sources.update(
                gen_python_files_in_dir(p, root, include_regex, exclude_regex, report)
                gen_python_files_in_dir(
                    p, root, include_regex, exclude_regex, report, get_gitignore(root)
                )
            )
        elif p.is_file() or s == "-":
            # if a file was explicitly given, we don't care about its extension
@@ -3453,12 +3456,23 @@ def get_imports_from_children(children: List[LN]) -> Generator[str, None, None]:
    return imports


@lru_cache()
def get_gitignore(root: Path) -> PathSpec:
    """ Return a PathSpec matching gitignore content if present."""
    gitignore = root / ".gitignore"
    if not gitignore.is_file():
        return PathSpec.from_lines("gitwildmatch", [])
    else:
        return PathSpec.from_lines("gitwildmatch", gitignore.open())


def gen_python_files_in_dir(
    path: Path,
    root: Path,
    include: Pattern[str],
    exclude: Pattern[str],
    report: "Report",
    gitignore: PathSpec,
) -> Iterator[Path]:
    """Generate all files under `path` whose paths are not excluded by the
    `exclude` regex, but are included by the `include` regex.
@@ -3469,6 +3483,12 @@ def gen_python_files_in_dir(
    """
    assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}"
    for child in path.iterdir():
        # First ignore files matching .gitignore
        if gitignore.match_file(child.as_posix()):
            report.path_ignored(child, f"matches the .gitignore file content")
            continue

        # Then ignore with `exclude` option.
        try:
            normalized_path = "/" + child.resolve().relative_to(root).as_posix()
        except ValueError:
@@ -3482,13 +3502,16 @@ def gen_python_files_in_dir(

        if child.is_dir():
            normalized_path += "/"

        exclude_match = exclude.search(normalized_path)
        if exclude_match and exclude_match.group(0):
            report.path_ignored(child, f"matches the --exclude regular expression")
            continue

        if child.is_dir():
            yield from gen_python_files_in_dir(child, root, include, exclude, report)
            yield from gen_python_files_in_dir(
                child, root, include, exclude, report, gitignore
            )

        elif child.is_file():
            include_match = include.search(normalized_path)
+1 −0
Changes for setup.py: 1 added line, 0 removed lines.
Original line number Diff line number Diff line
@@ -40,6 +40,7 @@ def get_long_description() -> str:
        "toml>=0.9.4",
        "typed-ast>=1.3.1",
        "regex",
        "pathspec>=0.6, <1",
    ],
    extras_require={"d": ["aiohttp>=3.3.2", "aiohttp-cors"]},
    test_suite="tests.test_black",
+52 −5
Changes for tests/test_black.py: 52 added lines, 5 removed lines.
Original line number Diff line number Diff line
@@ -29,6 +29,8 @@
else:
    has_blackd_deps = True

from pathspec import PathSpec

ff = partial(black.format_file_in_place, mode=black.FileMode(), fast=True)
fs = partial(black.format_str, mode=black.FileMode())
THIS_FILE = Path(__file__)
@@ -1392,6 +1394,7 @@ def test_include_exclude(self) -> None:
        include = re.compile(r"\.pyi?$")
        exclude = re.compile(r"/exclude/|/\.definitely_exclude/")
        report = black.Report()
        gitignore = PathSpec.from_lines("gitwildmatch", [])
        sources: List[Path] = []
        expected = [
            Path(path / "b/dont_exclude/a.py"),
@@ -1399,13 +1402,37 @@ def test_include_exclude(self) -> None:
        ]
        this_abs = THIS_DIR.resolve()
        sources.extend(
            black.gen_python_files_in_dir(path, this_abs, include, exclude, report)
            black.gen_python_files_in_dir(
                path, this_abs, include, exclude, report, gitignore
            )
        )
        self.assertEqual(sorted(expected), sorted(sources))

    def test_gitignore_exclude(self) -> None:
        path = THIS_DIR / "data" / "include_exclude_tests"
        include = re.compile(r"\.pyi?$")
        exclude = re.compile(r"")
        report = black.Report()
        gitignore = PathSpec.from_lines(
            "gitwildmatch", ["exclude/", ".definitely_exclude"]
        )
        sources: List[Path] = []
        expected = [
            Path(path / "b/dont_exclude/a.py"),
            Path(path / "b/dont_exclude/a.pyi"),
        ]
        this_abs = THIS_DIR.resolve()
        sources.extend(
            black.gen_python_files_in_dir(
                path, this_abs, include, exclude, report, gitignore
            )
        )
        self.assertEqual(sorted(expected), sorted(sources))

    def test_empty_include(self) -> None:
        path = THIS_DIR / "data" / "include_exclude_tests"
        report = black.Report()
        gitignore = PathSpec.from_lines("gitwildmatch", [])
        empty = re.compile(r"")
        sources: List[Path] = []
        expected = [
@@ -1422,7 +1449,12 @@ def test_empty_include(self) -> None:
        this_abs = THIS_DIR.resolve()
        sources.extend(
            black.gen_python_files_in_dir(
                path, this_abs, empty, re.compile(black.DEFAULT_EXCLUDES), report
                path,
                this_abs,
                empty,
                re.compile(black.DEFAULT_EXCLUDES),
                report,
                gitignore,
            )
        )
        self.assertEqual(sorted(expected), sorted(sources))
@@ -1430,6 +1462,7 @@ def test_empty_include(self) -> None:
    def test_empty_exclude(self) -> None:
        path = THIS_DIR / "data" / "include_exclude_tests"
        report = black.Report()
        gitignore = PathSpec.from_lines("gitwildmatch", [])
        empty = re.compile(r"")
        sources: List[Path] = []
        expected = [
@@ -1443,7 +1476,12 @@ def test_empty_exclude(self) -> None:
        this_abs = THIS_DIR.resolve()
        sources.extend(
            black.gen_python_files_in_dir(
                path, this_abs, re.compile(black.DEFAULT_INCLUDES), empty, report
                path,
                this_abs,
                re.compile(black.DEFAULT_INCLUDES),
                empty,
                report,
                gitignore,
            )
        )
        self.assertEqual(sorted(expected), sorted(sources))
@@ -1488,13 +1526,18 @@ def test_symlink_out_of_root_directory(self) -> None:
        include = re.compile(black.DEFAULT_INCLUDES)
        exclude = re.compile(black.DEFAULT_EXCLUDES)
        report = black.Report()
        gitignore = PathSpec.from_lines("gitwildmatch", [])
        # `child` should behave like a symlink which resolved path is clearly
        # outside of the `root` directory.
        path.iterdir.return_value = [child]
        child.resolve.return_value = Path("/a/b/c")
        child.is_symlink.return_value = True
        try:
            list(black.gen_python_files_in_dir(path, root, include, exclude, report))
            list(
                black.gen_python_files_in_dir(
                    path, root, include, exclude, report, gitignore
                )
            )
        except ValueError as ve:
            self.fail(f"`get_python_files_in_dir()` failed: {ve}")
        path.iterdir.assert_called_once()
@@ -1504,7 +1547,11 @@ def test_symlink_out_of_root_directory(self) -> None:
        # outside of the `root` directory.
        child.is_symlink.return_value = False
        with self.assertRaises(ValueError):
            list(black.gen_python_files_in_dir(path, root, include, exclude, report))
            list(
                black.gen_python_files_in_dir(
                    path, root, include, exclude, report, gitignore
                )
            )
        path.iterdir.assert_called()
        self.assertEqual(path.iterdir.call_count, 2)
        child.resolve.assert_called()