Commit cd3a93a1 authored by Zac-HD's avatar Zac-HD Committed by Łukasz Langa
Browse files

Property-based fuzz test

parent 205f3b67
Loading
Loading
Loading
Loading
+31 −0
Changes for .github/workflows/fuzz.yml: 31 added lines, 0 removed lines.
Original line number Diff line number Diff line
name: Fuzz

on: [push, pull_request]

jobs:
  build:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: [3.6, 3.7, 3.8]

    steps:
      - uses: actions/checkout@v2

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install --upgrade coverage
          python -m pip install --upgrade hypothesmith
          python -m pip install -e ".[d]"

      - name: Run fuzz tests
        run: |
          coverage run fuzz.py
          coverage report
+1 −0
Changes for .gitignore: 1 added line, 0 removed lines.
Original line number Diff line number Diff line
@@ -14,3 +14,4 @@ src/_black_version.py
.eggs
.dmypy.json
*.swp
.hypothesis/
+1 −0
Changes for README.md: 1 added line, 0 removed lines.
Original line number Diff line number Diff line
@@ -684,3 +684,4 @@ Multiple contributions by:
- Yazdan
- [Yngve Høiseth](mailto:yngve@hoiseth.net)
- [Yurii Karabas](mailto:1998uriyyo@gmail.com)
- [Zac Hatfield-Dodds](mailto:zac@zhd.dev)

fuzz.py

0 → 100644
+59 −0
Changes for fuzz.py: 59 added lines, 0 removed lines.
Original line number Diff line number Diff line
"""Property-based tests for Black.

By Zac Hatfield-Dodds, based on my Hypothesmith tool for source code
generation.  You can run this file with `python`, `pytest`, or (soon)
a coverage-guided fuzzer I'm working on.
"""

import hypothesmith
from hypothesis import HealthCheck, given, settings, strategies as st

import black


# This test uses the Hypothesis and Hypothesmith libraries to generate random
# syntatically-valid Python source code and run Black in odd modes.
@settings(
    max_examples=1000,  # roughly 1k tests/minute, or half that under coverage
    derandomize=True,  # deterministic mode to avoid CI flakiness
    deadline=None,  # ignore Hypothesis' health checks; we already know that
    suppress_health_check=HealthCheck.all(),  # this is slow and filter-heavy.
)
@given(
    # Note that while Hypothesmith might generate code unlike that written by
    # humans, it's a general test that should pass for any *valid* source code.
    # (so e.g. running it against code scraped of the internet might also help)
    src_contents=hypothesmith.from_grammar() | hypothesmith.from_node(),
    # Using randomly-varied modes helps us to exercise less common code paths.
    mode=st.builds(
        black.FileMode,
        line_length=st.just(88) | st.integers(0, 200),
        string_normalization=st.booleans(),
        is_pyi=st.booleans(),
    ),
)
def test_idempotent_any_syntatically_valid_python(
    src_contents: str, mode: black.FileMode
) -> None:
    # Before starting, let's confirm that the input string is valid Python:
    compile(src_contents, "<string>", "exec")  # else the bug is in hypothesmith

    # Then format the code...
    try:
        dst_contents = black.format_str(src_contents, mode=mode)
    except black.InvalidInput:
        # This is a bug - if it's valid Python code, as above, black should be
        # able to code with it.  See issues #970, #1012, #1358, and #1557.
        # TODO: remove this try-except block when issues are resolved.
        return

    # And check that we got equivalent and stable output.
    black.assert_equivalent(src_contents, dst_contents)
    black.assert_stable(src_contents, dst_contents, mode=mode)

    # Future test: check that pure-python and mypyc versions of black
    # give identical output for identical input?


if __name__ == "__main__":
    test_idempotent_any_syntatically_valid_python()