diff --git a/.github/mypyc-requirements.txt b/.github/mypyc-requirements.txt index 4542673174c..352d36c0070 100644 --- a/.github/mypyc-requirements.txt +++ b/.github/mypyc-requirements.txt @@ -1,4 +1,4 @@ -mypy == 0.920 +mypy == 0.971 # A bunch of packages for type information mypy-extensions >= 0.4.3 diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 97f5f01e1b5..fc94dea62d9 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -1,4 +1,4 @@ -name: Documentation Build +name: Documentation on: [push, pull_request] diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 4ee6c839b48..a2810e25f77 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -2,6 +2,10 @@ name: Fuzz on: [push, pull_request] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + permissions: contents: read diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1dd5ab5d35e..90c48013080 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,7 +16,7 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python + - name: Set up latest Python uses: actions/setup-python@v4 with: python-version: "*" @@ -27,9 +27,9 @@ jobs: python -m pip install -e '.[d]' python -m pip install tox - - name: Lint + - name: Run pre-commit hooks uses: pre-commit/action@v3.0.0 - - name: Run On Self + - name: Format ourselves run: | tox -e run_self diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index cda215aa5d6..31a83266345 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -1,4 +1,4 @@ -name: pypi_upload +name: Publish to PyPI on: release: @@ -8,14 +8,14 @@ permissions: contents: read jobs: - build: - name: PyPI Upload + main: + name: sdist + pure wheel runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python + - name: Set up latest Python uses: actions/setup-python@v4 with: python-version: "*" @@ -26,11 +26,51 @@ jobs: python -m pip install --upgrade build twine - name: Build wheel and source distributions - run: | - python -m build + run: python -m build - name: Upload to PyPI via Twine env: TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - run: | - twine upload --verbose -u '__token__' dist/* + run: twine upload --verbose -u '__token__' dist/* + + mypyc: + name: mypyc wheels (${{ matrix.name }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + name: linux-x86_64 + - os: windows-2019 + name: windows-amd64 + - os: macos-11 + name: macos-x86_64 + macos_arch: "x86_64" + - os: macos-11 + name: macos-arm64 + macos_arch: "arm64" + - os: macos-11 + name: macos-universal2 + macos_arch: "universal2" + + steps: + - uses: actions/checkout@v3 + + - name: Build wheels via cibuildwheel + uses: pypa/cibuildwheel@v2.8.1 + env: + CIBW_ARCHS_MACOS: "${{ matrix.macos_arch }}" + # This isn't supported in pyproject.toml which makes sense (but is annoying). + CIBW_PROJECT_REQUIRES_PYTHON: ">=3.6.2" + + - name: Upload wheels as workflow artifacts + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.name }}-mypyc-wheels + path: ./wheelhouse/*.whl + + - name: Upload wheels to PyPI via Twine + env: + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: pipx run twine upload --verbose -u '__token__' wheelhouse/*.whl diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7b4716c5493..7cc55d1bf76 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,8 +11,15 @@ on: - "docs/**" - "*.md" +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: - build: + main: # We want to run on external PRs, but not on our own internal PRs as they'll be run # by the push to the branch. Without this if check, checks are duplicated since # internal PRs match both the push and pull_request events. @@ -35,29 +42,23 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install tox run: | python -m pip install --upgrade pip python -m pip install --upgrade tox - name: Unit tests if: "!startsWith(matrix.python-version, 'pypy')" - run: | - tox -e ci-py -- -v --color=yes + run: tox -e ci-py -- -v --color=yes - - name: Unit tests pypy + - name: Unit tests (pypy) if: "startsWith(matrix.python-version, 'pypy')" - run: | - tox -e ci-pypy3 -- -v --color=yes + run: tox -e ci-pypy3 -- -v --color=yes - - name: Publish coverage to Coveralls - # If pushed / is a pull request against main repo AND + - name: Upload coverage to Coveralls + # Upload coverage if we are on the main repository and # we're running on Linux (this action only supports Linux) - if: - ((github.event_name == 'push' && github.repository == 'psf/black') || - github.event.pull_request.base.repo.full_name == 'psf/black') && matrix.os == - 'ubuntu-latest' - + if: github.repository == 'psf/black' && matrix.os == 'ubuntu-latest' uses: AndreMiras/coveralls-python-action@v20201129 with: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -66,17 +67,40 @@ jobs: debug: true coveralls-finish: - needs: build - # If pushed / is a pull request against main repo - if: - (github.event_name == 'push' && github.repository == 'psf/black') || - github.event.pull_request.base.repo.full_name == 'psf/black' + needs: main + if: github.repository == 'psf/black' runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Coveralls finished + - name: Send finished signal to Coveralls uses: AndreMiras/coveralls-python-action@v20201129 with: parallel-finished: true debug: true + + uvloop: + if: + github.event_name == 'push' || github.event.pull_request.head.repo.full_name != + github.repository + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macOS-latest] + + steps: + - uses: actions/checkout@v3 + + - name: Set up latest Python + uses: actions/setup-python@v4 + with: + python-version: "*" + + - name: Install black with uvloop + run: | + python -m pip install pip --upgrade --disable-pip-version-check + python -m pip install -e ".[uvloop]" + + - name: Format ourselves + run: python -m black --check src/ diff --git a/.github/workflows/upload_binary.yml b/.github/workflows/upload_binary.yml index ed5ed961e67..22535a64c67 100644 --- a/.github/workflows/upload_binary.yml +++ b/.github/workflows/upload_binary.yml @@ -1,16 +1,14 @@ -name: Upload self-contained binaries +name: Publish executables on: release: types: [published] permissions: - contents: read + contents: write # actions/upload-release-asset needs this. jobs: build: - permissions: - contents: write # for actions/upload-release-asset to upload release asset runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -38,15 +36,21 @@ jobs: with: python-version: "*" - - name: Install dependencies + - name: Install Black and PyInstaller run: | - python -m pip install --upgrade pip wheel setuptools - python -m pip install . + python -m pip install --upgrade pip wheel + python -m pip install .[colorama] python -m pip install pyinstaller - - name: Build binary + - name: Build executable with PyInstaller + run: > + python -m PyInstaller -F --name ${{ matrix.asset_name }} --add-data + 'src/blib2to3${{ matrix.pathsep }}blib2to3' src/black/__main__.py + + - name: Quickly test executable run: | - python -m PyInstaller -F --name ${{ matrix.asset_name }} --add-data 'src/blib2to3${{ matrix.pathsep }}blib2to3' src/black/__main__.py + ./dist/${{ matrix.asset_name }} --version + ./dist/${{ matrix.asset_name }} src --verbose - name: Upload binary as release asset uses: actions/upload-release-asset@v1 diff --git a/.github/workflows/uvloop_test.yml b/.github/workflows/uvloop_test.yml deleted file mode 100644 index 9f247826969..00000000000 --- a/.github/workflows/uvloop_test.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: test uvloop - -on: - push: - paths-ignore: - - "docs/**" - - "*.md" - - pull_request: - paths-ignore: - - "docs/**" - - "*.md" - -permissions: - contents: read - -jobs: - build: - # We want to run on external PRs, but not on our own internal PRs as they'll be run - # by the push to the branch. Without this if check, checks are duplicated since - # internal PRs match both the push and pull_request events. - if: - github.event_name == 'push' || github.event.pull_request.head.repo.full_name != - github.repository - - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macOS-latest] - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: "*" - - - name: Install latest pip - run: | - python -m pip install --upgrade pip - - - name: Test uvloop Extra Install - run: | - python -m pip install -e ".[uvloop]" - - - name: Format ourselves - run: | - python -m black --check src/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a6dedc44968..87bb6e62987 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,6 +23,11 @@ repos: files: '(CHANGES\.md|the_basics\.md)$' additional_dependencies: *version_check_dependencies + - repo: https://github.com/pycqa/isort + rev: 5.10.1 + hooks: + - id: isort + - repo: https://github.com/pycqa/flake8 rev: 4.0.1 hooks: @@ -33,7 +38,7 @@ repos: - flake8-simplify - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.940 + rev: v0.971 hooks: - id: mypy exclude: ^docs/conf.py @@ -46,13 +51,13 @@ repos: - platformdirs >= 2.1.0 - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.5.1 + rev: v2.7.1 hooks: - id: prettier exclude: \.github/workflows/diff_shades\.yml - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v4.3.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace diff --git a/AUTHORS.md b/AUTHORS.md index faa2b05840f..f30cd55a08b 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -20,6 +20,7 @@ Multiple contributions by: - [Adam Johnson](mailto:me@adamj.eu) - [Adam Williamson](mailto:adamw@happyassassin.net) - [Alexander Huynh](mailto:github@grande.coffee) +- [Alexandr Artemyev](mailto:mogost@gmail.com) - [Alex Vandiver](mailto:github@chmrr.net) - [Allan Simon](mailto:allan.simon@supinfo.com) - Anders-Petter Ljungquist diff --git a/CHANGES.md b/CHANGES.md index 77f292c278a..28e0507a375 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,9 @@ +- Fix an infinite loop when using `# fmt: on/off` in the middle of an expression or code + block (#3158) +- Fix incorrect handling of `# fmt: skip` on colon `:` lines. (#3148) - Comments are no longer deleted when a line had spaces removed around power operators (#2874) @@ -23,15 +26,21 @@ normalized as expected (#3168) - Implicitly concatenated strings inside a list, set, or function call are now wrapped inside parentheses (#3162) +- When using `--skip-magic-trailing-comma` or `-C`, trailing commas are stripped from + subscript expressions with more than 1 element (#3209) ### _Blackd_ +- `blackd` now supports preview style via `X-Preview` header (#3217) + ### Configuration +- Black now uses the presence of debug f-strings to detect target version. (#3215) + ### Documentation +- Vim plugin: prefix messages with `Black: ` so it's clear they come from Black (#3194) +- Docker: changed to a /opt/venv installation + added to PATH to be available to + non-root users (#3202) + ### Output - Change from deprecated `asyncio.get_event_loop()` to create our event loop which removes DeprecationWarning (#3164) +- Remove logging from internal `blib2to3` library since it regularily emits error logs + about failed caching that can and should be ignored (#3193) ### Packaging diff --git a/Dockerfile b/Dockerfile index c393e29f632..4e8f12f9798 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,16 +2,18 @@ FROM python:3-slim AS builder RUN mkdir /src COPY . /src/ -RUN pip install --no-cache-dir --upgrade pip setuptools wheel \ +ENV VIRTUAL_ENV=/opt/venv +RUN python -m venv $VIRTUAL_ENV +RUN . /opt/venv/bin/activate && pip install --no-cache-dir --upgrade pip setuptools wheel \ # Install build tools to compile dependencies that don't have prebuilt wheels && apt update && apt install -y git build-essential \ && cd /src \ - && pip install --user --no-cache-dir .[colorama,d] + && pip install --no-cache-dir .[colorama,d] FROM python:3-slim # copy only Python packages to limit the image size -COPY --from=builder /root/.local /root/.local -ENV PATH=/root/.local/bin:$PATH +COPY --from=builder /opt/venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" -CMD ["black"] +CMD ["/opt/venv/bin/black"] diff --git a/action/main.py b/action/main.py index d14b10f421d..cd920f5fe0d 100644 --- a/action/main.py +++ b/action/main.py @@ -2,7 +2,7 @@ import shlex import sys from pathlib import Path -from subprocess import run, PIPE, STDOUT +from subprocess import PIPE, STDOUT, run ACTION_PATH = Path(os.environ["GITHUB_ACTION_PATH"]) ENV_PATH = ACTION_PATH / ".black-env" diff --git a/autoload/black.vim b/autoload/black.vim index 6c381b431a3..ed657be7bd3 100644 --- a/autoload/black.vim +++ b/autoload/black.vim @@ -158,9 +158,9 @@ def Black(**kwargs): ) except black.NothingChanged: if not quiet: - print(f'Already well formatted, good job. (took {time.time() - start:.4f}s)') + print(f'Black: already well formatted, good job. (took {time.time() - start:.4f}s)') except Exception as exc: - print(exc) + print(f'Black: {exc}') else: current_buffer = vim.current.window.buffer cursors = [] @@ -177,7 +177,7 @@ def Black(**kwargs): except vim.error: window.cursor = (len(window.buffer), 0) if not quiet: - print(f'Reformatted in {time.time() - start:.4f}s.') + print(f'Black: reformatted in {time.time() - start:.4f}s.') def get_configs(): filename = vim.eval("@%") diff --git a/docs/requirements.txt b/docs/requirements.txt index 65387e05e7e..121df45e6c2 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,7 @@ # Used by ReadTheDocs; pinned requirements for stability. myst-parser==0.18.0 -Sphinx==5.0.2 +Sphinx==5.1.1 # Older versions break Sphinx even though they're declared to be supported. docutils==0.18.1 sphinxcontrib-programoutput==0.17 diff --git a/docs/usage_and_configuration/black_as_a_server.md b/docs/usage_and_configuration/black_as_a_server.md index fc9d1cab716..a2d4252109a 100644 --- a/docs/usage_and_configuration/black_as_a_server.md +++ b/docs/usage_and_configuration/black_as_a_server.md @@ -54,8 +54,11 @@ The headers controlling how source code is formatted are: command line flag. If present and its value is not the empty string, no string normalization will be performed. - `X-Skip-Magic-Trailing-Comma`: corresponds to the `--skip-magic-trailing-comma` - command line flag. If present and its value is not the empty string, trailing commas + command line flag. If present and its value is not an empty string, trailing commas will not be used as a reason to split lines. +- `X-Preview`: corresponds to the `--preview` command line flag. If present and its + value is not an empty string, experimental and potentially disruptive style changes + will be used. - `X-Fast-Or-Safe`: if set to `fast`, `blackd` will act as _Black_ does when passed the `--fast` command line flag. - `X-Python-Variant`: if set to `pyi`, `blackd` will act as _Black_ does when passed the diff --git a/gallery/gallery.py b/gallery/gallery.py index be4d81dc427..38e52e34795 100755 --- a/gallery/gallery.py +++ b/gallery/gallery.py @@ -10,15 +10,7 @@ from concurrent.futures import ThreadPoolExecutor from functools import lru_cache, partial from pathlib import Path -from typing import ( - Generator, - List, - NamedTuple, - Optional, - Tuple, - Union, - cast, -) +from typing import Generator, List, NamedTuple, Optional, Tuple, Union, cast from urllib.request import urlopen, urlretrieve PYPI_INSTANCE = "https://pypi.org/pypi" diff --git a/mypy.ini b/mypy.ini index 8ee9068d10b..244e8ae92f5 100644 --- a/mypy.ini +++ b/mypy.ini @@ -22,6 +22,7 @@ warn_no_return=True warn_redundant_casts=True warn_unused_ignores=True disallow_any_generics=True +no_implicit_optional=True # Unreachable blocks have been an issue when compiling mypyc, let's try # to avoid 'em in the first place. diff --git a/pyproject.toml b/pyproject.toml index 6df037c8a39..813e86b2e93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,13 +22,73 @@ extend-exclude = ''' # this off. preview = true -# Build system information below. +# Build system information and other project-specific configuration below. # NOTE: You don't need this in your own Black configuration. [build-system] requires = ["setuptools>=45.0", "setuptools_scm[toml]>=6.3.1", "wheel"] build-backend = "setuptools.build_meta" +[tool.cibuildwheel] +build-verbosity = 1 +# So these are the environments we target: +# - Python: CPython 3.6+ only +# - Architecture (64-bit only): amd64 / x86_64, universal2, and arm64 +# - OS: Linux (no musl), Windows, and macOS +build = "cp3*-*" +skip = ["*-manylinux_i686", "*-musllinux_*", "*-win32", "pp-*"] +before-build = ["pip install -r .github/mypyc-requirements.txt"] +# This is the bare minimum needed to run the test suite. Pulling in the full +# test_requirements.txt would download a bunch of other packages not necessary +# here and would slow down the testing step a fair bit. +test-requires = ["pytest>=6.1.1"] +test-command = 'pytest {project} -k "not incompatible_with_mypyc"' +test-extras = ["d"," jupyter"] +# Skip trying to test arm64 builds on Intel Macs. (so cross-compilation doesn't +# straight up crash) +test-skip = ["*-macosx_arm64", "*-macosx_universal2:arm64"] + +[tool.cibuildwheel.environment] +BLACK_USE_MYPYC = "1" +MYPYC_OPT_LEVEL = "3" +MYPYC_DEBUG_LEVEL = "0" +# The dependencies required to build wheels with mypyc aren't specified in +# [build-system].requires so we'll have to manage the build environment ourselves. +PIP_NO_BUILD_ISOLATION = "no" + +[tool.cibuildwheel.linux] +before-build = [ + "pip install -r .github/mypyc-requirements.txt", + "yum install -y clang", +] +# Newer images break the builds, not sure why. We'll need to investigate more later. +manylinux-x86_64-image = "quay.io/pypa/manylinux2014_x86_64:2021-11-20-f410d11" + +[tool.cibuildwheel.linux.environment] +BLACK_USE_MYPYC = "1" +MYPYC_OPT_LEVEL = "3" +MYPYC_DEBUG_LEVEL = "0" +PIP_NO_BUILD_ISOLATION = "no" +# Black needs Clang to compile successfully on Linux. +CC = "clang" + +[tool.cibuildwheel.windows] +# For some reason, (compiled) mypyc is failing to start up with "ImportError: DLL load +# failed: A dynamic link library (DLL) initialization routine failed." on Windows for +# at least 3.6. Let's just use interpreted mypy[c]. +# See also: https://github.com/mypyc/mypyc/issues/819. +before-build = [ + "pip install -r .github/mypyc-requirements.txt --no-binary mypy" +] + +[tool.isort] +atomic = true +profile = "black" +line_length = 88 +skip_gitignore = true +skip_glob = ["src/blib2to3", "tests/data", "profiling"] +known_first_party = ["black", "blib2to3", "blackd", "_black_version"] + [tool.pytest.ini_options] # Option below requires `tests/optional.py` addopts = "--strict-config --strict-markers" diff --git a/fuzz.py b/scripts/fuzz.py similarity index 97% rename from fuzz.py rename to scripts/fuzz.py index f5f655ea279..83e02f45152 100644 --- a/fuzz.py +++ b/scripts/fuzz.py @@ -8,7 +8,8 @@ import re import hypothesmith -from hypothesis import HealthCheck, given, settings, strategies as st +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st import black from blib2to3.pgen2.tokenize import TokenError @@ -78,6 +79,7 @@ def test_idempotent_any_syntatically_valid_python( # (if you want only bounded fuzzing, just use `pytest fuzz.py`) try: import sys + import atheris except ImportError: pass diff --git a/scripts/migrate-black.py b/scripts/migrate-black.py index 1183cb8a104..63cc5096a93 100755 --- a/scripts/migrate-black.py +++ b/scripts/migrate-black.py @@ -5,7 +5,7 @@ import logging import os import sys -from subprocess import check_output, run, Popen, PIPE +from subprocess import PIPE, Popen, check_output, run def git(*args: str) -> str: diff --git a/setup.py b/setup.py index 522a42a7ce2..bc0cc32352e 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,8 @@ # Copyright (C) 2020 Łukasz Langa -from setuptools import setup, find_packages -import sys import os +import sys + +from setuptools import find_packages, setup assert sys.version_info >= (3, 6, 2), "black requires Python 3.6.2+" from pathlib import Path # noqa E402 @@ -66,7 +67,10 @@ def find_python_files(base: Path) -> List[Path]: ] opt_level = os.getenv("MYPYC_OPT_LEVEL", "3") - ext_modules = mypycify(mypyc_targets, opt_level=opt_level, verbose=True) + debug_level = os.getenv("MYPYC_DEBUG_LEVEL", "3") + ext_modules = mypycify( + mypyc_targets, opt_level=opt_level, debug_level=debug_level, verbose=True + ) else: ext_modules = [] diff --git a/src/black/__init__.py b/src/black/__init__.py index 026d135ea0e..b46d7925120 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1,20 +1,20 @@ import asyncio -from json.decoder import JSONDecodeError -import json -from contextlib import contextmanager -from datetime import datetime -from enum import Enum import io -from multiprocessing import Manager, freeze_support +import json import os -from pathlib import Path -from pathspec.patterns.gitwildmatch import GitWildMatchPatternError import platform import re import signal import sys import tokenize import traceback +from contextlib import contextmanager +from dataclasses import replace +from datetime import datetime +from enum import Enum +from json.decoder import JSONDecodeError +from multiprocessing import Manager, freeze_support +from pathlib import Path from typing import ( TYPE_CHECKING, Any, @@ -34,48 +34,62 @@ import click from click.core import ParameterSource -from dataclasses import replace from mypy_extensions import mypyc_attr +from pathspec.patterns.gitwildmatch import GitWildMatchPatternError -from black.const import DEFAULT_LINE_LENGTH, DEFAULT_INCLUDES, DEFAULT_EXCLUDES -from black.const import STDIN_PLACEHOLDER -from black.nodes import STARS, syms, is_simple_decorator_expression -from black.nodes import is_string_token, is_number_token -from black.lines import Line, EmptyLineTracker -from black.linegen import transform_line, LineGenerator, LN +from _black_version import version as __version__ +from black.cache import Cache, filter_cached, get_cache_info, read_cache, write_cache from black.comments import normalize_fmt_off -from black.mode import FUTURE_FLAG_TO_FEATURE, Mode, TargetVersion -from black.mode import Feature, supports_feature, VERSION_TO_FEATURES -from black.cache import read_cache, write_cache, get_cache_info, filter_cached, Cache -from black.concurrency import cancel, shutdown, maybe_install_uvloop -from black.output import dump_to_file, ipynb_diff, diff, color_diff, out, err -from black.report import Report, Changed, NothingChanged +from black.concurrency import cancel, maybe_install_uvloop, shutdown +from black.const import ( + DEFAULT_EXCLUDES, + DEFAULT_INCLUDES, + DEFAULT_LINE_LENGTH, + STDIN_PLACEHOLDER, +) from black.files import ( find_project_root, find_pyproject_toml, - parse_pyproject_toml, find_user_pyproject_toml, + gen_python_files, + get_gitignore, + normalize_path_maybe_ignore, + parse_pyproject_toml, + wrap_stream_for_windows, ) -from black.files import gen_python_files, get_gitignore, normalize_path_maybe_ignore -from black.files import wrap_stream_for_windows -from black.parsing import InvalidInput # noqa F401 -from black.parsing import lib2to3_parse, parse_ast, stringify_ast from black.handle_ipynb_magics import ( - mask_cell, - unmask_cell, - remove_trailing_semicolon, - put_trailing_semicolon_back, - TRANSFORMED_MAGICS, PYTHON_CELL_MAGICS, + TRANSFORMED_MAGICS, jupyter_dependencies_are_installed, + mask_cell, + put_trailing_semicolon_back, + remove_trailing_semicolon, + unmask_cell, ) - - -# lib2to3 fork -from blib2to3.pytree import Node, Leaf +from black.linegen import LN, LineGenerator, transform_line +from black.lines import EmptyLineTracker, Line +from black.mode import ( + FUTURE_FLAG_TO_FEATURE, + VERSION_TO_FEATURES, + Feature, + Mode, + TargetVersion, + supports_feature, +) +from black.nodes import ( + STARS, + is_number_token, + is_simple_decorator_expression, + is_string_token, + syms, +) +from black.output import color_diff, diff, dump_to_file, err, ipynb_diff, out +from black.parsing import InvalidInput # noqa F401 +from black.parsing import lib2to3_parse, parse_ast, stringify_ast +from black.report import Changed, NothingChanged, Report +from black.trans import iter_fexpr_spans from blib2to3.pgen2 import token - -from _black_version import version as __version__ +from blib2to3.pytree import Leaf, Node if TYPE_CHECKING: from concurrent.futures import Executor @@ -772,7 +786,7 @@ def reformat_many( workers: Optional[int], ) -> None: """Reformat multiple files using a ProcessPoolExecutor.""" - from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor + from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor executor: Executor worker_count = workers if workers is not None else DEFAULT_WORKERS @@ -1229,6 +1243,7 @@ def get_features_used( # noqa: C901 Currently looking for: - f-strings; + - self-documenting expressions in f-strings (f"{x=}"); - underscores in numeric literals; - trailing commas after * or ** in function signatures and calls; - positional only arguments in function signatures and lambdas; @@ -1250,6 +1265,11 @@ def get_features_used( # noqa: C901 value_head = n.value[:2] if value_head in {'f"', 'F"', "f'", "F'", "rf", "fr", "RF", "FR"}: features.add(Feature.F_STRINGS) + if Feature.DEBUG_F_STRINGS not in features: + for span_beg, span_end in iter_fexpr_spans(n.value): + if n.value[span_beg : span_end - 1].rstrip().endswith("="): + features.add(Feature.DEBUG_F_STRINGS) + break elif is_number_token(n): if "_" in n.value: diff --git a/src/black/brackets.py b/src/black/brackets.py index c5ed4bf5b9f..3566f5b6c37 100644 --- a/src/black/brackets.py +++ b/src/black/brackets.py @@ -1,7 +1,7 @@ """Builds on top of nodes.py to track brackets.""" -from dataclasses import dataclass, field import sys +from dataclasses import dataclass, field from typing import Dict, Iterable, List, Optional, Tuple, Union if sys.version_info < (3, 8): @@ -9,12 +9,20 @@ else: from typing import Final -from blib2to3.pytree import Leaf, Node +from black.nodes import ( + BRACKET, + CLOSING_BRACKETS, + COMPARATORS, + LOGIC_OPERATORS, + MATH_OPERATORS, + OPENING_BRACKETS, + UNPACKING_PARENTS, + VARARGS_PARENTS, + is_vararg, + syms, +) from blib2to3.pgen2 import token - -from black.nodes import syms, is_vararg, VARARGS_PARENTS, UNPACKING_PARENTS -from black.nodes import BRACKET, OPENING_BRACKETS, CLOSING_BRACKETS -from black.nodes import MATH_OPERATORS, COMPARATORS, LOGIC_OPERATORS +from blib2to3.pytree import Leaf, Node # types LN = Union[Leaf, Node] diff --git a/src/black/cache.py b/src/black/cache.py index 552c248d2ad..9455ff44772 100644 --- a/src/black/cache.py +++ b/src/black/cache.py @@ -2,16 +2,14 @@ import os import pickle -from pathlib import Path import tempfile +from pathlib import Path from typing import Dict, Iterable, Set, Tuple from platformdirs import user_cache_dir -from black.mode import Mode - from _black_version import version as __version__ - +from black.mode import Mode # types Timestamp = float diff --git a/src/black/comments.py b/src/black/comments.py index 23bf87fca7c..dc58934f9d3 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -1,7 +1,7 @@ +import re import sys from dataclasses import dataclass from functools import lru_cache -import re from typing import Iterator, List, Optional, Union if sys.version_info >= (3, 8): @@ -9,11 +9,16 @@ else: from typing_extensions import Final -from blib2to3.pytree import Node, Leaf +from black.nodes import ( + CLOSING_BRACKETS, + STANDALONE_COMMENT, + WHITESPACE, + container_of, + first_leaf_column, + preceding_leaf, +) from blib2to3.pgen2 import token - -from black.nodes import first_leaf_column, preceding_leaf, container_of -from black.nodes import STANDALONE_COMMENT, WHITESPACE +from blib2to3.pytree import Leaf, Node, type_repr # types LN = Union[Leaf, Node] @@ -174,6 +179,11 @@ def convert_one_fmt_off_pair(node: Node, *, preview: bool) -> bool: first.prefix = prefix[comment.consumed :] if comment.value in FMT_SKIP: first.prefix = "" + standalone_comment_prefix = prefix + else: + standalone_comment_prefix = ( + prefix[:previous_consumed] + "\n" * comment.newlines + ) hidden_value = "".join(str(n) for n in ignored_nodes) if comment.value in FMT_OFF: hidden_value = comment.value + "\n" + hidden_value @@ -195,7 +205,7 @@ def convert_one_fmt_off_pair(node: Node, *, preview: bool) -> bool: Leaf( STANDALONE_COMMENT, hidden_value, - prefix=prefix[:previous_consumed] + "\n" * comment.newlines, + prefix=standalone_comment_prefix, ), ) return True @@ -211,26 +221,10 @@ def generate_ignored_nodes( If comment is skip, returns leaf only. Stops at the end of the block. """ - container: Optional[LN] = container_of(leaf) if comment.value in FMT_SKIP: - prev_sibling = leaf.prev_sibling - # Need to properly format the leaf prefix to compare it to comment.value, - # which is also formatted - comments = list_comments(leaf.prefix, is_endmarker=False, preview=preview) - if comments and comment.value == comments[0].value and prev_sibling is not None: - leaf.prefix = "" - siblings = [prev_sibling] - while ( - "\n" not in prev_sibling.prefix - and prev_sibling.prev_sibling is not None - ): - prev_sibling = prev_sibling.prev_sibling - siblings.insert(0, prev_sibling) - for sibling in siblings: - yield sibling - elif leaf.parent is not None: - yield leaf.parent + yield from _generate_ignored_nodes_from_fmt_skip(leaf, comment, preview=preview) return + container: Optional[LN] = container_of(leaf) while container is not None and container.type != token.ENDMARKER: if is_fmt_on(container, preview=preview): return @@ -238,6 +232,14 @@ def generate_ignored_nodes( # fix for fmt: on in children if contains_fmt_on_at_column(container, leaf.column, preview=preview): for child in container.children: + if isinstance(child, Leaf) and is_fmt_on(child, preview=preview): + if child.type in CLOSING_BRACKETS: + # This means `# fmt: on` is placed at a different bracket level + # than `# fmt: off`. This is an invalid use, but as a courtesy, + # we include this closing bracket in the ignored nodes. + # The alternative is to fail the formatting. + yield child + return if contains_fmt_on_at_column(child, leaf.column, preview=preview): return yield child @@ -246,6 +248,51 @@ def generate_ignored_nodes( container = container.next_sibling +def _generate_ignored_nodes_from_fmt_skip( + leaf: Leaf, comment: ProtoComment, *, preview: bool +) -> Iterator[LN]: + """Generate all leaves that should be ignored by the `# fmt: skip` from `leaf`.""" + prev_sibling = leaf.prev_sibling + parent = leaf.parent + # Need to properly format the leaf prefix to compare it to comment.value, + # which is also formatted + comments = list_comments(leaf.prefix, is_endmarker=False, preview=preview) + if not comments or comment.value != comments[0].value: + return + if prev_sibling is not None: + leaf.prefix = "" + siblings = [prev_sibling] + while "\n" not in prev_sibling.prefix and prev_sibling.prev_sibling is not None: + prev_sibling = prev_sibling.prev_sibling + siblings.insert(0, prev_sibling) + for sibling in siblings: + yield sibling + elif ( + parent is not None + and type_repr(parent.type) == "suite" + and leaf.type == token.NEWLINE + ): + # The `# fmt: skip` is on the colon line of the if/while/def/class/... + # statements. The ignored nodes should be previous siblings of the + # parent suite node. + leaf.prefix = "" + ignored_nodes: List[LN] = [] + parent_sibling = parent.prev_sibling + while parent_sibling is not None and type_repr(parent_sibling.type) != "suite": + ignored_nodes.insert(0, parent_sibling) + parent_sibling = parent_sibling.prev_sibling + # Special case for `async_stmt` where the ASYNC token is on the + # grandparent node. + grandparent = parent.parent + if ( + grandparent is not None + and grandparent.prev_sibling is not None + and grandparent.prev_sibling.type == token.ASYNC + ): + ignored_nodes.insert(0, grandparent.prev_sibling) + yield from iter(ignored_nodes) + + def is_fmt_on(container: LN, preview: bool) -> bool: """Determine whether formatting is switched on within a container. Determined by whether the last `# fmt:` comment is `on` or `off`. diff --git a/src/black/debug.py b/src/black/debug.py index 5143076ab35..150b44842dd 100644 --- a/src/black/debug.py +++ b/src/black/debug.py @@ -1,12 +1,11 @@ from dataclasses import dataclass from typing import Iterator, TypeVar, Union -from blib2to3.pytree import Node, Leaf, type_repr -from blib2to3.pgen2 import token - from black.nodes import Visitor from black.output import out from black.parsing import lib2to3_parse +from blib2to3.pgen2 import token +from blib2to3.pytree import Leaf, Node, type_repr LN = Union[Leaf, Node] T = TypeVar("T") diff --git a/src/black/files.py b/src/black/files.py index 0382397e8a2..17515d52b57 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -1,9 +1,10 @@ -from functools import lru_cache import io import os -from pathlib import Path import sys +from functools import lru_cache +from pathlib import Path from typing import ( + TYPE_CHECKING, Any, Dict, Iterable, @@ -14,7 +15,6 @@ Sequence, Tuple, Union, - TYPE_CHECKING, ) from mypy_extensions import mypyc_attr @@ -30,9 +30,9 @@ else: import tomli as tomllib +from black.handle_ipynb_magics import jupyter_dependencies_are_installed from black.output import err from black.report import Report -from black.handle_ipynb_magics import jupyter_dependencies_are_installed if TYPE_CHECKING: import colorama # noqa: F401 diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index a0ed56baafc..693f1a68bd4 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -1,22 +1,20 @@ """Functions to process IPython magics with.""" -from functools import lru_cache -import dataclasses import ast -from typing import Dict, List, Tuple, Optional - +import collections +import dataclasses import secrets import sys -import collections +from functools import lru_cache +from typing import Dict, List, Optional, Tuple if sys.version_info >= (3, 10): from typing import TypeGuard else: from typing_extensions import TypeGuard -from black.report import NothingChanged from black.output import out - +from black.report import NothingChanged TRANSFORMED_MAGICS = frozenset( ( @@ -90,11 +88,7 @@ def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: Mirrors the logic in `quiet` from `IPython.core.displayhook`, but uses ``tokenize_rt`` so that round-tripping works fine. """ - from tokenize_rt import ( - src_to_tokens, - tokens_to_src, - reversed_enumerate, - ) + from tokenize_rt import reversed_enumerate, src_to_tokens, tokens_to_src tokens = src_to_tokens(src) trailing_semicolon = False @@ -118,7 +112,7 @@ def put_trailing_semicolon_back(src: str, has_trailing_semicolon: bool) -> str: """ if not has_trailing_semicolon: return src - from tokenize_rt import src_to_tokens, tokens_to_src, reversed_enumerate + from tokenize_rt import reversed_enumerate, src_to_tokens, tokens_to_src tokens = src_to_tokens(src) for idx, token in reversed_enumerate(tokens): diff --git a/src/black/linegen.py b/src/black/linegen.py index 1f132b7888f..a2e41bf5912 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -1,38 +1,67 @@ """ Generating lines of code. """ -from functools import partial, wraps import sys +from functools import partial, wraps from typing import Collection, Iterator, List, Optional, Set, Union, cast -from black.nodes import WHITESPACE, RARROW, STATEMENT, STANDALONE_COMMENT -from black.nodes import ASSIGNMENTS, OPENING_BRACKETS, CLOSING_BRACKETS -from black.nodes import Visitor, syms, is_arith_like, ensure_visible +from black.brackets import COMMA_PRIORITY, DOT_PRIORITY, max_delimiter_priority_in_atom +from black.comments import FMT_OFF, generate_comments, list_comments +from black.lines import ( + Line, + append_leaves, + can_be_split, + can_omit_invisible_parens, + is_line_short_enough, + line_to_string, +) +from black.mode import Feature, Mode, Preview from black.nodes import ( + ASSIGNMENTS, + CLOSING_BRACKETS, + OPENING_BRACKETS, + RARROW, + STANDALONE_COMMENT, + STATEMENT, + WHITESPACE, + Visitor, + ensure_visible, + is_arith_like, + is_atom_with_invisible_parens, is_docstring, is_empty_tuple, - is_one_tuple, + is_lpar_token, + is_multiline_string, + is_name_token, is_one_sequence_between, + is_one_tuple, + is_rpar_token, + is_stub_body, + is_stub_suite, + is_vararg, + is_walrus_assignment, + is_yield, + syms, + wrap_in_parentheses, ) -from black.nodes import is_name_token, is_lpar_token, is_rpar_token -from black.nodes import is_walrus_assignment, is_yield, is_vararg, is_multiline_string -from black.nodes import is_stub_suite, is_stub_body, is_atom_with_invisible_parens -from black.nodes import wrap_in_parentheses -from black.brackets import max_delimiter_priority_in_atom -from black.brackets import DOT_PRIORITY, COMMA_PRIORITY -from black.lines import Line, line_to_string, is_line_short_enough -from black.lines import can_omit_invisible_parens, can_be_split, append_leaves -from black.comments import generate_comments, list_comments, FMT_OFF from black.numerics import normalize_numeric_literal -from black.strings import get_string_prefix, fix_docstring -from black.strings import normalize_string_prefix, normalize_string_quotes -from black.trans import Transformer, CannotTransform, StringMerger, StringSplitter -from black.trans import StringParenWrapper, StringParenStripper, hug_power_op -from black.mode import Mode, Feature, Preview - -from blib2to3.pytree import Node, Leaf +from black.strings import ( + fix_docstring, + get_string_prefix, + normalize_string_prefix, + normalize_string_quotes, +) +from black.trans import ( + CannotTransform, + StringMerger, + StringParenStripper, + StringParenWrapper, + StringSplitter, + Transformer, + hug_power_op, +) from blib2to3.pgen2 import token - +from blib2to3.pytree import Leaf, Node # types LeafID = int @@ -220,7 +249,9 @@ def visit_async_stmt(self, node: Node) -> Iterator[Line]: for child in children: yield from self.visit(child) - if child.type == token.ASYNC: + if child.type == token.ASYNC or child.type == STANDALONE_COMMENT: + # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async + # line. break internal_stmt = next(children) diff --git a/src/black/lines.py b/src/black/lines.py index 8b591c324a5..30622650d53 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -1,6 +1,6 @@ -from dataclasses import dataclass, field import itertools import sys +from dataclasses import dataclass, field from typing import ( Callable, Dict, @@ -13,16 +13,25 @@ cast, ) -from blib2to3.pytree import Node, Leaf -from blib2to3.pgen2 import token - -from black.brackets import BracketTracker, DOT_PRIORITY +from black.brackets import DOT_PRIORITY, BracketTracker from black.mode import Mode, Preview -from black.nodes import STANDALONE_COMMENT, TEST_DESCENDANTS -from black.nodes import BRACKETS, OPENING_BRACKETS, CLOSING_BRACKETS -from black.nodes import syms, whitespace, replace_child, child_towards -from black.nodes import is_multiline_string, is_import, is_type_comment -from black.nodes import is_one_sequence_between +from black.nodes import ( + BRACKETS, + CLOSING_BRACKETS, + OPENING_BRACKETS, + STANDALONE_COMMENT, + TEST_DESCENDANTS, + child_towards, + is_import, + is_multiline_string, + is_one_sequence_between, + is_type_comment, + replace_child, + syms, + whitespace, +) +from blib2to3.pgen2 import token +from blib2to3.pytree import Leaf, Node # types T = TypeVar("T") @@ -264,6 +273,8 @@ def has_magic_trailing_comma( - it's not a single-element subscript Additionally, if ensure_removable: - it's not from square bracket indexing + (specifically, single-element square bracket indexing with + Preview.skip_magic_trailing_comma_in_subscript) """ if not ( closing.type in CLOSING_BRACKETS @@ -292,8 +303,22 @@ def has_magic_trailing_comma( if not ensure_removable: return True + comma = self.leaves[-1] - return bool(comma.parent and comma.parent.type == syms.listmaker) + if comma.parent is None: + return False + if Preview.skip_magic_trailing_comma_in_subscript in self.mode: + return ( + comma.parent.type != syms.subscriptlist + or closing.opening_bracket is None + or not is_one_sequence_between( + closing.opening_bracket, + closing, + self.leaves, + brackets=(token.LSQB, token.RSQB), + ) + ) + return comma.parent.type == syms.listmaker if self.is_import: return True diff --git a/src/black/mode.py b/src/black/mode.py index cef69ab8547..1fdc90c05f4 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -4,11 +4,10 @@ chosen by the user. """ -from hashlib import sha256 import sys - from dataclasses import dataclass, field from enum import Enum, auto +from hashlib import sha256 from operator import attrgetter from typing import Dict, Set from warnings import warn @@ -50,6 +49,7 @@ class Feature(Enum): ANN_ASSIGN_EXTENDED_RHS = 13 EXCEPT_STAR = 14 VARIADIC_GENERICS = 15 + DEBUG_F_STRINGS = 16 FORCE_OPTIONAL_PARENTHESES = 50 # __future__ flags @@ -82,6 +82,7 @@ class Feature(Enum): }, TargetVersion.PY38: { Feature.F_STRINGS, + Feature.DEBUG_F_STRINGS, Feature.NUMERIC_UNDERSCORES, Feature.TRAILING_COMMA_IN_CALL, Feature.TRAILING_COMMA_IN_DEF, @@ -94,6 +95,7 @@ class Feature(Enum): }, TargetVersion.PY39: { Feature.F_STRINGS, + Feature.DEBUG_F_STRINGS, Feature.NUMERIC_UNDERSCORES, Feature.TRAILING_COMMA_IN_CALL, Feature.TRAILING_COMMA_IN_DEF, @@ -107,6 +109,7 @@ class Feature(Enum): }, TargetVersion.PY310: { Feature.F_STRINGS, + Feature.DEBUG_F_STRINGS, Feature.NUMERIC_UNDERSCORES, Feature.TRAILING_COMMA_IN_CALL, Feature.TRAILING_COMMA_IN_DEF, @@ -121,6 +124,7 @@ class Feature(Enum): }, TargetVersion.PY311: { Feature.F_STRINGS, + Feature.DEBUG_F_STRINGS, Feature.NUMERIC_UNDERSCORES, Feature.TRAILING_COMMA_IN_CALL, Feature.TRAILING_COMMA_IN_DEF, @@ -152,6 +156,7 @@ class Preview(Enum): remove_block_trailing_newline = auto() remove_redundant_parens = auto() string_processing = auto() + skip_magic_trailing_comma_in_subscript = auto() class Deprecated(UserWarning): diff --git a/src/black/nodes.py b/src/black/nodes.py index 12f24b96687..8f341ab35d6 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -3,16 +3,7 @@ """ import sys -from typing import ( - Generic, - Iterator, - List, - Optional, - Set, - Tuple, - TypeVar, - Union, -) +from typing import Generic, Iterator, List, Optional, Set, Tuple, TypeVar, Union if sys.version_info >= (3, 8): from typing import Final @@ -25,14 +16,11 @@ from mypy_extensions import mypyc_attr -# lib2to3 fork -from blib2to3.pytree import Node, Leaf, type_repr, NL -from blib2to3 import pygram -from blib2to3.pgen2 import token - from black.cache import CACHE_DIR from black.strings import has_triple_quotes - +from blib2to3 import pygram +from blib2to3.pgen2 import token +from blib2to3.pytree import NL, Leaf, Node, type_repr pygram.initialize(CACHE_DIR) syms: Final = pygram.python_symbols diff --git a/src/black/output.py b/src/black/output.py index 9561d4b57d2..f4c17f28ea4 100644 --- a/src/black/output.py +++ b/src/black/output.py @@ -4,11 +4,11 @@ """ import json -from typing import Any, Optional -from mypy_extensions import mypyc_attr import tempfile +from typing import Any, Optional from click import echo, style +from mypy_extensions import mypyc_attr @mypyc_attr(patchable=True) diff --git a/src/black/parsing.py b/src/black/parsing.py index 859281cec52..57bee330240 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -11,16 +11,14 @@ else: from typing import Final -# lib2to3 fork -from blib2to3.pytree import Node, Leaf +from black.mode import Feature, TargetVersion, supports_feature +from black.nodes import syms from blib2to3 import pygram from blib2to3.pgen2 import driver from blib2to3.pgen2.grammar import Grammar from blib2to3.pgen2.parse import ParseError from blib2to3.pgen2.tokenize import TokenError - -from black.mode import TargetVersion, Feature, supports_feature -from black.nodes import syms +from blib2to3.pytree import Leaf, Node ast3: Any diff --git a/src/black/report.py b/src/black/report.py index 43b942c9e3c..a507671e4c0 100644 --- a/src/black/report.py +++ b/src/black/report.py @@ -7,7 +7,7 @@ from click import style -from black.output import out, err +from black.output import err, out class Changed(Enum): diff --git a/src/black/rusty.py b/src/black/rusty.py index 822e3d7858a..84a80b5a2c2 100644 --- a/src/black/rusty.py +++ b/src/black/rusty.py @@ -4,7 +4,6 @@ """ from typing import Generic, TypeVar, Union - T = TypeVar("T") E = TypeVar("E", bound=Exception) diff --git a/src/black/trans.py b/src/black/trans.py index 811fe2e2aa8..5d797c16e07 100644 --- a/src/black/trans.py +++ b/src/black/trans.py @@ -1,10 +1,11 @@ """ String transformers that can split and merge strings. """ +import re +import sys from abc import ABC, abstractmethod from collections import defaultdict from dataclasses import dataclass -import re from typing import ( Any, Callable, @@ -21,29 +22,38 @@ TypeVar, Union, ) -import sys if sys.version_info < (3, 8): - from typing_extensions import Literal, Final + from typing_extensions import Final, Literal else: from typing import Literal, Final from mypy_extensions import trait -from black.rusty import Result, Ok, Err - -from black.mode import Feature -from black.nodes import syms, replace_child, parent_type -from black.nodes import is_empty_par, is_empty_lpar, is_empty_rpar -from black.nodes import OPENING_BRACKETS, CLOSING_BRACKETS, STANDALONE_COMMENT -from black.lines import Line, append_leaves from black.brackets import BracketMatchError from black.comments import contains_pragma_comment -from black.strings import has_triple_quotes, get_string_prefix, assert_is_leaf_string -from black.strings import normalize_string_quotes - -from blib2to3.pytree import Leaf, Node +from black.lines import Line, append_leaves +from black.mode import Feature +from black.nodes import ( + CLOSING_BRACKETS, + OPENING_BRACKETS, + STANDALONE_COMMENT, + is_empty_lpar, + is_empty_par, + is_empty_rpar, + parent_type, + replace_child, + syms, +) +from black.rusty import Err, Ok, Result +from black.strings import ( + assert_is_leaf_string, + get_string_prefix, + has_triple_quotes, + normalize_string_quotes, +) from blib2to3.pgen2 import token +from blib2to3.pytree import Leaf, Node class CannotTransform(Exception): diff --git a/src/blackd/__init__.py b/src/blackd/__init__.py index 0463f169e19..e52a9917cf3 100644 --- a/src/blackd/__init__.py +++ b/src/blackd/__init__.py @@ -8,6 +8,7 @@ try: from aiohttp import web + from .middlewares import cors except ImportError as ie: raise ImportError( @@ -16,11 +17,11 @@ + "to obtain aiohttp_cors: `pip install black[d]`" ) from None -import black -from black.concurrency import maybe_install_uvloop import click +import black from _black_version import version as __version__ +from black.concurrency import maybe_install_uvloop # This is used internally by tests to shut down the server prematurely _stop_signal = asyncio.Event() @@ -31,6 +32,7 @@ PYTHON_VARIANT_HEADER = "X-Python-Variant" SKIP_STRING_NORMALIZATION_HEADER = "X-Skip-String-Normalization" SKIP_MAGIC_TRAILING_COMMA = "X-Skip-Magic-Trailing-Comma" +PREVIEW = "X-Preview" FAST_OR_SAFE_HEADER = "X-Fast-Or-Safe" DIFF_HEADER = "X-Diff" @@ -40,6 +42,7 @@ PYTHON_VARIANT_HEADER, SKIP_STRING_NORMALIZATION_HEADER, SKIP_MAGIC_TRAILING_COMMA, + PREVIEW, FAST_OR_SAFE_HEADER, DIFF_HEADER, ] @@ -108,6 +111,7 @@ async def handle(request: web.Request, executor: Executor) -> web.Response: skip_magic_trailing_comma = bool( request.headers.get(SKIP_MAGIC_TRAILING_COMMA, False) ) + preview = bool(request.headers.get(PREVIEW, False)) fast = False if request.headers.get(FAST_OR_SAFE_HEADER, "safe") == "fast": fast = True @@ -117,6 +121,7 @@ async def handle(request: web.Request, executor: Executor) -> web.Response: line_length=line_length, string_normalization=not skip_string_normalization, magic_trailing_comma=not skip_magic_trailing_comma, + preview=preview, ) req_bytes = await request.content.read() charset = request.charset if request.charset is not None else "utf8" diff --git a/src/blackd/middlewares.py b/src/blackd/middlewares.py index 97994ecc1df..7abde525bfd 100644 --- a/src/blackd/middlewares.py +++ b/src/blackd/middlewares.py @@ -1,7 +1,8 @@ -from typing import Iterable, Awaitable, Callable -from aiohttp.web_response import StreamResponse -from aiohttp.web_request import Request +from typing import Awaitable, Callable, Iterable + from aiohttp.web_middlewares import middleware +from aiohttp.web_request import Request +from aiohttp.web_response import StreamResponse Handler = Callable[[Request], Awaitable[StreamResponse]] Middleware = Callable[[Request, Handler], Awaitable[StreamResponse]] diff --git a/src/blib2to3/pgen2/driver.py b/src/blib2to3/pgen2/driver.py index 8fe820651da..daf271dfa9a 100644 --- a/src/blib2to3/pgen2/driver.py +++ b/src/blib2to3/pgen2/driver.py @@ -263,14 +263,13 @@ def load_grammar( logger = logging.getLogger(__name__) gp = _generate_pickle_name(gt) if gp is None else gp if force or not _newer(gp, gt): - logger.info("Generating grammar tables from %s", gt) g: grammar.Grammar = pgen.generate_grammar(gt) if save: - logger.info("Writing grammar tables to %s", gp) try: g.dump(gp) - except OSError as e: - logger.info("Writing failed: %s", e) + except OSError: + # Ignore error, caching is not vital. + pass else: g = grammar.Grammar() g.load(gp) diff --git a/src/blib2to3/pgen2/parse.py b/src/blib2to3/pgen2/parse.py index a9dc11f39ce..d6deaac6964 100644 --- a/src/blib2to3/pgen2/parse.py +++ b/src/blib2to3/pgen2/parse.py @@ -114,7 +114,7 @@ def add_token(self, tok_type: int, tok_val: Text, raw: bool = False) -> None: args.insert(0, ilabel) func(*args) - def determine_route(self, value: Text = None, force: bool = False) -> Optional[int]: + def determine_route(self, value: Optional[Text] = None, force: bool = False) -> Optional[int]: alive_ilabels = self.ilabels if len(alive_ilabels) == 0: *_, most_successful_ilabel = self._dead_ilabels diff --git a/tests/data/miscellaneous/docstring_preview_no_string_normalization.py b/tests/data/miscellaneous/docstring_preview_no_string_normalization.py index 0957231eb9c..338cc01d33e 100644 --- a/tests/data/miscellaneous/docstring_preview_no_string_normalization.py +++ b/tests/data/miscellaneous/docstring_preview_no_string_normalization.py @@ -3,8 +3,8 @@ def do_not_touch_this_prefix(): def do_not_touch_this_prefix2(): - F'There was a bug where docstring prefixes would be normalized even with -S.' + FR'There was a bug where docstring prefixes would be normalized even with -S.' def do_not_touch_this_prefix3(): - uR'''There was a bug where docstring prefixes would be normalized even with -S.''' + u'''There was a bug where docstring prefixes would be normalized even with -S.''' diff --git a/tests/data/preview/skip_magic_trailing_comma.py b/tests/data/preview/skip_magic_trailing_comma.py new file mode 100644 index 00000000000..e98174af427 --- /dev/null +++ b/tests/data/preview/skip_magic_trailing_comma.py @@ -0,0 +1,34 @@ +# We should not remove the trailing comma in a single-element subscript. +a: tuple[int,] +b = tuple[int,] + +# But commas in multiple element subscripts should be removed. +c: tuple[int, int,] +d = tuple[int, int,] + +# Remove commas for non-subscripts. +small_list = [1,] +list_of_types = [tuple[int,],] +small_set = {1,} +set_of_types = {tuple[int,],} + +# Except single element tuples +small_tuple = (1,) + +# output +# We should not remove the trailing comma in a single-element subscript. +a: tuple[int,] +b = tuple[int,] + +# But commas in multiple element subscripts should be removed. +c: tuple[int, int] +d = tuple[int, int] + +# Remove commas for non-subscripts. +small_list = [1] +list_of_types = [tuple[int,]] +small_set = {1} +set_of_types = {tuple[int,]} + +# Except single element tuples +small_tuple = (1,) diff --git a/tests/data/preview_310/remove_newline_after match.py b/tests/data/preview_310/remove_newline_after_match.py similarity index 100% rename from tests/data/preview_310/remove_newline_after match.py rename to tests/data/preview_310/remove_newline_after_match.py diff --git a/tests/data/simple_cases/fmtonoff5.py b/tests/data/simple_cases/fmtonoff5.py new file mode 100644 index 00000000000..746aa41f4e4 --- /dev/null +++ b/tests/data/simple_cases/fmtonoff5.py @@ -0,0 +1,36 @@ +# Regression test for https://github.com/psf/black/issues/3129. +setup( + entry_points={ + # fmt: off + "console_scripts": [ + "foo-bar" + "=foo.bar.:main", + # fmt: on + ] # Includes an formatted indentation. + }, +) + + +# Regression test for https://github.com/psf/black/issues/2015. +run( + # fmt: off + [ + "ls", + "-la", + ] + # fmt: on + + path, + check=True, +) + + +# Regression test for https://github.com/psf/black/issues/3026. +def test_func(): + # yapf: disable + if unformatted( args ): + return True + # yapf: enable + elif b: + return True + + return False diff --git a/tests/data/simple_cases/fmtskip8.py b/tests/data/simple_cases/fmtskip8.py new file mode 100644 index 00000000000..38e9c2a9f47 --- /dev/null +++ b/tests/data/simple_cases/fmtskip8.py @@ -0,0 +1,62 @@ +# Make sure a leading comment is not removed. +def some_func( unformatted, args ): # fmt: skip + print("I am some_func") + return 0 + # Make sure this comment is not removed. + + +# Make sure a leading comment is not removed. +async def some_async_func( unformatted, args): # fmt: skip + print("I am some_async_func") + await asyncio.sleep(1) + + +# Make sure a leading comment is not removed. +class SomeClass( Unformatted, SuperClasses ): # fmt: skip + def some_method( self, unformatted, args ): # fmt: skip + print("I am some_method") + return 0 + + async def some_async_method( self, unformatted, args ): # fmt: skip + print("I am some_async_method") + await asyncio.sleep(1) + + +# Make sure a leading comment is not removed. +if unformatted_call( args ): # fmt: skip + print("First branch") + # Make sure this is not removed. +elif another_unformatted_call( args ): # fmt: skip + print("Second branch") +else : # fmt: skip + print("Last branch") + + +while some_condition( unformatted, args ): # fmt: skip + print("Do something") + + +for i in some_iter( unformatted, args ): # fmt: skip + print("Do something") + + +async def test_async_for(): + async for i in some_async_iter( unformatted, args ): # fmt: skip + print("Do something") + + +try : # fmt: skip + some_call() +except UnformattedError as ex: # fmt: skip + handle_exception() +finally : # fmt: skip + finally_call() + + +with give_me_context( unformatted, args ): # fmt: skip + print("Do something") + + +async def test_async_with(): + async with give_me_async_context( unformatted, args ): # fmt: skip + print("Do something") diff --git a/tests/optional.py b/tests/optional.py index a4e9441ef1c..853ecaa2a43 100644 --- a/tests/optional.py +++ b/tests/optional.py @@ -14,11 +14,11 @@ Adapted from https://pypi.org/project/pytest-optional-tests/, (c) 2019 Reece Hart """ -from functools import lru_cache import itertools import logging import re -from typing import FrozenSet, List, Set, TYPE_CHECKING +from functools import lru_cache +from typing import TYPE_CHECKING, FrozenSet, List, Set import pytest @@ -32,8 +32,8 @@ if TYPE_CHECKING: - from _pytest.config.argparsing import Parser from _pytest.config import Config + from _pytest.config.argparsing import Parser from _pytest.mark.structures import MarkDecorator from _pytest.nodes import Node diff --git a/tests/test_black.py b/tests/test_black.py index 6b7ed99f970..9d3fd8940a2 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -6,6 +6,7 @@ import logging import multiprocessing import os +import re import sys import types import unittest @@ -31,7 +32,6 @@ import click import pytest -import re from click import unstyle from click.testing import CliRunner from pathspec import PathSpec @@ -59,8 +59,8 @@ dump_to_stderr, ff, fs, - read_data, get_case_path, + read_data, read_data_from_file, ) @@ -310,6 +310,26 @@ def test_detect_pos_only_arguments(self) -> None: versions = black.detect_target_versions(root) self.assertIn(black.TargetVersion.PY38, versions) + def test_detect_debug_f_strings(self) -> None: + root = black.lib2to3_parse("""f"{x=}" """) + features = black.get_features_used(root) + self.assertIn(black.Feature.DEBUG_F_STRINGS, features) + versions = black.detect_target_versions(root) + self.assertIn(black.TargetVersion.PY38, versions) + + root = black.lib2to3_parse( + """f"{x}"\nf'{"="}'\nf'{(x:=5)}'\nf'{f(a="3=")}'\nf'{x:=10}'\n""" + ) + features = black.get_features_used(root) + self.assertNotIn(black.Feature.DEBUG_F_STRINGS, features) + + # We don't yet support feature version detection in nested f-strings + root = black.lib2to3_parse( + """f"heard a rumour that { f'{1+1=}' } ... seems like it could be true" """ + ) + features = black.get_features_used(root) + self.assertNotIn(black.Feature.DEBUG_F_STRINGS, features) + @patch("black.dump_to_file", dump_to_stderr) def test_string_quotes(self) -> None: source, expected = read_data("miscellaneous", "string_quotes") diff --git a/tests/test_blackd.py b/tests/test_blackd.py index 75d756705be..1d12113a3f3 100644 --- a/tests/test_blackd.py +++ b/tests/test_blackd.py @@ -2,15 +2,16 @@ from typing import Any from unittest.mock import patch -from click.testing import CliRunner import pytest +from click.testing import CliRunner -from tests.util import read_data, DETERMINISTIC_HEADER +from tests.util import DETERMINISTIC_HEADER, read_data try: - import blackd - from aiohttp.test_utils import AioHTTPTestCase from aiohttp import web + from aiohttp.test_utils import AioHTTPTestCase + + import blackd except ImportError as e: raise RuntimeError("Please install Black with the 'd' extra") from e @@ -166,6 +167,13 @@ async def test_blackd_invalid_line_length(self) -> None: ) self.assertEqual(response.status, 400) + @unittest_run_loop + async def test_blackd_preview(self) -> None: + response = await self.client.post( + "/", data=b'print("hello")\n', headers={blackd.PREVIEW: "true"} + ) + self.assertEqual(response.status, 204) + @unittest_run_loop async def test_blackd_response_black_version_header(self) -> None: response = await self.client.post("/") diff --git a/tests/test_format.py b/tests/test_format.py index 7a099fb9f33..01cd61eef63 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -8,10 +8,10 @@ from tests.util import ( DEFAULT_MODE, PY36_VERSIONS, + all_data_cases, assert_format, dump_to_stderr, read_data, - all_data_cases, ) @@ -36,7 +36,12 @@ def test_simple_format(filename: str) -> None: @pytest.mark.parametrize("filename", all_data_cases("preview")) def test_preview_format(filename: str) -> None: - check_file("preview", filename, black.Mode(preview=True)) + magic_trailing_comma = filename != "skip_magic_trailing_comma" + check_file( + "preview", + filename, + black.Mode(preview=True, magic_trailing_comma=magic_trailing_comma), + ) @pytest.mark.parametrize("filename", all_data_cases("preview_39")) diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index e1d7dd88dcb..7aa2e91dd00 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -1,23 +1,24 @@ import contextlib -from dataclasses import replace import pathlib import re from contextlib import ExitStack as does_not_raise +from dataclasses import replace from typing import ContextManager +import pytest +from _pytest.monkeypatch import MonkeyPatch from click.testing import CliRunner -from black.handle_ipynb_magics import jupyter_dependencies_are_installed + from black import ( - main, + Mode, NothingChanged, format_cell, format_file_contents, format_file_in_place, + main, ) -import pytest -from black import Mode -from _pytest.monkeypatch import MonkeyPatch -from tests.util import DATA_DIR, read_jupyter_notebook, get_case_path +from black.handle_ipynb_magics import jupyter_dependencies_are_installed +from tests.util import DATA_DIR, get_case_path, read_jupyter_notebook with contextlib.suppress(ModuleNotFoundError): import IPython diff --git a/tests/test_no_ipynb.py b/tests/test_no_ipynb.py index a3c897760fb..3e0b1593bf0 100644 --- a/tests/test_no_ipynb.py +++ b/tests/test_no_ipynb.py @@ -1,10 +1,11 @@ -import pytest import pathlib -from tests.util import get_case_path -from black import main, jupyter_dependencies_are_installed +import pytest from click.testing import CliRunner +from black import jupyter_dependencies_are_installed, main +from tests.util import get_case_path + pytestmark = pytest.mark.no_jupyter runner = CliRunner() diff --git a/tests/test_trans.py b/tests/test_trans.py index a1666a9c166..dce8a939677 100644 --- a/tests/test_trans.py +++ b/tests/test_trans.py @@ -1,4 +1,5 @@ from typing import List, Tuple + from black.trans import iter_fexpr_spans diff --git a/tox.ini b/tox.ini index 7af9e48d6f0..51ff4872db0 100644 --- a/tox.ini +++ b/tox.ini @@ -59,7 +59,7 @@ deps = commands = pip install -e .[d] coverage erase - coverage run fuzz.py + coverage run {toxinidir}/scripts/fuzz.py coverage report [testenv:run_self]