diff --git a/.biobuddies/autoformat-excludes b/.biobuddies/autoformat-excludes new file mode 100644 index 0000000..e865a4f --- /dev/null +++ b/.biobuddies/autoformat-excludes @@ -0,0 +1,14 @@ +# Cookiecutter outputs +^\.biobuddies/ +^\.editorconfig$ +^\.github/copilot-instructions\.md$ +^\.github/pull_request_template\.md$ +^\.gitignore$ +^\.prettierrc\.toml$ +^\.shellcheckrc$ +^\.yamllint\.yaml$ +# Markdown with intentional trailing whitespace +^README\.md$ +# Presentations +^documentation/djangocon-us-2024\.md$ +^documentation/self2024-python-defined-infrastructure\.md$ diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index 440de46..0f412d0 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -7,23 +7,36 @@ env: CLOUDFLARE_ACCOUNT_ID: 0123456789abcdef0123456789abcdef00 CLOUDFLARE_ZONE_ID: 0123456789abcdef0123456789abcdef01 INSH_TF: tofu - PATH: .venv/bin:/home/runner/.local/bin:/home/runner/.asdf/shims:/usr/bin:/bin + PATH: .venv/bin:/home/runner/.local/bin:/home/runner/.local/share/mise/shims:/usr/bin:/bin jobs: check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - run: bash .biobuddies/includes.bash forceready - - - run: .venv/bin/just pcam --color always --show-diff-on-failure - - run: .venv/bin/just hti demo && tofu -chdir=deploys/demo/terraform validate - - run: .venv/bin/just test + # Real hash is head_ref on pull_request, ref on push + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.head_ref || github.ref }} + - uses: jdx/mise-action@v3 + with: + experimental: true + - run: mise run set-environment + - run: mise run pre-commit + - run: mise tvalid demo + - run: mise test - run: .venv/bin/just build_twine - summarize: + tabularize: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - run: bash .biobuddies/includes.bash summarize + # Real hash is head_ref on pull_request, ref on push + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.head_ref || github.ref }} + - uses: jdx/mise-action@v3 + with: + experimental: true + - run: mise run tabularize on: # yamllint disable-line rule:truthy pull_request: diff --git a/.python-version b/.python-version deleted file mode 100644 index 871f80a..0000000 --- a/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.12.3 diff --git a/.tofu-version b/.tofu-version index f8e233b..e6dbb7c 100644 --- a/.tofu-version +++ b/.tofu-version @@ -1 +1 @@ -1.9.0 +1.11.5 diff --git a/.tool-versions b/.tool-versions deleted file mode 100644 index 05427ec..0000000 --- a/.tool-versions +++ /dev/null @@ -1,3 +0,0 @@ -glow 2.1.0 -uv 0.9.18 -tenv 4.4.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7355b46..43fb33e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,7 +36,7 @@ simplest: - Split files around 500 lines * Alphabetize, sometimes within sections (header worth a comment) * Use Python 3.12+ idioms like: - - `from pytest import parametrize` + - `from pytest import mark; @mark.parametrize()` - `from pathlib import Path; Path('a') / 'b'` - `from subprocess import check_call, check_output; check_call(...); check_output(...)` - Usually easiest to not `re.compile` at all than worry about aliasing the builtin diff --git a/deploys/demo/terraform/.terraform.lock.hcl b/deploys/demo/terraform/.terraform.lock.hcl index 4f993e7..d8ad47e 100644 --- a/deploys/demo/terraform/.terraform.lock.hcl +++ b/deploys/demo/terraform/.terraform.lock.hcl @@ -5,6 +5,7 @@ provider "registry.opentofu.org/hashicorp/null" { version = "3.2.3" hashes = [ "h1:LF8arSzHfhbyQSFtTMTYEqCM34klzrbAQBJMHYCs9d8=", + "h1:LN7WjQlMDIYGsXlum1kvMk5M8XzS2gzPTHmbEkxB6B0=", "zh:1d57d25084effd3fdfd902eca00020b34b1fb020253b84d7dd471301606015ac", "zh:65b7f9799b88464d9c2ec529713b7f52ea744275b61a8dc86cdedab1b2dcb933", "zh:80d3e9c95b7b4ae7c54005cd127cae82e5c53d2b7023ef24c147337bac9dadd9", diff --git a/helicopyter/__init__.py b/helicopyter/__init__.py index a57b408..2327534 100644 --- a/helicopyter/__init__.py +++ b/helicopyter/__init__.py @@ -6,10 +6,10 @@ from os import environ from pathlib import Path from re import sub -from subprocess import check_output +from shutil import which +from subprocess import PIPE, CalledProcessError, check_output from typing import Any, TypeVar -from cdk8s import Chart from cdktf import App, TerraformElement, TerraformStack from constructs import Construct, Node from tap import Tap @@ -17,12 +17,6 @@ environ['JSII_SILENCE_WARNING_UNTESTED_NODE_VERSION'] = '1' -class HeliChart(Chart): - def __init__(self, cona: str) -> None: - super().__init__(App(), cona) - self.cona = cona - - class HeliStack(TerraformStack): def __init__(self, cona: str) -> None: # Something is automatically creating outdir, which is cdktf.out by default @@ -139,9 +133,17 @@ def multisynth( raise if hashicorp_configuration_language: unformatted = '# AUTOGENERATED by helicopyter\n\n' + stack.to_hcl_terraform()['hcl'] - autoformatted = check_output( # noqa: S603 - [format_with, 'fmt', '-'], input=unformatted.encode() - ).decode() + try: + autoformatted = check_output( # noqa: S603 + [format_with, 'fmt', '-'], input=unformatted.encode(), stderr=PIPE + ).decode() + except CalledProcessError as error: + print(f'which {format_with}: {which(format_with)}') + print(f'{format_with} fmt stderr: {error.stderr}') + print( + f'{format_with} --version: {check_output([format_with, "--version"], stderr=PIPE)}' + ) # noqa: S603 + raise formatted = sub( r'\n{3,}', '\n\n', diff --git a/mise.toml b/mise.toml new file mode 100644 index 0000000..1d99dbc --- /dev/null +++ b/mise.toml @@ -0,0 +1,321 @@ +[settings] +experimental = true + +[settings.python] +uv_venv_auto = 'create|source' + +[env] +TENV_AUTO_INSTALL = 'true' +_.path = ['./node_modules/.bin'] +_.python.venv = { path = '.venv' } + +[hooks] +postinstall = """ +set -o errexit -o nounset +npm clean-install --no-audit --no-fund & +npm_pid=$! +uv venv --allow-existing --no-progress +uv pip sync requirements.txt +wait "$npm_pid" +""" + +# Prerequisite tasks +[tasks.cona] +run = """ +#!/usr/bin/env bash +set -o errexit -o nounset -o pipefail +if [[ ${GITHUB_REPOSITORY-} ]]; then echo "${GITHUB_REPOSITORY##*/}" +elif [[ ${VIRTUAL_ENV-} ]]; then basename "${VIRTUAL_ENV%/.venv}" +else basename "$PWD" +fi +""" + +[tasks.envi] +run = """ +#!/usr/bin/env bash +set -o errexit -o nounset -o pipefail +if [[ ${ENVI-} ]]; then echo "$ENVI" +elif [[ ${GITHUB_ACTIONS-} ]]; then echo github +else echo local +fi +""" + +[tasks.giha] +run = 'git describe --abbrev=40 --always --dirty --match=-' + +[tasks.orgn] +run = ''' +#!/usr/bin/env bash +set -o errexit -o nounset -o pipefail +if [[ ${GITHUB_REPOSITORY_OWNER-} ]]; then echo "$GITHUB_REPOSITORY_OWNER" +else git remote get-url origin | sed -E 's,.+github.com[:/]([^/]+).+,\1,' +fi +''' + +[tasks.run-on-sources] +quiet = true +run = ''' +#!/usr/bin/env bash +set -o errexit -o nounset -o pipefail +tool=${1:?missing executable} +eval "glob=(${2:?missing source glob})" +shift 2 +command_args=$* +set -- $( + git ls-files -- "${glob[@]}" \ + | grep --invert-match --extended-regexp --file <( + grep --invert-match '^#' .biobuddies/autoformat-excludes + ) || [[ $? -eq 1 ]] +) +[[ $# -eq 0 ]] && exit 0 +[[ -z $command_args ]] || set -- $command_args "$@" +"$tool" "$@" +''' + +[tasks.tabr] +run = ''' +#!/usr/bin/env bash +set -o errexit -o nounset -o pipefail +if [[ ${GITHUB_HEAD_REF-} ]]; then echo "$GITHUB_HEAD_REF" +elif [[ ${GITHUB_REF_NAME-} ]]; then echo "$GITHUB_REF_NAME" +else + git describe --all --dirty --exact-match 2>/dev/null \ + | sed -En '/-dirty$/ q; s,(remotes/[^/]+|heads|tags)/,,p' +fi +''' + +[tasks.tinit] +run = """ +#!/usr/bin/env bash +set -o errexit -o nounset -o pipefail +cona=${1:?missing cona} +shift +tf=${INSH_TF:-terraform} +"$tf" -chdir="deploys/$cona/terraform" init "$@" +python -m helicopyter --format_with="$tf" "$cona" +""" + +# Leaf tasks +[tasks.actionlint] +run = 'actionlint "$@"' + +[tasks.bash] +run = 'env bash --login -o errexit -o pipefail' + +[tasks.basedpyright] +run = 'mise run-on-sources basedpyright "*.py{,i}" --level error "$@"' + +[tasks.cookiecutter] +run = ''' +#!/usr/bin/env bash +set -o errexit -o nounset -o pipefail +cookiecutter --config-file .cookiecutter.yaml --no-input --overwrite-if-exists "$( + [[ $(mise orgn)/$(mise cona) == biobuddies/helicopyter ]] \ + && echo . \ + || echo https://github.com/biobuddies/helicopyter.git +)" +''' + +[tasks.django-upgrade] +run = 'mise run-on-sources django-upgrade \*.py "$@"' + +[tasks.djlint-django] +run = 'mise run-on-sources djlint \*.dj.html --profile=django --reformat "$@"' + +[tasks.djlint-jinja] +run = 'mise run-on-sources djlint \*.j2.html --profile=jinja --reformat "$@"' + +[tasks.end-of-file-fixer] +run = 'mise run-on-sources end-of-file-fixer . "$@"' + +[tasks.hadolint] +run = 'mise run-on-sources hadolint "*Dockerfile*" "$@"' + +[tasks.helicopyter] +run = 'python -m helicopyter --format_with=${INSH_TF:-terraform} all "$@"' + +[tasks.mailmap] +run = """ +#!/usr/bin/env bash +set -o errexit -o nounset -o pipefail +emails_names=$(git log --format='%aE%x09%aN' --use-mailmap "$@" | sort --unique) +return_code=0 +for repeat in $(echo "$emails_names" | cut -f 1 | uniq --repeated); do + echo "$emails_names" | grep "$repeat" + echo + return_code=1 +done +[[ $return_code == 0 ]] || echo 'Multiple names are associated with the same email address.' +exit $return_code +""" + +[tasks.prettier-write] +run = 'mise run-on-sources prettier "*.{html,js,json,jsx,toml,ts,tsx,yaml}" --write "$@"' + +[tasks.ruff-check-fix] +run = 'mise run-on-sources ruff "*.py{,i}" check --force-exclude --fix "$@"' + +[tasks.ruff-format] +run = 'mise run-on-sources ruff "*.py{,i}" format --force-exclude "$@"' + +[tasks.set-environment] +run = """ +#!/usr/bin/env bash +set -o errexit -o nounset -o pipefail +cat <=14.0.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.3" - } - }, "node_modules/blake3-wasm": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/blake3-wasm/-/blake3-wasm-2.1.5.tgz", diff --git a/package.json b/package.json index 52b9d64..9360c5a 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,5 @@ { "devDependencies": { - "basedpyright": "*", - "cdk8s-cli": "*", "prettier": "*", "prettier-plugin-jinja-template": "*", "prettier-plugin-organize-attributes": "*", diff --git a/pyproject.toml b/pyproject.toml index 292411b..cf54d77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,16 +5,17 @@ backend-path = [''] [project] classifiers = [ - 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.12', 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)', 'Operating System :: OS Independent', 'Topic :: System :: Systems Administration', 'Topic :: System :: Installation/Setup', ] description = 'Python Terraform JSON configuration generator' -dependencies = ['cdk8s', 'cdktf', 'cdktf-cdktf-provider-null', 'typed-argument-parser'] +dependencies = ['cdktf', 'typed-argument-parser'] dynamic = ['version'] name = 'helicopyter' +requires-python = '>=3.12' readme = 'README.md' [project.optional-dependencies] @@ -26,9 +27,11 @@ demos = [ 'cdktf-cdktf-provider-cloudflare==11.*', 'cdktf-cdktf-provider-docker', 'cdktf-cdktf-provider-github', + 'cdktf-cdktf-provider-null', ] precommit = [ 'actionlint-py', + 'basedpyright', 'codespell>=2.3', # For codespell inline ignore comment support 'cookiecutter', 'django-upgrade', diff --git a/requirements.txt b/requirements.txt index 9d4e8bd..3d2f2a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,14 +10,14 @@ attrs==25.3.0 # via # cattrs # jsii +basedpyright==1.38.2 + # via helicopyter (pyproject.toml) binaryornot==0.4.4 # via cookiecutter build==1.2.2.post1 # via helicopyter (pyproject.toml) cattrs==24.1.3 # via jsii -cdk8s==2.69.65 - # via helicopyter (pyproject.toml) cdktf==0.20.12 # via # helicopyter (pyproject.toml) @@ -56,7 +56,6 @@ colorama==0.4.6 # via djlint constructs==10.4.2 # via - # cdk8s # cdktf # cdktf-cdktf-provider-aws # cdktf-cdktf-provider-cloudflare @@ -127,9 +126,8 @@ jsbeautifier==1.15.4 # via # cssbeautifier # djlint -jsii==1.111.0 +jsii==1.127.0 # via - # cdk8s # cdktf # cdktf-cdktf-provider-aws # cdktf-cdktf-provider-cloudflare @@ -163,6 +161,8 @@ nh3==0.2.21 # via readme-renderer nodeenv==1.9.1 # via pre-commit +nodejs-wheel-binaries==24.14.0 + # via basedpyright packaging==25.0 # via # build @@ -192,7 +192,6 @@ ptyprocess==0.7.0 # via pexpect publication==0.0.3 # via - # cdk8s # cdktf # cdktf-cdktf-provider-aws # cdktf-cdktf-provider-cloudflare @@ -290,7 +289,6 @@ typed-argument-parser==1.10.1 # via helicopyter (pyproject.toml) typeguard==2.13.3 # via - # cdk8s # cdktf # cdktf-cdktf-provider-aws # cdktf-cdktf-provider-cloudflare diff --git a/test_tooling.py b/test_tooling.py index d2f28f6..7a303ff 100644 --- a/test_tooling.py +++ b/test_tooling.py @@ -1,54 +1,115 @@ """Integration tests for tools and configuration.""" -from os import getenv +from json import loads +from os import environ, getenv from pathlib import Path from re import match from subprocess import check_output -from tempfile import NamedTemporaryFile -import try_repo +from pytest import mark def test_just(): - assert check_output(['.venv/bin/just', 'cona']) == b'helicopyter\n' # noqa:S603 + assert check_output(['.venv/bin/just', 'cona']) == b'helicopyter\n' # noqa: S603 assert ( - check_output(['.venv/bin/just', 'envi']) == b'github\n' # noqa:S603 + check_output(['.venv/bin/just', 'envi']) == b'github\n' # noqa: S603 if getenv('GITHUB_ACTIONS') else b'local\n' ) - giha = check_output(['.venv/bin/just', 'giha']) # noqa:S603 + giha = check_output(['.venv/bin/just', 'giha']) # noqa: S603 assert match(rb'^[0-9a-f]{40}(-dirty)?\n$', giha) - if check_output(['git', 'status', '--porcelain']): # noqa:S603 + if check_output(['git', 'status', '--porcelain', '--untracked-files=no']): # noqa: S603 assert giha.endswith(b'-dirty\n') else: assert not giha.endswith(b'-dirty\n') - assert check_output(['.venv/bin/just', 'orgn']) == b'biobuddies\n' # noqa:S603 + assert check_output(['.venv/bin/just', 'orgn']) == b'biobuddies\n' # noqa: S603 + + +def test_mise(): + assert check_output(['mise', 'cona']) == b'helicopyter\n' # noqa: S603 + + assert ( + check_output(['mise', 'envi']) == b'github\n' # noqa: S603 + if getenv('GITHUB_ACTIONS') + else b'local\n' + ) + + giha = check_output(['mise', 'giha']) # noqa: S603 + assert match(rb'^[0-9a-f]{40}(-dirty)?\n$', giha) + is_dirty = bool(check_output(['git', 'status', '--porcelain', '--untracked-files=no'])) # noqa: S603 + assert giha.endswith(b'-dirty\n') == is_dirty + + assert check_output(['mise', 'orgn']) == b'biobuddies\n' # noqa: S603 + + tabr_env = { + 'MISE_TRUSTED_CONFIG_PATHS': getenv('MISE_TRUSTED_CONFIG_PATHS', ''), + 'PATH': environ['PATH'], + } + assert check_output(['mise', 'tabr'], env=tabr_env) == ( # noqa: S603 + b'' + if is_dirty + else check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() + b'\n' # noqa: S603 + ) def test_prettier(): - with NamedTemporaryFile('w+', delete_on_close=False, dir='.', suffix='.j2.html') as test_file: - test_file.write( - '\n{% for item in items %}
{{item}}
{% endfor %}\n\n' - ) - test_file.close() - try_repo.pre_commit.main.main( - ('try-repo', '.', 'prettier-write', '--files', str(test_file.name), '--verbose') - ) - assert Path(test_file.name).read_text() == ( + test_path = Path('tmp-test-prettier.j2.html') + test_path.write_text( + '\n{% for item in items %}
{{item}}
{% endfor %}\n\n' + ) + check_output(['git', 'add', str(test_path)]) # noqa: S603 + try: + check_output(['mise', 'prettier-write']) # noqa: S603 + assert test_path.read_text() == ( '\n' ' \n' ' {% for item in items %}
{{ item }}
{% endfor %}\n' ' \n' '\n' ) + finally: + check_output(['git', 'rm', '--force', '--quiet', str(test_path)]) # noqa: S603 -def test_typos(tmp_path: Path): - input_path = tmp_path / 'wxperiment-\xb5.yml' # noqa: typos - input_path.write_text('wxperiment:\n - \xb5\n yml') # noqa: typos - try_repo.pre_commit.main.main(('try-repo', '.', 'typos', '--files', str(input_path))) - assert (tmp_path / 'experiment-\u03bc.yaml').read_text() == 'experiment:\n - \u03bc\n yaml' +def test_typos(): + input_path = Path('wxperiment-\xb5.yml') # noqa: RUF100 # noqa: typos + input_path.write_text('wxperiment:\n - \xb5\n yml') # noqa: RUF100 # noqa: typos + output_path = Path('experiment-\u03bc.yaml') + check_output(['git', 'add', str(input_path)]) # noqa: S603 + try: + check_output(['mise', 'typos']) # noqa: S603 # noqa: typos + assert output_path.read_text() == 'experiment:\n - \u03bc\n yaml' + finally: + input_path.unlink(missing_ok=True) + output_path.unlink(missing_ok=True) + check_output(['git', 'rm', '--force', '--quiet', str(input_path)]) # noqa: S603 # TODO also the html escape sequence µ -> μ + + +@mark.parametrize( + ('git_describe', 'tabr'), + ( + ('remotes/origin/mybranch', 'mybranch'), + ('heads/mybranch', 'mybranch'), + ('tags/v2025.02.03', 'v2025.02.03'), + ('heads/mybranch-dirty', ''), + ), +) +def test_tabr_git_describe_mocked(git_describe: str, tabr: str): + original = loads( + check_output(['mise', 'tasks', 'info', 'tabr', '--json']) # noqa: S603 + )['run'][0].replace('\\n', '\n') + target = 'git describe --all --dirty --exact-match' + assert target in original + mocked = original.replace(target, f'echo "{git_describe}"') + output = ( + check_output( # noqa: S603 + ['/usr/bin/env', 'bash', '-c', mocked], env={} + ) + .decode() + .strip() + ) + assert output == tabr