diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index efcf6c6e3e245..afac835f2bb96 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -937,6 +937,12 @@ repos: pass_filenames: true files: \.py$ exclude: ^airflow/providers|^dev/.*\.py$|^scripts/.*\.py$|^tests/|^docker_tests/|^docs/.*\.py$|^airflow/utils/helpers.py$ + - id: check-deferrable-default-value + name: Check default value of deferrable attribute + language: python + entry: ./scripts/ci/pre_commit/pre_commit_check_deferrable_default.py + pass_filenames: false + files: ^airflow/.*/sensors/.*\.py$|^airflow/.*/operators/.*\.py$ ## ADD MOST PRE-COMMITS ABOVE THAT LINE # The below pre-commits are those requiring CI image to be built - id: mypy-dev diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst index 381dc53c93fb8..a9ec8c106aa14 100644 --- a/STATIC_CODE_CHECKS.rst +++ b/STATIC_CODE_CHECKS.rst @@ -171,6 +171,8 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-decorated-operator-implements-custom-name | Check @task decorator implements custom_operator_name | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ +| check-deferrable-default-value | Check default value of deferrable attribute | | ++-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-docstring-param-types | Check that docstrings do not specify param types | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-example-dags-urls | Check that example dags url include provider versions | | diff --git a/airflow/providers/google/cloud/operators/datafusion.py b/airflow/providers/google/cloud/operators/datafusion.py index b214c36d7844d..8ae64e1a4b720 100644 --- a/airflow/providers/google/cloud/operators/datafusion.py +++ b/airflow/providers/google/cloud/operators/datafusion.py @@ -24,6 +24,7 @@ from googleapiclient.errors import HttpError from airflow import AirflowException +from airflow.configuration import conf from airflow.providers.google.cloud.hooks.datafusion import SUCCESS_STATES, DataFusionHook, PipelineStates from airflow.providers.google.cloud.links.datafusion import ( DataFusionInstanceLink, @@ -759,7 +760,7 @@ def __init__( gcp_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, asynchronous=False, - deferrable=False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), poll_interval=3.0, **kwargs, ) -> None: diff --git a/airflow/providers/snowflake/operators/snowflake.py b/airflow/providers/snowflake/operators/snowflake.py index 4087397d83042..61e60b5781fb7 100644 --- a/airflow/providers/snowflake/operators/snowflake.py +++ b/airflow/providers/snowflake/operators/snowflake.py @@ -23,6 +23,7 @@ from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, Sequence, SupportsAbs, cast from airflow import AirflowException +from airflow.configuration import conf from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.common.sql.operators.sql import ( SQLCheckOperator, @@ -30,9 +31,7 @@ SQLIntervalCheckOperator, SQLValueCheckOperator, ) -from airflow.providers.snowflake.hooks.snowflake_sql_api import ( - SnowflakeSqlApiHook, -) +from airflow.providers.snowflake.hooks.snowflake_sql_api import SnowflakeSqlApiHook from airflow.providers.snowflake.triggers.snowflake_trigger import SnowflakeSqlApiTrigger if TYPE_CHECKING: @@ -451,7 +450,7 @@ def __init__( token_life_time: timedelta = LIFETIME, token_renewal_delta: timedelta = RENEWAL_DELTA, bindings: dict[str, Any] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs: Any, ) -> None: self.snowflake_conn_id = snowflake_conn_id diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py index 1349a962107e2..ed7a06d4a4666 100644 --- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py +++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py @@ -39,6 +39,7 @@ "check-core-deprecation-classes", "check-daysago-import-from-utils", "check-decorated-operator-implements-custom-name", + "check-deferrable-default-value", "check-docstring-param-types", "check-example-dags-urls", "check-executables-have-shebangs", diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 241dc9bd43f6c..88363d44e93c8 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -60,7 +60,7 @@ setup:version:123b462a421884dc2320ffc5e54b2478 setup:201c30ea237ea013a9a209a77092a2e8 shell:13f90c5749811e2f00e24d95e44e946d start-airflow:22c118d58b13a9d190e966bed5bb8ed8 -static-checks:a0f6ae35129b99b88ce65aede2c3a150 +static-checks:521edd8e7669b666b288ad0243b45be7 testing:docker-compose-tests:70167e67853cacd9ca784695d65a7846 testing:helm-tests:936cf28fd84ce4ff5113795fdae9624b testing:integration-tests:35f0ac57157bf8fe227fd080cf216622 diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index f313cb8abe781..e5b2130124d5d 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -35,8 +35,8 @@ .breeze-help-r1 { fill: #c5c8c6;font-weight: bold } .breeze-help-r2 { fill: #c5c8c6 } .breeze-help-r3 { fill: #d0b344;font-weight: bold } -.breeze-help-r4 { fill: #868887 } -.breeze-help-r5 { fill: #68a0b3;font-weight: bold } +.breeze-help-r4 { fill: #68a0b3;font-weight: bold } +.breeze-help-r5 { fill: #868887 } .breeze-help-r6 { fill: #98a84b;font-weight: bold } .breeze-help-r7 { fill: #8d7b39 } @@ -211,57 +211,57 @@ -Usage: breeze [OPTIONS] COMMAND [ARGS]... +Usage: breeze [OPTIONS] COMMAND [ARGS]... -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ -│[default: 3.8]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(all | all-testable | cassandra | celery | kafka | kerberos | mongo | otel | pinot |     │ -│statsd | statsd | trino)                                                                 │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ -│(INTEGER RANGE)                                                                       │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Developer commands ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ -│if contents of www directory changed.                                                           │ -│static-checks     Run static checks.                                                                              │ -│build-docs        Build documentation in the container.                                                           │ -│down              Stop running breeze environment.                                                                │ -│shell             Enter breeze environment. this is the default command use when no other is selected.            │ -│exec              Joins the interactive shell of running airflow container.                                       │ -│compile-www-assetsCompiles www assets.                                                                            │ -│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Testing commands ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│testing        Tools that developers can use to run tests                                                         │ -│k8s            Tools that developers use to run Kubernetes tests                                                  │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Image commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│ci-image         Tools that developers can use to manually manage CI images                                       │ -│prod-image       Tools that developers can use to manually manage PROD images                                     │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Release management commands ────────────────────────────────────────────────────────────────────────────────────────╮ -│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ -│sbom                   Tools that release managers can use to prepare sbom information                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Other commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│setup     Tools that developers can use to configure Breeze                                                       │ -│ci        Tools that CI workflows use to cleanup/manage CI environment                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ +│[default: 3.8]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                            │ +│(all | all-testable | cassandra | celery | kafka | kerberos | mongo | otel | pinot |     │ +│statsd | statsd | trino)                                                                 │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ +│(INTEGER RANGE)                                                                       │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Developer commands ─────────────────────────────────────────────────────────────────────────────────────────────────╮ +│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ +│if contents of www directory changed.                                                           │ +│static-checks     Run static checks.                                                                              │ +│build-docs        Build documentation in the container.                                                           │ +│down              Stop running breeze environment.                                                                │ +│shell             Enter breeze environment. this is the default command use when no other is selected.            │ +│exec              Joins the interactive shell of running airflow container.                                       │ +│compile-www-assetsCompiles www assets.                                                                            │ +│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Testing commands ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│testing        Tools that developers can use to run tests                                                         │ +│k8s            Tools that developers use to run Kubernetes tests                                                  │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Image commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ci-image         Tools that developers can use to manually manage CI images                                       │ +│prod-image       Tools that developers can use to manually manage PROD images                                     │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Release management commands ────────────────────────────────────────────────────────────────────────────────────────╮ +│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ +│sbom                   Tools that release managers can use to prepare sbom information                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Other commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│setup     Tools that developers can use to configure Breeze                                                       │ +│ci        Tools that CI workflows use to cleanup/manage CI environment                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_setup.svg b/images/breeze/output_setup.svg index 566cad3ff843e..3698b389cb23b 100644 --- a/images/breeze/output_setup.svg +++ b/images/breeze/output_setup.svg @@ -35,8 +35,8 @@ .breeze-setup-r1 { fill: #c5c8c6;font-weight: bold } .breeze-setup-r2 { fill: #c5c8c6 } .breeze-setup-r3 { fill: #d0b344;font-weight: bold } -.breeze-setup-r4 { fill: #868887 } -.breeze-setup-r5 { fill: #68a0b3;font-weight: bold } +.breeze-setup-r4 { fill: #68a0b3;font-weight: bold } +.breeze-setup-r5 { fill: #868887 } .breeze-setup-r6 { fill: #98a84b;font-weight: bold } @@ -108,23 +108,23 @@ -Usage: breeze setup [OPTIONS] COMMAND [ARGS]... +Usage: breeze setup [OPTIONS] COMMAND [ARGS]... Tools that developers can use to configure Breeze -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Setup ──────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│autocomplete                    Enables autocompletion of breeze commands.                                        │ -│self-upgrade                    Self upgrade Breeze.                                                              │ -│config                          Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).                │ -│regenerate-command-images       Regenerate breeze command images.                                                 │ -│version                         Print information about version of apache-airflow-breeze.                         │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│check-all-params-in-groups                 Check that all parameters are put in groups.                           │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Setup ──────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│autocomplete                    Enables autocompletion of breeze commands.                                        │ +│self-upgrade                    Self upgrade Breeze.                                                              │ +│config                          Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).                │ +│regenerate-command-images       Regenerate breeze command images.                                                 │ +│version                         Print information about version of apache-airflow-breeze.                         │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│check-all-params-in-groups                 Check that all parameters are put in groups.                           │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_setup_check-all-params-in-groups.svg b/images/breeze/output_setup_check-all-params-in-groups.svg index fbdca6ae752fb..c684f857c9f94 100644 --- a/images/breeze/output_setup_check-all-params-in-groups.svg +++ b/images/breeze/output_setup_check-all-params-in-groups.svg @@ -35,8 +35,8 @@ .breeze-setup-check-all-params-in-groups-r1 { fill: #c5c8c6;font-weight: bold } .breeze-setup-check-all-params-in-groups-r2 { fill: #c5c8c6 } .breeze-setup-check-all-params-in-groups-r3 { fill: #d0b344;font-weight: bold } -.breeze-setup-check-all-params-in-groups-r4 { fill: #868887 } -.breeze-setup-check-all-params-in-groups-r5 { fill: #68a0b3;font-weight: bold } +.breeze-setup-check-all-params-in-groups-r4 { fill: #68a0b3;font-weight: bold } +.breeze-setup-check-all-params-in-groups-r5 { fill: #868887 } .breeze-setup-check-all-params-in-groups-r6 { fill: #8d7b39 } .breeze-setup-check-all-params-in-groups-r7 { fill: #98a84b;font-weight: bold } @@ -142,34 +142,34 @@ -Usage: breeze setup check-all-params-in-groups [OPTIONS] +Usage: breeze setup check-all-params-in-groups [OPTIONS] Check that all parameters are put in groups. -╭─ Check all params in groups flags ───────────────────────────────────────────────────────────────────────────────────╮ -│--commandCommand(s) to regenerate images for (optional, might be repeated)                                      │ -│(main | build-docs | ci:fix-ownership | ci:free-space | ci:get-workflow-info | ci:resource-check |     │ -│ci:selective-check | ci | ci-image:build | ci-image:pull | ci-image:verify | ci-image | cleanup |      │ -│compile-www-assets | down | exec | k8s:build-k8s-image | k8s:configure-cluster | k8s:create-cluster |  │ -│k8s:delete-cluster | k8s:deploy-airflow | k8s:k9s | k8s:logs | k8s:run-complete-tests | k8s:setup-env |│ -│k8s:shell | k8s:status | k8s:tests | k8s:upload-k8s-image | k8s | prod-image:build | prod-image:pull | │ -│prod-image:verify | prod-image | release-management:create-minor-branch |                              │ -│release-management:generate-constraints | release-management:generate-issue-content-providers |        │ -│release-management:generate-providers-metadata | release-management:install-provider-packages |        │ -│release-management:prepare-airflow-package | release-management:prepare-provider-documentation |       │ -│release-management:prepare-provider-packages | release-management:release-prod-images |                │ -│release-management:start-rc-process | release-management:start-release |                               │ -│release-management:verify-provider-packages | release-management | sbom:generate-provider-requirements │ -│| sbom:update-sbom-information | sbom | setup:autocomplete | setup:check-all-params-in-groups |        │ -│setup:config | setup:regenerate-command-images | setup:self-upgrade | setup:version | setup | shell |  │ -│start-airflow | static-checks | testing:docker-compose-tests | testing:helm-tests |                    │ -│testing:integration-tests | testing:tests | testing)                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Check all params in groups flags ───────────────────────────────────────────────────────────────────────────────────╮ +│--commandCommand(s) to regenerate images for (optional, might be repeated)                                      │ +│(main | build-docs | ci:fix-ownership | ci:free-space | ci:get-workflow-info | ci:resource-check |     │ +│ci:selective-check | ci | ci-image:build | ci-image:pull | ci-image:verify | ci-image | cleanup |      │ +│compile-www-assets | down | exec | k8s:build-k8s-image | k8s:configure-cluster | k8s:create-cluster |  │ +│k8s:delete-cluster | k8s:deploy-airflow | k8s:k9s | k8s:logs | k8s:run-complete-tests | k8s:setup-env |│ +│k8s:shell | k8s:status | k8s:tests | k8s:upload-k8s-image | k8s | prod-image:build | prod-image:pull | │ +│prod-image:verify | prod-image | release-management:create-minor-branch |                              │ +│release-management:generate-constraints | release-management:generate-issue-content-providers |        │ +│release-management:generate-providers-metadata | release-management:install-provider-packages |        │ +│release-management:prepare-airflow-package | release-management:prepare-provider-documentation |       │ +│release-management:prepare-provider-packages | release-management:release-prod-images |                │ +│release-management:start-rc-process | release-management:start-release |                               │ +│release-management:verify-provider-packages | release-management | sbom:generate-provider-requirements │ +│| sbom:update-sbom-information | sbom | setup:autocomplete | setup:check-all-params-in-groups |        │ +│setup:config | setup:regenerate-command-images | setup:self-upgrade | setup:version | setup | shell |  │ +│start-airflow | static-checks | testing:docker-compose-tests | testing:helm-tests |                    │ +│testing:integration-tests | testing:tests | testing)                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_setup_regenerate-command-images.svg b/images/breeze/output_setup_regenerate-command-images.svg index 00dc4b90232d0..dd7ee821952b1 100644 --- a/images/breeze/output_setup_regenerate-command-images.svg +++ b/images/breeze/output_setup_regenerate-command-images.svg @@ -35,8 +35,8 @@ .breeze-setup-regenerate-command-images-r1 { fill: #c5c8c6;font-weight: bold } .breeze-setup-regenerate-command-images-r2 { fill: #c5c8c6 } .breeze-setup-regenerate-command-images-r3 { fill: #d0b344;font-weight: bold } -.breeze-setup-regenerate-command-images-r4 { fill: #868887 } -.breeze-setup-regenerate-command-images-r5 { fill: #68a0b3;font-weight: bold } +.breeze-setup-regenerate-command-images-r4 { fill: #68a0b3;font-weight: bold } +.breeze-setup-regenerate-command-images-r5 { fill: #868887 } .breeze-setup-regenerate-command-images-r6 { fill: #8d7b39 } .breeze-setup-regenerate-command-images-r7 { fill: #98a84b;font-weight: bold } @@ -154,38 +154,38 @@ -Usage: breeze setup regenerate-command-images [OPTIONS] +Usage: breeze setup regenerate-command-images [OPTIONS] Regenerate breeze command images. -╭─ Image regeneration option ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--forceForces regeneration of all images│ -│--commandCommand(s) to regenerate images for (optional, might be repeated)                                   │ -│(main | build-docs | ci:fix-ownership | ci:free-space | ci:get-workflow-info | ci:resource-check |  │ -│ci:selective-check | ci | ci-image:build | ci-image:pull | ci-image:verify | ci-image | cleanup |   │ -│compile-www-assets | down | exec | k8s:build-k8s-image | k8s:configure-cluster | k8s:create-cluster │ -│| k8s:delete-cluster | k8s:deploy-airflow | k8s:k9s | k8s:logs | k8s:run-complete-tests |           │ -│k8s:setup-env | k8s:shell | k8s:status | k8s:tests | k8s:upload-k8s-image | k8s | prod-image:build |│ -│prod-image:pull | prod-image:verify | prod-image | release-management:create-minor-branch |         │ -│release-management:generate-constraints | release-management:generate-issue-content-providers |     │ -│release-management:generate-providers-metadata | release-management:install-provider-packages |     │ -│release-management:prepare-airflow-package | release-management:prepare-provider-documentation |    │ -│release-management:prepare-provider-packages | release-management:release-prod-images |             │ -│release-management:start-rc-process | release-management:start-release |                            │ -│release-management:verify-provider-packages | release-management |                                  │ -│sbom:generate-provider-requirements | sbom:update-sbom-information | sbom | setup:autocomplete |    │ -│setup:check-all-params-in-groups | setup:config | setup:regenerate-command-images |                 │ -│setup:self-upgrade | setup:version | setup | shell | start-airflow | static-checks |                │ -│testing:docker-compose-tests | testing:helm-tests | testing:integration-tests | testing:tests |     │ -│testing)                                                                                            │ -│--check-onlyOnly check if some images need to be regenerated. Return 0 if no need or 1 if needed. Cannot be used│ -│together with --command flag or --force.                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Image regeneration option ──────────────────────────────────────────────────────────────────────────────────────────╮ +│--forceForces regeneration of all images│ +│--commandCommand(s) to regenerate images for (optional, might be repeated)                                   │ +│(main | build-docs | ci:fix-ownership | ci:free-space | ci:get-workflow-info | ci:resource-check |  │ +│ci:selective-check | ci | ci-image:build | ci-image:pull | ci-image:verify | ci-image | cleanup |   │ +│compile-www-assets | down | exec | k8s:build-k8s-image | k8s:configure-cluster | k8s:create-cluster │ +│| k8s:delete-cluster | k8s:deploy-airflow | k8s:k9s | k8s:logs | k8s:run-complete-tests |           │ +│k8s:setup-env | k8s:shell | k8s:status | k8s:tests | k8s:upload-k8s-image | k8s | prod-image:build |│ +│prod-image:pull | prod-image:verify | prod-image | release-management:create-minor-branch |         │ +│release-management:generate-constraints | release-management:generate-issue-content-providers |     │ +│release-management:generate-providers-metadata | release-management:install-provider-packages |     │ +│release-management:prepare-airflow-package | release-management:prepare-provider-documentation |    │ +│release-management:prepare-provider-packages | release-management:release-prod-images |             │ +│release-management:start-rc-process | release-management:start-release |                            │ +│release-management:verify-provider-packages | release-management |                                  │ +│sbom:generate-provider-requirements | sbom:update-sbom-information | sbom | setup:autocomplete |    │ +│setup:check-all-params-in-groups | setup:config | setup:regenerate-command-images |                 │ +│setup:self-upgrade | setup:version | setup | shell | start-airflow | static-checks |                │ +│testing:docker-compose-tests | testing:helm-tests | testing:integration-tests | testing:tests |     │ +│testing)                                                                                            │ +│--check-onlyOnly check if some images need to be regenerated. Return 0 if no need or 1 if needed. Cannot be used│ +│together with --command flag or --force.                                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_static-checks.svg b/images/breeze/output_static-checks.svg index e735b2afc3064..b372524f7a480 100644 --- a/images/breeze/output_static-checks.svg +++ b/images/breeze/output_static-checks.svg @@ -1,4 +1,4 @@ - + - + @@ -243,9 +243,12 @@ + + + - Command: static-checks + Command: static-checks @@ -256,72 +259,73 @@ -Usage: breeze static-checks [OPTIONS] [PRECOMMIT_ARGS]... +Usage: breeze static-checks [OPTIONS] [PRECOMMIT_ARGS]... Run static checks. -╭─ Pre-commit flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--type-tType(s) of the static checks to run.                                             │ -│(all | black | blacken-docs | check-aiobotocore-optional |                       │ -│check-airflow-config-yaml-consistent | check-airflow-provider-compatibility |    │ -│check-apache-license-rat | check-base-operator-partial-arguments |               │ -│check-base-operator-usage | check-boring-cyborg-configuration |                  │ -│check-breeze-top-dependencies-limited | check-builtin-literals |                 │ -│check-changelog-has-no-duplicates | check-core-deprecation-classes |             │ -│check-daysago-import-from-utils | check-decorated-operator-implements-custom-name│ -│| check-docstring-param-types | check-example-dags-urls |                        │ -│check-executables-have-shebangs | check-extra-packages-references |              │ -│check-extras-order | check-for-inclusive-language | check-hooks-apply |          │ -│check-incorrect-use-of-LoggingMixin | check-init-decorator-arguments |           │ -│check-lazy-logging | check-links-to-example-dags-do-not-use-hardcoded-versions | │ -│check-merge-conflict | check-newsfragments-are-valid |                           │ -│check-no-airflow-deprecation-in-providers | check-no-providers-in-core-examples |│ -│check-no-relative-imports | check-only-new-session-with-provide-session |        │ -│check-persist-credentials-disabled-in-github-workflows |                         │ -│check-pre-commit-information-consistent | check-provide-create-sessions-imports |│ -│check-provider-yaml-valid | check-providers-init-file-missing |                  │ -│check-providers-subpackages-init-file-exist | check-pydevd-left-in-code |        │ -│check-revision-heads-map | check-safe-filter-usage-in-html | check-setup-order | │ -│check-start-date-not-used-in-defaults | check-system-tests-present |             │ -│check-system-tests-tocs | check-tests-unittest-testcase |                        │ -│check-urlparse-usage-in-code | check-usage-of-re2-over-re | check-xml | codespell│ -│| compile-www-assets | compile-www-assets-dev |                                  │ -│create-missing-init-py-files-tests | debug-statements | detect-private-key |     │ -│doctoc | end-of-file-fixer | fix-encoding-pragma | flynt | identity |            │ -│insert-license | lint-chart-schema | lint-css | lint-dockerfile | lint-helm-chart│ -│| lint-json-schema | lint-markdown | lint-openapi | mixed-line-ending | mypy-core│ -│| mypy-dev | mypy-docs | mypy-providers | pretty-format-json | python-no-log-warn│ -│| replace-bad-characters | rst-backticks | ruff | shellcheck |                   │ -│trailing-whitespace | ts-compile-format-lint-www | update-black-version |        │ -│update-breeze-cmd-output | update-breeze-readme-config-hash |                    │ -│update-common-sql-api-stubs | update-er-diagram | update-extras |                │ -│update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |            │ -│update-installed-providers-to-be-sorted | update-local-yml-file |                │ -│update-migration-references | update-providers-dependencies |                    │ -│update-spelling-wordlist-to-be-sorted | update-supported-versions |              │ -│update-vendored-in-k8s-json-schema | update-version | yamllint)                  │ -│--show-diff-on-failure-sShow diff for files modified by the checks.│ -│--initialize-environmentInitialize environment before running checks.│ -│--max-initialization-attemptsMaximum number of attempts to initialize environment before giving up.│ -│(INTEGER RANGE)                                                       │ -│[default: 3; 1<=x<=10]                                                │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Selecting files to run the checks on ───────────────────────────────────────────────────────────────────────────────╮ -│--file-fList of files to run the checks on.(PATH)│ -│--all-files-aRun checks on all files.│ -│--commit-ref-rRun checks for this commit reference only (can be any git commit-ish reference). Mutually    │ -│exclusive with --last-commit.                                                                │ -│(TEXT)                                                                                       │ -│--last-commit-cRun checks for all files in last commit. Mutually exclusive with --commit-ref.│ -│--only-my-changes-mRun checks for commits belonging to my PR only: for all commits between merge base to `main` │ -│branch and HEAD of your branch.                                                              │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Pre-commit flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--type-tType(s) of the static checks to run.                                             │ +│(all | black | blacken-docs | check-aiobotocore-optional |                       │ +│check-airflow-config-yaml-consistent | check-airflow-provider-compatibility |    │ +│check-apache-license-rat | check-base-operator-partial-arguments |               │ +│check-base-operator-usage | check-boring-cyborg-configuration |                  │ +│check-breeze-top-dependencies-limited | check-builtin-literals |                 │ +│check-changelog-has-no-duplicates | check-core-deprecation-classes |             │ +│check-daysago-import-from-utils | check-decorated-operator-implements-custom-name│ +│| check-deferrable-default-value | check-docstring-param-types |                 │ +│check-example-dags-urls | check-executables-have-shebangs |                      │ +│check-extra-packages-references | check-extras-order |                           │ +│check-for-inclusive-language | check-hooks-apply |                               │ +│check-incorrect-use-of-LoggingMixin | check-init-decorator-arguments |           │ +│check-lazy-logging | check-links-to-example-dags-do-not-use-hardcoded-versions | │ +│check-merge-conflict | check-newsfragments-are-valid |                           │ +│check-no-airflow-deprecation-in-providers | check-no-providers-in-core-examples |│ +│check-no-relative-imports | check-only-new-session-with-provide-session |        │ +│check-persist-credentials-disabled-in-github-workflows |                         │ +│check-pre-commit-information-consistent | check-provide-create-sessions-imports |│ +│check-provider-yaml-valid | check-providers-init-file-missing |                  │ +│check-providers-subpackages-init-file-exist | check-pydevd-left-in-code |        │ +│check-revision-heads-map | check-safe-filter-usage-in-html | check-setup-order | │ +│check-start-date-not-used-in-defaults | check-system-tests-present |             │ +│check-system-tests-tocs | check-tests-unittest-testcase |                        │ +│check-urlparse-usage-in-code | check-usage-of-re2-over-re | check-xml | codespell│ +│| compile-www-assets | compile-www-assets-dev |                                  │ +│create-missing-init-py-files-tests | debug-statements | detect-private-key |     │ +│doctoc | end-of-file-fixer | fix-encoding-pragma | flynt | identity |            │ +│insert-license | lint-chart-schema | lint-css | lint-dockerfile | lint-helm-chart│ +│| lint-json-schema | lint-markdown | lint-openapi | mixed-line-ending | mypy-core│ +│| mypy-dev | mypy-docs | mypy-providers | pretty-format-json | python-no-log-warn│ +│| replace-bad-characters | rst-backticks | ruff | shellcheck |                   │ +│trailing-whitespace | ts-compile-format-lint-www | update-black-version |        │ +│update-breeze-cmd-output | update-breeze-readme-config-hash |                    │ +│update-common-sql-api-stubs | update-er-diagram | update-extras |                │ +│update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |            │ +│update-installed-providers-to-be-sorted | update-local-yml-file |                │ +│update-migration-references | update-providers-dependencies |                    │ +│update-spelling-wordlist-to-be-sorted | update-supported-versions |              │ +│update-vendored-in-k8s-json-schema | update-version | yamllint)                  │ +│--show-diff-on-failure-sShow diff for files modified by the checks.│ +│--initialize-environmentInitialize environment before running checks.│ +│--max-initialization-attemptsMaximum number of attempts to initialize environment before giving up.│ +│(INTEGER RANGE)                                                       │ +│[default: 3; 1<=x<=10]                                                │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Selecting files to run the checks on ───────────────────────────────────────────────────────────────────────────────╮ +│--file-fList of files to run the checks on.(PATH)│ +│--all-files-aRun checks on all files.│ +│--commit-ref-rRun checks for this commit reference only (can be any git commit-ish reference). Mutually    │ +│exclusive with --last-commit.                                                                │ +│(TEXT)                                                                                       │ +│--last-commit-cRun checks for all files in last commit. Mutually exclusive with --commit-ref.│ +│--only-my-changes-mRun checks for commits belonging to my PR only: for all commits between merge base to `main` │ +│branch and HEAD of your branch.                                                              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/ci/pre_commit/pre_commit_check_deferrable_default.py b/scripts/ci/pre_commit/pre_commit_check_deferrable_default.py new file mode 100755 index 0000000000000..784d25d5221e2 --- /dev/null +++ b/scripts/ci/pre_commit/pre_commit_check_deferrable_default.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import ast +import glob +import itertools +import os +import sys +from typing import Iterator + +ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir)) + +DEFERRABLE_DOC = ( + "https://github.com/apache/airflow/blob/main/docs/apache-airflow/" + "authoring-and-scheduling/deferring.rst#writing-deferrable-operators" +) + + +def _is_valid_deferrable_default(default: ast.AST) -> bool: + """Check whether default is 'conf.getboolean("operators", "default_deferrable", fallback=False)'""" + if not isinstance(default, ast.Call): + return False # Not a function call. + + # Check the function callee is exactly 'conf.getboolean'. + call_to_conf_getboolean = ( + isinstance(default.func, ast.Attribute) + and isinstance(default.func.value, ast.Name) + and default.func.value.id == "conf" + and default.func.attr == "getboolean" + ) + if not call_to_conf_getboolean: + return False + + # Check arguments. + return ( + len(default.args) == 2 + and isinstance(default.args[0], ast.Constant) + and default.args[0].value == "operators" + and isinstance(default.args[1], ast.Constant) + and default.args[1].value == "default_deferrable" + and len(default.keywords) == 1 + and default.keywords[0].arg == "fallback" + and isinstance(default.keywords[0].value, ast.Constant) + and default.keywords[0].value.value is False + ) + + +def iter_check_deferrable_default_errors(module_filename: str) -> Iterator[str]: + ast_obj = ast.parse(open(module_filename).read()) + cls_nodes = (node for node in ast.iter_child_nodes(ast_obj) if isinstance(node, ast.ClassDef)) + init_method_nodes = ( + node + for cls_node in cls_nodes + for node in ast.iter_child_nodes(cls_node) + if isinstance(node, ast.FunctionDef) and node.name == "__init__" + ) + + for node in init_method_nodes: + args = node.args + arguments = reversed([*args.args, *args.kwonlyargs]) + defaults = reversed([*args.defaults, *args.kw_defaults]) + for argument, default in itertools.zip_longest(arguments, defaults, fillvalue=None): + if argument is None or default is None: + continue + if argument.arg != "deferrable" or _is_valid_deferrable_default(default): + continue + yield f"{module_filename}:{default.lineno}" + + +def main() -> int: + modules = itertools.chain( + glob.glob(f"{ROOT_DIR}/airflow/**/sensors/**.py", recursive=True), + glob.glob(f"{ROOT_DIR}/airflow/**/operators/**.py", recursive=True), + ) + + errors = [error for module in modules for error in iter_check_deferrable_default_errors(module)] + if errors: + print("Incorrect deferrable default values detected at:") + for error in errors: + print(f" {error}") + print( + """Please set the default value of deferrbale to """ + """"conf.getboolean("operators", "default_deferrable", fallback=False)"\n""" + f"See: {DEFERRABLE_DOC}\n" + ) + + return len(errors) + + +if __name__ == "__main__": + sys.exit(main())