Add pre-commit to check that aiobotocore is an optional dependency

.pre-commit-config.yaml

@@ -610,6 +610,14 @@ repos:
         language: python
         files: ^.*LICENSE.*$|^.*LICENCE.*$
         pass_filenames: false
+      - id: check-aiobotocore-optional
+        name: Check if aiobotocore is an optional dependency only
+        entry: ./scripts/ci/pre_commit/pre_commit_check_aiobotocore_optional.py
+        language: python
+        files: ^airflow/providers/.*/provider\.yaml$
+        pass_filenames: true
+        additional_dependencies: ['click', 'rich>=12.4.4', 'pyyaml']
+        require_serial: true
       - id: check-airflow-config-yaml-consistent
         name: Checks for consistency between config.yml and default_config.cfg
         language: python

STATIC_CODE_CHECKS.rst

@@ -144,6 +144,8 @@ require Breeze Docker image to be build locally.
 +-----------------------------------------------------------+------------------------------------------------------------------+---------+
 | blacken-docs                                              | Run black on Python code blocks in documentation files           |         |
 +-----------------------------------------------------------+------------------------------------------------------------------+---------+
+| check-aiobotocore-optional                                | Check if aiobotocore is an optional dependency only              |         |
++-----------------------------------------------------------+------------------------------------------------------------------+---------+
 | check-airflow-config-yaml-consistent                      | Checks for consistency between config.yml and default_config.cfg |         |
 +-----------------------------------------------------------+------------------------------------------------------------------+---------+
 | check-airflow-provider-compatibility                      | Check compatibility of Providers with Airflow                    |         |

dev/breeze/src/airflow_breeze/pre_commit_ids.py

@@ -26,6 +26,7 @@
     "all",
     "black",
     "blacken-docs",
+    "check-aiobotocore-optional",
     "check-airflow-config-yaml-consistent",
     "check-airflow-provider-compatibility",
     "check-apache-license-rat",

images/breeze/output-commands-hash.txt

@@ -55,7 +55,7 @@ setup:version:123b462a421884dc2320ffc5e54b2478
 setup:26f37743534e14f5aad5300aad920301
 shell:bd3e004a92ebcec8feb40fc5cd95872d
 start-airflow:ee5066f1420a489864b48bc4e5e472da
-static-checks:cb1029ee6292860bb9fc425ef67b656d
+static-checks:c22e24fc35d2f82135b30998e4f352c4
 stop:e5aa686b4e53707ced4039d8414d5cd6
 testing:docker-compose-tests:b86c044b24138af0659a05ed6331576c
 testing:helm-tests:936cf28fd84ce4ff5113795fdae9624b

scripts/ci/pre_commit/pre_commit_check_aiobotocore_optional.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import yaml
+from rich.console import Console
+
+AIRFLOW_SOURCES = Path(__file__).parents[3]
+
+console = Console(color_system="standard", width=200)
+
+if __name__ == "__main__":
+    errors = []
+    for file in sys.argv[1:]:
+        console.print(f"[blue]Checking[/blue]: {file}")
+        provider_yaml_content = yaml.safe_load(Path(file).read_text())
+        dependencies = provider_yaml_content.get("dependencies")
+        if dependencies and any(dependency.startswith("aiobotocore") for dependency in dependencies):
+            errors.append(
+                f"\n[red]Error: the aibotocore cannot be a required dependency, "
+                f"because it restricts botocore too much[/]\n\n"
+                f"The [magenta]{file}[/] file has aiobotocore dependency set at top level.\n\n"
+                f"[yellow]Please remove it and make sure it is added only as "
+                f"an optional dependency in additional-extras[/n]\n"
+            )
+    if errors:
+        for error in errors:
+            console.print(error)
+        sys.exit(1)