Skip to content

Fix test to test if we miss tests (AKA 'who tests the tests') #35443

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eladkal merged 1 commit into from
Nov 4, 2023
Merged

Fix test to test if we miss tests (AKA 'who tests the tests') #35443

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
183 changes: 175 additions & 8 deletions tests/always/test_project_structure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,21 +57,182 @@ def test_providers_modules_should_have_tests(self):
"""
Assert every module in /airflow/providers has a corresponding test_ file in tests/airflow/providers.
"""
# The test below had a but for quite a while and we missed a lot of modules to have tess
# We should make sure that one goes to 0
OVERLOOKED_TESTS = [
"tests/providers/amazon/aws/executors/ecs/test_boto_schema.py",
"tests/providers/amazon/aws/executors/ecs/test_ecs_executor_config.py",
"tests/providers/amazon/aws/executors/ecs/test_utils.py",
"tests/providers/amazon/aws/fs/test_s3.py",
"tests/providers/amazon/aws/hooks/test_dms.py",
"tests/providers/amazon/aws/links/test_base_aws.py",
"tests/providers/amazon/aws/links/test_batch.py",
"tests/providers/amazon/aws/links/test_emr.py",
"tests/providers/amazon/aws/links/test_glue.py",
"tests/providers/amazon/aws/links/test_logs.py",
"tests/providers/amazon/aws/operators/test_dms.py",
"tests/providers/amazon/aws/operators/test_emr.py",
"tests/providers/amazon/aws/operators/test_s3.py",
"tests/providers/amazon/aws/operators/test_sagemaker.py",
"tests/providers/amazon/aws/sensors/test_dms.py",
"tests/providers/amazon/aws/sensors/test_emr.py",
"tests/providers/amazon/aws/sensors/test_s3.py",
"tests/providers/amazon/aws/sensors/test_sagemaker.py",
"tests/providers/amazon/aws/test_exceptions.py",
"tests/providers/amazon/aws/triggers/test_athena.py",
"tests/providers/amazon/aws/triggers/test_batch.py",
"tests/providers/amazon/aws/triggers/test_eks.py",
"tests/providers/amazon/aws/triggers/test_emr.py",
"tests/providers/amazon/aws/triggers/test_glue_crawler.py",
"tests/providers/amazon/aws/triggers/test_lambda_function.py",
"tests/providers/amazon/aws/triggers/test_rds.py",
"tests/providers/amazon/aws/triggers/test_redshift_cluster.py",
"tests/providers/amazon/aws/triggers/test_step_function.py",
"tests/providers/amazon/aws/utils/test_rds.py",
"tests/providers/amazon/aws/utils/test_sagemaker.py",
"tests/providers/amazon/aws/utils/test_sqs.py",
"tests/providers/amazon/aws/utils/test_tags.py",
"tests/providers/amazon/aws/waiters/test_base_waiter.py",
"tests/providers/apache/cassandra/hooks/test_cassandra.py",
"tests/providers/apache/druid/operators/test_druid_check.py",
"tests/providers/apache/hdfs/hooks/test_hdfs.py",
"tests/providers/apache/hdfs/log/test_hdfs_task_handler.py",
"tests/providers/apache/hdfs/sensors/test_hdfs.py",
"tests/providers/apache/hive/plugins/test_hive.py",
"tests/providers/apache/kafka/hooks/test_base.py",
"tests/providers/apache/kafka/hooks/test_client.py",
"tests/providers/apache/kafka/hooks/test_consume.py",
"tests/providers/apache/kafka/hooks/test_produce.py",
"tests/providers/celery/executors/test_celery_executor_utils.py",
"tests/providers/celery/executors/test_default_celery.py",
"tests/providers/cncf/kubernetes/backcompat/test_backwards_compat_converters.py",
"tests/providers/cncf/kubernetes/executors/test_kubernetes_executor_types.py",
"tests/providers/cncf/kubernetes/executors/test_kubernetes_executor_utils.py",
"tests/providers/cncf/kubernetes/operators/test_kubernetes_pod.py",
"tests/providers/cncf/kubernetes/test_k8s_model.py",
"tests/providers/cncf/kubernetes/test_kube_client.py",
"tests/providers/cncf/kubernetes/test_kube_config.py",
"tests/providers/cncf/kubernetes/test_pod_generator_deprecated.py",
"tests/providers/cncf/kubernetes/test_pod_launcher_deprecated.py",
"tests/providers/cncf/kubernetes/test_python_kubernetes_script.py",
"tests/providers/cncf/kubernetes/test_secret.py",
"tests/providers/cncf/kubernetes/triggers/test_kubernetes_pod.py",
"tests/providers/cncf/kubernetes/utils/test_delete_from.py",
"tests/providers/cncf/kubernetes/utils/test_k8s_hashlib_wrapper.py",
"tests/providers/cncf/kubernetes/utils/test_xcom_sidecar.py",
"tests/providers/common/io/operators/test_file_transfer.py",
"tests/providers/daskexecutor/executors/test_dask_executor.py",
"tests/providers/databricks/hooks/test_databricks_base.py",
"tests/providers/dbt/cloud/hooks/test_dbt.py",
"tests/providers/dbt/cloud/operators/test_dbt.py",
"tests/providers/dbt/cloud/sensors/test_dbt.py",
"tests/providers/dbt/cloud/triggers/test_dbt.py",
"tests/providers/docker/test_exceptions.py",
"tests/providers/elasticsearch/log/test_es_json_formatter.py",
"tests/providers/elasticsearch/log/test_es_response.py",
"tests/providers/google/cloud/fs/test_gcs.py",
"tests/providers/google/cloud/links/test_automl.py",
"tests/providers/google/cloud/links/test_base.py",
"tests/providers/google/cloud/links/test_bigquery.py",
"tests/providers/google/cloud/links/test_bigquery_dts.py",
"tests/providers/google/cloud/links/test_bigtable.py",
"tests/providers/google/cloud/links/test_cloud_build.py",
"tests/providers/google/cloud/links/test_cloud_functions.py",
"tests/providers/google/cloud/links/test_cloud_memorystore.py",
"tests/providers/google/cloud/links/test_cloud_sql.py",
"tests/providers/google/cloud/links/test_cloud_storage_transfer.py",
"tests/providers/google/cloud/links/test_cloud_tasks.py",
"tests/providers/google/cloud/links/test_compute.py",
"tests/providers/google/cloud/links/test_data_loss_prevention.py",
"tests/providers/google/cloud/links/test_datacatalog.py",
"tests/providers/google/cloud/links/test_dataflow.py",
"tests/providers/google/cloud/links/test_dataform.py",
"tests/providers/google/cloud/links/test_datafusion.py",
"tests/providers/google/cloud/links/test_dataplex.py",
"tests/providers/google/cloud/links/test_dataprep.py",
"tests/providers/google/cloud/links/test_dataproc.py",
"tests/providers/google/cloud/links/test_datastore.py",
"tests/providers/google/cloud/links/test_kubernetes_engine.py",
"tests/providers/google/cloud/links/test_life_sciences.py",
"tests/providers/google/cloud/links/test_mlengine.py",
"tests/providers/google/cloud/links/test_pubsub.py",
"tests/providers/google/cloud/links/test_spanner.py",
"tests/providers/google/cloud/links/test_stackdriver.py",
"tests/providers/google/cloud/links/test_vertex_ai.py",
"tests/providers/google/cloud/links/test_workflows.py",
"tests/providers/google/cloud/operators/vertex_ai/test_auto_ml.py",
"tests/providers/google/cloud/operators/vertex_ai/test_batch_prediction_job.py",
"tests/providers/google/cloud/operators/vertex_ai/test_custom_job.py",
"tests/providers/google/cloud/operators/vertex_ai/test_dataset.py",
"tests/providers/google/cloud/operators/vertex_ai/test_endpoint_service.py",
"tests/providers/google/cloud/operators/vertex_ai/test_hyperparameter_tuning_job.py",
"tests/providers/google/cloud/operators/vertex_ai/test_model_service.py",
"tests/providers/google/cloud/operators/vertex_ai/test_pipeline_job.py",
"tests/providers/google/cloud/sensors/test_dataform.py",
"tests/providers/google/cloud/transfers/test_bigquery_to_sql.py",
"tests/providers/google/cloud/transfers/test_presto_to_gcs.py",
"tests/providers/google/cloud/transfers/test_trino_to_gcs.py",
"tests/providers/google/cloud/triggers/test_cloud_composer.py",
"tests/providers/google/cloud/utils/test_bigquery.py",
"tests/providers/google/cloud/utils/test_bigquery_get_data.py",
"tests/providers/google/cloud/utils/test_dataform.py",
"tests/providers/google/common/links/test_storage.py",
"tests/providers/google/common/test_consts.py",
"tests/providers/google/test_go_module_utils.py",
"tests/providers/microsoft/azure/fs/test_adls.py",
"tests/providers/microsoft/azure/hooks/test_batch.py",
"tests/providers/microsoft/azure/hooks/test_container_instance.py",
"tests/providers/microsoft/azure/hooks/test_container_registry.py",
"tests/providers/microsoft/azure/hooks/test_container_volume.py",
"tests/providers/microsoft/azure/hooks/test_cosmos.py",
"tests/providers/microsoft/azure/hooks/test_data_factory.py",
"tests/providers/microsoft/azure/hooks/test_data_lake.py",
"tests/providers/microsoft/azure/hooks/test_fileshare.py",
"tests/providers/microsoft/azure/hooks/test_synapse.py",
"tests/providers/microsoft/azure/operators/test_adls.py",
"tests/providers/microsoft/azure/operators/test_batch.py",
"tests/providers/microsoft/azure/operators/test_container_instances.py",
"tests/providers/microsoft/azure/operators/test_cosmos.py",
"tests/providers/microsoft/azure/operators/test_data_factory.py",
"tests/providers/microsoft/azure/operators/test_synapse.py",
"tests/providers/microsoft/azure/secrets/test_key_vault.py",
"tests/providers/microsoft/azure/sensors/test_cosmos.py",
"tests/providers/microsoft/azure/sensors/test_data_factory.py",
"tests/providers/microsoft/azure/transfers/test_azure_blob_to_gcs.py",
"tests/providers/microsoft/azure/triggers/test_data_factory.py",
"tests/providers/microsoft/azure/triggers/test_wasb.py",
"tests/providers/mongo/sensors/test_mongo.py",
"tests/providers/openlineage/extractors/test_base.py",
"tests/providers/openlineage/extractors/test_bash.py",
"tests/providers/openlineage/extractors/test_manager.py",
"tests/providers/openlineage/extractors/test_python.py",
"tests/providers/openlineage/plugins/test_adapter.py",
"tests/providers/openlineage/plugins/test_facets.py",
"tests/providers/openlineage/plugins/test_macros.py",
"tests/providers/openlineage/test_sqlparser.py",
"tests/providers/presto/transfers/test_gcs_to_presto.py",
"tests/providers/redis/operators/test_redis_publish.py",
"tests/providers/redis/sensors/test_redis_key.py",
"tests/providers/slack/notifications/test_slack_notifier.py",
"tests/providers/snowflake/triggers/test_snowflake_trigger.py",
"tests/providers/trino/transfers/test_gcs_to_trino.py",
]

# TODO: Should we extend this test to cover other directories?
modules_files = glob.glob(f"{ROOT_FOLDER}/airflow/providers/**/*.py", recursive=True)
modules_files = list(glob.glob(f"{ROOT_FOLDER}/airflow/providers/**/*.py", recursive=True))

# Make path relative
modules_files = (os.path.relpath(f, ROOT_FOLDER) for f in modules_files)
modules_files = list(os.path.relpath(f, ROOT_FOLDER) for f in modules_files)
# Exclude example_dags
modules_files = (f for f in modules_files if "/example_dags/" not in f)
modules_files = list(f for f in modules_files if "/example_dags/" not in f)
# Exclude __init__.py
modules_files = (f for f in modules_files if not f.endswith("__init__.py"))
modules_files = list(f for f in modules_files if not f.endswith("__init__.py"))
# Change airflow/ to tests/
expected_test_files = (
expected_test_files = list(
f'tests/{f.partition("/")[2]}' for f in modules_files if not f.endswith("__init__.py")
)
# Add test_ prefix to filename
expected_test_files = (
expected_test_files = list(
f'{f.rpartition("/")[0]}/test_{f.rpartition("/")[2]}'
for f in expected_test_files
if not f.endswith("__init__.py")
Expand All @@ -84,12 +245,18 @@ def test_providers_modules_should_have_tests(self):
current_test_files = (f for f in current_test_files if not f.endswith("__init__.py"))

modules_files = set(modules_files)
expected_test_files = set(expected_test_files)
expected_test_files = set(expected_test_files) - set(OVERLOOKED_TESTS)
current_test_files = set(current_test_files)

missing_tests_files = expected_test_files - expected_test_files.intersection(current_test_files)

assert set() == missing_tests_files, "Detect missing tests in providers module"
assert set() == missing_tests_files, "Detect missing tests in providers module - please add tests"

added_test_files = current_test_files.intersection(OVERLOOKED_TESTS)
assert set() == added_test_files, (
"Detect added tests in providers module - please remove the tests "
"from OVERLOOKED_TESTS list above"
)


def get_imports_from_file(filepath: str):
Expand Down