apache · chethanuk · Jun 5, 2022 · Jul 18, 2022 · Jul 20, 2022 · Jul 20, 2022
diff --git a/docs/apache-airflow-providers-airbyte/index.rst b/docs/apache-airflow-providers-airbyte/index.rst
@@ -38,7 +38,7 @@ Content
     :maxdepth: 1
     :caption: Resources
 
-    Example DAGs <https://github.com/apache/airflow/tree/providers-airbyte/3.0.0/airflow/providers/airbyte/example_dags>
+    Example DAGs <https://github.com/apache/airflow/tree/providers-airbyte/3.0.0/tests/system/providers/airbyte>
     PyPI Repository <https://pypi.org/project/apache-airflow-providers-airbyte/>
     Installing from sources <installing-providers-from-sources>
 

diff --git a/docs/apache-airflow-providers-airbyte/operators/airbyte.rst b/docs/apache-airflow-providers-airbyte/operators/airbyte.rst
@@ -45,14 +45,14 @@ return the ``job_id`` that should be pass to the AirbyteSensor.
 
 An example using the synchronous way:
 
-.. exampleinclude:: /../../airflow/providers/airbyte/example_dags/example_airbyte_trigger_job.py
+.. exampleinclude:: /../../tests/system/providers/airbyte/example_airbyte_trigger_job.py
     :language: python
     :start-after: [START howto_operator_airbyte_synchronous]
     :end-before: [END howto_operator_airbyte_synchronous]
 
 An example using the async way:
 
-.. exampleinclude:: /../../airflow/providers/airbyte/example_dags/example_airbyte_trigger_job.py
+.. exampleinclude:: /../../tests/system/providers/airbyte/example_airbyte_trigger_job.py
     :language: python
     :start-after: [START howto_operator_airbyte_asynchronous]
     :end-before: [END howto_operator_airbyte_asynchronous]
diff --git a/docs/apache-airflow-providers-amazon/operators/s3.rst b/docs/apache-airflow-providers-amazon/operators/s3.rst
@@ -38,7 +38,7 @@ Create an Amazon S3 bucket
 To create an Amazon S3 bucket you can use
 :class:`~airflow.providers.amazon.aws.operators.s3.S3CreateBucketOperator`.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_create_bucket]
@@ -52,7 +52,7 @@ Delete an Amazon S3 bucket
 To delete an Amazon S3 bucket you can use
 :class:`~airflow.providers.amazon.aws.operators.s3.S3DeleteBucketOperator`.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_delete_bucket]
@@ -66,7 +66,7 @@ Set the tags for an Amazon S3 bucket
 To set the tags for an Amazon S3 bucket you can use
 :class:`~airflow.providers.amazon.aws.operators.s3.S3PutBucketTaggingOperator`.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_put_bucket_tagging]
@@ -80,7 +80,7 @@ Get the tag of an Amazon S3 bucket
 To get the tag set associated with an Amazon S3 bucket you can use
 :class:`~airflow.providers.amazon.aws.operators.s3.S3GetBucketTaggingOperator`.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_get_bucket_tagging]
@@ -94,7 +94,7 @@ Delete the tags of an Amazon S3 bucket
 To delete the tags of an Amazon S3 bucket you can use
 :class:`~airflow.providers.amazon.aws.operators.s3.S3DeleteBucketTaggingOperator`.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_delete_bucket_tagging]
@@ -108,7 +108,7 @@ Create an Amazon S3 object
 To create a new (or replace) Amazon S3 object you can use
 :class:`~airflow.providers.amazon.aws.operators.s3.S3CreateObjectOperator`.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_create_object]
@@ -123,7 +123,7 @@ To copy an Amazon S3 object from one bucket to another you can use
 :class:`~airflow.providers.amazon.aws.operators.s3.S3CopyObjectOperator`.
 The Amazon S3 connection used here needs to have access to both source and destination bucket/key.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_copy_object]
@@ -137,7 +137,7 @@ Delete Amazon S3 objects
 To delete one or multiple Amazon S3 objects you can use
 :class:`~airflow.providers.amazon.aws.operators.s3.S3DeleteObjectsOperator`.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_delete_objects]
@@ -153,7 +153,7 @@ To transform the data from one Amazon S3 object and save it to another object yo
 You can also apply an optional [Amazon S3 Select expression](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-glacier-select-sql-reference-select.html)
 to select the data you want to retrieve from ``source_s3_key`` using ``select_expression``.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_file_transform]
@@ -169,7 +169,7 @@ To list all Amazon S3 prefixes within an Amazon S3 bucket you can use
 See `here <https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-prefixes.html>`__
 for more information about Amazon S3 prefixes.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_list_prefixes]
@@ -184,7 +184,7 @@ To list all Amazon S3 objects within an Amazon S3 bucket you can use
 :class:`~airflow.providers.amazon.aws.operators.s3.S3ListOperator`.
 You can specify a ``prefix`` to filter the objects whose name begins with such prefix.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_list]
@@ -208,15 +208,15 @@ Please keep in mind, especially when used to check a large volume of keys, that
 
 To check one file:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_sensor_s3_key_single_key]
     :end-before: [END howto_sensor_s3_key_single_key]
 
 To check multiple files:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_sensor_s3_key_multiple_keys]
@@ -236,13 +236,13 @@ multiple files can match one key. The list of matched S3 object attributes conta
 
     [{"Size": int}]
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_sensor_s3_key_function_definition]
     :end-before: [END howto_sensor_s3_key_function_definition]
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_sensor_s3_key_function]
@@ -259,7 +259,7 @@ the inactivity period has passed with no increase in the number of objects you c
 Note, this sensor will not behave correctly in reschedule mode,
 as the state of the listed objects in the Amazon S3 bucket will be lost between rescheduled invocations.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_sensor_s3_keys_unchanged]

diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/dynamodb_to_s3.rst b/docs/apache-airflow-providers-amazon/operators/transfer/dynamodb_to_s3.rst
@@ -48,7 +48,7 @@ To get more information visit:
 
 Example usage:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_dynamodb_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_dynamodb_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_dynamodb_to_s3]
@@ -57,7 +57,7 @@ Example usage:
 To parallelize the replication, users can create multiple ``DynamoDBToS3Operator`` tasks using the
 ``TotalSegments`` parameter.  For instance to replicate with parallelism of 2, create two tasks:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_dynamodb_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_dynamodb_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_dynamodb_to_s3_segmented]

diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/ftp_to_s3.rst b/docs/apache-airflow-providers-amazon/operators/transfer/ftp_to_s3.rst
@@ -41,7 +41,7 @@ To get more information about this operator visit:
 
 Example usage:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_ftp_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_ftp_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_ftp_to_s3]

diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/gcs_to_s3.rst b/docs/apache-airflow-providers-amazon/operators/transfer/gcs_to_s3.rst
@@ -39,7 +39,7 @@ To copy data from a Google Cloud Storage bucket to an Amazon S3 bucket you can u
 
 Example usage:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_gcs_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_gcs_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_gcs_to_s3]

diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/google_api_to_s3.rst b/docs/apache-airflow-providers-amazon/operators/transfer/google_api_to_s3.rst
@@ -38,7 +38,7 @@ Google Sheets to Amazon S3 transfer operator
 
 This example loads data from Google Sheets and save it to an Amazon S3 file.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_google_api_sheets_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_google_api_sheets_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_google_api_sheets_to_s3]
@@ -57,7 +57,7 @@ It searches for up to 50 videos (due to pagination) in a given time range
 (``YOUTUBE_VIDEO_PUBLISHED_AFTER``, ``YOUTUBE_VIDEO_PUBLISHED_BEFORE``) on a YouTube channel (``YOUTUBE_CHANNEL_ID``)
 saves the response in Amazon S3 and also pushes the data to xcom.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_google_api_youtube_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_google_api_youtube_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_google_api_youtube_search_to_s3]
@@ -66,7 +66,7 @@ saves the response in Amazon S3 and also pushes the data to xcom.
 It passes over the YouTube IDs to the next request which then gets the
 information (``YOUTUBE_VIDEO_FIELDS``) for the requested videos and saves them in Amazon S3 (``S3_BUCKET_NAME``).
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_google_api_youtube_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_google_api_youtube_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_google_api_youtube_list_to_s3]

diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/imap_attachment_to_s3.rst b/docs/apache-airflow-providers-amazon/operators/transfer/imap_attachment_to_s3.rst
@@ -38,7 +38,7 @@ Imap Attachment To Amazon S3 transfer operator
 To save an email attachment via IMAP protocol from an email server to an Amazon S3 Bucket you can use
 :class:`~airflow.providers.amazon.aws.transfers.imap_attachment_to_s3.ImapAttachmentToS3Operator`
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_imap_attachment_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_imap_attachment_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_imap_attachment_to_s3]

diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/local_to_s3.rst b/docs/apache-airflow-providers-amazon/operators/transfer/local_to_s3.rst
@@ -42,7 +42,7 @@ To get more information about this operator visit:
 
 Example usage:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_local_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_local_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_local_to_s3]

diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/mongo_to_s3.rst b/docs/apache-airflow-providers-amazon/operators/transfer/mongo_to_s3.rst
@@ -43,7 +43,7 @@ To get more information about this operator visit:
 
 Example usage:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_mongo_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_mongo_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_mongo_to_s3]

diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/s3_to_ftp.rst b/docs/apache-airflow-providers-amazon/operators/transfer/s3_to_ftp.rst
@@ -42,7 +42,7 @@ To get more information about this operator visit:
 
 Example usage:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3_to_ftp.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3_to_ftp.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_s3_to_ftp]

diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/s3_to_redshift.rst b/docs/apache-airflow-providers-amazon/operators/transfer/s3_to_redshift.rst
@@ -42,7 +42,7 @@ To get more information about this operator visit:
 
 Example usage:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3_to_redshift.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3_to_redshift.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_s3_to_redshift]

diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/s3_to_sftp.rst b/docs/apache-airflow-providers-amazon/operators/transfer/s3_to_sftp.rst
@@ -43,7 +43,7 @@ To get more information about this operator visit:
 
 Example usage:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3_to_sftp.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3_to_sftp.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_s3_to_sftp]

diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/salesforce_to_s3.rst b/docs/apache-airflow-providers-amazon/operators/transfer/salesforce_to_s3.rst
@@ -38,7 +38,7 @@ Extract data from Salesforce to Amazon S3 transfer operator
 The following example demonstrates a use case of extracting account data from a Salesforce
 instance and upload to an Amazon S3 bucket.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_salesforce_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_salesforce_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_salesforce_to_s3]

diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/sftp_to_s3.rst b/docs/apache-airflow-providers-amazon/operators/transfer/sftp_to_s3.rst
@@ -42,7 +42,7 @@ To get more information about this operator visit:
 
 Example usage:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_sftp_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_sftp_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_sftp_to_s3]

diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/sql_to_s3.rst b/docs/apache-airflow-providers-amazon/operators/transfer/sql_to_s3.rst
@@ -44,7 +44,7 @@ To get more information about this operator visit:
 
 Example usage:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_sql_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_sql_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_sql_to_s3]

diff --git a/...roviders/airbyte/example_dags/__init__.py → tests/system/providers/airbyte/__init__.py b/...roviders/airbyte/example_dags/__init__.py → tests/system/providers/airbyte/__init__.py
diff --git a/...ample_dags/example_airbyte_trigger_job.py → ...rs/airbyte/example_airbyte_trigger_job.py b/...ample_dags/example_airbyte_trigger_job.py → ...rs/airbyte/example_airbyte_trigger_job.py
@@ -18,14 +18,19 @@
 
 """Example DAG demonstrating the usage of the AirbyteTriggerSyncOperator."""
 
+import os
 from datetime import datetime, timedelta
 
 from airflow import DAG
 from airflow.providers.airbyte.operators.airbyte import AirbyteTriggerSyncOperator
 from airflow.providers.airbyte.sensors.airbyte import AirbyteJobSensor
 
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
+DAG_ID = "example_airbyte_operator"
+CONN_ID = '15bc3800-82e4-48c3-a32d-620661273f28'
+
 with DAG(
-    dag_id='example_airbyte_operator',
+    dag_id=DAG_ID,
     schedule_interval=None,
     start_date=datetime(2021, 1, 1),
     dagrun_timeout=timedelta(minutes=60),
@@ -36,14 +41,14 @@
     # [START howto_operator_airbyte_synchronous]
     sync_source_destination = AirbyteTriggerSyncOperator(
         task_id='airbyte_sync_source_dest_example',
-        connection_id='15bc3800-82e4-48c3-a32d-620661273f28',
+        connection_id=CONN_ID,
     )
     # [END howto_operator_airbyte_synchronous]
 
     # [START howto_operator_airbyte_asynchronous]
     async_source_destination = AirbyteTriggerSyncOperator(
         task_id='airbyte_async_source_dest_example',
-        connection_id='15bc3800-82e4-48c3-a32d-620661273f28',
+        connection_id=CONN_ID,
         asynchronous=True,
     )
 
@@ -55,3 +60,8 @@
 
     # Task dependency created via `XComArgs`:
     #   async_source_destination >> airbyte_sensor
+
+from tests.system.utils import get_test_run  # noqa: E402
+
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)
diff --git a/...perators/test_s3_bucket_tagging_system.py → ...ystem/providers/amazon/aws/s3/__init__.py b/...perators/test_s3_bucket_tagging_system.py → ...ystem/providers/amazon/aws/s3/__init__.py
@@ -15,24 +15,3 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-import pytest
-
-from tests.test_utils.amazon_system_helpers import AWS_DAG_FOLDER, AmazonSystemTest, provide_aws_context
-
-BUCKET_NAME = "AmazonSystemTestBucket"
-
-
-@pytest.mark.backend("mysql", "postgres")
-@pytest.mark.system("amazon.aws")
-class ExampleDagsSystemTest(AmazonSystemTest):
-    @provide_aws_context()
-    def setUp(self):
-        super().setUp()
-
-    @provide_aws_context()
-    def tearDown(self):
-        super().tearDown()
-
-    @provide_aws_context()
-    def test_run_example_dag_bucket_tagging(self):
-        self.run_dag('s3_bucket_tagging_dag', AWS_DAG_FOLDER)