Skip to content

Sample Skipping Update #21247

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
scbedd merged 3 commits into from
Oct 13, 2021
Merged

Sample Skipping Update #21247

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b44dddd
allow specific skipping by python specifier set
scbedd Oct 13, 2021
25d3364
remove test exclude
scbedd Oct 13, 2021
d515712
move code into common. super useful
scbedd Oct 13, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions scripts/devops_tasks/common_tasks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import io
import re
import fnmatch
import platform

# Assumes the presence of setuptools
from pkg_resources import parse_version, parse_requirements, Requirement, WorkingSet, working_set
Expand Down Expand Up @@ -212,6 +213,13 @@ def filter_for_compatibility(package_set):
return collected_packages


def compare_python_version(version_spec):
current_sys_version = parse(platform.python_version())
spec_set = SpecifierSet(version_spec)

return current_sys_version in spec_set


# this function is where a glob string gets translated to a list of packages
# It is called by both BUILD (package) and TEST. In the future, this function will be the central location
# for handling targeting of release packages
Expand Down
103 changes: 54 additions & 49 deletions scripts/devops_tasks/test_run_samples.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,19 @@
import os
import logging
from fnmatch import fnmatch

try:
from subprocess import TimeoutExpired, check_call, CalledProcessError
except ImportError:
from subprocess32 import TimeoutExpired, check_call, CalledProcessError
from common_tasks import (
run_check_call,
process_glob_string,
)
from common_tasks import run_check_call, compare_python_version

logging.getLogger().setLevel(logging.INFO)

root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", ".."))

MINIMUM_TESTED_PYTHON_VERSION = ">=3.7.0"

"""
Some samples may "run forever" or need to be timed out after a period of time. Add them here in the following format:
TIMEOUT_SAMPLES = {
Expand Down Expand Up @@ -54,15 +54,15 @@
"recv_with_custom_starting_position_async.py": (10),
"sample_code_eventhub_async.py": (10),
"send_and_receive_amqp_annotated_message.py": (10),
"send_and_receive_amqp_annotated_message_async.py": (10)
"send_and_receive_amqp_annotated_message_async.py": (10),
},
"azure-eventhub-checkpointstoreblob": {
"receive_events_using_checkpoint_store.py": (10),
"receive_events_using_checkpoint_store_storage_api_version.py": (10)
"receive_events_using_checkpoint_store_storage_api_version.py": (10),
},
"azure-eventhub-checkpointstoreblob-aio": {
"receive_events_using_checkpoint_store_async.py": (10),
"receive_events_using_checkpoint_store_storage_api_version_async.py": (10)
"receive_events_using_checkpoint_store_storage_api_version_async.py": (10),
},
"azure-servicebus": {
"failure_and_recovery.py": (10),
Expand All @@ -71,8 +71,8 @@
"session_pool_receive.py": (20),
"receive_iterator_queue_async.py": (10),
"sample_code_servicebus_async.py": (30),
"session_pool_receive_async.py": (20)
}
"session_pool_receive_async.py": (20),
},
}


Expand All @@ -81,17 +81,16 @@
"azure-eventgrid": [
"__init__.py",
"consume_cloud_events_from_eventhub.py",
"consume_eventgrid_events_from_service_bus_queue.py"],
"consume_eventgrid_events_from_service_bus_queue.py",
],
"azure-eventhub": [
"connection_to_custom_endpoint_address.py",
"proxy.py",
"connection_to_custom_endpoint_address_async.py",
"iot_hub_connection_string_receive_async.py",
"proxy_async.py"
],
"azure-eventhub-checkpointstoretable":[
"receive_events_using_checkpoint_store.py"
"proxy_async.py",
],
"azure-eventhub-checkpointstoretable": ["receive_events_using_checkpoint_store.py"],
"azure-servicebus": [
"mgmt_queue.py",
"mgmt_rule.py",
Expand All @@ -104,19 +103,19 @@
"mgmt_subscription_async.py",
"mgmt_topic_async.py",
"proxy_async.py",
"receive_deferred_message_queue_async.py"
"receive_deferred_message_queue_async.py",
],
"azure-communication-chat": [
"chat_client_sample_async.py",
"chat_client_sample.py",
"chat_thread_client_sample_async.py",
"chat_thread_client_sample.py"
"chat_thread_client_sample.py",
],
"azure-communication-phonenumbers": [
"purchase_phone_number_sample_async.py",
"purchase_phone_number_sample.py",
"release_phone_number_sample_async.py",
"release_phone_number_sample.py"
"release_phone_number_sample.py",
],
"azure-ai-translation-document": [
"sample_list_document_statuses_with_filters_async.py",
Expand All @@ -130,19 +129,16 @@
"sample_manage_custom_models.py",
"sample_manage_custom_models_async.py",
],
"azure-ai-language-questionanswering": [
"sample_chat.py"
]
"azure-ai-language-questionanswering": ["sample_chat.py"],
}


def run_check_call_with_timeout(
command_array,
working_directory,
timeout,
pass_if_timeout,
acceptable_return_codes=[],
always_exit=False
always_exit=False,
):
"""This is copied from common_tasks.py with some additions.
Don't want to break anyone that's using the original code.
Expand All @@ -163,13 +159,9 @@ def run_check_call_with_timeout(
return err
except TimeoutExpired as err:
if pass_if_timeout:
logging.info(
"Sample timed out successfully"
)
logging.info("Sample timed out successfully")
else:
logging.info(
"Fail: Sample timed out"
)
logging.info("Fail: Sample timed out")
return err


Expand All @@ -180,9 +172,7 @@ def execute_sample(sample, samples_errors, timed):
if sys.version_info < (3, 5) and sample.endswith("_async.py"):
return

logging.info(
"Testing {}".format(sample)
)
logging.info("Testing {}".format(sample))
command_array = [sys.executable, sample]

if not timed:
Expand All @@ -195,13 +185,24 @@ def execute_sample(sample, samples_errors, timed):
sample_name = os.path.basename(sample)
if errors:
samples_errors.append(sample_name)
logging.info(
"ERROR: {}".format(sample_name)
)
logging.info("ERROR: {}".format(sample_name))
else:
logging.info(
"SUCCESS: {}.".format(sample_name)
)
logging.info("SUCCESS: {}.".format(sample_name))


def resolve_sample_ignore(sample_file, package_name):
ignored_files = [
(f, ">=2.7") if not isinstance(f, tuple) else f
for f in IGNORED_SAMPLES.get(package_name, [])
]
ignored_files_dict = {key: value for (key, value) in ignored_files}

if sample_file in ignored_files_dict and compare_python_version(
ignored_files_dict[sample_file]
):
return False
else:
return True


def run_samples(targeted_package):
Expand All @@ -219,7 +220,7 @@ def run_samples(targeted_package):
try:
with open(samples_dir_path + "/sample_dev_requirements.txt") as sample_dev_reqs:
for dep in sample_dev_reqs.readlines():
check_call([sys.executable, '-m', 'pip', 'install', dep])
check_call([sys.executable, "-m", "pip", "install", dep])
except IOError:
pass

Expand All @@ -232,14 +233,18 @@ def run_samples(targeted_package):
timeout, pass_if_timeout = timeout
else:
pass_if_timeout = True
timed_sample_paths.append((os.path.abspath(os.path.join(path, name)), timeout, pass_if_timeout))
elif fnmatch(name, "*.py") and name not in IGNORED_SAMPLES.get(package_name, []):
timed_sample_paths.append(
(
os.path.abspath(os.path.join(path, name)),
timeout,
pass_if_timeout,
)
)
elif fnmatch(name, "*.py") and resolve_sample_ignore(name, package_name):
sample_paths.append(os.path.abspath(os.path.join(path, name)))

if not sample_paths and not timed_sample_paths:
logging.info(
"No samples found in {}".format(targeted_package)
)
logging.info("No samples found in {}".format(targeted_package))
exit(0)

for sample in sample_paths:
Expand All @@ -252,9 +257,7 @@ def run_samples(targeted_package):
logging.error("Sample(s) that ran with errors: {}".format(samples_errors))
exit(1)

logging.info(
"All samples ran successfully in {}".format(targeted_package)
)
logging.info("All samples ran successfully in {}".format(targeted_package))


if __name__ == "__main__":
Expand All @@ -275,6 +278,8 @@ def run_samples(targeted_package):
service_dir = os.path.join("sdk", args.target_package)
target_dir = os.path.join(root_dir, service_dir)

logging.info("User opted to run samples")

run_samples(target_dir)
if compare_python_version(MINIMUM_TESTED_PYTHON_VERSION):
logging.info(
"User opted to run samples, and python version is greater than minimum supported."
)
run_samples(target_dir)