From 06e9f6baee6c41ee741ace35fe375d380ede9018 Mon Sep 17 00:00:00 2001 From: Saw-jan Date: Wed, 2 Oct 2024 17:11:02 +0545 Subject: [PATCH 1/2] test: retry failed scenario --- .drone.star | 7 ++++- test/gui/shared/scripts/bdd_hooks.py | 21 +++++++++++++-- .../shared/scripts/helpers/ConfigHelper.py | 27 ++++++++++++------- .../scripts/helpers/SetupClientHelper.py | 6 ++++- 4 files changed, 47 insertions(+), 14 deletions(-) diff --git a/.drone.star b/.drone.star index add6235b5ee..5301e22d9d8 100644 --- a/.drone.star +++ b/.drone.star @@ -209,8 +209,13 @@ def gui_test_pipeline(ctx): "--tags ~@skipOnLinux", ] - if not "full-ci" in ctx.build.title.lower() and ctx.build.event == "pull_request": + # '--retry' and '--abortOnFail' are mutually exclusive + if "debug-gui-test" in ctx.build.title.lower() or "full-ci" in ctx.build.title.lower() or ctx.build.event in ("tag", "cron"): + # retry failed tests once + squish_parameters.append("--retry 1") + elif not "full-ci" in ctx.build.title.lower() and ctx.build.event == "pull_request": squish_parameters.append("--abortOnFail") + pass if params.get("skip", False): continue diff --git a/test/gui/shared/scripts/bdd_hooks.py b/test/gui/shared/scripts/bdd_hooks.py index a464c93ba18..a5902b27772 100644 --- a/test/gui/shared/scripts/bdd_hooks.py +++ b/test/gui/shared/scripts/bdd_hooks.py @@ -50,6 +50,7 @@ # this will reset in every test suite PREVIOUS_FAIL_RESULT_COUNT = 0 PREVIOUS_ERROR_RESULT_COUNT = 0 +PREVIOUS_SCENARIO = "" # runs before a feature @@ -65,6 +66,11 @@ def hook(context): def hook(context): unlock_keyring() clear_scenario_config() + global PREVIOUS_SCENARIO + if PREVIOUS_SCENARIO == context.title: + test.log("[INFO] Retrying this failed scenario...") + set_config("retrying", True) + PREVIOUS_SCENARIO = context.title # runs before every scenario @@ -178,7 +184,7 @@ def save_screenrecord(filename): file_parts = filename.rsplit(".", 1) filename = f"{file_parts[0]}_{idx+1}.{file_parts[1]}" shutil.move(video, os.path.join(screenrecords_dir, filename)) - + set_config("videoRecordCount", get_config("videoRecordCount") + 1) shutil.rmtree(prefix_path_namespace(video_dir)) @@ -202,7 +208,11 @@ def hook(context): test.log("Failed to save screenshot") # check video report - if get_config("screenRecordOnFailure"): + if ( + get_config("screenRecordOnFailure") + or get_config("retrying") + and get_config("videoRecordCount") < get_config("videoRecordLimit") + ): filename = get_screenrecord_name(context.title) save_screenrecord(filename) @@ -250,6 +260,13 @@ def hook(context): delete_created_users() +@OnFeatureEnd +def hook(context): + test.log("----------------") + test.log(str(get_config("videoRecordCount"))) + test.log("----------------") + + def teardown_client(): # Cleanup user accounts from UI for Windows platform # It is not needed for Linux so skipping it in order to save CI time diff --git a/test/gui/shared/scripts/helpers/ConfigHelper.py b/test/gui/shared/scripts/helpers/ConfigHelper.py index a853737bb29..407d3283c1c 100644 --- a/test/gui/shared/scripts/helpers/ConfigHelper.py +++ b/test/gui/shared/scripts/helpers/ConfigHelper.py @@ -79,6 +79,8 @@ def get_default_home_dir(): DEFAULT_PATH_CONFIG = { 'custom_lib': os.path.abspath('../shared/scripts/custom_lib'), 'home_dir': get_default_home_dir(), + # allow to record first 5 videos + 'videoRecordLimit': 5, } # default config values @@ -96,11 +98,15 @@ def get_default_home_dir(): 'guiTestReportDir': os.path.abspath('../reports'), 'ocis': False, 'screenRecordOnFailure': False, + 'retrying': False, + 'videoRecordCount': 0, } CONFIG.update(DEFAULT_PATH_CONFIG) READONLY_CONFIG = list(CONFIG_ENV_MAP.keys()) + list(DEFAULT_PATH_CONFIG.keys()) +SCENARIO_CONFIGS = {} + def read_cfg_file(cfg_path): cfg = ConfigParser() @@ -135,6 +141,8 @@ def init_config(): # Set the default values if empty for key, value in CONFIG.items(): + if key == 'videoRecordCount': + CONFIG[key] = get_config('videoRecordCount') if key in ('maxSyncTimeout', 'minSyncTimeout'): CONFIG[key] = builtins.int(value) elif key in ('localBackendUrl', 'middlewareUrl', 'secureLocalBackendUrl'): @@ -154,22 +162,21 @@ def init_config(): CONFIG[key] = value.rstrip('/') + '/' -def get_config(key=None): - if key: - return CONFIG[key] - return CONFIG +def get_config(key): + return CONFIG[key] def set_config(key, value): if key in READONLY_CONFIG: raise KeyError(f'Cannot set read-only config: {key}') + # save the initial config value + # 'videoRecordCount' is a special case, it is not a scenario config + # but is a global config throughout the test run + if key not in SCENARIO_CONFIGS and not key == 'videoRecordCount': + SCENARIO_CONFIGS[key] = CONFIG.get(key) CONFIG[key] = value def clear_scenario_config(): - global CONFIG - initial_config = {} - for key in READONLY_CONFIG: - initial_config[key] = CONFIG[key] - - CONFIG = initial_config + for key, value in SCENARIO_CONFIGS.items(): + CONFIG[key] = value diff --git a/test/gui/shared/scripts/helpers/SetupClientHelper.py b/test/gui/shared/scripts/helpers/SetupClientHelper.py index ea6ebf2ed9d..337c8fa25e6 100644 --- a/test/gui/shared/scripts/helpers/SetupClientHelper.py +++ b/test/gui/shared/scripts/helpers/SetupClientHelper.py @@ -103,7 +103,11 @@ def start_client(): + ' --logdebug' + ' --logflush' ) - if get_config('screenRecordOnFailure'): + if ( + get_config('screenRecordOnFailure') + or get_config('retrying') + and get_config('videoRecordCount') < get_config('videoRecordLimit') + ): test.startVideoCapture() From 3b4d16ed987a60b5984dfed4ea856afc066e74a7 Mon Sep 17 00:00:00 2001 From: Saw-jan Date: Thu, 3 Oct 2024 15:41:43 +0545 Subject: [PATCH 2/2] test: report helper --- .drone.star | 5 +- test/gui/.pylintrc | 1 + test/gui/config.sample.ini | 2 +- test/gui/shared/scripts/bdd_hooks.py | 80 +++---------------- .../shared/scripts/helpers/ConfigHelper.py | 16 ++-- .../shared/scripts/helpers/ReportHelper.py | 75 +++++++++++++++++ .../scripts/helpers/SetupClientHelper.py | 8 +- 7 files changed, 101 insertions(+), 86 deletions(-) create mode 100644 test/gui/shared/scripts/helpers/ReportHelper.py diff --git a/.drone.star b/.drone.star index 5301e22d9d8..be738ad8e9b 100644 --- a/.drone.star +++ b/.drone.star @@ -210,12 +210,11 @@ def gui_test_pipeline(ctx): ] # '--retry' and '--abortOnFail' are mutually exclusive - if "debug-gui-test" in ctx.build.title.lower() or "full-ci" in ctx.build.title.lower() or ctx.build.event in ("tag", "cron"): + if "full-ci" in ctx.build.title.lower() or ctx.build.event in ("tag", "cron"): # retry failed tests once squish_parameters.append("--retry 1") elif not "full-ci" in ctx.build.title.lower() and ctx.build.event == "pull_request": squish_parameters.append("--abortOnFail") - pass if params.get("skip", False): continue @@ -339,7 +338,7 @@ def gui_tests(squish_parameters = "", server_type = "oc10"): "STACKTRACE_FILE": "%s/stacktrace.log" % dir["guiTestReport"], "PLAYWRIGHT_BROWSERS_PATH": "%s/.playwright" % dir["base"], "OWNCLOUD_CORE_DUMP": 1, - "SCREEN_RECORD_ON_FAILURE": False, + "RECORD_VIDEO_ON_FAILURE": False, # allow to use any available pnpm version "COREPACK_ENABLE_STRICT": 0, }, diff --git a/test/gui/.pylintrc b/test/gui/.pylintrc index 146f973f5ed..11c27e4440c 100644 --- a/test/gui/.pylintrc +++ b/test/gui/.pylintrc @@ -18,6 +18,7 @@ ignore-paths=^tst_.*/test.py$, shared/scripts/custom_lib ignored-modules= squish, + squishinfo, object, objectmaphelper, test, diff --git a/test/gui/config.sample.ini b/test/gui/config.sample.ini index 1c374b19dc1..23dc09f61f7 100644 --- a/test/gui/config.sample.ini +++ b/test/gui/config.sample.ini @@ -11,4 +11,4 @@ TEMP_FOLDER_PATH= CLIENT_CONFIG_DIR= GUI_TEST_REPORT_DIR= OCIS=false -SCREEN_RECORD_ON_FAILURE=false \ No newline at end of file +RECORD_VIDEO_ON_FAILURE=false \ No newline at end of file diff --git a/test/gui/shared/scripts/bdd_hooks.py b/test/gui/shared/scripts/bdd_hooks.py index a5902b27772..992cdf4bb70 100644 --- a/test/gui/shared/scripts/bdd_hooks.py +++ b/test/gui/shared/scripts/bdd_hooks.py @@ -17,7 +17,6 @@ # manual for a complete reference of the available API. import shutil import os -import glob from urllib import request, error from datetime import datetime @@ -36,8 +35,9 @@ ) from helpers.api.utils import url_join from helpers.FilesHelper import prefix_path_namespace, cleanup_created_paths -from pageObjects.Toolbar import Toolbar +from helpers.ReportHelper import save_video_recording, take_screenshot +from pageObjects.Toolbar import Toolbar from pageObjects.AccountSetting import AccountSetting from pageObjects.AccountConnectionWizard import AccountConnectionWizard @@ -148,44 +148,9 @@ def scenario_failed(): ) -def get_screenshot_name(title): - return title.replace(" ", "_").replace("/", "_").strip(".") + ".png" - - -def get_screenrecord_name(title): - return title.replace(" ", "_").replace("/", "_").strip(".") + ".mp4" - - -def save_screenrecord(filename): - try: - # do not throw if stopVideoCapture() fails - test.stopVideoCapture() - except: - test.log("Failed to stop screen recording") - - if not (video_dir := squishinfo.resultDir): - video_dir = squishinfo.testCase - else: - test_case = "/".join(squishinfo.testCase.split("/")[-2:]) - video_dir = os.path.join(video_dir, test_case) - video_dir = os.path.join(video_dir, "attachments") - - if scenario_failed(): - video_files = glob.glob(f"{video_dir}/**/*.mp4", recursive=True) - screenrecords_dir = os.path.join( - get_config("guiTestReportDir"), "screenrecords" - ) - if not os.path.exists(screenrecords_dir): - os.makedirs(screenrecords_dir) - # reverse the list to get the latest video first - video_files.reverse() - for idx, video in enumerate(video_files): - if idx: - file_parts = filename.rsplit(".", 1) - filename = f"{file_parts[0]}_{idx+1}.{file_parts[1]}" - shutil.move(video, os.path.join(screenrecords_dir, filename)) - set_config("videoRecordCount", get_config("videoRecordCount") + 1) - shutil.rmtree(prefix_path_namespace(video_dir)) +def scenario_title_to_filename(title): + # scenario name can have "/" which is invalid filename + return title.replace(" ", "_").replace("/", "_").strip(".") # runs after every scenario @@ -195,26 +160,14 @@ def hook(context): clear_waited_after_sync() close_socket_connection() - # capture a screenshot if there is error or test failure in the current scenario execution - if scenario_failed() and os.getenv("CI") and is_linux(): - # scenario name can have "/" which is invalid filename - filename = get_screenshot_name(context.title) - directory = os.path.join(get_config("guiTestReportDir"), "screenshots") - if not os.path.exists(directory): - os.makedirs(directory) - try: - squish.saveDesktopScreenshot(os.path.join(directory, filename)) - except: - test.log("Failed to save screenshot") - - # check video report - if ( - get_config("screenRecordOnFailure") - or get_config("retrying") - and get_config("videoRecordCount") < get_config("videoRecordLimit") - ): - filename = get_screenrecord_name(context.title) - save_screenrecord(filename) + # generate screenshot and video reports + if is_linux(): + filename = scenario_title_to_filename(context.title) + if scenario_failed(): + take_screenshot(f"{filename}.png") + + if get_config("videoRecordingStarted"): + save_video_recording(f"{filename}.mp4", scenario_failed()) # teardown accounts and configs teardown_client() @@ -260,13 +213,6 @@ def hook(context): delete_created_users() -@OnFeatureEnd -def hook(context): - test.log("----------------") - test.log(str(get_config("videoRecordCount"))) - test.log("----------------") - - def teardown_client(): # Cleanup user accounts from UI for Windows platform # It is not needed for Linux so skipping it in order to save CI time diff --git a/test/gui/shared/scripts/helpers/ConfigHelper.py b/test/gui/shared/scripts/helpers/ConfigHelper.py index 407d3283c1c..83b09184665 100644 --- a/test/gui/shared/scripts/helpers/ConfigHelper.py +++ b/test/gui/shared/scripts/helpers/ConfigHelper.py @@ -73,7 +73,7 @@ def get_default_home_dir(): 'clientConfigDir': 'CLIENT_CONFIG_DIR', 'guiTestReportDir': 'GUI_TEST_REPORT_DIR', 'ocis': 'OCIS', - 'screenRecordOnFailure': 'SCREEN_RECORD_ON_FAILURE', + 'recordVideoOnFailure': 'RECORD_VIDEO_ON_FAILURE', } DEFAULT_PATH_CONFIG = { @@ -97,9 +97,9 @@ def get_default_home_dir(): 'clientConfigDir': get_config_home(), 'guiTestReportDir': os.path.abspath('../reports'), 'ocis': False, - 'screenRecordOnFailure': False, + 'recordVideoOnFailure': False, 'retrying': False, - 'videoRecordCount': 0, + 'videoRecordingStarted': False, } CONFIG.update(DEFAULT_PATH_CONFIG) @@ -114,7 +114,7 @@ def read_cfg_file(cfg_path): for key, _ in CONFIG.items(): if key in CONFIG_ENV_MAP: if value := cfg.get('DEFAULT', CONFIG_ENV_MAP[key]): - if key in ('ocis', 'screenRecordOnFailure'): + if key in ('ocis', 'recordVideoOnFailure'): CONFIG[key] = value == 'true' else: CONFIG[key] = value @@ -134,15 +134,13 @@ def init_config(): # read and override configs from environment variables for key, value in CONFIG_ENV_MAP.items(): if os.environ.get(value): - if key in ('ocis', 'screenRecordOnFailure'): + if key in ('ocis', 'recordVideoOnFailure'): CONFIG[key] = os.environ.get(value) == 'true' else: CONFIG[key] = os.environ.get(value) # Set the default values if empty for key, value in CONFIG.items(): - if key == 'videoRecordCount': - CONFIG[key] = get_config('videoRecordCount') if key in ('maxSyncTimeout', 'minSyncTimeout'): CONFIG[key] = builtins.int(value) elif key in ('localBackendUrl', 'middlewareUrl', 'secureLocalBackendUrl'): @@ -170,9 +168,7 @@ def set_config(key, value): if key in READONLY_CONFIG: raise KeyError(f'Cannot set read-only config: {key}') # save the initial config value - # 'videoRecordCount' is a special case, it is not a scenario config - # but is a global config throughout the test run - if key not in SCENARIO_CONFIGS and not key == 'videoRecordCount': + if key not in SCENARIO_CONFIGS: SCENARIO_CONFIGS[key] = CONFIG.get(key) CONFIG[key] = value diff --git a/test/gui/shared/scripts/helpers/ReportHelper.py b/test/gui/shared/scripts/helpers/ReportHelper.py new file mode 100644 index 00000000000..fe778105820 --- /dev/null +++ b/test/gui/shared/scripts/helpers/ReportHelper.py @@ -0,0 +1,75 @@ +import os +import glob +import shutil +import test +import squish +import squishinfo + +from helpers.ConfigHelper import get_config +from helpers.FilesHelper import prefix_path_namespace + + +def get_screenrecords_path(): + return os.path.join(get_config("guiTestReportDir"), "screenrecords") + + +def get_screenshots_path(): + return os.path.join(get_config("guiTestReportDir"), "screenshots") + + +def is_video_enabled(): + return ( + get_config("recordVideoOnFailure") + or get_config("retrying") + and not reached_video_limit() + ) + + +def reached_video_limit(): + video_report_dir = get_screenrecords_path() + if not os.path.exists(video_report_dir): + return False + entries = [f for f in os.scandir(video_report_dir) if f.is_file()] + return len(entries) >= get_config("videoRecordLimit") + + +def save_video_recording(filename, test_failed): + try: + # do not throw if stopVideoCapture() fails + test.stopVideoCapture() + except: + test.log("Failed to stop screen recording") + + if not (video_dir := squishinfo.resultDir): + video_dir = squishinfo.testCase + else: + test_case = "/".join(squishinfo.testCase.split("/")[-2:]) + video_dir = os.path.join(video_dir, test_case) + video_dir = os.path.join(video_dir, "attachments") + + # if the test failed + # move videos to the screenrecords directory + if test_failed: + video_files = glob.glob(f"{video_dir}/**/*.mp4", recursive=True) + screenrecords_dir = get_screenrecords_path() + if not os.path.exists(screenrecords_dir): + os.makedirs(screenrecords_dir) + # reverse the list to get the latest video first + video_files.reverse() + for idx, video in enumerate(video_files): + if idx: + file_parts = filename.rsplit(".", 1) + filename = f"{file_parts[0]}_{idx+1}.{file_parts[1]}" + shutil.move(video, os.path.join(screenrecords_dir, filename)) + # remove the video directory + shutil.rmtree(prefix_path_namespace(video_dir)) + + +def take_screenshot(filename): + directory = get_screenshots_path() + if not os.path.exists(directory): + os.makedirs(directory) + try: + squish.saveDesktopScreenshot(os.path.join(directory, filename)) + except: + test.log("Failed to save screenshot") diff --git a/test/gui/shared/scripts/helpers/SetupClientHelper.py b/test/gui/shared/scripts/helpers/SetupClientHelper.py index 337c8fa25e6..594b1ae6419 100644 --- a/test/gui/shared/scripts/helpers/SetupClientHelper.py +++ b/test/gui/shared/scripts/helpers/SetupClientHelper.py @@ -13,6 +13,7 @@ from helpers.SyncHelper import listen_sync_status_for_item from helpers.api.utils import url_join from helpers.UserHelper import get_displayname_for_user +from helpers.ReportHelper import is_video_enabled def substitute_inline_codes(value): @@ -103,12 +104,9 @@ def start_client(): + ' --logdebug' + ' --logflush' ) - if ( - get_config('screenRecordOnFailure') - or get_config('retrying') - and get_config('videoRecordCount') < get_config('videoRecordLimit') - ): + if is_video_enabled(): test.startVideoCapture() + set_config('videoRecordingStarted', True) def get_polling_interval():