From 8a3fb52bcf0f4acaecf2219db170a2714a89bbcb Mon Sep 17 00:00:00 2001 From: Vladyslav Podorozhnyi Date: Fri, 8 Dec 2023 20:57:03 +0100 Subject: [PATCH 01/11] Update full-integration-tests.yaml --- .github/workflows/full-integration-tests.yaml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.github/workflows/full-integration-tests.yaml b/.github/workflows/full-integration-tests.yaml index 5413602..823c95f 100644 --- a/.github/workflows/full-integration-tests.yaml +++ b/.github/workflows/full-integration-tests.yaml @@ -52,7 +52,7 @@ jobs: OUTPUT_FILE="integration-testsuites.json" TESTSUITE_DIR="./testsuite/Magento" CODE_DIR="../../../app/code" - MAX_DIRS_PER_LINE=15 + MAX_DIRS_PER_LINE=1 # Initialize variables dir_count=0 @@ -158,17 +158,6 @@ jobs: run_magento_integration_tests_real_suite: 0 base_directory: "./main" - # TODO: this have to be removed after we analyse and fix all tests - # added now because it kills too many tests that are going to be running - - name: Skip invalid tests - working-directory: ./main - run: | - echo "Patch file dev/tests/integration/testsuite/Magento/Downloadable/Block/Sales/Order/Email/Items/Order/DownloadableTest.php" - echo "Add new line after line 47" - sed -i '47 a $this->markTestSkipped('"'"'TODO: skipped temporary due to execution errors'"'"');' dev/tests/integration/testsuite/Magento/Downloadable/Block/Sales/Order/Email/Items/Order/DownloadableTest.php - echo "Patch complete" - # here bash code that perform the patch ends - - name: Create Mage-OS testsuite working-directory: ./main/dev/tests/integration run: | From 61f98766c31797fada366ae1aee9aaa803cd46ce Mon Sep 17 00:00:00 2001 From: Vlad Podorozhnyi Date: Fri, 22 Dec 2023 13:48:47 +0100 Subject: [PATCH 02/11] Add optional input on integration test directory. --- .github/workflows/full-integration-tests.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/full-integration-tests.yaml b/.github/workflows/full-integration-tests.yaml index 823c95f..14b74a7 100644 --- a/.github/workflows/full-integration-tests.yaml +++ b/.github/workflows/full-integration-tests.yaml @@ -12,6 +12,10 @@ on: type: string description: "head SHA" required: true + test_directory: + type: string + description: "Test directory to run integration tests" + required: false permissions: contents: write @@ -49,6 +53,16 @@ jobs: name: Calculate Matrix for testsuite working-directory: dev/tests/integration run: | + + # check if ${{ github.event.inputs.test_directory }} is set + # then set ${{ github.event.inputs.test_directory }} as output and exit + # old output: echo "testsuite_dirs=$(jq -c .testsuites integration-testsuites.json)" >> "$GITHUB_OUTPUT" + + if [ -n "${{ github.event.inputs.test_directory }}" ]; then + echo "testsuite_dirs=['${{ github.event.inputs.test_directory }}']" >> "$GITHUB_OUTPUT" + exit 0 + fi + OUTPUT_FILE="integration-testsuites.json" TESTSUITE_DIR="./testsuite/Magento" CODE_DIR="../../../app/code" From fe6af88b6de4c6cfd9c8d47381ba6d7ec3c27a8c Mon Sep 17 00:00:00 2001 From: Vlad Podorozhnyi Date: Tue, 9 Jan 2024 23:01:50 +0100 Subject: [PATCH 03/11] Add exclusion list - directories from exclusion list should be also splited on multiple batches --- .github/workflows/full-integration-tests.yaml | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/.github/workflows/full-integration-tests.yaml b/.github/workflows/full-integration-tests.yaml index 14b74a7..7acc732 100644 --- a/.github/workflows/full-integration-tests.yaml +++ b/.github/workflows/full-integration-tests.yaml @@ -54,10 +54,6 @@ jobs: working-directory: dev/tests/integration run: | - # check if ${{ github.event.inputs.test_directory }} is set - # then set ${{ github.event.inputs.test_directory }} as output and exit - # old output: echo "testsuite_dirs=$(jq -c .testsuites integration-testsuites.json)" >> "$GITHUB_OUTPUT" - if [ -n "${{ github.event.inputs.test_directory }}" ]; then echo "testsuite_dirs=['${{ github.event.inputs.test_directory }}']" >> "$GITHUB_OUTPUT" exit 0 @@ -67,6 +63,9 @@ jobs: TESTSUITE_DIR="./testsuite/Magento" CODE_DIR="../../../app/code" MAX_DIRS_PER_LINE=1 + + ## variable with array of exclusion list of directories - they will be handled separately + EXCLUSION_LIST=("./testsuite/Magento/Catalog" "./testsuite/Magento/Bundle") # Initialize variables dir_count=0 @@ -94,7 +93,11 @@ jobs: # Iterate over the directories and populate the JSON content while IFS= read -r -d '' dir; do - add_dir_to_line "${dir}" + # if dir is in EXCLUSION_LIST then skip it + if [[ " ${EXCLUSION_LIST[@]} " =~ " ${dir} " ]]; then + continue + fi + add_dir_to_line "${dir}" done < <(find "$TESTSUITE_DIR" -mindepth 1 -maxdepth 1 -type d -print0) # Add app/code integration test directories @@ -102,7 +105,23 @@ jobs: relative_dir="${dir}" # Convert absolute path to relative add_dir_to_line "$relative_dir" done < <(find "$CODE_DIR" -mindepth 4 -maxdepth 4 -type d -name 'Integration' -print0) - + + # Handle exclusion list + # run over EXCLUSION_LIST, find all "*Test.php" files, and put them in array. Exclude directories. + EXCLUSION_LIST_FILES=() + for dir in "${EXCLUSION_LIST[@]}"; do + while IFS= read -r -d '' file; do + EXCLUSION_LIST_FILES+=("$file") + done < <(find "$dir" -mindepth 1 -type f -name '*Test.php' -print0) + done + + ## run over EXCLUSION_LIST_FILES and call add_dir_to_line for each file + MAX_DIRS_PER_LINE=30 + dir_count=0 + for file in "${EXCLUSION_LIST_FILES[@]}"; do + add_dir_to_line "$file" + done + # Handle the last line if it's not empty if [ -n "$current_line" ]; then json_content="$json_content$current_line" From a020ed0717ff32e47306223dfd23ee56824e1994 Mon Sep 17 00:00:00 2001 From: Vlad Podorozhnyi Date: Tue, 9 Jan 2024 23:13:42 +0100 Subject: [PATCH 04/11] Add exclusion list - directories from exclusion list should be also splited on multiple batches --- .github/workflows/full-integration-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/full-integration-tests.yaml b/.github/workflows/full-integration-tests.yaml index 7acc732..ba2a3ae 100644 --- a/.github/workflows/full-integration-tests.yaml +++ b/.github/workflows/full-integration-tests.yaml @@ -116,7 +116,7 @@ jobs: done ## run over EXCLUSION_LIST_FILES and call add_dir_to_line for each file - MAX_DIRS_PER_LINE=30 + MAX_DIRS_PER_LINE=10 dir_count=0 for file in "${EXCLUSION_LIST_FILES[@]}"; do add_dir_to_line "$file" From 03ccf900d78295ae9212d17fe59b9deb984d0241 Mon Sep 17 00:00:00 2001 From: Vlad Podorozhnyi Date: Wed, 10 Jan 2024 00:14:42 +0100 Subject: [PATCH 05/11] Add exclusion list - directories from exclusion list should be also splited on multiple batches --- .github/workflows/full-integration-tests.yaml | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/full-integration-tests.yaml b/.github/workflows/full-integration-tests.yaml index ba2a3ae..8266ea6 100644 --- a/.github/workflows/full-integration-tests.yaml +++ b/.github/workflows/full-integration-tests.yaml @@ -64,9 +64,12 @@ jobs: CODE_DIR="../../../app/code" MAX_DIRS_PER_LINE=1 - ## variable with array of exclusion list of directories - they will be handled separately + ## variable with array of exclusion list of directories - they will be handled separately and splitted on more batches EXCLUSION_LIST=("./testsuite/Magento/Catalog" "./testsuite/Magento/Bundle") + ## variable with array of exclusion list of files - separate run will be executed for each file (separate place in tests matrix) + STANDALONE_TESTS=("./testsuite/Magento/AsynchronousOperations/Model/MassScheduleTest.php") + # Initialize variables dir_count=0 json_content="{\n\t\"testsuites\": [\n\t\t\"" @@ -111,15 +114,26 @@ jobs: EXCLUSION_LIST_FILES=() for dir in "${EXCLUSION_LIST[@]}"; do while IFS= read -r -d '' file; do + # check if file is in STANDALONE_TESTS + if [[ " ${STANDALONE_TESTS[@]} " =~ " ${file} " ]]; then + continue + fi EXCLUSION_LIST_FILES+=("$file") - done < <(find "$dir" -mindepth 1 -type f -name '*Test.php' -print0) + done < <(find "$dir" -mindepth 1 -type f -name '*Test.php' -print0) done ## run over EXCLUSION_LIST_FILES and call add_dir_to_line for each file MAX_DIRS_PER_LINE=10 dir_count=0 for file in "${EXCLUSION_LIST_FILES[@]}"; do - add_dir_to_line "$file" + add_dir_to_line "$file" + done + + ## run over STANDALONE_TESTS and call add_dir_to_line for each file + MAX_DIRS_PER_LINE=1 + dir_count=0 + for file in "${STANDALONE_TESTS[@]}"; do + add_dir_to_line "$file" done # Handle the last line if it's not empty From 1ff7ae56f257d43ee78db197d10bd35dd92b5ddf Mon Sep 17 00:00:00 2001 From: Vlad Podorozhnyi Date: Wed, 10 Jan 2024 00:17:32 +0100 Subject: [PATCH 06/11] Add exclusion list - directories from exclusion list should be also splited on multiple batches --- .github/workflows/full-integration-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/full-integration-tests.yaml b/.github/workflows/full-integration-tests.yaml index 8266ea6..a977128 100644 --- a/.github/workflows/full-integration-tests.yaml +++ b/.github/workflows/full-integration-tests.yaml @@ -65,7 +65,7 @@ jobs: MAX_DIRS_PER_LINE=1 ## variable with array of exclusion list of directories - they will be handled separately and splitted on more batches - EXCLUSION_LIST=("./testsuite/Magento/Catalog" "./testsuite/Magento/Bundle") + EXCLUSION_LIST=("./testsuite/Magento/Catalog" "./testsuite/Magento/Bundle" "./testsuite/Magento/AsynchronousOperations") ## variable with array of exclusion list of files - separate run will be executed for each file (separate place in tests matrix) STANDALONE_TESTS=("./testsuite/Magento/AsynchronousOperations/Model/MassScheduleTest.php") From d8fa78e26ab1980a64f7c947f372c11fb24bc3cb Mon Sep 17 00:00:00 2001 From: Vlad Podorozhnyi Date: Wed, 10 Jan 2024 00:41:27 +0100 Subject: [PATCH 07/11] Add exclusion list - directories from exclusion list should be also splited on multiple batches --- .github/workflows/full-integration-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/full-integration-tests.yaml b/.github/workflows/full-integration-tests.yaml index a977128..4be42b4 100644 --- a/.github/workflows/full-integration-tests.yaml +++ b/.github/workflows/full-integration-tests.yaml @@ -68,7 +68,7 @@ jobs: EXCLUSION_LIST=("./testsuite/Magento/Catalog" "./testsuite/Magento/Bundle" "./testsuite/Magento/AsynchronousOperations") ## variable with array of exclusion list of files - separate run will be executed for each file (separate place in tests matrix) - STANDALONE_TESTS=("./testsuite/Magento/AsynchronousOperations/Model/MassScheduleTest.php") + STANDALONE_TESTS=("./testsuite/Magento/AsynchronousOperations/Model/MassScheduleTest.php" "./testsuite/Magento/Catalog/Observer/SwitchPriceAttributeScopeOnConfigChangeTest.php") # Initialize variables dir_count=0 From 7b3964e1abb46c6884d160839ce468d65b2895b3 Mon Sep 17 00:00:00 2001 From: Vlad Podorozhnyi Date: Wed, 10 Jan 2024 01:26:58 +0100 Subject: [PATCH 08/11] Add exclusion list - directories from exclusion list should be also splited on multiple batches --- .github/workflows/full-integration-tests.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/full-integration-tests.yaml b/.github/workflows/full-integration-tests.yaml index 4be42b4..abba2dc 100644 --- a/.github/workflows/full-integration-tests.yaml +++ b/.github/workflows/full-integration-tests.yaml @@ -124,14 +124,16 @@ jobs: ## run over EXCLUSION_LIST_FILES and call add_dir_to_line for each file MAX_DIRS_PER_LINE=10 - dir_count=0 + dir_count=MAX_DIRS_PER_LINE-1 + add_dir_to_line "" for file in "${EXCLUSION_LIST_FILES[@]}"; do add_dir_to_line "$file" done ## run over STANDALONE_TESTS and call add_dir_to_line for each file MAX_DIRS_PER_LINE=1 - dir_count=0 + dir_count=MAX_DIRS_PER_LINE-1 + add_dir_to_line "" for file in "${STANDALONE_TESTS[@]}"; do add_dir_to_line "$file" done From cf28b67b69ab850fcec879396fa711317b6405a1 Mon Sep 17 00:00:00 2001 From: Vlad Podorozhnyi Date: Wed, 10 Jan 2024 09:04:31 +0100 Subject: [PATCH 09/11] Add exclusion list - directories from exclusion list should be also splited on multiple batches --- .github/workflows/full-integration-tests.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/full-integration-tests.yaml b/.github/workflows/full-integration-tests.yaml index abba2dc..cbb8b44 100644 --- a/.github/workflows/full-integration-tests.yaml +++ b/.github/workflows/full-integration-tests.yaml @@ -53,6 +53,8 @@ jobs: name: Calculate Matrix for testsuite working-directory: dev/tests/integration run: | + + ### TODO: rebuild all that hell to node-js if [ -n "${{ github.event.inputs.test_directory }}" ]; then echo "testsuite_dirs=['${{ github.event.inputs.test_directory }}']" >> "$GITHUB_OUTPUT" @@ -65,7 +67,7 @@ jobs: MAX_DIRS_PER_LINE=1 ## variable with array of exclusion list of directories - they will be handled separately and splitted on more batches - EXCLUSION_LIST=("./testsuite/Magento/Catalog" "./testsuite/Magento/Bundle" "./testsuite/Magento/AsynchronousOperations") + EXCLUSION_LIST=("./testsuite/Magento/Catalog" "./testsuite/Magento/Bundle" "./testsuite/Magento/AsynchronousOperations" "./testsuite/Magento/Sales") ## variable with array of exclusion list of files - separate run will be executed for each file (separate place in tests matrix) STANDALONE_TESTS=("./testsuite/Magento/AsynchronousOperations/Model/MassScheduleTest.php" "./testsuite/Magento/Catalog/Observer/SwitchPriceAttributeScopeOnConfigChangeTest.php") @@ -123,7 +125,7 @@ jobs: done ## run over EXCLUSION_LIST_FILES and call add_dir_to_line for each file - MAX_DIRS_PER_LINE=10 + MAX_DIRS_PER_LINE=20 dir_count=MAX_DIRS_PER_LINE-1 add_dir_to_line "" for file in "${EXCLUSION_LIST_FILES[@]}"; do From 7b222e281f15fa58d7322e5dd2c785280a0c16ac Mon Sep 17 00:00:00 2001 From: Vlad Podorozhnyi Date: Wed, 10 Jan 2024 10:01:52 +0100 Subject: [PATCH 10/11] Add exclusion list - directories from exclusion list should be also splited on multiple batches --- .github/workflows/full-integration-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/full-integration-tests.yaml b/.github/workflows/full-integration-tests.yaml index cbb8b44..4016f9f 100644 --- a/.github/workflows/full-integration-tests.yaml +++ b/.github/workflows/full-integration-tests.yaml @@ -125,7 +125,7 @@ jobs: done ## run over EXCLUSION_LIST_FILES and call add_dir_to_line for each file - MAX_DIRS_PER_LINE=20 + MAX_DIRS_PER_LINE=10 dir_count=MAX_DIRS_PER_LINE-1 add_dir_to_line "" for file in "${EXCLUSION_LIST_FILES[@]}"; do From bb9ce5281f9f9d4ba5cbebff5d573ab56f18f3c7 Mon Sep 17 00:00:00 2001 From: Vladyslav Podorozhnyi Date: Wed, 24 Jan 2024 10:11:15 +0100 Subject: [PATCH 11/11] Update full-integration-tests.yaml --- .github/workflows/full-integration-tests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/full-integration-tests.yaml b/.github/workflows/full-integration-tests.yaml index 4016f9f..0b49d7f 100644 --- a/.github/workflows/full-integration-tests.yaml +++ b/.github/workflows/full-integration-tests.yaml @@ -1,3 +1,5 @@ +## Workflow is only for Magento 2 Main repo - app/code and dev/tests/integration are supported only +## 3rd party modules are not supported yet name: Integration Tests - Full Test Suite run-name: ${{ github.actor }} is running Full Integration Test Suite