From be569ba4accf8a4e08ab3e810d6420e501ec6595 Mon Sep 17 00:00:00 2001 From: davidlion Date: Tue, 8 Jul 2025 21:20:29 -0400 Subject: [PATCH 1/7] fix(taskfiles)!: Check the specified FILE_SHA256 matches the downloaded file in `remote:curl`. --- exports/taskfiles/utils/remote.yaml | 8 +++++--- taskfiles/remote/tests.yaml | 27 ++++++++++++++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/exports/taskfiles/utils/remote.yaml b/exports/taskfiles/utils/remote.yaml index c7f1fa3..116de4c 100644 --- a/exports/taskfiles/utils/remote.yaml +++ b/exports/taskfiles/utils/remote.yaml @@ -23,9 +23,8 @@ tasks: status: - >- diff - <(echo "{{.FILE_SHA256}}") - <(openssl dgst -sha256 "{{.OUTPUT_FILE}}" - | awk '{print $2}') + <(echo "{{.FILE_SHA256}}") + <(openssl dgst -sha256 "{{.OUTPUT_FILE}}" | awk '{print $2}') cmds: - |- mkdir -p "{{dir .OUTPUT_FILE}}" @@ -50,6 +49,9 @@ tasks: echo "Failed to download after $max_attempts attempts." exit 1 fi + diff \ + <(echo "{{.FILE_SHA256}}") \ + <(openssl dgst -sha256 "{{.OUTPUT_FILE}}" | awk '{print $2}') # Uses curl to download a tar file from the given URL and extracts its contents. # diff --git a/taskfiles/remote/tests.yaml b/taskfiles/remote/tests.yaml index 59cffa7..0275695 100644 --- a/taskfiles/remote/tests.yaml +++ b/taskfiles/remote/tests.yaml @@ -4,21 +4,38 @@ includes: remote: "../../exports/taskfiles/utils/remote.yaml" vars: - G_EXTRACTED_ZIP_CODEOWNERS_PATH: "yscope-dev-utils-main/.github/CODEOWNERS" - G_EXTRACTED_ZIP_LICENSE_PATH: "yscope-dev-utils-main/LICENSE" + G_EXTRACTED_ROOT_DIR: "yscope-dev-utils-fd7c42dd7b59f8f4ab0eccba5078393e10cddb00" + G_EXTRACTED_ZIP_CODEOWNERS_PATH: "{{.G_EXTRACTED_ROOT_DIR}}/.github/CODEOWNERS" + G_EXTRACTED_ZIP_LICENSE_PATH: "{{.G_EXTRACTED_ROOT_DIR}}/LICENSE" G_EXTRACTED_ZIP_PULL_REQUEST_TEMPLATE_PATH: >- - yscope-dev-utils-main/.github/PULL_REQUEST_TEMPLATE.md - G_TEST_ZIP_FILE_SHA256: "2c9a21f83484e004c41c28be759451dbdc787190eb044365ba58f7bb846418f6" - G_TEST_ZIP_FILE_URL: "https://github.com/y-scope/yscope-dev-utils/archive/refs/heads/main.zip" + {{.G_EXTRACTED_ROOT_DIR}}/.github/PULL_REQUEST_TEMPLATE.md + G_TEST_ZIP_FILE_SHA256: "141e807e9b4b9e28c254165c5a402ff54c0c9d3f9153178dfcff5354ace0c3d4" + G_TEST_ZIP_FILE_URL: "https://github.com/y-scope/yscope-dev-utils/archive/fd7c42d.zip" tasks: default: internal: true cmds: + - task: "curl-test-success" - task: "download-and-extract-zip-test-basic" - task: "download-and-extract-zip-test-exclusions" - task: "download-and-extract-zip-test-inclusions" + curl-test-success: + vars: + OUTPUT_DIR: "{{.G_OUTPUT_DIR}}/{{.TASK | replace \":\" \"#\"}}" + OUTPUT_FILE: "{{.OUTPUT_DIR}}.zip" + cmds: + - task: "remote:curl" + vars: + FILE_SHA256: "{{.G_TEST_ZIP_FILE_SHA256}}" + OUTPUT_FILE: "{{.OUTPUT_FILE}}" + URL: "{{.G_TEST_ZIP_FILE_URL}}" + - |- + diff \ + <(echo "{{.G_TEST_ZIP_FILE_SHA256}}") \ + <(openssl dgst -sha256 "{{.OUTPUT_FILE}}" | awk '{print $2}') + download-and-extract-zip-test-basic: vars: OUTPUT_DIR: "{{.G_OUTPUT_DIR}}/{{.TASK | replace \":\" \"#\"}}" From 0023c0f68479965872d6d9938cb2eb8a494a032a Mon Sep 17 00:00:00 2001 From: davidlion Date: Sat, 12 Jul 2025 01:13:53 -0400 Subject: [PATCH 2/7] Reorganize test variables. --- taskfiles/remote/tests.yaml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/taskfiles/remote/tests.yaml b/taskfiles/remote/tests.yaml index 0275695..6ffc9a4 100644 --- a/taskfiles/remote/tests.yaml +++ b/taskfiles/remote/tests.yaml @@ -4,13 +4,17 @@ includes: remote: "../../exports/taskfiles/utils/remote.yaml" vars: - G_EXTRACTED_ROOT_DIR: "yscope-dev-utils-fd7c42dd7b59f8f4ab0eccba5078393e10cddb00" - G_EXTRACTED_ZIP_CODEOWNERS_PATH: "{{.G_EXTRACTED_ROOT_DIR}}/.github/CODEOWNERS" - G_EXTRACTED_ZIP_LICENSE_PATH: "{{.G_EXTRACTED_ROOT_DIR}}/LICENSE" - G_EXTRACTED_ZIP_PULL_REQUEST_TEMPLATE_PATH: >- - {{.G_EXTRACTED_ROOT_DIR}}/.github/PULL_REQUEST_TEMPLATE.md + # Test zip file info + G_TEST_COMMIT_HASH: "fd7c42dd7b59f8f4ab0eccba5078393e10cddb00" G_TEST_ZIP_FILE_SHA256: "141e807e9b4b9e28c254165c5a402ff54c0c9d3f9153178dfcff5354ace0c3d4" - G_TEST_ZIP_FILE_URL: "https://github.com/y-scope/yscope-dev-utils/archive/fd7c42d.zip" + G_TEST_ZIP_FILE_URL: "https://github.com/y-scope/yscope-dev-utils/archive/{{.G_TEST_COMMIT_HASH}}.zip" + + # Extracted test zip file contents + G_EXTRACTED_ZIP_DIR: "yscope-dev-utils-{{.G_TEST_COMMIT_HASH}}" + G_EXTRACTED_ZIP_CODEOWNERS_PATH: "{{.G_EXTRACTED_ZIP_DIR}}/.github/CODEOWNERS" + G_EXTRACTED_ZIP_LICENSE_PATH: "{{.G_EXTRACTED_ZIP_DIR}}/LICENSE" + G_EXTRACTED_ZIP_PULL_REQUEST_TEMPLATE_PATH: >- + {{.G_EXTRACTED_ZIP_DIR}}/.github/PULL_REQUEST_TEMPLATE.md tasks: default: From 6cad6605c721936fae531f60c62e73e1b5b851d3 Mon Sep 17 00:00:00 2001 From: davidlion Date: Sat, 12 Jul 2025 01:27:22 -0400 Subject: [PATCH 3/7] Fix success test; Add skip test; rename cleaner. --- exports/taskfiles/utils/remote.yaml | 6 ++--- taskfiles/remote/tests.yaml | 42 ++++++++++++++++++++++++----- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/exports/taskfiles/utils/remote.yaml b/exports/taskfiles/utils/remote.yaml index 116de4c..13d500c 100644 --- a/exports/taskfiles/utils/remote.yaml +++ b/exports/taskfiles/utils/remote.yaml @@ -21,9 +21,9 @@ tasks: vars: ["FILE_SHA256", "URL"] generates: ["{{.OUTPUT_FILE}}"] status: - - >- - diff - <(echo "{{.FILE_SHA256}}") + - |- + diff \ + <(echo "{{.FILE_SHA256}}") \ <(openssl dgst -sha256 "{{.OUTPUT_FILE}}" | awk '{print $2}') cmds: - |- diff --git a/taskfiles/remote/tests.yaml b/taskfiles/remote/tests.yaml index 6ffc9a4..070f532 100644 --- a/taskfiles/remote/tests.yaml +++ b/taskfiles/remote/tests.yaml @@ -30,6 +30,34 @@ tasks: OUTPUT_DIR: "{{.G_OUTPUT_DIR}}/{{.TASK | replace \":\" \"#\"}}" OUTPUT_FILE: "{{.OUTPUT_DIR}}.zip" cmds: + - task: "remote-test-cleaner" + vars: + OUTPUT_DIR: "{{.OUTPUT_DIR}}" + - task: "remote:curl" + vars: + FILE_SHA256: "{{.G_TEST_ZIP_FILE_SHA256}}" + OUTPUT_FILE: "{{.OUTPUT_FILE}}" + URL: "{{.G_TEST_ZIP_FILE_URL}}" + - |- + diff \ + <(echo "{{.G_TEST_ZIP_FILE_SHA256}}") \ + <(openssl dgst -sha256 "{{.OUTPUT_FILE}}" | awk '{print $2}') + + curl-test-success-skip: + vars: + OUTPUT_DIR: "{{.G_OUTPUT_DIR}}/{{.TASK | replace \":\" \"#\"}}" + OUTPUT_FILE: "{{.OUTPUT_DIR}}.zip" + OUTPUT_FILE_MOD_TS: "{{.CHECKSUM_FILE}}-mod-ts.txt" + cmds: + - task: "remote-test-cleaner" + vars: + OUTPUT_DIR: "{{.OUTPUT_DIR}}" + - task: "remote:curl" + vars: + FILE_SHA256: "{{.G_TEST_ZIP_FILE_SHA256}}" + OUTPUT_FILE: "{{.OUTPUT_FILE}}" + URL: "{{.G_TEST_ZIP_FILE_URL}}" + - "date -r '{{.OUTPUT_FILE}}' > '{{.OUTPUT_FILE_MOD_TS}}'" - task: "remote:curl" vars: FILE_SHA256: "{{.G_TEST_ZIP_FILE_SHA256}}" @@ -39,12 +67,13 @@ tasks: diff \ <(echo "{{.G_TEST_ZIP_FILE_SHA256}}") \ <(openssl dgst -sha256 "{{.OUTPUT_FILE}}" | awk '{print $2}') + - "diff '{{.OUTPUT_FILE_MOD_TS}}' <(date -r '{{.OUTPUT_FILE}}')" download-and-extract-zip-test-basic: vars: OUTPUT_DIR: "{{.G_OUTPUT_DIR}}/{{.TASK | replace \":\" \"#\"}}" cmds: - - task: "download-and-extract-zip-test-cleaner" + - task: "remote-test-cleaner" vars: OUTPUT_DIR: "{{.OUTPUT_DIR}}" - task: "remote:download-and-extract-zip" @@ -64,7 +93,7 @@ tasks: vars: OUTPUT_DIR: "{{.G_OUTPUT_DIR}}/{{.TASK | replace \":\" \"#\"}}" cmds: - - task: "download-and-extract-zip-test-cleaner" + - task: "remote-test-cleaner" vars: OUTPUT_DIR: "{{.OUTPUT_DIR}}" - task: "remote:download-and-extract-zip" @@ -87,7 +116,7 @@ tasks: vars: OUTPUT_DIR: "{{.G_OUTPUT_DIR}}/{{.TASK | replace \":\" \"#\"}}" cmds: - - task: "download-and-extract-zip-test-cleaner" + - task: "remote-test-cleaner" vars: OUTPUT_DIR: "{{.OUTPUT_DIR}}" - task: "remote:download-and-extract-zip" @@ -104,11 +133,10 @@ tasks: - "test -e '{{.OUTPUT_DIR}}/{{.G_EXTRACTED_ZIP_CODEOWNERS_PATH}}'" - "test -e '{{.OUTPUT_DIR}}/{{.G_EXTRACTED_ZIP_PULL_REQUEST_TEMPLATE_PATH}}'" - # Cleans up the files output by download-and-extract-zip (assuming their default paths weren't - # changed). + # Cleans up the files output by remote tasks (assuming their default paths weren't changed). # - # @param {string} OUTPUT_DIR Output directory passed to download-and-extract-zip. - download-and-extract-zip-test-cleaner: + # @param {string} OUTPUT_DIR Output directory passed to remote tasks. + remote-test-cleaner: internal: true requires: vars: ["OUTPUT_DIR"] From 23b4fd13df7bf83b46ea7d97f074a0f17c64dc43 Mon Sep 17 00:00:00 2001 From: davidlion Date: Sat, 12 Jul 2025 01:30:57 -0400 Subject: [PATCH 4/7] Fix long line. --- taskfiles/remote/tests.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/taskfiles/remote/tests.yaml b/taskfiles/remote/tests.yaml index 070f532..c7ac0d2 100644 --- a/taskfiles/remote/tests.yaml +++ b/taskfiles/remote/tests.yaml @@ -7,7 +7,8 @@ vars: # Test zip file info G_TEST_COMMIT_HASH: "fd7c42dd7b59f8f4ab0eccba5078393e10cddb00" G_TEST_ZIP_FILE_SHA256: "141e807e9b4b9e28c254165c5a402ff54c0c9d3f9153178dfcff5354ace0c3d4" - G_TEST_ZIP_FILE_URL: "https://github.com/y-scope/yscope-dev-utils/archive/{{.G_TEST_COMMIT_HASH}}.zip" + G_TEST_ZIP_FILE_URL: >- + https://github.com/y-scope/yscope-dev-utils/archive/{{.G_TEST_COMMIT_HASH}}.zip # Extracted test zip file contents G_EXTRACTED_ZIP_DIR: "yscope-dev-utils-{{.G_TEST_COMMIT_HASH}}" From 60eee1670f2837d1a39a7ae0f72c6d85b60a89cb Mon Sep 17 00:00:00 2001 From: davidlion Date: Sat, 12 Jul 2025 01:32:20 -0400 Subject: [PATCH 5/7] Missing file name update. --- taskfiles/remote/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfiles/remote/tests.yaml b/taskfiles/remote/tests.yaml index c7ac0d2..c1ba406 100644 --- a/taskfiles/remote/tests.yaml +++ b/taskfiles/remote/tests.yaml @@ -48,7 +48,7 @@ tasks: vars: OUTPUT_DIR: "{{.G_OUTPUT_DIR}}/{{.TASK | replace \":\" \"#\"}}" OUTPUT_FILE: "{{.OUTPUT_DIR}}.zip" - OUTPUT_FILE_MOD_TS: "{{.CHECKSUM_FILE}}-mod-ts.txt" + OUTPUT_FILE_MOD_TS: "{{.OUTPUT_FILE}}-mod-ts.txt" cmds: - task: "remote-test-cleaner" vars: From 4bda198765e2c7362475767b07e616b9f141e736 Mon Sep 17 00:00:00 2001 From: davidlion Date: Sat, 12 Jul 2025 01:45:51 -0400 Subject: [PATCH 6/7] Update taskfiles/remote/tests.yaml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- taskfiles/remote/tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/taskfiles/remote/tests.yaml b/taskfiles/remote/tests.yaml index c1ba406..a04c1a2 100644 --- a/taskfiles/remote/tests.yaml +++ b/taskfiles/remote/tests.yaml @@ -22,6 +22,7 @@ tasks: internal: true cmds: - task: "curl-test-success" + - task: "curl-test-success-skip" - task: "download-and-extract-zip-test-basic" - task: "download-and-extract-zip-test-exclusions" - task: "download-and-extract-zip-test-inclusions" From cc69b1b11edbcc3676bd0a87f5c9df8f43076b51 Mon Sep 17 00:00:00 2001 From: davidlion Date: Sun, 13 Jul 2025 14:05:55 -0400 Subject: [PATCH 7/7] Add docstring for success-skip test. --- taskfiles/remote/tests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/taskfiles/remote/tests.yaml b/taskfiles/remote/tests.yaml index a04c1a2..7d05dd0 100644 --- a/taskfiles/remote/tests.yaml +++ b/taskfiles/remote/tests.yaml @@ -45,6 +45,8 @@ tasks: <(echo "{{.G_TEST_ZIP_FILE_SHA256}}") \ <(openssl dgst -sha256 "{{.OUTPUT_FILE}}" | awk '{print $2}') + # Tests that re-running the curl task won't re-download the file if it already exists and matches + # the expected checksum. curl-test-success-skip: vars: OUTPUT_DIR: "{{.G_OUTPUT_DIR}}/{{.TASK | replace \":\" \"#\"}}"