From 1fea2dfd12d3bdf894c51ec0ce305a8ef55ab746 Mon Sep 17 00:00:00 2001 From: davidlion Date: Thu, 14 Nov 2024 11:40:12 -0500 Subject: [PATCH 01/13] Add cmake and remote utils; change checksum data path to an array of paths. --- taskfiles/utils.yml | 185 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 173 insertions(+), 12 deletions(-) diff --git a/taskfiles/utils.yml b/taskfiles/utils.yml index 13bdded..828912e 100644 --- a/taskfiles/utils.yml +++ b/taskfiles/utils.yml @@ -5,10 +5,10 @@ tasks: # CHECKSUM UTILS # === - # @param {string} DATA_DIR The directory to compute the checksum for. + # @param {[]string} [DATA_PATHS] List of paths to compute the checksum for. # @param {string} OUTPUT_FILE - # @param {[]string} [EXCLUDE_PATHS] A list of paths, relative to `DATA_DIR`, to exclude from the - # checksum. + # @param {[]string} [EXCLUDE_PATHS] List of paths, relative to any `DATA_PATHS`, to exclude from + # the checksum. compute-checksum: desc: "Tries to compute a checksum for the given directory and output it to a file." internal: true @@ -18,7 +18,6 @@ tasks: cmds: - >- tar cf - - --directory "{{.DATA_DIR}}" --group 0 --mtime "UTC 1970-01-01" --numeric-owner @@ -27,15 +26,18 @@ tasks: {{- range .EXCLUDE_PATHS}} --exclude="{{.}}" {{- end}} - . 2> /dev/null + {{- range .DATA_PATHS}} + "{{.}}" + {{- end}} + 2> /dev/null | md5sum > {{.OUTPUT_FILE}} # Ignore errors so that dependent tasks don't fail ignore_error: true - # @param {string} DATA_DIR The directory to validate the checksum for. + # @param {[]string} [DATA_PATHS] List of paths to validate the checksum for. # @param {string} OUTPUT_FILE - # @param {[]string} [EXCLUDE_PATHS] A list of paths, relative to `DATA_DIR`, to exclude from the - # checksum. + # @param {[]string} [EXCLUDE_PATHS] List of paths, relative to any `DATA_PATHS`, to exclude from + # the checksum. validate-checksum: desc: "Validates the checksum of the given directory matches the checksum in the given file, or deletes the checksum file otherwise." @@ -48,16 +50,19 @@ tasks: cmds: - task: "compute-checksum" vars: - DATA_DIR: "{{.DATA_DIR}}" + DATA_PATHS: + ref: ".DATA_PATHS" EXCLUDE_PATHS: - ref: "default (list) .EXCLUDE_PATHS" + ref: ".EXCLUDE_PATHS" OUTPUT_FILE: "{{.TMP_CHECKSUM_FILE}}" - defer: "rm -f '{{.TMP_CHECKSUM_FILE}}'" # Check that the directory exists and the checksum matches; otherwise delete the checksum file - >- ( - test -d "{{.DATA_DIR}}" - && diff -q '{{.TMP_CHECKSUM_FILE}}' '{{.CHECKSUM_FILE}}' 2> /dev/null + {{- range .DATA_PATHS}} + test -e "{{.}}" && + {{- end}} + diff -q '{{.TMP_CHECKSUM_FILE}}' '{{.CHECKSUM_FILE}}' 2> /dev/null ) || rm -f '{{.CHECKSUM_FILE}}' # === @@ -141,3 +146,159 @@ tasks: \( -iname "*.cpp" -o -iname "*.h" -o -iname "*.hpp" \) \ -print0 | \ xargs -0 clang-tidy {{.FLAGS}} + + # === + # CMAKE UTILS + # === + + # Runs cmake configure and build steps for the given source and build directories. + # + # @param {string} BUILD_DIR Cmake build directory to create. + # @param {string} SOURCE_DIR Project source directory containing the CMakeLists.txt file. + # @param {string}[optional] CHECKSUM_FILE Path to store the checksum of built files. + # @param {string}[optional] CMAKE_ARGS Any additional arguments to pass to cmake configure. + cmake-build: + label: "cmake-build: {{.SOURCE_DIR}} {{.BUILD_DIR}}" + internal: true + vars: + CHECKSUM_FILE: '{{default (printf "%s.md5" .BUILD_DIR) .CHECKSUM_FILE}}' + CMAKE_ARGS: "{{default nil .CMAKE_ARGS}}" + requires: + vars: ["BUILD_DIR", "SOURCE_DIR"] + sources: + - "{{.SOURCE_DIR}}/**/*" + - exclude: "{{.SOURCE_DIR}}/.cache/**/*" + - exclude: "{{.SOURCE_DIR}}/compile_commands.json" + generates: ["{{.CHECKSUM_FILE}}"] + deps: + - task: "validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_PATHS: ["{{.BUILD_DIR}}"] + EXCLUDE_PATHS: ["install_manifest.txt"] + cmds: + - >- + cmake + -S "{{.SOURCE_DIR}}" + -B "{{.BUILD_DIR}}" + {{.CMAKE_ARGS}} + - >- + cmake + --build "{{.BUILD_DIR}}" + --parallel + # This command must be last + - task: "compute-checksum" + vars: + DATA_PATHS: ["{{.BUILD_DIR}}"] + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + EXCLUDE_PATHS: ["install_manifest.txt"] + + # Runs cmake install step for the given build directory. + # + # @param {string} BUILD_DIR Cmake build directory. + # @param {string} INSTALL_PREFIX Path prefix for installing the project. + # @param {string}[optional] CHECKSUM_FILE Path to store the checksum of installed files. + # @param {[]string}[optional] DATA_PATHS Paths to compute the the checksum for. Overrides using + # the install prefix path. + cmake-install: + label: "cmake-install: {{.BUILD_DIR}} {{.INSTALL_PREFIX}}" + internal: true + vars: + CHECKSUM_FILE: '{{default (printf "%s.md5" .INSTALL_PREFIX) .CHECKSUM_FILE}}' + # Convert the install prefix to a single element array for the default case where DATA_PATHS + # is not manually set. + _INSTALL_PREFIX_ARRAY: ["{{.INSTALL_PREFIX}}"] + DATA_PATHS: + ref: "default ._INSTALL_PREFIX_ARRAY .DATA_PATHS" + requires: + vars: ["BUILD_DIR", "INSTALL_PREFIX"] + sources: + - "{{.BUILD_DIR}}/**/*" + - exclude: "{{.BUILD_DIR}}/install_manifest.txt" + generates: ["{{.CHECKSUM_FILE}}"] + deps: + - task: "validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_PATHS: + ref: ".DATA_PATHS" + cmds: + - >- + cmake + --install "{{.BUILD_DIR}}" + --prefix "{{.INSTALL_PREFIX}}" + # This command must be last + - task: "compute-checksum" + vars: + DATA_PATHS: + ref: ".DATA_PATHS" + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + + # === + # REMOTE UTILS + # === + + # Runs curl to download the provided URL. + # + # @param {string} URL + # @param {string} URL_SHA256 Content hash to verify downloaded file against. + # @param {string}[optional] OUTPUT_FILE File path to store the download file. + curl: + label: "curl: {{.OUTPUT_FILE}}" + internal: true + vars: + OUTPUT_FILE: "{{default (base .URL) .OUTPUT_FILE}}" + requires: + vars: ["URL", "URL_SHA256"] + generates: + - "{{.OUTPUT_FILE}}" + status: + - >- + diff + <(echo "{{.URL_SHA256}}") + <(openssl dgst -sha256 "{{.OUTPUT_FILE}}" + | awk '{print $2}') + cmds: + - |- + mkdir -p "{{dir .OUTPUT_FILE}}" + curl -L "{{.URL}}" -o "{{.OUTPUT_FILE}}" + + # Runs curl to download the provided URL and tar to extract its contents. + # + # @param {string} OUTPUT_DIR Path to extract downloaded tar file contents to. + # @param {string} URL + # @param {string} URL_SHA256 Content hash to verify downloaded tar file against. + # @param {string}[optional] CHECKSUM_FILE File path to store the checksum of downloaded tar file. + # @param {int}[optional] STRIP Number of leading components to strip from file names for tar. + # @param {string}[optional] TAR_FILE File path to store the downloaded tar file. + fetch-src: + label: "fetch-src: {{.OUTPUT_DIR}}" + internal: true + vars: + CHECKSUM_FILE: '{{default (printf "%s.md5" .OUTPUT_DIR) .CHECKSUM_FILE}}' + STRIP: "{{default 1 .STRIP}}" + TAR_FILE: '{{default (printf "%s.tar.gz" .OUTPUT_DIR) .TAR_FILE}}' + requires: + vars: ["OUTPUT_DIR", "URL", "URL_SHA256"] + sources: ["{{.TASKFILE}}"] + generates: ["{{.CHECKSUM_FILE}}", "{{.TAR_FILE}}"] + deps: + - task: "curl" + vars: + URL: "{{.URL}}" + URL_SHA256: "{{.URL_SHA256}}" + OUTPUT_FILE: "{{.TAR_FILE}}" + - task: "validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_PATHS: ["{{.OUTPUT_DIR}}"] + cmds: + - |- + rm -rf "{{.OUTPUT_DIR}}" + mkdir -p "{{.OUTPUT_DIR}}" + tar -x --strip-components="{{.STRIP}}" -C "{{.OUTPUT_DIR}}" -f "{{.TAR_FILE}}" + # This command must be last + - task: "compute-checksum" + vars: + DATA_PATHS: ["{{.OUTPUT_DIR}}"] + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" From 419938fa340365cb25176c369a7fc59dbe6502b4 Mon Sep 17 00:00:00 2001 From: davidlion Date: Thu, 14 Nov 2024 12:11:01 -0500 Subject: [PATCH 02/13] Update taskfiles/utils.yml Co-authored-by: Junhao Liao --- taskfiles/utils.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfiles/utils.yml b/taskfiles/utils.yml index 828912e..945ecfe 100644 --- a/taskfiles/utils.yml +++ b/taskfiles/utils.yml @@ -56,7 +56,7 @@ tasks: ref: ".EXCLUDE_PATHS" OUTPUT_FILE: "{{.TMP_CHECKSUM_FILE}}" - defer: "rm -f '{{.TMP_CHECKSUM_FILE}}'" - # Check that the directory exists and the checksum matches; otherwise delete the checksum file + # Check that the path exists and the checksum matches; otherwise delete the checksum file - >- ( {{- range .DATA_PATHS}} From 123418c494a7b7eaf0f1b659740f066518dc706f Mon Sep 17 00:00:00 2001 From: davidlion Date: Thu, 14 Nov 2024 12:11:33 -0500 Subject: [PATCH 03/13] Update taskfiles/utils.yml Co-authored-by: Junhao Liao --- taskfiles/utils.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfiles/utils.yml b/taskfiles/utils.yml index 945ecfe..dc709ac 100644 --- a/taskfiles/utils.yml +++ b/taskfiles/utils.yml @@ -53,7 +53,7 @@ tasks: DATA_PATHS: ref: ".DATA_PATHS" EXCLUDE_PATHS: - ref: ".EXCLUDE_PATHS" + ref: "default (list) .EXCLUDE_PATHS" OUTPUT_FILE: "{{.TMP_CHECKSUM_FILE}}" - defer: "rm -f '{{.TMP_CHECKSUM_FILE}}'" # Check that the path exists and the checksum matches; otherwise delete the checksum file From c9e6c0216425926a2c527342d8b044c833e10623 Mon Sep 17 00:00:00 2001 From: davidlion Date: Thu, 14 Nov 2024 18:59:45 -0500 Subject: [PATCH 04/13] Fix incorrect DATA_DIR names in requires statements. --- taskfiles/utils.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/taskfiles/utils.yml b/taskfiles/utils.yml index dc709ac..f2bcc4e 100644 --- a/taskfiles/utils.yml +++ b/taskfiles/utils.yml @@ -14,7 +14,7 @@ tasks: internal: true silent: true requires: - vars: ["DATA_DIR", "OUTPUT_FILE"] + vars: ["DATA_PATHS", "OUTPUT_FILE"] cmds: - >- tar cf - @@ -46,7 +46,7 @@ tasks: vars: TMP_CHECKSUM_FILE: "{{.CHECKSUM_FILE}}.tmp" requires: - vars: ["CHECKSUM_FILE", "DATA_DIR"] + vars: ["CHECKSUM_FILE", "DATA_PATHS"] cmds: - task: "compute-checksum" vars: @@ -162,7 +162,7 @@ tasks: internal: true vars: CHECKSUM_FILE: '{{default (printf "%s.md5" .BUILD_DIR) .CHECKSUM_FILE}}' - CMAKE_ARGS: "{{default nil .CMAKE_ARGS}}" + CMAKE_ARGS: "{{default "" .CMAKE_ARGS}}" requires: vars: ["BUILD_DIR", "SOURCE_DIR"] sources: From d09b8fbc523cafabdfc0b40a3140d392109f7875 Mon Sep 17 00:00:00 2001 From: davidlion Date: Thu, 14 Nov 2024 21:37:33 -0500 Subject: [PATCH 05/13] Add rabbitai fixes; update doc strings; update quotes; download-and-extract-tar needs to handle filtering the tar ball. --- taskfiles/utils.yml | 83 ++++++++++++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 31 deletions(-) diff --git a/taskfiles/utils.yml b/taskfiles/utils.yml index f2bcc4e..613ba5a 100644 --- a/taskfiles/utils.yml +++ b/taskfiles/utils.yml @@ -5,9 +5,9 @@ tasks: # CHECKSUM UTILS # === - # @param {[]string} [DATA_PATHS] List of paths to compute the checksum for. + # @param {string[]} DATA_PATHS List of paths to compute the checksum for. # @param {string} OUTPUT_FILE - # @param {[]string} [EXCLUDE_PATHS] List of paths, relative to any `DATA_PATHS`, to exclude from + # @param {string[]} [EXCLUDE_PATHS] List of paths, relative to any `DATA_PATHS`, to exclude from # the checksum. compute-checksum: desc: "Tries to compute a checksum for the given directory and output it to a file." @@ -34,9 +34,9 @@ tasks: # Ignore errors so that dependent tasks don't fail ignore_error: true - # @param {[]string} [DATA_PATHS] List of paths to validate the checksum for. + # @param {string[]} DATA_PATHS List of paths to validate the checksum for. # @param {string} OUTPUT_FILE - # @param {[]string} [EXCLUDE_PATHS] List of paths, relative to any `DATA_PATHS`, to exclude from + # @param {string[]} [EXCLUDE_PATHS] List of paths, relative to any `DATA_PATHS`, to exclude from # the checksum. validate-checksum: desc: "Validates the checksum of the given directory matches the checksum in the given file, or @@ -62,8 +62,8 @@ tasks: {{- range .DATA_PATHS}} test -e "{{.}}" && {{- end}} - diff -q '{{.TMP_CHECKSUM_FILE}}' '{{.CHECKSUM_FILE}}' 2> /dev/null - ) || rm -f '{{.CHECKSUM_FILE}}' + diff -q "{{.TMP_CHECKSUM_FILE}}" "{{.CHECKSUM_FILE}}" 2> /dev/null + ) || rm -f "{{.CHECKSUM_FILE}}" # === # STRING UTILS @@ -80,7 +80,7 @@ tasks: # 2. We can't use `--regexp` instead of `-E` since `--regexp` is not supported on macOS src="{{.FILE_PATH}}" dst="{{.FILE_PATH}}.tmp" - sed -E '{{.SED_EXP}}' "${src}" > "${dst}" + sed -E "{{.SED_EXP}}" "${src}" > "${dst}" mv "${dst}" "${src}" # === @@ -116,7 +116,7 @@ tasks: # Runs clang-format on C++ files at the given paths. # # @param {string} FLAGS Any flags to pass to clang-format. - # @param {[]string} SRC_PATHS The paths on which to run clang-format. + # @param {string[]} SRC_PATHS The paths on which to run clang-format. # @param {string} VENV_DIR Python virtual environment where clang-format is installed. clang-format: internal: true @@ -133,7 +133,7 @@ tasks: # Runs clang-tidy on C++ files at the given paths. # # @param {string} FLAGS Any flags to pass to clang-tidy. - # @param {[]string} SRC_PATHS The paths on which to run clang-tidy. + # @param {string[]} SRC_PATHS The paths on which to run clang-tidy. # @param {string} VENV_DIR Python virtual environment where clang-tidy is installed. clang-tidy: internal: true @@ -155,14 +155,16 @@ tasks: # # @param {string} BUILD_DIR Cmake build directory to create. # @param {string} SOURCE_DIR Project source directory containing the CMakeLists.txt file. - # @param {string}[optional] CHECKSUM_FILE Path to store the checksum of built files. - # @param {string}[optional] CMAKE_ARGS Any additional arguments to pass to cmake configure. + # @param {string={{.BUILD_DIR}}.md5} [CHECKSUM_FILE] Path to store the checksum of built files. + # @param {string=""} [CMAKE_ARGS] Any additional arguments to pass to cmake configure. cmake-build: - label: "cmake-build: {{.SOURCE_DIR}} {{.BUILD_DIR}}" + label: "{{.TASK}}-{{.SOURCE_DIR}}-{{.BUILD_DIR}}" internal: true vars: - CHECKSUM_FILE: '{{default (printf "%s.md5" .BUILD_DIR) .CHECKSUM_FILE}}' - CMAKE_ARGS: "{{default "" .CMAKE_ARGS}}" + CHECKSUM_FILE: >- + {{default (printf "%s.md5" .BUILD_DIR) .CHECKSUM_FILE}} + CMAKE_ARGS: >- + {{default "" .CMAKE_ARGS}} requires: vars: ["BUILD_DIR", "SOURCE_DIR"] sources: @@ -197,16 +199,17 @@ tasks: # # @param {string} BUILD_DIR Cmake build directory. # @param {string} INSTALL_PREFIX Path prefix for installing the project. - # @param {string}[optional] CHECKSUM_FILE Path to store the checksum of installed files. - # @param {[]string}[optional] DATA_PATHS Paths to compute the the checksum for. Overrides using - # the install prefix path. + # @param {string={{.INSTALL_PREFIX}}.md5} [CHECKSUM_FILE] Path to store the checksum of installed + # files. + # @param {string[]=[{{.INSTALL_PREFIX}}]} [DATA_PATHS] Paths to compute the the checksum for. cmake-install: - label: "cmake-install: {{.BUILD_DIR}} {{.INSTALL_PREFIX}}" + label: "{{.TASK}}-{{.BUILD_DIR}}-{{.INSTALL_PREFIX}}" internal: true vars: - CHECKSUM_FILE: '{{default (printf "%s.md5" .INSTALL_PREFIX) .CHECKSUM_FILE}}' - # Convert the install prefix to a single element array for the default case where DATA_PATHS - # is not manually set. + CHECKSUM_FILE: >- + {{default (printf "%s.md5" .INSTALL_PREFIX) .CHECKSUM_FILE}} + # Convert the install prefix to a single element array in the default case where DATA_PATHS + # has not been set. _INSTALL_PREFIX_ARRAY: ["{{.INSTALL_PREFIX}}"] DATA_PATHS: ref: "default ._INSTALL_PREFIX_ARRAY .DATA_PATHS" @@ -242,9 +245,9 @@ tasks: # # @param {string} URL # @param {string} URL_SHA256 Content hash to verify downloaded file against. - # @param {string}[optional] OUTPUT_FILE File path to store the download file. + # @param {string={{(base .URL)}}} [OUTPUT_FILE] File path to store the download file. curl: - label: "curl: {{.OUTPUT_FILE}}" + label: "{{.TASK}}-{{.OUTPUT_FILE}}" internal: true vars: OUTPUT_FILE: "{{default (base .URL) .OUTPUT_FILE}}" @@ -261,23 +264,41 @@ tasks: cmds: - |- mkdir -p "{{dir .OUTPUT_FILE}}" - curl -L "{{.URL}}" -o "{{.OUTPUT_FILE}}" + max_attempts=3 + attempt=1 + while [ $attempt -le $max_attempts ]; do + if curl --fail --location --show-error --connect-timeout 10 --max-time 300 \ + --location "{{.URL}}" --output "{{.OUTPUT_FILE}}"; + then + break + fi + echo "Attempt $attempt failed. Retrying..." + attempt=$((attempt + 1)) + sleep 5 + done + if [ $attempt -gt $max_attempts ]; then + echo "Failed to download after $max_attempts attempts" + exit 1 + fi # Runs curl to download the provided URL and tar to extract its contents. # # @param {string} OUTPUT_DIR Path to extract downloaded tar file contents to. # @param {string} URL # @param {string} URL_SHA256 Content hash to verify downloaded tar file against. - # @param {string}[optional] CHECKSUM_FILE File path to store the checksum of downloaded tar file. - # @param {int}[optional] STRIP Number of leading components to strip from file names for tar. - # @param {string}[optional] TAR_FILE File path to store the downloaded tar file. - fetch-src: - label: "fetch-src: {{.OUTPUT_DIR}}" + # @param {string={{.OUTPUT_DIR}}.md5} [CHECKSUM_FILE] File path to store the checksum of + # downloaded tar file. + # @param {int=1} [STRIP] Number of leading components to strip from file names for tar. + # @param {string={{.OUTPUT_DIR}}.tar.gz} [TAR_FILE] File path to store the downloaded tar file. + download-and-extract-tar: + label: "{{.TASK}}-{{.OUTPUT_DIR}}" internal: true vars: - CHECKSUM_FILE: '{{default (printf "%s.md5" .OUTPUT_DIR) .CHECKSUM_FILE}}' + CHECKSUM_FILE: >- + {{default (printf "%s.md5" .OUTPUT_DIR) .CHECKSUM_FILE}} STRIP: "{{default 1 .STRIP}}" - TAR_FILE: '{{default (printf "%s.tar.gz" .OUTPUT_DIR) .TAR_FILE}}' + TAR_FILE: >- + {{default (printf "%s.tar.gz" .OUTPUT_DIR) .TAR_FILE}} requires: vars: ["OUTPUT_DIR", "URL", "URL_SHA256"] sources: ["{{.TASKFILE}}"] From dd4c1e62490fd68f5f471f4ce2a35103126ae641 Mon Sep 17 00:00:00 2001 From: davidlion Date: Fri, 15 Nov 2024 00:18:06 -0500 Subject: [PATCH 06/13] Add optional exlucde and include paths parameters to download-and-extract-tar. --- taskfiles/utils.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/taskfiles/utils.yml b/taskfiles/utils.yml index 613ba5a..577cf6e 100644 --- a/taskfiles/utils.yml +++ b/taskfiles/utils.yml @@ -10,7 +10,7 @@ tasks: # @param {string[]} [EXCLUDE_PATHS] List of paths, relative to any `DATA_PATHS`, to exclude from # the checksum. compute-checksum: - desc: "Tries to compute a checksum for the given directory and output it to a file." + desc: "Tries to compute a checksum for the given paths and output it to a file." internal: true silent: true requires: @@ -288,6 +288,8 @@ tasks: # @param {string} URL_SHA256 Content hash to verify downloaded tar file against. # @param {string={{.OUTPUT_DIR}}.md5} [CHECKSUM_FILE] File path to store the checksum of # downloaded tar file. + # @param {string[]=[]} [EXCLUDE_PATHS] Wildcard patterns excluded from the tar archive. + # @param {string[]=[]} [INCLUDE_PATHS] Wildcard patterns included from the tar archive. # @param {int=1} [STRIP] Number of leading components to strip from file names for tar. # @param {string={{.OUTPUT_DIR}}.tar.gz} [TAR_FILE] File path to store the downloaded tar file. download-and-extract-tar: @@ -296,6 +298,10 @@ tasks: vars: CHECKSUM_FILE: >- {{default (printf "%s.md5" .OUTPUT_DIR) .CHECKSUM_FILE}} + EXCLUDE_PATHS: + ref: "default (list) .EXCLUDE_PATHS" + INCLUDE_PATHS: + ref: "default (list) .INCLUDE_PATHS" STRIP: "{{default 1 .STRIP}}" TAR_FILE: >- {{default (printf "%s.tar.gz" .OUTPUT_DIR) .TAR_FILE}} @@ -317,7 +323,15 @@ tasks: - |- rm -rf "{{.OUTPUT_DIR}}" mkdir -p "{{.OUTPUT_DIR}}" - tar -x --strip-components="{{.STRIP}}" -C "{{.OUTPUT_DIR}}" -f "{{.TAR_FILE}}" + - >- + tar --extract + --strip-components="{{.STRIP}}" + --directory "{{.OUTPUT_DIR}}" + --file "{{.TAR_FILE}}" + --wildcards + --no-anchored + {{- range .INCLUDE_PATHS}} "{{.}}" {{- end}} + {{- range .EXCLUDE_PATHS}} --exclude="{{.}}" {{- end}} # This command must be last - task: "compute-checksum" vars: From 4cd555be9771c7962ee8e45b86b696b9a80e3256 Mon Sep 17 00:00:00 2001 From: davidlion Date: Mon, 18 Nov 2024 10:28:41 -0500 Subject: [PATCH 07/13] Apply suggestions from code review. Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- taskfiles/utils.yml | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/taskfiles/utils.yml b/taskfiles/utils.yml index 577cf6e..688dd42 100644 --- a/taskfiles/utils.yml +++ b/taskfiles/utils.yml @@ -56,7 +56,7 @@ tasks: ref: "default (list) .EXCLUDE_PATHS" OUTPUT_FILE: "{{.TMP_CHECKSUM_FILE}}" - defer: "rm -f '{{.TMP_CHECKSUM_FILE}}'" - # Check that the path exists and the checksum matches; otherwise delete the checksum file + # Check that the paths exist and the checksum matches; otherwise delete the checksum file. - >- ( {{- range .DATA_PATHS}} @@ -151,12 +151,12 @@ tasks: # CMAKE UTILS # === - # Runs cmake configure and build steps for the given source and build directories. + # Runs CMake's configure and build steps for the given source and build directories. # - # @param {string} BUILD_DIR Cmake build directory to create. + # @param {string} BUILD_DIR CMake build directory to create. # @param {string} SOURCE_DIR Project source directory containing the CMakeLists.txt file. # @param {string={{.BUILD_DIR}}.md5} [CHECKSUM_FILE] Path to store the checksum of built files. - # @param {string=""} [CMAKE_ARGS] Any additional arguments to pass to cmake configure. + # @param {string=""} [CMAKE_ARGS] Any additional arguments to pass to CMake's configure step. cmake-build: label: "{{.TASK}}-{{.SOURCE_DIR}}-{{.BUILD_DIR}}" internal: true @@ -192,13 +192,13 @@ tasks: - task: "compute-checksum" vars: DATA_PATHS: ["{{.BUILD_DIR}}"] - OUTPUT_FILE: "{{.CHECKSUM_FILE}}" EXCLUDE_PATHS: ["install_manifest.txt"] + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" - # Runs cmake install step for the given build directory. + # Runs the CMake install step for the given build directory. # - # @param {string} BUILD_DIR Cmake build directory. - # @param {string} INSTALL_PREFIX Path prefix for installing the project. + # @param {string} BUILD_DIR CMake build directory. + # @param {string} INSTALL_PREFIX Path prefix of where the project should be installed. # @param {string={{.INSTALL_PREFIX}}.md5} [CHECKSUM_FILE] Path to store the checksum of installed # files. # @param {string[]=[{{.INSTALL_PREFIX}}]} [DATA_PATHS] Paths to compute the the checksum for. @@ -208,11 +208,8 @@ tasks: vars: CHECKSUM_FILE: >- {{default (printf "%s.md5" .INSTALL_PREFIX) .CHECKSUM_FILE}} - # Convert the install prefix to a single element array in the default case where DATA_PATHS - # has not been set. - _INSTALL_PREFIX_ARRAY: ["{{.INSTALL_PREFIX}}"] DATA_PATHS: - ref: "default ._INSTALL_PREFIX_ARRAY .DATA_PATHS" + ref: "default (list .INSTALL_PREFIX) .DATA_PATHS" requires: vars: ["BUILD_DIR", "INSTALL_PREFIX"] sources: @@ -241,11 +238,11 @@ tasks: # REMOTE UTILS # === - # Runs curl to download the provided URL. + # Runs curl to download a file from the given URL. # # @param {string} URL # @param {string} URL_SHA256 Content hash to verify downloaded file against. - # @param {string={{(base .URL)}}} [OUTPUT_FILE] File path to store the download file. + # @param {string={{(base .URL)}}} [OUTPUT_FILE] Path where the file should be stored. curl: label: "{{.TASK}}-{{.OUTPUT_FILE}}" internal: true @@ -253,8 +250,7 @@ tasks: OUTPUT_FILE: "{{default (base .URL) .OUTPUT_FILE}}" requires: vars: ["URL", "URL_SHA256"] - generates: - - "{{.OUTPUT_FILE}}" + generates: ["{{.OUTPUT_FILE}}"] status: - >- diff @@ -277,21 +273,21 @@ tasks: sleep 5 done if [ $attempt -gt $max_attempts ]; then - echo "Failed to download after $max_attempts attempts" + echo "Failed to download after $max_attempts attempts." exit 1 fi - # Runs curl to download the provided URL and tar to extract its contents. + # Runs curl to download the tarball from the given URL and extracts its contents. # - # @param {string} OUTPUT_DIR Path to extract downloaded tar file contents to. + # @param {string} OUTPUT_DIR Directory in which to extract the tarball. # @param {string} URL # @param {string} URL_SHA256 Content hash to verify downloaded tar file against. # @param {string={{.OUTPUT_DIR}}.md5} [CHECKSUM_FILE] File path to store the checksum of # downloaded tar file. - # @param {string[]=[]} [EXCLUDE_PATHS] Wildcard patterns excluded from the tar archive. - # @param {string[]=[]} [INCLUDE_PATHS] Wildcard patterns included from the tar archive. - # @param {int=1} [STRIP] Number of leading components to strip from file names for tar. - # @param {string={{.OUTPUT_DIR}}.tar.gz} [TAR_FILE] File path to store the downloaded tar file. + # @param {string[]=[]} [EXCLUDE_PATHS] Wildcard patterns for paths that shouldn't be extracted. + # @param {string[]=[]} [INCLUDE_PATHS] Wildcard patterns for paths to extract. + # @param {int=1} [STRIP] Number of leading path components to strip from the extracted files. + # @param {string={{.OUTPUT_DIR}}.tar.gz} [TAR_FILE] Path where the tarball should be stored. download-and-extract-tar: label: "{{.TASK}}-{{.OUTPUT_DIR}}" internal: true From b482d6d9d8cd034cf8b87db70d2d48cd855dab9f Mon Sep 17 00:00:00 2001 From: davidlion Date: Mon, 18 Nov 2024 11:27:32 -0500 Subject: [PATCH 08/13] Address review comments. --- taskfiles/utils.yml | 68 +++++++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/taskfiles/utils.yml b/taskfiles/utils.yml index 688dd42..490abb0 100644 --- a/taskfiles/utils.yml +++ b/taskfiles/utils.yml @@ -7,7 +7,8 @@ tasks: # @param {string[]} DATA_PATHS List of paths to compute the checksum for. # @param {string} OUTPUT_FILE - # @param {string[]} [EXCLUDE_PATHS] List of paths, relative to any `DATA_PATHS`, to exclude from + # @param {string[]} [EXCLUDE_PATTERNS] Path wildcard patterns, relative to any `DATA_PATHS`, to + # exclude from the checksum. # the checksum. compute-checksum: desc: "Tries to compute a checksum for the given paths and output it to a file." @@ -23,7 +24,9 @@ tasks: --numeric-owner --owner 0 --sort name - {{- range .EXCLUDE_PATHS}} + --no-anchored + --wildcards + {{- range .EXCLUDE_PATTERNS}} --exclude="{{.}}" {{- end}} {{- range .DATA_PATHS}} @@ -36,8 +39,8 @@ tasks: # @param {string[]} DATA_PATHS List of paths to validate the checksum for. # @param {string} OUTPUT_FILE - # @param {string[]} [EXCLUDE_PATHS] List of paths, relative to any `DATA_PATHS`, to exclude from - # the checksum. + # @param {string[]} [EXCLUDE_PATTERNS] List of paths, relative to any `DATA_PATHS`, to exclude + # from the checksum. validate-checksum: desc: "Validates the checksum of the given directory matches the checksum in the given file, or deletes the checksum file otherwise." @@ -52,8 +55,8 @@ tasks: vars: DATA_PATHS: ref: ".DATA_PATHS" - EXCLUDE_PATHS: - ref: "default (list) .EXCLUDE_PATHS" + EXCLUDE_PATTERNS: + ref: "default (list) .EXCLUDE_PATTERNS" OUTPUT_FILE: "{{.TMP_CHECKSUM_FILE}}" - defer: "rm -f '{{.TMP_CHECKSUM_FILE}}'" # Check that the paths exist and the checksum matches; otherwise delete the checksum file. @@ -157,7 +160,7 @@ tasks: # @param {string} SOURCE_DIR Project source directory containing the CMakeLists.txt file. # @param {string={{.BUILD_DIR}}.md5} [CHECKSUM_FILE] Path to store the checksum of built files. # @param {string=""} [CMAKE_ARGS] Any additional arguments to pass to CMake's configure step. - cmake-build: + cmake-config-and-build: label: "{{.TASK}}-{{.SOURCE_DIR}}-{{.BUILD_DIR}}" internal: true vars: @@ -169,15 +172,13 @@ tasks: vars: ["BUILD_DIR", "SOURCE_DIR"] sources: - "{{.SOURCE_DIR}}/**/*" - - exclude: "{{.SOURCE_DIR}}/.cache/**/*" - - exclude: "{{.SOURCE_DIR}}/compile_commands.json" generates: ["{{.CHECKSUM_FILE}}"] deps: - task: "validate-checksum" vars: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" DATA_PATHS: ["{{.BUILD_DIR}}"] - EXCLUDE_PATHS: ["install_manifest.txt"] + EXCLUDE_PATTERNS: ["install_manifest.txt"] cmds: - >- cmake @@ -241,7 +242,7 @@ tasks: # Runs curl to download a file from the given URL. # # @param {string} URL - # @param {string} URL_SHA256 Content hash to verify downloaded file against. + # @param {string} FILE_SHA256 Content hash to verify downloaded file against. # @param {string={{(base .URL)}}} [OUTPUT_FILE] Path where the file should be stored. curl: label: "{{.TASK}}-{{.OUTPUT_FILE}}" @@ -249,12 +250,12 @@ tasks: vars: OUTPUT_FILE: "{{default (base .URL) .OUTPUT_FILE}}" requires: - vars: ["URL", "URL_SHA256"] + vars: ["URL", "FILE_SHA256"] generates: ["{{.OUTPUT_FILE}}"] status: - >- diff - <(echo "{{.URL_SHA256}}") + <(echo "{{.FILE_SHA256}}") <(openssl dgst -sha256 "{{.OUTPUT_FILE}}" | awk '{print $2}') cmds: @@ -263,8 +264,14 @@ tasks: max_attempts=3 attempt=1 while [ $attempt -le $max_attempts ]; do - if curl --fail --location --show-error --connect-timeout 10 --max-time 300 \ - --location "{{.URL}}" --output "{{.OUTPUT_FILE}}"; + if curl \ + --fail \ + --location \ + --show-error \ + --connect-timeout 10 \ + --max-time 300 \ + "{{.URL}}" \ + --output "{{.OUTPUT_FILE}}"; then break fi @@ -281,12 +288,13 @@ tasks: # # @param {string} OUTPUT_DIR Directory in which to extract the tarball. # @param {string} URL - # @param {string} URL_SHA256 Content hash to verify downloaded tar file against. + # @param {string} FILE_SHA256 Content hash to verify downloaded tar file against. # @param {string={{.OUTPUT_DIR}}.md5} [CHECKSUM_FILE] File path to store the checksum of # downloaded tar file. # @param {string[]=[]} [EXCLUDE_PATHS] Wildcard patterns for paths that shouldn't be extracted. # @param {string[]=[]} [INCLUDE_PATHS] Wildcard patterns for paths to extract. - # @param {int=1} [STRIP] Number of leading path components to strip from the extracted files. + # @param {int=1} [NUM_COMPONENTS_TO_STRIP] Number of leading path components to strip from the + # extracted files. # @param {string={{.OUTPUT_DIR}}.tar.gz} [TAR_FILE] Path where the tarball should be stored. download-and-extract-tar: label: "{{.TASK}}-{{.OUTPUT_DIR}}" @@ -294,22 +302,22 @@ tasks: vars: CHECKSUM_FILE: >- {{default (printf "%s.md5" .OUTPUT_DIR) .CHECKSUM_FILE}} - EXCLUDE_PATHS: - ref: "default (list) .EXCLUDE_PATHS" + EXCLUDE_PATTERNS: + ref: "default (list) .EXCLUDE_PATTERNS" INCLUDE_PATHS: ref: "default (list) .INCLUDE_PATHS" - STRIP: "{{default 1 .STRIP}}" + NUM_COMPONENTS_TO_STRIP: "{{default 1 .NUM_COMPONENTS_TO_STRIP}}" TAR_FILE: >- {{default (printf "%s.tar.gz" .OUTPUT_DIR) .TAR_FILE}} requires: - vars: ["OUTPUT_DIR", "URL", "URL_SHA256"] + vars: ["OUTPUT_DIR", "URL", "FILE_SHA256"] sources: ["{{.TASKFILE}}"] generates: ["{{.CHECKSUM_FILE}}", "{{.TAR_FILE}}"] deps: - task: "curl" vars: URL: "{{.URL}}" - URL_SHA256: "{{.URL_SHA256}}" + FILE_SHA256: "{{.FILE_SHA256}}" OUTPUT_FILE: "{{.TAR_FILE}}" - task: "validate-checksum" vars: @@ -320,14 +328,20 @@ tasks: rm -rf "{{.OUTPUT_DIR}}" mkdir -p "{{.OUTPUT_DIR}}" - >- - tar --extract - --strip-components="{{.STRIP}}" + tar + --extract + --strip-components="{{.NUM_COMPONENTS_TO_STRIP}}" --directory "{{.OUTPUT_DIR}}" --file "{{.TAR_FILE}}" - --wildcards --no-anchored - {{- range .INCLUDE_PATHS}} "{{.}}" {{- end}} - {{- range .EXCLUDE_PATHS}} --exclude="{{.}}" {{- end}} + --wildcards + {{- range .EXCLUDE_PATTERNS}} + --exclude="{{.}}" + {{- end}} + {{- range .DATA_PATHS}} + "{{.}}" + {{- end}} + 2> /dev/null # This command must be last - task: "compute-checksum" vars: From 80d47730d4b6459c17a46549b68b2b73be2ea998 Mon Sep 17 00:00:00 2001 From: davidlion Date: Thu, 21 Nov 2024 10:25:16 -0500 Subject: [PATCH 09/13] Remove checksum usage from cmake tasks; fix some naming. --- taskfiles/utils.yml | 88 ++++++++++++--------------------------------- 1 file changed, 22 insertions(+), 66 deletions(-) diff --git a/taskfiles/utils.yml b/taskfiles/utils.yml index 490abb0..77b7a9e 100644 --- a/taskfiles/utils.yml +++ b/taskfiles/utils.yml @@ -5,17 +5,16 @@ tasks: # CHECKSUM UTILS # === - # @param {string[]} DATA_PATHS List of paths to compute the checksum for. + # @param {string[]} DATA_PATTERNS Path wildcard patterns to compute the checksum for. # @param {string} OUTPUT_FILE - # @param {string[]} [EXCLUDE_PATTERNS] Path wildcard patterns, relative to any `DATA_PATHS`, to + # @param {string[]} [EXCLUDE_PATTERNS] Path wildcard patterns, relative to any `DATA_PATTERNS`, to # exclude from the checksum. - # the checksum. compute-checksum: desc: "Tries to compute a checksum for the given paths and output it to a file." internal: true silent: true requires: - vars: ["DATA_PATHS", "OUTPUT_FILE"] + vars: ["DATA_PATTERNS", "OUTPUT_FILE"] cmds: - >- tar cf - @@ -29,7 +28,7 @@ tasks: {{- range .EXCLUDE_PATTERNS}} --exclude="{{.}}" {{- end}} - {{- range .DATA_PATHS}} + {{- range .DATA_PATTERNS}} "{{.}}" {{- end}} 2> /dev/null @@ -37,10 +36,10 @@ tasks: # Ignore errors so that dependent tasks don't fail ignore_error: true - # @param {string[]} DATA_PATHS List of paths to validate the checksum for. + # @param {string[]} DATA_PATTERNS Path wildcard patterns to validate the checksum for. # @param {string} OUTPUT_FILE - # @param {string[]} [EXCLUDE_PATTERNS] List of paths, relative to any `DATA_PATHS`, to exclude - # from the checksum. + # @param {string[]} [EXCLUDE_PATTERNS] Path wildcard patterns, relative to any `DATA_PATTERNS`, to + # exclude from the checksum. validate-checksum: desc: "Validates the checksum of the given directory matches the checksum in the given file, or deletes the checksum file otherwise." @@ -49,12 +48,12 @@ tasks: vars: TMP_CHECKSUM_FILE: "{{.CHECKSUM_FILE}}.tmp" requires: - vars: ["CHECKSUM_FILE", "DATA_PATHS"] + vars: ["CHECKSUM_FILE", "DATA_PATTERNS"] cmds: - task: "compute-checksum" vars: - DATA_PATHS: - ref: ".DATA_PATHS" + DATA_PATTERNS: + ref: ".DATA_PATTERNS" EXCLUDE_PATTERNS: ref: "default (list) .EXCLUDE_PATTERNS" OUTPUT_FILE: "{{.TMP_CHECKSUM_FILE}}" @@ -62,7 +61,7 @@ tasks: # Check that the paths exist and the checksum matches; otherwise delete the checksum file. - >- ( - {{- range .DATA_PATHS}} + {{- range .DATA_PATTERNS}} test -e "{{.}}" && {{- end}} diff -q "{{.TMP_CHECKSUM_FILE}}" "{{.CHECKSUM_FILE}}" 2> /dev/null @@ -158,27 +157,15 @@ tasks: # # @param {string} BUILD_DIR CMake build directory to create. # @param {string} SOURCE_DIR Project source directory containing the CMakeLists.txt file. - # @param {string={{.BUILD_DIR}}.md5} [CHECKSUM_FILE] Path to store the checksum of built files. # @param {string=""} [CMAKE_ARGS] Any additional arguments to pass to CMake's configure step. cmake-config-and-build: label: "{{.TASK}}-{{.SOURCE_DIR}}-{{.BUILD_DIR}}" internal: true vars: - CHECKSUM_FILE: >- - {{default (printf "%s.md5" .BUILD_DIR) .CHECKSUM_FILE}} CMAKE_ARGS: >- {{default "" .CMAKE_ARGS}} requires: vars: ["BUILD_DIR", "SOURCE_DIR"] - sources: - - "{{.SOURCE_DIR}}/**/*" - generates: ["{{.CHECKSUM_FILE}}"] - deps: - - task: "validate-checksum" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - DATA_PATHS: ["{{.BUILD_DIR}}"] - EXCLUDE_PATTERNS: ["install_manifest.txt"] cmds: - >- cmake @@ -189,51 +176,21 @@ tasks: cmake --build "{{.BUILD_DIR}}" --parallel - # This command must be last - - task: "compute-checksum" - vars: - DATA_PATHS: ["{{.BUILD_DIR}}"] - EXCLUDE_PATHS: ["install_manifest.txt"] - OUTPUT_FILE: "{{.CHECKSUM_FILE}}" - # Runs the CMake install step for the given build directory. + # Unconditionally runs the CMake install step for the given build directory. # # @param {string} BUILD_DIR CMake build directory. # @param {string} INSTALL_PREFIX Path prefix of where the project should be installed. - # @param {string={{.INSTALL_PREFIX}}.md5} [CHECKSUM_FILE] Path to store the checksum of installed - # files. - # @param {string[]=[{{.INSTALL_PREFIX}}]} [DATA_PATHS] Paths to compute the the checksum for. cmake-install: label: "{{.TASK}}-{{.BUILD_DIR}}-{{.INSTALL_PREFIX}}" internal: true - vars: - CHECKSUM_FILE: >- - {{default (printf "%s.md5" .INSTALL_PREFIX) .CHECKSUM_FILE}} - DATA_PATHS: - ref: "default (list .INSTALL_PREFIX) .DATA_PATHS" requires: vars: ["BUILD_DIR", "INSTALL_PREFIX"] - sources: - - "{{.BUILD_DIR}}/**/*" - - exclude: "{{.BUILD_DIR}}/install_manifest.txt" - generates: ["{{.CHECKSUM_FILE}}"] - deps: - - task: "validate-checksum" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - DATA_PATHS: - ref: ".DATA_PATHS" cmds: - >- cmake --install "{{.BUILD_DIR}}" --prefix "{{.INSTALL_PREFIX}}" - # This command must be last - - task: "compute-checksum" - vars: - DATA_PATHS: - ref: ".DATA_PATHS" - OUTPUT_FILE: "{{.CHECKSUM_FILE}}" # === # REMOTE UTILS @@ -275,7 +232,6 @@ tasks: then break fi - echo "Attempt $attempt failed. Retrying..." attempt=$((attempt + 1)) sleep 5 done @@ -284,18 +240,18 @@ tasks: exit 1 fi - # Runs curl to download the tarball from the given URL and extracts its contents. + # Runs curl to download the tarball file from the given URL and extracts its contents. # - # @param {string} OUTPUT_DIR Directory in which to extract the tarball. + # @param {string} OUTPUT_DIR Directory in which to extract the tarball file. # @param {string} URL # @param {string} FILE_SHA256 Content hash to verify downloaded tar file against. # @param {string={{.OUTPUT_DIR}}.md5} [CHECKSUM_FILE] File path to store the checksum of # downloaded tar file. - # @param {string[]=[]} [EXCLUDE_PATHS] Wildcard patterns for paths that shouldn't be extracted. - # @param {string[]=[]} [INCLUDE_PATHS] Wildcard patterns for paths to extract. + # @param {string[]=[]} [EXCLUDE_PATTERNS] Path wildcard patterns that should not be extracted. + # @param {string[]=[]} [DATA_PATTERNS] Path wildcard patterns to extract. # @param {int=1} [NUM_COMPONENTS_TO_STRIP] Number of leading path components to strip from the # extracted files. - # @param {string={{.OUTPUT_DIR}}.tar.gz} [TAR_FILE] Path where the tarball should be stored. + # @param {string={{.OUTPUT_DIR}}.tar.gz} [TAR_FILE] Path where the tarball file should be stored. download-and-extract-tar: label: "{{.TASK}}-{{.OUTPUT_DIR}}" internal: true @@ -304,8 +260,8 @@ tasks: {{default (printf "%s.md5" .OUTPUT_DIR) .CHECKSUM_FILE}} EXCLUDE_PATTERNS: ref: "default (list) .EXCLUDE_PATTERNS" - INCLUDE_PATHS: - ref: "default (list) .INCLUDE_PATHS" + DATA_PATTERNS: + ref: "default (list) .DATA_PATTERNS" NUM_COMPONENTS_TO_STRIP: "{{default 1 .NUM_COMPONENTS_TO_STRIP}}" TAR_FILE: >- {{default (printf "%s.tar.gz" .OUTPUT_DIR) .TAR_FILE}} @@ -322,7 +278,7 @@ tasks: - task: "validate-checksum" vars: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - DATA_PATHS: ["{{.OUTPUT_DIR}}"] + DATA_PATTERNS: ["{{.OUTPUT_DIR}}"] cmds: - |- rm -rf "{{.OUTPUT_DIR}}" @@ -338,12 +294,12 @@ tasks: {{- range .EXCLUDE_PATTERNS}} --exclude="{{.}}" {{- end}} - {{- range .DATA_PATHS}} + {{- range .DATA_PATTERNS}} "{{.}}" {{- end}} 2> /dev/null # This command must be last - task: "compute-checksum" vars: - DATA_PATHS: ["{{.OUTPUT_DIR}}"] + DATA_PATTERNS: ["{{.OUTPUT_DIR}}"] OUTPUT_FILE: "{{.CHECKSUM_FILE}}" From 7223ea5049f19d55a75ef6dc7a8a6d28c406723c Mon Sep 17 00:00:00 2001 From: davidlion Date: Fri, 22 Nov 2024 10:26:14 -0500 Subject: [PATCH 10/13] Apply suggestions from code review. Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- taskfiles/utils.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/taskfiles/utils.yml b/taskfiles/utils.yml index 77b7a9e..c5e6a90 100644 --- a/taskfiles/utils.yml +++ b/taskfiles/utils.yml @@ -177,7 +177,7 @@ tasks: --build "{{.BUILD_DIR}}" --parallel - # Unconditionally runs the CMake install step for the given build directory. + # Runs the CMake install step for the given build directory. # # @param {string} BUILD_DIR CMake build directory. # @param {string} INSTALL_PREFIX Path prefix of where the project should be installed. @@ -199,7 +199,7 @@ tasks: # Runs curl to download a file from the given URL. # # @param {string} URL - # @param {string} FILE_SHA256 Content hash to verify downloaded file against. + # @param {string} FILE_SHA256 Content hash to verify the downloaded file against. # @param {string={{(base .URL)}}} [OUTPUT_FILE] Path where the file should be stored. curl: label: "{{.TASK}}-{{.OUTPUT_FILE}}" @@ -240,18 +240,18 @@ tasks: exit 1 fi - # Runs curl to download the tarball file from the given URL and extracts its contents. + # Runs curl to download the tar file from the given URL and extracts its contents. # - # @param {string} OUTPUT_DIR Directory in which to extract the tarball file. + # @param {string} OUTPUT_DIR Directory in which to extract the tar file. # @param {string} URL - # @param {string} FILE_SHA256 Content hash to verify downloaded tar file against. + # @param {string} FILE_SHA256 Content hash to verify the downloaded tar file against. # @param {string={{.OUTPUT_DIR}}.md5} [CHECKSUM_FILE] File path to store the checksum of # downloaded tar file. # @param {string[]=[]} [EXCLUDE_PATTERNS] Path wildcard patterns that should not be extracted. # @param {string[]=[]} [DATA_PATTERNS] Path wildcard patterns to extract. # @param {int=1} [NUM_COMPONENTS_TO_STRIP] Number of leading path components to strip from the # extracted files. - # @param {string={{.OUTPUT_DIR}}.tar.gz} [TAR_FILE] Path where the tarball file should be stored. + # @param {string={{.OUTPUT_DIR}}.tar.gz} [TAR_FILE] Path where the tar file should be stored. download-and-extract-tar: label: "{{.TASK}}-{{.OUTPUT_DIR}}" internal: true From f4324e5978595a38ef2212300eb1d2757ec9a60b Mon Sep 17 00:00:00 2001 From: davidlion Date: Fri, 22 Nov 2024 10:33:11 -0500 Subject: [PATCH 11/13] ABCs --- taskfiles/utils.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/taskfiles/utils.yml b/taskfiles/utils.yml index c5e6a90..f8e0020 100644 --- a/taskfiles/utils.yml +++ b/taskfiles/utils.yml @@ -207,7 +207,7 @@ tasks: vars: OUTPUT_FILE: "{{default (base .URL) .OUTPUT_FILE}}" requires: - vars: ["URL", "FILE_SHA256"] + vars: ["FILE_SHA256", "URL"] generates: ["{{.OUTPUT_FILE}}"] status: - >- @@ -258,23 +258,23 @@ tasks: vars: CHECKSUM_FILE: >- {{default (printf "%s.md5" .OUTPUT_DIR) .CHECKSUM_FILE}} - EXCLUDE_PATTERNS: - ref: "default (list) .EXCLUDE_PATTERNS" DATA_PATTERNS: ref: "default (list) .DATA_PATTERNS" + EXCLUDE_PATTERNS: + ref: "default (list) .EXCLUDE_PATTERNS" NUM_COMPONENTS_TO_STRIP: "{{default 1 .NUM_COMPONENTS_TO_STRIP}}" TAR_FILE: >- {{default (printf "%s.tar.gz" .OUTPUT_DIR) .TAR_FILE}} requires: - vars: ["OUTPUT_DIR", "URL", "FILE_SHA256"] + vars: ["FILE_SHA256", "OUTPUT_DIR", "URL"] sources: ["{{.TASKFILE}}"] generates: ["{{.CHECKSUM_FILE}}", "{{.TAR_FILE}}"] deps: - task: "curl" vars: - URL: "{{.URL}}" FILE_SHA256: "{{.FILE_SHA256}}" OUTPUT_FILE: "{{.TAR_FILE}}" + URL: "{{.URL}}" - task: "validate-checksum" vars: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" From 7074e3864c16a6940d91eeb47478fcd3d935c681 Mon Sep 17 00:00:00 2001 From: davidlion Date: Wed, 4 Dec 2024 16:02:30 -0500 Subject: [PATCH 12/13] Fix checksum check to support ** patterns; remove quotations to support wildcards; add set/shopt flags globally. --- taskfiles/utils.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/taskfiles/utils.yml b/taskfiles/utils.yml index f8e0020..66a242c 100644 --- a/taskfiles/utils.yml +++ b/taskfiles/utils.yml @@ -1,5 +1,8 @@ version: "3" +set: ["u", "pipefail"] +shopt: ["globstar"] + tasks: # === # CHECKSUM UTILS @@ -29,7 +32,7 @@ tasks: --exclude="{{.}}" {{- end}} {{- range .DATA_PATTERNS}} - "{{.}}" + {{.}} {{- end}} 2> /dev/null | md5sum > {{.OUTPUT_FILE}} @@ -58,13 +61,14 @@ tasks: ref: "default (list) .EXCLUDE_PATTERNS" OUTPUT_FILE: "{{.TMP_CHECKSUM_FILE}}" - defer: "rm -f '{{.TMP_CHECKSUM_FILE}}'" - # Check that the paths exist and the checksum matches; otherwise delete the checksum file. - - >- - ( - {{- range .DATA_PATTERNS}} - test -e "{{.}}" && - {{- end}} - diff -q "{{.TMP_CHECKSUM_FILE}}" "{{.CHECKSUM_FILE}}" 2> /dev/null + # Check that all paths exist and the checksum matches; otherwise delete the checksum file. + - |- + ( {{- range .DATA_PATTERNS}} + for path in {{.}}; do + test -e "$path" + done + {{- end}} + diff -q "{{.TMP_CHECKSUM_FILE}}" "{{.CHECKSUM_FILE}}" 2> /dev/null ) || rm -f "{{.CHECKSUM_FILE}}" # === From 68baceed94790b31aaa8dc19c2caa13f8b82dc00 Mon Sep 17 00:00:00 2001 From: davidlion Date: Wed, 4 Dec 2024 16:02:48 -0500 Subject: [PATCH 13/13] Reorder label field to match guidelines. --- taskfiles/utils.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/taskfiles/utils.yml b/taskfiles/utils.yml index 66a242c..2309bee 100644 --- a/taskfiles/utils.yml +++ b/taskfiles/utils.yml @@ -15,6 +15,7 @@ tasks: compute-checksum: desc: "Tries to compute a checksum for the given paths and output it to a file." internal: true + label: "{{.TASK}}-{{.OUTPUT_FILE}}" silent: true requires: vars: ["DATA_PATTERNS", "OUTPUT_FILE"] @@ -47,6 +48,7 @@ tasks: desc: "Validates the checksum of the given directory matches the checksum in the given file, or deletes the checksum file otherwise." internal: true + label: "{{.TASK}}-{{.CHECKSUM_FILE}}" silent: true vars: TMP_CHECKSUM_FILE: "{{.CHECKSUM_FILE}}.tmp" @@ -163,8 +165,8 @@ tasks: # @param {string} SOURCE_DIR Project source directory containing the CMakeLists.txt file. # @param {string=""} [CMAKE_ARGS] Any additional arguments to pass to CMake's configure step. cmake-config-and-build: - label: "{{.TASK}}-{{.SOURCE_DIR}}-{{.BUILD_DIR}}" internal: true + label: "{{.TASK}}-{{.SOURCE_DIR}}-{{.BUILD_DIR}}" vars: CMAKE_ARGS: >- {{default "" .CMAKE_ARGS}} @@ -186,8 +188,8 @@ tasks: # @param {string} BUILD_DIR CMake build directory. # @param {string} INSTALL_PREFIX Path prefix of where the project should be installed. cmake-install: - label: "{{.TASK}}-{{.BUILD_DIR}}-{{.INSTALL_PREFIX}}" internal: true + label: "{{.TASK}}-{{.BUILD_DIR}}-{{.INSTALL_PREFIX}}" requires: vars: ["BUILD_DIR", "INSTALL_PREFIX"] cmds: @@ -206,8 +208,8 @@ tasks: # @param {string} FILE_SHA256 Content hash to verify the downloaded file against. # @param {string={{(base .URL)}}} [OUTPUT_FILE] Path where the file should be stored. curl: - label: "{{.TASK}}-{{.OUTPUT_FILE}}" internal: true + label: "{{.TASK}}-{{.OUTPUT_FILE}}" vars: OUTPUT_FILE: "{{default (base .URL) .OUTPUT_FILE}}" requires: @@ -257,8 +259,8 @@ tasks: # extracted files. # @param {string={{.OUTPUT_DIR}}.tar.gz} [TAR_FILE] Path where the tar file should be stored. download-and-extract-tar: - label: "{{.TASK}}-{{.OUTPUT_DIR}}" internal: true + label: "{{.TASK}}-{{.OUTPUT_DIR}}" vars: CHECKSUM_FILE: >- {{default (printf "%s.md5" .OUTPUT_DIR) .CHECKSUM_FILE}}