diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bca53d..ed4a48b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ on: defaults: run: - shell: bash + shell: bash --noprofile --norc -CeEuxo pipefail {0} concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 331087b..f074b82 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,7 @@ on: defaults: run: - shell: bash + shell: bash --noprofile --norc -CeEuxo pipefail {0} jobs: create-release: diff --git a/.markdownlint-cli2.yaml b/.markdownlint-cli2.yaml index 35e5227..2782f6d 100644 --- a/.markdownlint-cli2.yaml +++ b/.markdownlint-cli2.yaml @@ -1,9 +1,10 @@ -# https://github.com/DavidAnson/markdownlint#rules--aliases +# https://github.com/DavidAnson/markdownlint/blob/HEAD/doc/Rules.md config: - line-length: false - no-duplicate-heading: false - no-inline-html: false - no-emphasis-as-heading: false + line-length: false # MD013 + no-duplicate-heading: false # MD024 + no-blanks-blockquote: false # MD028 + no-inline-html: false # MD033 + no-emphasis-as-heading: false # MD036 # https://github.com/DavidAnson/markdownlint-cli2#markdownlint-cli2jsonc noBanner: true diff --git a/.shellcheckrc b/.shellcheckrc index 9de7d94..339847e 100644 --- a/.shellcheckrc +++ b/.shellcheckrc @@ -2,13 +2,34 @@ # https://github.com/koalaman/shellcheck/blob/HEAD/shellcheck.1.md#rc-files # See also: -# https://www.shellcheck.net/wiki/Optional +# https://github.com/koalaman/shellcheck/wiki/Optional # https://google.github.io/styleguide/shellguide.html -# https://www.shellcheck.net/wiki/SC2292 +# https://github.com/koalaman/shellcheck/wiki/SC2249 +# enable=add-default-case + +# https://github.com/koalaman/shellcheck/wiki/SC2244 +enable=avoid-nullary-conditions + +# https://github.com/koalaman/shellcheck/wiki/SC2312 +# enable=check-extra-masked-returns + +# https://github.com/koalaman/shellcheck/wiki/SC2310 +# https://github.com/koalaman/shellcheck/wiki/SC2311 +# enable=check-set-e-suppressed + +# enable=check-unassigned-uppercase + +# https://github.com/koalaman/shellcheck/wiki/SC2230 +enable=deprecate-which + +# https://github.com/koalaman/shellcheck/wiki/SC2248 +enable=quote-safe-variables + +# https://github.com/koalaman/shellcheck/wiki/SC2292 # https://google.github.io/styleguide/shellguide.html#s6.3-tests enable=require-double-brackets -# https://www.shellcheck.net/wiki/SC2250 +# https://github.com/koalaman/shellcheck/wiki/SC2250 # https://google.github.io/styleguide/shellguide.html#s5.6-variable-expansion enable=require-variable-braces diff --git a/main.sh b/main.sh index ecea3f9..a86d270 100755 --- a/main.sh +++ b/main.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 OR MIT -set -eEuo pipefail +set -CeEuo pipefail IFS=$'\n\t' retry() { @@ -14,24 +14,24 @@ retry() { "$@" } bail() { - echo "::error::$*" + printf '::error::%s\n' "$*" exit 1 } warn() { - echo "::warning::$*" + printf '::warning::%s\n' "$*" } download_and_checksum() { local url="${1:?}" local checksum="${2:?}" retry curl --proto '=https' --tlsv1.2 -fsSL --retry 10 "${url}" -o tmp - if type -P sha256sum &>/dev/null; then + if type -P sha256sum >/dev/null; then sha256sum -c - >/dev/null <<<"${checksum} *tmp" - elif type -P shasum &>/dev/null; then + elif type -P shasum >/dev/null; then # GitHub-hosted macOS runner does not install GNU Coreutils by default. # https://github.com/actions/runner-images/issues/90 shasum -a 256 -c - >/dev/null <<<"${checksum} *tmp" else - warn "checksum requires 'sha256sum' or 'shasum' command; consider installing one of them; skipped checksum for $(basename "${url}")" + warn "checksum requires 'sha256sum' or 'shasum' command; consider installing one of them; skipped checksum for $(basename -- "${url}")" fi } @@ -77,7 +77,7 @@ fi version="${tag}" # extract the portion of the tag matching the prefix pattern if [[ -n "${prefix}" ]]; then - prefix=$(grep <<<"${tag}" -Eo "^${prefix}") + prefix=$(grep -Eo "^${prefix}" <<<"${tag}") prefix="${prefix%-}" version="${tag#"${prefix}"}" version="${version#-}" @@ -110,6 +110,7 @@ if [[ -n "${branch}" ]]; then fi fi +notes='' if [[ -n "${changelog}" ]]; then # https://github.com/taiki-e/install-action/blob/HEAD/manifests/parse-changelog.json parse_changelog_version='0.6.10' @@ -140,9 +141,9 @@ if [[ -n "${changelog}" ]]; then *) bail "unrecognized OS type '$(uname -s)'" ;; esac action_dir="${HOME}/.create-gh-release-action" - mkdir -p "${action_dir}/bin" + mkdir -p -- "${action_dir}/bin" ( - cd "${action_dir}/bin" + cd -- "${action_dir}/bin" download_and_checksum "https://github.com/taiki-e/parse-changelog/releases/download/v${parse_changelog_version}/parse-changelog-${parse_changelog_target}.tar.gz" "${parse_changelog_checksum}" tar xzf tmp ) @@ -150,12 +151,12 @@ if [[ -n "${changelog}" ]]; then # If allow_missing_changelog is true then default to empty value if version not found if [[ "${allow_missing_changelog}" == "true" ]]; then - notes=$("${action_dir}/bin/parse-changelog${exe}" "${parse_changelog_options[@]}" || echo "") + notes=$("${action_dir}/bin/parse-changelog${exe}" "${parse_changelog_options[@]}" || true) else notes=$("${action_dir}/bin/parse-changelog${exe}" "${parse_changelog_options[@]}") fi - rm -rf "${action_dir}" + rm -rf -- "${action_dir}" fi # https://cli.github.com/manual/gh_release_view @@ -165,16 +166,16 @@ if GITHUB_TOKEN="${token}" gh release view "${tag}" &>/dev/null; then fi # https://cli.github.com/manual/gh_release_create -GITHUB_TOKEN="${token}" retry gh release create "${release_options[@]}" --title "${title}" --notes "${notes:-}" +GITHUB_TOKEN="${token}" retry gh release create "${release_options[@]}" --title "${title}" --notes "${notes}" # Set (computed) prefix and version outputs for future step use. computed_prefix=${tag%"${version}"} if [[ -n "${GITHUB_OUTPUT:-}" ]]; then - echo "computed-prefix=${computed_prefix}" >>"${GITHUB_OUTPUT}" - echo "version=${version}" >>"${GITHUB_OUTPUT}" + printf 'computed-prefix=%s\n' "${computed_prefix}" >>"${GITHUB_OUTPUT}" + printf 'version=%s\n' "${version}" >>"${GITHUB_OUTPUT}" else # Self-hosted runner may not set GITHUB_OUTPUT. warn "GITHUB_OUTPUT is not set; skip setting 'computed-prefix' and 'version' outputs" - echo "computed-prefix: ${computed_prefix}" - echo "version: ${version}" + printf 'computed-prefix: %s\n' "${computed_prefix}" + printf 'version: %s\n' "${version}" fi diff --git a/tools/publish.sh b/tools/publish.sh index 6787bdd..dc2b399 100755 --- a/tools/publish.sh +++ b/tools/publish.sh @@ -1,11 +1,9 @@ #!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 OR MIT -set -eEuo pipefail +set -CeEuo pipefail IFS=$'\n\t' -cd "$(dirname "$0")"/.. - -# shellcheck disable=SC2154 -trap 's=$?; echo >&2 "$0: error on line "${LINENO}": ${BASH_COMMAND}"; exit ${s}' ERR +trap -- 's=$?; printf >&2 "%s\n" "${0##*/}:${LINENO}: \`${BASH_COMMAND}\` exit with ${s}"; exit ${s}' ERR +cd -- "$(dirname -- "$0")"/.. # Publish a new release. # @@ -26,7 +24,7 @@ retry() { "$@" } bail() { - echo >&2 "error: $*" + printf >&2 'error: %s\n' "$*" exit 1 } @@ -41,6 +39,11 @@ fi if [[ $# -gt 1 ]]; then bail "invalid argument '$2'" fi +if { sed --help 2>&1 || true; } | grep -Eq -e '-i extension'; then + in_place=(-i '') +else + in_place=(-i) +fi # Make sure there is no uncommitted change. git diff --exit-code @@ -52,12 +55,15 @@ if gh release view "${tag}" &>/dev/null; then fi # Make sure that the release was created from an allowed branch. -if ! git branch | grep -q '\* main$'; then +if ! git branch | grep -Eq '\* main$'; then bail "current branch is not 'main'" fi +if ! git remote -v | grep -F origin | grep -Eq 'github\.com[:/]taiki-e/'; then + bail "cannot publish a new release from fork repository" +fi release_date=$(date -u '+%Y-%m-%d') -tags=$(git --no-pager tag | (grep -E "^${tag_prefix}[0-9]+" || true)) +tags=$(git --no-pager tag | { grep -E "^${tag_prefix}[0-9]+" || true; }) if [[ -n "${tags}" ]]; then # Make sure the same release does not exist in changelog. if grep -Eq "^## \\[${version//./\\.}\\]" "${changelog}"; then @@ -67,11 +73,12 @@ if [[ -n "${tags}" ]]; then bail "link to ${version} already exist in ${changelog}" fi # Update changelog. - remote_url=$(grep -E '^\[Unreleased\]: https://' "${changelog}" | sed 's/^\[Unreleased\]: //; s/\.\.\.HEAD$//') + remote_url=$(grep -E '^\[Unreleased\]: https://' "${changelog}" | sed -E 's/^\[Unreleased\]: //; s/\.\.\.HEAD$//') prev_tag="${remote_url#*/compare/}" remote_url="${remote_url%/compare/*}" - sed -i "s/^## \\[Unreleased\\]/## [Unreleased]\\n\\n## [${version}] - ${release_date}/" "${changelog}" - sed -i "s#^\[Unreleased\]: https://.*#[Unreleased]: ${remote_url}/compare/${tag}...HEAD\\n[${version}]: ${remote_url}/compare/${prev_tag}...${tag}#" "${changelog}" + sed -E "${in_place[@]}" \ + -e "s/^## \\[Unreleased\\]/## [Unreleased]\\n\\n## [${version}] - ${release_date}/" \ + -e "s#^\[Unreleased\]: https://.*#[Unreleased]: ${remote_url}/compare/${tag}...HEAD\\n[${version}]: ${remote_url}/compare/${prev_tag}...${tag}#" "${changelog}" if ! grep -Eq "^## \\[${version//./\\.}\\] - ${release_date}$" "${changelog}"; then bail "failed to update ${changelog}" fi @@ -94,9 +101,9 @@ changes=$(parse-changelog "${changelog}" "${version}") if [[ -z "${changes}" ]]; then bail "changelog for ${version} has no body" fi -echo "============== CHANGELOG ==============" -echo "${changes}" -echo "=======================================" +printf '============== CHANGELOG ==============\n' +printf '%s\n' "${changes}" +printf '=======================================\n' if [[ -n "${tags}" ]]; then # Create a release commit. diff --git a/tools/tidy.sh b/tools/tidy.sh index 7abfb45..c7ba387 100755 --- a/tools/tidy.sh +++ b/tools/tidy.sh @@ -1,23 +1,23 @@ #!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 OR MIT # shellcheck disable=SC2046 -set -eEuo pipefail +set -CeEuo pipefail IFS=$'\n\t' -cd "$(dirname "$0")"/.. - -# shellcheck disable=SC2154 -trap 's=$?; echo >&2 "$0: error on line "${LINENO}": ${BASH_COMMAND}"; exit ${s}' ERR +trap -- 's=$?; printf >&2 "%s\n" "${0##*/}:${LINENO}: \`${BASH_COMMAND}\` exit with ${s}"; exit ${s}' ERR +trap -- 'printf >&2 "%s\n" "${0##*/}: trapped SIGINT"; exit 1' SIGINT +cd -- "$(dirname -- "$0")"/.. # USAGE: # ./tools/tidy.sh # # Note: This script requires the following tools: +# - git +# - jq 1.6+ +# - npm (node 18+) +# - python 3.6+ # - shfmt # - shellcheck -# - npm -# - jq -# - python -# - rustup (if Rust code exists) +# - cargo, rustfmt (if Rust code exists) # - clang-format (if C/C++ code exists) # # This script is shared with other repositories, so there may also be @@ -40,21 +40,58 @@ check_config() { error "could not found $1 in the repository root" fi } -info() { - echo >&2 "info: $*" +check_install() { + for tool in "$@"; do + if ! type -P "${tool}" >/dev/null; then + if [[ "${tool}" == "python3" ]]; then + if type -P python >/dev/null; then + continue + fi + fi + error "'${tool}' is required to run this check" + return 1 + fi + done +} +retry() { + for i in {1..10}; do + if "$@"; then + return 0 + else + sleep "${i}" + fi + done + "$@" } error() { if [[ -n "${GITHUB_ACTIONS:-}" ]]; then - echo "::error::$*" + printf '::error::%s\n' "$*" else - echo >&2 "error: $*" + printf >&2 'error: %s\n' "$*" fi should_fail=1 } -venv() { - local bin="$1" - shift - "${venv_bin}/${bin}${exe}" "$@" +warn() { + if [[ -n "${GITHUB_ACTIONS:-}" ]]; then + printf '::warning::%s\n' "$*" + else + printf >&2 'warning: %s\n' "$*" + fi +} +info() { + printf >&2 'info: %s\n' "$*" +} +sed_rhs_escape() { + sed 's/\\/\\\\/g; s/\&/\\\&/g; s/\//\\\//g' <<<"$1" +} +venv_install_yq() { + if [[ ! -e "${venv_bin}/yq${exe}" ]]; then + if [[ ! -d .venv ]]; then + "python${py_suffix}" -m venv .venv >&2 + fi + info "installing yq to .venv using pip${py_suffix}" + "${venv_bin}/pip${py_suffix}${exe}" install yq >&2 + fi } if [[ $# -gt 0 ]]; then @@ -65,127 +102,235 @@ EOF exit 1 fi +exe='' +py_suffix='' +if type -P python3 >/dev/null; then + py_suffix='3' +fi +venv_bin=.venv/bin +yq() { + venv_install_yq + "${venv_bin}/yq${exe}" "$@" +} +tomlq() { + venv_install_yq + "${venv_bin}/tomlq${exe}" "$@" +} +case "$(uname -s)" in + Linux) + if [[ "$(uname -o)" == "Android" ]]; then + ostype=android + else + ostype=linux + fi + ;; + Darwin) ostype=macos ;; + FreeBSD) ostype=freebsd ;; + NetBSD) ostype=netbsd ;; + OpenBSD) ostype=openbsd ;; + DragonFly) ostype=dragonfly ;; + SunOS) + if [[ "$(/usr/bin/uname -o)" == "illumos" ]]; then + ostype=illumos + else + ostype=solaris + # Solaris /usr/bin/* are not POSIX-compliant (e.g., grep has no -q, -E, -F), + # and POSIX-compliant commands are in /usr/xpg{4,6,7}/bin. + # https://docs.oracle.com/cd/E88353_01/html/E37853/xpg-7.html + if [[ "${PATH}" != *"/usr/xpg4/bin"* ]]; then + export PATH="/usr/xpg4/bin:${PATH}" + fi + # GNU/BSD grep/sed is required to run some checks, but most checks are okay with other POSIX grep/sed. + # Solaris /usr/xpg4/bin/grep has -q, -E, -F, but no -o (non-POSIX). + # Solaris /usr/xpg4/bin/sed has no -E (POSIX.1-2024) yet. + if type -P ggrep >/dev/null; then + grep() { ggrep "$@"; } + fi + if type -P gsed >/dev/null; then + sed() { gsed "$@"; } + fi + fi + ;; + MINGW* | MSYS* | CYGWIN* | Windows_NT) + ostype=windows + exe=.exe + venv_bin=.venv/Scripts + if type -P jq >/dev/null; then + # https://github.com/jqlang/jq/issues/1854 + _tmp=$(jq -r .a <<<'{}') + if [[ "${_tmp}" != "null" ]]; then + _tmp=$(jq -b -r .a 2>/dev/null <<<'{}' || true) + if [[ "${_tmp}" == "null" ]]; then + jq() { command jq -b "$@"; } + else + jq() { command jq "$@" | tr -d '\r'; } + fi + yq() { + venv_install_yq + "${venv_bin}/yq${exe}" "$@" | tr -d '\r' + } + tomlq() { + venv_install_yq + "${venv_bin}/tomlq${exe}" "$@" | tr -d '\r' + } + fi + fi + ;; + *) error "unrecognized os type '$(uname -s)' for \`\$(uname -s)\`" ;; +esac + +check_install git +exclude_from_ls_files=() +# - `find` lists symlinks. `! ( -name -prune )` (.i.e., ignore ) are manually listed from .gitignore. +# - `git submodule status` lists submodules. Use sed to remove the first character indicates status ( |+|-). +# - `git ls-files --deleted` lists removed files. +while IFS=$'\n' read -r line; do exclude_from_ls_files+=("${line}"); done < <({ + find . \! \( -name .git -prune \) \! \( -name target -prune \) \! \( -name .venv -prune \) \! \( -name tmp -prune \) -type l | cut -c3- + git submodule status | sed 's/^.//' | cut -d' ' -f2 + git ls-files --deleted +} | LC_ALL=C sort -u) +exclude_from_ls_files_no_symlink=() +while IFS=$'\n' read -r line; do exclude_from_ls_files_no_symlink+=("${line}"); done < <({ + git submodule status | sed 's/^.//' | cut -d' ' -f2 + git ls-files --deleted +} | LC_ALL=C sort -u) +ls_files() { + if [[ "${1:-}" == "--include-symlink" ]]; then + shift + comm -23 <(git ls-files "$@" | LC_ALL=C sort) <(printf '%s\n' ${exclude_from_ls_files_no_symlink[@]+"${exclude_from_ls_files_no_symlink[@]}"}) + else + comm -23 <(git ls-files "$@" | LC_ALL=C sort) <(printf '%s\n' ${exclude_from_ls_files[@]+"${exclude_from_ls_files[@]}"}) + fi +} + # Rust (if exists) -if [[ -n "$(git ls-files '*.rs')" ]]; then +if [[ -n "$(ls_files '*.rs')" ]]; then info "checking Rust code style" check_config .rustfmt.toml - if type -P rustup &>/dev/null; then + if check_install cargo jq python3; then # `cargo fmt` cannot recognize files not included in the current workspace and modules # defined inside macros, so run rustfmt directly. # We need to use nightly rustfmt because we use the unstable formatting options of rustfmt. - rustc_version=$(rustc -vV | grep '^release:' | cut -d' ' -f2) - if [[ "${rustc_version}" == *"nightly"* ]] || [[ "${rustc_version}" == *"dev"* ]]; then - rustup component add rustfmt &>/dev/null - echo "+ rustfmt \$(git ls-files '*.rs')" - rustfmt $(git ls-files '*.rs') + rustc_version=$(rustc -vV | grep -E '^release:' | cut -d' ' -f2) + if [[ "${rustc_version}" =~ nightly|dev ]] || ! type -P rustup >/dev/null; then + if type -P rustup >/dev/null; then + retry rustup component add rustfmt &>/dev/null + fi + info "running \`rustfmt \$(git ls-files '*.rs')\`" + rustfmt $(ls_files '*.rs') else - rustup component add rustfmt --toolchain nightly &>/dev/null - echo "+ rustfmt +nightly \$(git ls-files '*.rs')" - rustfmt +nightly $(git ls-files '*.rs') - fi - check_diff $(git ls-files '*.rs') - else - error "'rustup' is not installed; skipped Rust code style check" - fi - cast_without_turbofish=$(grep -n -E '\.cast\(\)' $(git ls-files '*.rs') || true) - if [[ -n "${cast_without_turbofish}" ]]; then - error "please replace \`.cast()\` with \`.cast::()\`:" - echo "${cast_without_turbofish}" - fi - # Sync readme and crate-level doc. - first='1' - for readme in $(git ls-files '*README.md'); do - if ! grep -q '^' "${readme}"; then - continue - fi - lib="$(dirname "${readme}")/src/lib.rs" - if [[ -n "${first}" ]]; then - first='' - info "checking readme and crate-level doc are synchronized" - fi - if ! grep -q '^' "${readme}"; then - bail "missing '' comment in ${readme}" - fi - if ! grep -q '^' "${lib}"; then - bail "missing '' comment in ${lib}" - fi - if ! grep -q '^' "${lib}"; then - bail "missing '' comment in ${lib}" - fi - new=$(tr <"${readme}" '\n' '\a' | grep -o '.*' | sed 's/\&/\\\&/g; s/\\/\\\\/g') - new=$(tr <"${lib}" '\n' '\a' | awk -v new="${new}" 'gsub(".*",new)' | tr '\a' '\n') - echo "${new}" >"${lib}" - check_diff "${lib}" - done - # Make sure that public Rust crates don't contain executables and binaries. - executables='' - binaries='' - metadata=$(cargo metadata --format-version=1 --no-deps) - has_public_crate='' - for id in $(jq <<<"${metadata}" '.workspace_members[]'); do - pkg=$(jq <<<"${metadata}" ".packages[] | select(.id == ${id})") - publish=$(jq <<<"${pkg}" -r '.publish') - manifest_path=$(jq <<<"${pkg}" -r '.manifest_path') - if ! grep -q '^\[lints\]' "${manifest_path}" && ! grep -q '^\[lints\.rust\]' "${manifest_path}"; then - error "no [lints] table in ${manifest_path} please add '[lints]' with 'workspace = true'" + if type -P rustup >/dev/null; then + retry rustup component add rustfmt --toolchain nightly &>/dev/null + fi + info "running \`rustfmt +nightly \$(git ls-files '*.rs')\`" + rustfmt +nightly $(ls_files '*.rs') fi - # Publishing is unrestricted if null, and forbidden if an empty array. - if [[ "${publish}" == "[]" ]]; then - continue + check_diff $(ls_files '*.rs') + cast_without_turbofish=$(grep -Fn '.cast()' $(ls_files '*.rs') || true) + if [[ -n "${cast_without_turbofish}" ]]; then + error "please replace \`.cast()\` with \`.cast::()\`:" + printf '%s\n' "${cast_without_turbofish}" fi - has_public_crate='1' - done - if [[ -n "${has_public_crate}" ]]; then - info "checking public crates don't contain executables and binaries" + # Sync readme and crate-level doc. + first=1 + for readme in $(ls_files '*README.md'); do + if ! grep -Eq '^' "${readme}"; then + continue + fi + lib="$(dirname -- "${readme}")/src/lib.rs" + if [[ -n "${first}" ]]; then + first='' + info "checking readme and crate-level doc are synchronized" + fi + if ! grep -Eq '^' "${readme}"; then + bail "missing '' comment in ${readme}" + fi + if ! grep -Eq '^' "${lib}"; then + bail "missing '' comment in ${lib}" + fi + if ! grep -Eq '^' "${lib}"; then + bail "missing '' comment in ${lib}" + fi + new=$(tr '\n' '\a' <"${readme}" | grep -Eo '.*') + new=$(tr '\n' '\a' <"${lib}" | sed "s/.*/$(sed_rhs_escape "${new}")/" | tr '\a' '\n') + printf '%s\n' "${new}" >|"${lib}" + check_diff "${lib}" + done + # Make sure that public Rust crates don't contain executables and binaries. + executables='' + binaries='' + metadata=$(cargo metadata --format-version=1 --no-deps) + root_manifest='' if [[ -f Cargo.toml ]]; then root_manifest=$(cargo locate-project --message-format=plain --manifest-path Cargo.toml) - root_pkg=$(jq <<<"${metadata}" ".packages[] | select(.manifest_path == \"${root_manifest}\")") - if [[ -n "${root_pkg}" ]]; then - publish=$(jq <<<"${root_pkg}" -r '.publish') - # Publishing is unrestricted if null, and forbidden if an empty array. - if [[ "${publish}" != "[]" ]]; then - if ! grep -Eq '^exclude = \[.*"/\.\*".*\]' Cargo.toml; then - error "top-level Cargo.toml of non-virtual workspace should have 'exclude' field with \"/.*\"" - fi - if [[ -e tools ]] && ! grep -Eq '^exclude = \[.*"/tools".*\]' Cargo.toml; then - error "top-level Cargo.toml of non-virtual workspace should have 'exclude' field with \"/tools\" if it exists" - fi - if [[ -e target-specs ]] && ! grep -Eq '^exclude = \[.*"/target-specs".*\]' Cargo.toml; then - error "top-level Cargo.toml of non-virtual workspace should have 'exclude' field with \"/target-specs\" if it exists" - fi - fi - fi fi - for p in $(git ls-files); do - # Skip directories. - if [[ -d "${p}" ]]; then - continue + exclude='' + has_public_crate='' + has_root_crate='' + for pkg in $(jq -c '. as $metadata | .workspace_members[] as $id | $metadata.packages[] | select(.id == $id)' <<<"${metadata}"); do + eval "$(jq -r '@sh "publish=\(.publish) manifest_path=\(.manifest_path)"' <<<"${pkg}")" + if [[ "$(tomlq -c '.lints' "${manifest_path}")" == "null" ]]; then + error "no [lints] table in ${manifest_path} please add '[lints]' with 'workspace = true'" fi - # Top-level hidden files/directories and tools/* are excluded from crates.io (ensured by the above check). - # TODO: fully respect exclude field in Cargo.toml. - case "${p}" in - .* | tools/* | target-specs/*) continue ;; - esac - if [[ -x "${p}" ]]; then - executables+="${p}"$'\n' + # Publishing is unrestricted if null, and forbidden if an empty array. + if [[ -z "${publish}" ]]; then + continue fi - # Use diff instead of file because file treats an empty file as a binary - # https://unix.stackexchange.com/questions/275516/is-there-a-convenient-way-to-classify-files-as-binary-or-text#answer-402870 - if (diff .gitattributes "${p}" || true) | grep -q '^Binary file'; then - binaries+="${p}"$'\n' + has_public_crate=1 + if [[ "${manifest_path}" == "${root_manifest}" ]]; then + has_root_crate=1 + exclude=$(tomlq -r '.package.exclude[]' "${manifest_path}") + if ! grep -Eq '^/\.\*$' <<<"${exclude}"; then + error "top-level Cargo.toml of non-virtual workspace should have 'exclude' field with \"/.*\"" + fi + if [[ -e tools ]] && ! grep -Eq '^/tools$' <<<"${exclude}"; then + error "top-level Cargo.toml of non-virtual workspace should have 'exclude' field with \"/tools\" if it exists" + fi + if [[ -e target-specs ]] && ! grep -Eq '^/target-specs$' <<<"${exclude}"; then + error "top-level Cargo.toml of non-virtual workspace should have 'exclude' field with \"/target-specs\" if it exists" + fi fi done - if [[ -n "${executables}" ]]; then - error "file-permissions-check failed: executables are only allowed to be present in directories that are excluded from crates.io" - echo "=======================================" - echo -n "${executables}" - echo "=======================================" - fi - if [[ -n "${binaries}" ]]; then - error "file-permissions-check failed: binaries are only allowed to be present in directories that are excluded from crates.io" - echo "=======================================" - echo -n "${binaries}" - echo "=======================================" + if [[ -n "${has_public_crate}" ]]; then + info "checking public crates don't contain executables and binaries" + for p in $(ls_files --include-symlink); do + # Skip directories. + if [[ -d "${p}" ]]; then + continue + fi + # Top-level hidden files/directories and tools/* are excluded from crates.io (ensured by the above check). + # TODO: fully respect exclude field in Cargo.toml. + case "${p}" in + .* | tools/* | target-specs/*) continue ;; + */*) ;; + *) + # If there is no crate at root, executables at the repository root directory if always okay. + if [[ -z "${has_root_crate}" ]]; then + continue + fi + ;; + esac + if [[ -x "${p}" ]]; then + executables+="${p}"$'\n' + fi + # Use `diff` instead of `file` because `file` treats an empty file as a binary. + # https://unix.stackexchange.com/questions/275516/is-there-a-convenient-way-to-classify-files-as-binary-or-text#answer-402870 + if { diff .gitattributes "${p}" || true; } | grep -Eq '^Binary file'; then + binaries+="${p}"$'\n' + fi + done + if [[ -n "${executables}" ]]; then + error "file-permissions-check failed: executables are only allowed to be present in directories that are excluded from crates.io" + printf '=======================================\n' + printf '%s' "${executables}" + printf '=======================================\n' + fi + if [[ -n "${binaries}" ]]; then + error "file-permissions-check failed: binaries are only allowed to be present in directories that are excluded from crates.io" + printf '=======================================\n' + printf '%s' "${binaries}" + printf '=======================================\n' + fi fi fi elif [[ -e .rustfmt.toml ]]; then @@ -193,148 +338,357 @@ elif [[ -e .rustfmt.toml ]]; then fi # C/C++ (if exists) -if [[ -n "$(git ls-files '*.c' '*.h' '*.cpp' '*.hpp')" ]]; then +clang_format_ext=('*.c' '*.h' '*.cpp' '*.hpp') +if [[ -n "$(ls_files "${clang_format_ext[@]}")" ]]; then info "checking C/C++ code style" check_config .clang-format - if type -P clang-format &>/dev/null; then - echo "+ clang-format -i \$(git ls-files '*.c' '*.h' '*.cpp' '*.hpp')" - clang-format -i $(git ls-files '*.c' '*.h' '*.cpp' '*.hpp') - check_diff $(git ls-files '*.c' '*.h' '*.cpp' '*.hpp') - else - error "'clang-format' is not installed; skipped C/C++ code style check" + if check_install clang-format; then + IFS=' ' + info "running \`clang-format -i \$(git ls-files ${clang_format_ext[*]})\`" + IFS=$'\n\t' + clang-format -i $(ls_files "${clang_format_ext[@]}") + check_diff $(ls_files "${clang_format_ext[@]}") fi elif [[ -e .clang-format ]]; then error ".clang-format is unused" fi +# https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html +cpp_alt_ext=('*.cc' '*.cp' '*.cxx' '*.C' '*.CPP' '*.c++') +hpp_alt_ext=('*.hh' '*.hp' '*.hxx' '*.H' '*.HPP' '*.h++') +if [[ -n "$(ls_files "${cpp_alt_ext[@]}")" ]]; then + error "please use '.cpp' for consistency" + printf '=======================================\n' + ls_files "${cpp_alt_ext[@]}" + printf '=======================================\n' +fi +if [[ -n "$(ls_files "${hpp_alt_ext[@]}")" ]]; then + error "please use '.hpp' for consistency" + printf '=======================================\n' + ls_files "${hpp_alt_ext[@]}" + printf '=======================================\n' +fi # YAML/JavaScript/JSON (if exists) -if [[ -n "$(git ls-files '*.yml' '*.yaml' '*.js' '*.json')" ]]; then +prettier_ext=('*.yml' '*.yaml' '*.js' '*.json') +if [[ -n "$(ls_files "${prettier_ext[@]}")" ]]; then info "checking YAML/JavaScript/JSON code style" check_config .editorconfig - if type -P npm &>/dev/null; then - echo "+ npx -y prettier -l -w \$(git ls-files '*.yml' '*.yaml' '*.js' '*.json')" - npx -y prettier -l -w $(git ls-files '*.yml' '*.yaml' '*.js' '*.json') - check_diff $(git ls-files '*.yml' '*.yaml' '*.js' '*.json') - else - error "'npm' is not installed; skipped YAML/JavaScript/JSON code style check" - fi - # Check GitHub workflows. - if [[ -d .github/workflows ]]; then - info "checking GitHub workflows" - if type -P jq &>/dev/null; then - if type -P python3 &>/dev/null || type -P python &>/dev/null; then - py_suffix='' - if type -P python3 &>/dev/null; then - py_suffix='3' - fi - exe='' - venv_bin='.venv/bin' - case "$(uname -s)" in - MINGW* | MSYS* | CYGWIN* | Windows_NT) - exe='.exe' - venv_bin='.venv/Scripts' - ;; - esac - if [[ ! -d .venv ]]; then - "python${py_suffix}" -m venv .venv - fi - if [[ ! -e "${venv_bin}/yq${exe}" ]]; then - venv "pip${py_suffix}" install yq - fi - for workflow in .github/workflows/*.yml; do - # The top-level permissions must be weak as they are referenced by all jobs. - permissions=$(venv yq -c '.permissions' "${workflow}") - case "${permissions}" in - '{"contents":"read"}' | '{"contents":"none"}') ;; - null) error "${workflow}: top level permissions not found; it must be 'contents: read' or weaker permissions" ;; - *) error "${workflow}: only 'contents: read' and weaker permissions are allowed at top level; if you want to use stronger permissions, please set job-level permissions" ;; - esac - # Make sure the 'needs' section is not out of date. - if grep -q '# tidy:needs' "${workflow}" && ! grep -Eq '# *needs: \[' "${workflow}"; then - # shellcheck disable=SC2207 - jobs_actual=($(venv yq '.jobs' "${workflow}" | jq -r 'keys_unsorted[]')) - unset 'jobs_actual[${#jobs_actual[@]}-1]' - # shellcheck disable=SC2207 - jobs_expected=($(venv yq -r '.jobs."ci-success".needs[]' "${workflow}")) - if [[ "${jobs_actual[*]}" != "${jobs_expected[*]+"${jobs_expected[*]}"}" ]]; then - printf -v jobs '%s, ' "${jobs_actual[@]}" - sed -i "s/needs: \[.*\] # tidy:needs/needs: [${jobs%, }] # tidy:needs/" "${workflow}" - check_diff "${workflow}" - error "${workflow}: please update 'needs' section in 'ci-success' job" - fi - fi - done - else - error "'python3' is not installed; skipped GitHub workflow check" - fi - else - error "'jq' is not installed; skipped GitHub workflow check" - fi + if [[ "${ostype}" == "solaris" ]] && [[ -n "${CI:-}" ]] && ! type -P npm >/dev/null; then + warn "this check is skipped on Solaris due to no node 18+ in upstream package manager" + elif check_install npm; then + IFS=' ' + info "running \`npx -y prettier -l -w \$(git ls-files ${prettier_ext[*]})\`" + IFS=$'\n\t' + npx -y prettier -l -w $(ls_files "${prettier_ext[@]}") + check_diff $(ls_files "${prettier_ext[@]}") fi fi -if [[ -n "$(git ls-files '*.yaml' | (grep -v .markdownlint-cli2.yaml || true))" ]]; then +if [[ -n "$(ls_files '*.yaml' | { grep -Fv '.markdownlint-cli2.yaml' || true; })" ]]; then error "please use '.yml' instead of '.yaml' for consistency" - git ls-files '*.yaml' | (grep -v .markdownlint-cli2.yaml || true) + printf '=======================================\n' + ls_files '*.yaml' | { grep -Fv '.markdownlint-cli2.yaml' || true; } + printf '=======================================\n' fi # TOML (if exists) -if [[ -n "$(git ls-files '*.toml' | (grep -v .taplo.toml || true))" ]]; then +if [[ -n "$(ls_files '*.toml' | { grep -Fv '.taplo.toml' || true; })" ]]; then info "checking TOML style" check_config .taplo.toml - if type -P npm &>/dev/null; then - echo "+ npx -y @taplo/cli fmt \$(git ls-files '*.toml')" - RUST_LOG=warn npx -y @taplo/cli fmt $(git ls-files '*.toml') - check_diff $(git ls-files '*.toml') - else - error "'npm' is not installed; skipped TOML style check" + if [[ "${ostype}" == "solaris" ]] && [[ -n "${CI:-}" ]] && ! type -P npm >/dev/null; then + warn "this check is skipped on Solaris due to no node 18+ in upstream package manager" + elif check_install npm; then + info "running \`npx -y @taplo/cli fmt \$(git ls-files '*.toml')\`" + RUST_LOG=warn npx -y @taplo/cli fmt $(ls_files '*.toml') + check_diff $(ls_files '*.toml') fi elif [[ -e .taplo.toml ]]; then error ".taplo.toml is unused" fi # Markdown (if exists) -if [[ -n "$(git ls-files '*.md')" ]]; then +if [[ -n "$(ls_files '*.md')" ]]; then info "checking Markdown style" check_config .markdownlint-cli2.yaml - if type -P npm &>/dev/null; then - echo "+ npx -y markdownlint-cli2 \$(git ls-files '*.md')" - npx -y markdownlint-cli2 $(git ls-files '*.md') - else - error "'npm' is not installed; skipped Markdown style check" + if [[ "${ostype}" == "solaris" ]] && [[ -n "${CI:-}" ]] && ! type -P npm >/dev/null; then + warn "this check is skipped on Solaris due to no node 18+ in upstream package manager" + elif check_install npm; then + info "running \`npx -y markdownlint-cli2 \$(git ls-files '*.md')\`" + if ! npx -y markdownlint-cli2 $(ls_files '*.md'); then + error "check failed; please resolve the above markdownlint error(s)" + fi fi elif [[ -e .markdownlint-cli2.yaml ]]; then error ".markdownlint-cli2.yaml is unused" fi -if [[ -n "$(git ls-files '*.markdown')" ]]; then +if [[ -n "$(ls_files '*.markdown')" ]]; then error "please use '.md' instead of '.markdown' for consistency" - git ls-files '*.markdown' + printf '=======================================\n' + ls_files '*.markdown' + printf '=======================================\n' fi # Shell scripts info "checking Shell scripts" -if type -P shfmt &>/dev/null; then +shell_files=() +docker_files=() +bash_files=() +grep_ere_files=() +sed_ere_files=() +for p in $(ls_files '*.sh' '*Dockerfile*'); do + case "${p##*/}" in + *.sh) + shell_files+=("${p}") + if [[ "$(head -1 "${p}")" =~ ^#!/.*bash ]]; then + bash_files+=("${p}") + fi + ;; + *Dockerfile*) + docker_files+=("${p}") + bash_files+=("${p}") # TODO + ;; + esac + if grep -Eq '(^|[^0-9A-Za-z\."'\''-])(grep) -[A-Za-z]*E[^\)]' "${p}"; then + grep_ere_files+=("${p}") + fi + if grep -Eq '(^|[^0-9A-Za-z\."'\''-])(sed) -[A-Za-z]*E[^\)]' "${p}"; then + sed_ere_files+=("${p}") + fi +done +# TODO: .cirrus.yml +workflows=() +actions=() +if [[ -d .github/workflows ]]; then + for p in .github/workflows/*.yml; do + workflows+=("${p}") + bash_files+=("${p}") # TODO + done +fi +if [[ -n "$(ls_files '*action.yml')" ]]; then + for p in $(ls_files '*action.yml'); do + if [[ "${p##*/}" == "action.yml" ]]; then + actions+=("${p}") + if ! grep -Fq 'shell: sh' "${p}"; then + bash_files+=("${p}") + fi + fi + done +fi +# correctness +res=$({ grep -En '(\[\[ .* ]]|(^|[^\$])\(\(.*\)\))( +#| *$)' "${bash_files[@]}" || true; } | { grep -Ev '^[^ ]+: *(#|//)' || true; } | LC_ALL=C sort) +if [[ -n "${res}" ]]; then + error "bare [[ ]] and (( )) may not work as intended: see https://github.com/koalaman/shellcheck/issues/2360 for more" + printf '=======================================\n' + printf '%s\n' "${res}" + printf '=======================================\n' +fi +# TODO: chmod|chown +res=$({ grep -En '(^|[^0-9A-Za-z\."'\''-])(basename|cat|cd|cp|dirname|ln|ls|mkdir|mv|pushd|rm|rmdir|tee|touch)( +-[0-9A-Za-z]+)* +[^<>\|-]' "${bash_files[@]}" || true; } | { grep -Ev '^[^ ]+: *(#|//)' || true; } | LC_ALL=C sort) +if [[ -n "${res}" ]]; then + error "use \`--\` before path(s): see https://github.com/koalaman/shellcheck/issues/2707 / https://github.com/koalaman/shellcheck/issues/2612 / https://github.com/koalaman/shellcheck/issues/2305 / https://github.com/koalaman/shellcheck/issues/2157 / https://github.com/koalaman/shellcheck/issues/2121 / https://github.com/koalaman/shellcheck/issues/314 for more" + printf '=======================================\n' + printf '%s\n' "${res}" + printf '=======================================\n' +fi +res=$({ grep -En '(^|[^0-9A-Za-z\."'\''-])(LINES|RANDOM|PWD)=' "${bash_files[@]}" || true; } | { grep -Ev '^[^ ]+: *(#|//)' || true; } | LC_ALL=C sort) +if [[ -n "${res}" ]]; then + error "do not modify these built-in bash variables: see https://github.com/koalaman/shellcheck/issues/2160 / https://github.com/koalaman/shellcheck/issues/2559 for more" + printf '=======================================\n' + printf '%s\n' "${res}" + printf '=======================================\n' +fi +# perf +res=$({ grep -En '(^|[^\\])\$\((cat) ' "${bash_files[@]}" || true; } | { grep -Ev '^[^ ]+: *(#|//)' || true; } | LC_ALL=C sort) +if [[ -n "${res}" ]]; then + error "use faster \`\$(' "${bash_files[@]}" || true; } | { grep -Ev '^[^ ]+: *(#|//)' || true; } | LC_ALL=C sort) +if [[ -n "${res}" ]]; then + error "\`type -P\` doesn't output to stderr; use \`>\` instead of \`&>\`" + printf '=======================================\n' + printf '%s\n' "${res}" + printf '=======================================\n' +fi +# TODO: multi-line case +res=$({ grep -En '(^|[^0-9A-Za-z\."'\''-])(echo|printf )[^;)]* \|[^\|]' "${bash_files[@]}" || true; } | { grep -Ev '^[^ ]+: *(#|//)' || true; } | LC_ALL=C sort) +if [[ -n "${res}" ]]; then + error "use faster \`<<<...\` instead of \`echo ... |\`/\`printf ... |\`: see https://github.com/koalaman/shellcheck/issues/2593 for more" + printf '=======================================\n' + printf '%s\n' "${res}" + printf '=======================================\n' +fi +# style +if [[ ${#grep_ere_files[@]} -gt 0 ]]; then + # We intentionally do not check for occurrences in any other order (e.g., -iE, -i -E) here. + # This enforces the style and makes it easier to search. + res=$({ grep -En '(^|[^0-9A-Za-z\."'\''-])(grep) +([^-]|-[^EFP-]|--[^hv])' "${grep_ere_files[@]}" || true; } | { grep -Ev '^[^ ]+: *(#|//)' || true; } | LC_ALL=C sort) + if [[ -n "${res}" ]]; then + error "please always use ERE (grep -E) instead of BRE for code consistency within a file" + printf '=======================================\n' + printf '%s\n' "${res}" + printf '=======================================\n' + fi +fi +if [[ ${#sed_ere_files[@]} -gt 0 ]]; then + res=$({ grep -En '(^|[^0-9A-Za-z\."'\''-])(sed) +([^-]|-[^E-]|--[^hv])' "${sed_ere_files[@]}" || true; } | { grep -Ev '^[^ ]+: *(#|//)' || true; } | LC_ALL=C sort) + if [[ -n "${res}" ]]; then + error "please always use ERE (sed -E) instead of BRE for code consistency within a file" + printf '=======================================\n' + printf '%s\n' "${res}" + printf '=======================================\n' + fi +fi +if check_install shfmt; then check_config .editorconfig - echo "+ shfmt -l -w \$(git ls-files '*.sh')" - shfmt -l -w $(git ls-files '*.sh') - check_diff $(git ls-files '*.sh') -else - error "'shfmt' is not installed; skipped Shell scripts style check" + info "running \`shfmt -l -w \$(git ls-files '*.sh')\`" + if ! shfmt -l -w "${shell_files[@]}"; then + error "check failed; please resolve the shfmt error(s)" + fi + check_diff "${shell_files[@]}" fi -if type -P shellcheck &>/dev/null; then +if [[ "${ostype}" == "solaris" ]] && [[ -n "${CI:-}" ]] && ! type -P shellcheck >/dev/null; then + warn "this check is skipped on Solaris due to no haskell/shellcheck in upstream package manager" +elif check_install shellcheck; then check_config .shellcheckrc - echo "+ shellcheck \$(git ls-files '*.sh')" - if ! shellcheck $(git ls-files '*.sh'); then - should_fail=1 + info "running \`shellcheck \$(git ls-files '*.sh')\`" + if ! shellcheck "${shell_files[@]}"; then + error "check failed; please resolve the above shellcheck error(s)" fi - if [[ -n "$(git ls-files '*Dockerfile')" ]]; then + if [[ ${#docker_files[@]} -gt 0 ]]; then # SC2154 doesn't seem to work on dockerfile. - echo "+ shellcheck -e SC2148,SC2154,SC2250 \$(git ls-files '*Dockerfile')" - if ! shellcheck -e SC2148,SC2154,SC2250 $(git ls-files '*Dockerfile'); then - should_fail=1 + # SC2250 may not correct on dockerfile because $v and ${v} is sometime different: https://github.com/moby/moby/issues/42863 + info "running \`shellcheck --shell bash --exclude SC2154,SC2250 \$(git ls-files '*Dockerfile*')\`" + if ! shellcheck --shell bash --exclude SC2154,SC2250 "${docker_files[@]}"; then + error "check failed; please resolve the above shellcheck error(s)" + fi + fi + # Check scripts in other files. + if [[ ${#workflows[@]} -gt 0 ]] || [[ ${#actions[@]} -gt 0 ]]; then + info "running \`shellcheck --exclude SC2086,SC2096,SC2129\` for scripts in .github/workflows/*.yml and **/action.yml" + if [[ "${ostype}" == "windows" ]]; then + # No such file or directory: '/proc/N/fd/N' + warn "this check is skipped on Windows due to upstream bug (failed to found fd created by <())" + elif [[ "${ostype}" == "dragonfly" ]]; then + warn "this check is skipped on DragonFly BSD due to upstream bug (hang)" + elif check_install jq python3; then + shellcheck_for_gha() { + local text=$1 + local shell=$2 + local display_path=$3 + if [[ "${text}" == "null" ]]; then + return + fi + case "${shell}" in + bash* | sh*) ;; + *) return ;; + esac + # Use python because sed doesn't support .*?. + text=$( + "python${py_suffix}" - <(printf '%s\n%s' "#!/usr/bin/env ${shell%' {0}'}" "${text}") </dev/null; then + if [[ "${ostype}" == "solaris" ]] && [[ -n "${CI:-}" ]] && ! type -P npm >/dev/null; then + warn "this check is skipped on Solaris due to no node 18+ in upstream package manager" + elif [[ "${ostype}" == "illumos" ]]; then + warn "this check is skipped on illumos due to upstream bug (dictionaries are not loaded correctly)" + elif check_install npm jq python3; then has_rust='' - if [[ -n "$(git ls-files '*Cargo.toml')" ]]; then - has_rust='1' + if [[ -n "$(ls_files '*Cargo.toml')" ]]; then + has_rust=1 dependencies='' - for manifest_path in $(git ls-files '*Cargo.toml'); do - if [[ "${manifest_path}" != "Cargo.toml" ]] && ! grep -Eq '\[workspace\]' "${manifest_path}"; then + for manifest_path in $(ls_files '*Cargo.toml'); do + if [[ "${manifest_path}" != "Cargo.toml" ]] && [[ "$(tomlq -c '.workspace' "${manifest_path}")" == "null" ]]; then continue fi - metadata=$(cargo metadata --format-version=1 --no-deps --manifest-path "${manifest_path}") - for id in $(jq <<<"${metadata}" '.workspace_members[]'); do - dependencies+="$(jq <<<"${metadata}" ".packages[] | select(.id == ${id})" | jq -r '.dependencies[].name')"$'\n' - done + m=$(cargo metadata --format-version=1 --no-deps --manifest-path "${manifest_path}" || true) + if [[ -z "${m}" ]]; then + continue # Ignore broken manifest + fi + dependencies+="$(jq -r '. as $metadata | .workspace_members[] as $id | $metadata.packages[] | select(.id == $id) | .dependencies[].name' <<<"${m}")"$'\n' done - # shellcheck disable=SC2001 - dependencies=$(sed <<<"${dependencies}" 's/[0-9_-]/\n/g' | LC_ALL=C sort -f -u) + dependencies=$(LC_ALL=C sort -f -u <<<"${dependencies//[0-9_-]/$'\n'}") fi config_old=$(<.cspell.json) - config_new=$(grep <<<"${config_old}" -v '^ *//' | jq 'del(.dictionaries[] | select(index("organization-dictionary") | not))' | jq 'del(.dictionaryDefinitions[] | select(.name == "organization-dictionary" | not))') - trap -- 'echo "${config_old}" >.cspell.json; echo >&2 "$0: trapped SIGINT"; exit 1' SIGINT - echo "${config_new}" >.cspell.json + config_new=$(grep -Ev '^ *//' <<<"${config_old}" | jq 'del(.dictionaries[] | select(index("organization-dictionary") | not)) | del(.dictionaryDefinitions[] | select(.name == "organization-dictionary" | not))') + trap -- 'printf "%s\n" "${config_old}" >|.cspell.json; printf >&2 "%s\n" "${0##*/}: trapped SIGINT"; exit 1' SIGINT + printf '%s\n' "${config_new}" >|.cspell.json + dependencies_words='' if [[ -n "${has_rust}" ]]; then - dependencies_words=$(npx <<<"${dependencies}" -y cspell stdin --no-progress --no-summary --words-only --unique || true) + dependencies_words=$(npx -y cspell stdin --no-progress --no-summary --words-only --unique <<<"${dependencies}" || true) fi - all_words=$(npx -y cspell --no-progress --no-summary --words-only --unique $(git ls-files | (grep -v "${project_dictionary//\./\\.}" || true)) || true) - echo "${config_old}" >.cspell.json - trap - SIGINT - cat >.github/.cspell/rust-dependencies.txt <|.cspell.json + trap -- 'printf >&2 "%s\n" "${0##*/}: trapped SIGINT"; exit 1' SIGINT + cat >|.github/.cspell/rust-dependencies.txt <>.github/.cspell/rust-dependencies.txt + if [[ -n "${dependencies_words}" ]]; then + LC_ALL=C sort -f >>.github/.cspell/rust-dependencies.txt <<<"${dependencies_words}"$'\n' fi if [[ -z "${REMOVE_UNUSED_WORDS:-}" ]]; then check_diff .github/.cspell/rust-dependencies.txt @@ -424,9 +783,12 @@ EOF error "you may want to mark .github/.cspell/rust-dependencies.txt linguist-generated" fi - echo "+ npx -y cspell --no-progress --no-summary \$(git ls-files)" - if ! npx -y cspell --no-progress --no-summary $(git ls-files); then - error "spellcheck failed: please fix uses of above words or add to ${project_dictionary} if correct" + info "running \`npx -y cspell --no-progress --no-summary \$(git ls-files)\`" + if ! npx -y cspell --no-progress --no-summary $(ls_files); then + error "spellcheck failed: please fix uses of below words or add to ${project_dictionary} if correct" + printf '=======================================\n' + { npx -y cspell --no-progress --no-summary --words-only $(git ls-files) || true; } | LC_ALL=C sort -f -u + printf '=======================================\n\n' fi # Make sure the project-specific dictionary does not contain duplicated words. @@ -434,12 +796,16 @@ EOF if [[ "${dictionary}" == "${project_dictionary}" ]]; then continue fi - dup=$(sed '/^$/d' "${project_dictionary}" "${dictionary}" | LC_ALL=C sort -f | uniq -d -i | (grep -v '//.*' || true)) + case "${ostype}" in + # NetBSD uniq doesn't support -i flag. + netbsd) dup=$(sed '/^$/d; /^\/\//d' "${project_dictionary}" "${dictionary}" | LC_ALL=C sort -f | tr '[:upper:]' '[:lower:]' | LC_ALL=C uniq -d) ;; + *) dup=$(sed '/^$/d; /^\/\//d' "${project_dictionary}" "${dictionary}" | LC_ALL=C sort -f | LC_ALL=C uniq -d -i) ;; + esac if [[ -n "${dup}" ]]; then error "duplicated words in dictionaries; please remove the following words from ${project_dictionary}" - echo "=======================================" - echo "${dup}" - echo "=======================================" + printf '=======================================\n' + printf '%s\n' "${dup}" + printf '=======================================\n\n' fi done