diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6e0f7da8cd3..b872e9fe3cd0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,8 +32,8 @@ jobs: code: - '!documentation/**' - rust-format: - name: Check Rust Code Format + structure-check: + name: Repository Structure Checks runs-on: ubuntu-latest needs: changes if: needs.changes.outputs.code == 'true' || github.event_name != 'pull_request' @@ -41,10 +41,13 @@ jobs: - name: Checkout Code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # pin@v4 - - uses: actions-rust-lang/setup-rust-toolchain@v1 + - name: Set up Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 - - name: Run cargo fmt - run: cargo fmt --check + - name: Run structure checks + run: | + source ./bin/activate-hermit + just structure-check rust-build-and-test: name: Build and Test Rust Project diff --git a/Justfile b/Justfile index ce16e0a622b4..3009c0f8b848 100644 --- a/Justfile +++ b/Justfile @@ -4,19 +4,50 @@ default: @just --list +# Run repository structure checks +structure-check: + #!/usr/bin/env bash + set -e + echo "Running repository structure checks..." + + # Find all executable scripts + # Use -perm for BSD/macOS compatibility (GNU find uses -executable) + mapfile -t scripts < <(find scripts/ci/structure -name "*.sh" -type f \( -perm -u=x -o -perm -g=x -o -perm -o=x \) 2>/dev/null | sort) + + if [ ${#scripts[@]} -eq 0 ]; then + echo "ERROR: No structure check scripts found in scripts/ci/structure/" + exit 1 + fi + + for script in "${scripts[@]}"; do + echo "" + echo "========================================" + echo "Running: $script" + echo "========================================" + "$script" || exit 1 + done + + echo "" + echo "All structure checks passed" + # Run all style checks and formatting (precommit validation) check-everything: - @echo "🔧 RUNNING ALL STYLE CHECKS..." - @echo " → Formatting Rust code..." - cargo fmt --all - @echo " → Running clippy linting..." + #!/usr/bin/env bash + set -e + echo "RUNNING ALL STYLE CHECKS..." + echo "" + just structure-check + echo "" + echo "Running clippy linting..." ./scripts/clippy-lint.sh - @echo " → Checking UI code formatting..." + echo "" + echo "Checking UI code formatting..." cd ui/desktop && npm run lint:check - @echo " → Validating OpenAPI schema..." + echo "" + echo "Validating OpenAPI schema..." ./scripts/check-openapi-schema.sh - @echo "" - @echo "✅ All style checks passed!" + echo "" + echo "All style checks passed!" # Default release command release-binary: diff --git a/scripts/ci/structure/README.md b/scripts/ci/structure/README.md new file mode 100644 index 000000000000..05fa12f7a925 --- /dev/null +++ b/scripts/ci/structure/README.md @@ -0,0 +1,63 @@ +# Repository Structure Checks + +This directory contains scripts that perform static analysis and structural validation of the repository **before** any builds are executed. These checks enforce repository policies and prevent issues from being committed. + +## How It Works + +All executable shell scripts (`*.sh`) in this directory are automatically run by: +- **CI**: `.github/workflows/ci.yml` - `structure-check` job +- **Local**: `just check-everything` command + +Scripts are executed in alphabetical order. All scripts must pass for the check to succeed. + +## Adding a New Check + +1. Create a new executable shell script in this directory: + ```bash + touch scripts/ci/structure/check-your-thing.sh + chmod +x scripts/ci/structure/check-your-thing.sh + ``` + +2. Write your check script following this template: + ```bash + #!/usr/bin/env bash + set -e + + # Brief description of what this checks + + echo "Checking your thing..." + + # Your validation logic here + if ! validation_passes; then + echo "" + echo "ERROR: Your thing validation failed" + echo "" + echo "To fix this issue:" + echo " some-command-to-fix-it" + echo "" + exit 1 + fi + + echo "Your thing is valid" + ``` + +3. Test locally: + ```bash + ./scripts/ci/structure/check-your-thing.sh + just check-everything + ``` + +4. Commit and push - CI will automatically run your new check + +## Existing Checks + +- **check-prebuilt-binaries.sh** - Prevents prebuilt executables from being committed (security) +- **check-rust-format.sh** - Validates Rust code formatting + +## Guidelines + +- **Keep checks fast** - These run on every PR +- **Make errors actionable** - Tell users how to fix issues +- **Exit codes** - Exit 0 for success, non-zero for failure +- **Echo progress** - Let users know what's being checked +- **Use allowlists sparingly** - Only for temporary exceptions diff --git a/scripts/ci/structure/check-prebuilt-binaries.sh b/scripts/ci/structure/check-prebuilt-binaries.sh new file mode 100755 index 000000000000..ca24201219e1 --- /dev/null +++ b/scripts/ci/structure/check-prebuilt-binaries.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +set -e + +# Check for prebuilt executables/binaries in the repository +# This is a security measure to prevent supply chain attacks via checked-in binaries +# +# Similar checks in other projects: +# - Rust compiler: https://github.com/rust-lang/rust/blob/master/src/tools/tidy/src/bins.rs +# - OSSF Scorecard: https://github.com/ossf/scorecard/blob/main/checks/binary_artifact.go + +echo "Checking for prebuilt executables and binaries..." + +# Allowlist for existing binaries (TEMPORARY - to be removed) +# These binaries should be removed and built from source or downloaded at build time +# Added in: https://github.com/block/goose/pull/880 (commit cfd3ee8fd9c) +declare -A allowlist=( + ["ui/desktop/src/platform/windows/bin/libgcc_s_seh-1.dll"]=1 + ["ui/desktop/src/platform/windows/bin/libstdc++-6.dll"]=1 + ["ui/desktop/src/platform/windows/bin/libwinpthread-1.dll"]=1 + ["ui/desktop/src/platform/windows/bin/uv.exe"]=1 + ["ui/desktop/src/platform/windows/bin/uvx.exe"]=1 +) + +# Denylist of file type patterns that indicate binary executables or libraries +# These patterns match against the output of the 'file' command +denylist_patterns=( + "PE32.*executable" + "PE32.*DLL" + "MS-DOS executable" + "ELF.*executable" + "ELF.*shared object" + "Mach-O.*executable" + "Mach-O.*dynamically linked shared library" + "Mach-O.*bundle" + "Java archive data" + "compiled Java class" + "python.*byte-compiled" + "WebAssembly" + "current ar archive" +) + +# Associative arrays for fast O(1) lookups +declare -A violations +declare -A allowlisted_files +declare -A checked_files + +echo "Scanning git-tracked files..." + +# Collect all git-tracked files +mapfile -t all_files < <(git ls-files) + +echo "Checking ${#all_files[@]} tracked files..." + +# First pass: Quick extension check (no file command needed) +extension_regex='\.(exe|dll|so|dylib|a|o|bin|app|wasm|jar|class|pyc|pyd|pyo|lib)$' + +for file in "${all_files[@]}"; do + # Skip if not a regular file or if it's a symlink + [ -f "$file" ] && [ ! -L "$file" ] || continue + + # Use bash regex matching (no grep subprocess) + if [[ "$file" =~ $extension_regex ]]; then + checked_files["$file"]=1 + + if [[ -n "${allowlist[$file]}" ]]; then + allowlisted_files["$file"]=1 + else + violations["$file"]=1 + fi + fi +done + +echo "Found ${#violations[@]} files with suspicious extensions" + +# Second pass: Batch check remaining files with ONE file command +# Build list of files not yet checked +files_to_check=() +for file in "${all_files[@]}"; do + # Skip if not a regular file or if it's a symlink + [ -f "$file" ] && [ ! -L "$file" ] || continue + [[ -z "${checked_files[$file]}" ]] || continue + files_to_check+=("$file") +done + +if [ ${#files_to_check[@]} -gt 0 ]; then + echo "Deep scanning ${#files_to_check[@]} additional files..." + + # Use xargs to batch process files (handles ARG_MAX limits, processes in chunks of 100) + # Use process substitution instead of pipe to avoid subshell + while IFS=: read -r filepath description; do + # Check if description matches any denylist pattern + is_binary=false + for pattern in "${denylist_patterns[@]}"; do + if [[ "$description" =~ $pattern ]]; then + is_binary=true + break + fi + done + + if [ "$is_binary" = true ]; then + if [[ -n "${allowlist[$filepath]}" ]]; then + allowlisted_files["$filepath"]=1 + else + violations["$filepath"]=1 + fi + fi + done < <(printf '%s\n' "${files_to_check[@]}" | xargs -n 100 -P 1 file 2>/dev/null) +fi + +# Report allowlisted files +if [ ${#allowlisted_files[@]} -gt 0 ]; then + echo "" + echo "WARNING: The following binaries are temporarily allowlisted:" + echo "" + + for file in "${!allowlisted_files[@]}"; do + echo " [ALLOWLISTED] $file" + if [ -f "$file" ]; then + echo " $(file -b "$file")" + echo " Size: $(du -h "$file" | cut -f1)" + fi + echo "" + done + + echo "These files should be removed and replaced with build-time solutions." + echo "" +fi + +# Report violations +if [ ${#violations[@]} -gt 0 ]; then + echo "" + echo "ERROR: PREBUILT BINARIES DETECTED!" + echo "" + echo "The following prebuilt executables or binary files were found in the repository:" + echo "" + + for file in "${!violations[@]}"; do + echo " [VIOLATION] $file" + echo " $(file -b "$file")" + if [ -f "$file" ]; then + echo " Size: $(du -h "$file" | cut -f1)" + fi + echo "" + done + + echo "========================================================================" + echo "" + echo "POLICY: Prebuilt binaries are not allowed in this repository" + echo "" + echo "This is a security measure to prevent supply chain attacks." + echo "All executables must be built from source in CI/CD pipelines." + echo "" + echo "If you need platform-specific binaries:" + echo " 1. Build them in the GitHub Actions workflow" + echo " 2. Use package managers (npm, cargo, apt, brew, etc.)" + echo " 3. Download from trusted sources during build time" + echo " 4. Add checksums/signatures verification" + echo "" + echo "To remove these files:" + for file in "${!violations[@]}"; do + echo " git rm \"$file\"" + done + echo "" + echo "If this is a false positive, add the file to the allowlist in:" + echo " scripts/ci/structure/check-prebuilt-binaries.sh" + echo "" + echo "========================================================================" + + exit 1 +fi + +echo "No new prebuilt binaries detected (${#allowlisted_files[@]} allowlisted)" diff --git a/scripts/ci/structure/check-rust-format.sh b/scripts/ci/structure/check-rust-format.sh new file mode 100755 index 000000000000..46ee5b9e7fd1 --- /dev/null +++ b/scripts/ci/structure/check-rust-format.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -e + +# Check Rust code formatting + +echo "Checking Rust code formatting..." + +if ! cargo fmt --all --check; then + echo "" + echo "ERROR: Rust code is not properly formatted" + echo "" + echo "To fix this issue:" + echo " cargo fmt --all" + echo "" + exit 1 +fi + +echo "Rust code formatting is correct"