Skip to content

Fix eval with exclusions #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Oct 9, 2025
Merged

Fix eval with exclusions #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
763b469
imp: add exclusions for flake_urls; also filter draft releases
johnalotoski Oct 9, 2025
a0a88c3
recipe: add filter-packages
johnalotoski Oct 9, 2025
f01d4f9
imp: filter exclusions from packages.json
johnalotoski Oct 9, 2025
bc5b63a
Update 2025-10-09T01:48:52Z
actions-user Oct 9, 2025
5e03297
recipes: sort and add descriptions to all
johnalotoski Oct 9, 2025
47162d7
exclusions: add eval failure reason
johnalotoski Oct 9, 2025
5ccf334
exclusions: remove go pkging fails now that go dep has been rebuilt t…
johnalotoski Oct 9, 2025
2d1ff82
exclusions: add the remaining eval failure
johnalotoski Oct 9, 2025
8ae1bd3
Update 2025-10-09T03:21:15Z
actions-user Oct 9, 2025
78c6c00
exclusions: more missing zw3rk artifacts
johnalotoski Oct 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 47 additions & 40 deletions Justfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,54 +4,16 @@ list:

secret_key := env_var_or_default('NIX_SIGNING_KEY_FILE', "hydra_key")

# Based on projects.json, upload the CA contents and update packages.json
packages *ARGS:
./packages.cr \
--to "s3://devx?secret-key={{secret_key}}&endpoint=${S3_ENDPOINT}&region=auto&compression=zstd" \
--from-store https://cache.iog.io \
--systems x86_64-linux

# Based on projects.json, upload the CA contents and update packages.json
ci:
@just -v cache-download
@just -v packages
@just -v cache-upload

# download and uncompress the cache folder
# Download and uncompress the cache folder
cache-download:
@just -v rclone copyto s3://devx/capkgs/cache.tar.zst cache.tar.zst
tar xf cache.tar.zst

# compress and upload the cache folder
# Compress and upload the cache folder
cache-upload:
tar cfa cache.tar.zst cache
@just -v rclone copyto cache.tar.zst s3://devx/capkgs/cache.tar.zst

rclone *ARGS:
#!/usr/bin/env nu
if $env.CI? == "true" { $env.HOME = $env.PWD }

if $env.AWS_PROFILE? == null {
if $env.AWS_ACCESS_KEY_ID? == null and $env.AWS_SECRET_ACCESS_KEY? == null {
print "Both AWS_PROFILE and AWS keypair (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY) environment variables are not set."
print "Set appropriate AWS env vars and try again."
exit 1
}
}

if $env.S3_ENDPOINT? == null {
print "The S3_ENDPOINT environment variable is unset. Please set and try again."
exit 1
}

mkdir .config/rclone
rclone config create s3 s3 env_auth=true | save -f .config/rclone/rclone.conf
rclone --s3-provider Cloudflare --s3-region auto --s3-endpoint $env.S3_ENDPOINT --verbose {{ARGS}}

push:
git add packages.json
git commit -m 'Update packages.json'

# Attempt to build all packages from this flake
check:
#!/usr/bin/env nu
Expand Down Expand Up @@ -84,3 +46,48 @@ check:
print "Build results:"
$builds | table -e

# Based on projects.json, upload the CA contents and update packages.json
ci:
@just -v cache-download
@just -v packages
@just -v cache-upload

# Filter exclusion flake urls from packages.json
filter-packages:
jq --slurpfile ex exclusions.json 'with_entries(select(.key as $k | ($ex[0] | has($k) | not)))' packages.json > packages.tmp
mv packages.tmp packages.json

# Based on projects.json, upload the CA contents and update packages.json
packages *ARGS:
./packages.cr \
--to "s3://devx?secret-key={{secret_key}}&endpoint=${S3_ENDPOINT}&region=auto&compression=zstd" \
--from-store https://cache.iog.io \
--systems x86_64-linux

# Commit updates to packages.json
push:
git add packages.json
git commit -m 'Update packages.json'


# rclone {{ARGS}} -- for simplified rcloning with auto-config setup
rclone *ARGS:
#!/usr/bin/env nu
if $env.CI? == "true" { $env.HOME = $env.PWD }

if $env.AWS_PROFILE? == null {
if $env.AWS_ACCESS_KEY_ID? == null and $env.AWS_SECRET_ACCESS_KEY? == null {
print "Both AWS_PROFILE and AWS keypair (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY) environment variables are not set."
print "Set appropriate AWS env vars and try again."
exit 1
}
}

if $env.S3_ENDPOINT? == null {
print "The S3_ENDPOINT environment variable is unset. Please set and try again."
exit 1
}

mkdir .config/rclone
rclone config create s3 s3 env_auth=true | save -f .config/rclone/rclone.conf
rclone --s3-provider Cloudflare --s3-region auto --s3-endpoint $env.S3_ENDPOINT --verbose {{ARGS}}
60 changes: 60 additions & 0 deletions exclusions.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
{
"github:cardano-foundation/cardano-wallet/3f9055373c272a4d7b1f9f6bacc1df3b6dc906af#packages.x86_64-linux.cardano-address": "Not writing modified lock file",
"github:cardano-foundation/cardano-wallet/3f9055373c272a4d7b1f9f6bacc1df3b6dc906af#packages.x86_64-linux.cardano-wallet": "Not writing modified lock file",
"github:cardano-foundation/cardano-wallet/d372d86b65edd833ba11e485cbab19694afdbb1c#packages.x86_64-linux.cardano-wallet": "Missing zw3rk artifact",
"github:cardano-foundation/cardano-wallet/e69616ba776f4a52173ead423141d234e03d468f#packages.x86_64-linux.cardano-wallet": "Missing zw3rk artifact",
"github:cardano-scaling/hydra/1bbac9c14377983a87c9ffc28ab6146057b388aa#packages.x86_64-linux.hydra-node": "Missing zw3rk artifact",
"github:cardano-scaling/hydra/1bbac9c14377983a87c9ffc28ab6146057b388aa#packages.x86_64-linux.hydra-tui": "Missing zw3rk artifact",
"github:cardano-scaling/hydra/2baae13fdfb7dec6071501ed5290f7a54a86315f#packages.x86_64-linux.hydra-node": "Missing zw3rk artifact",
"github:cardano-scaling/hydra/2baae13fdfb7dec6071501ed5290f7a54a86315f#packages.x86_64-linux.hydra-tui": "Missing zw3rk artifact",
"github:cardano-scaling/hydra/9f1027e0fdff6765f5233f19c4639fdaa3558bfa#packages.x86_64-linux.hydra-node": "Missing zw3rk artifact",
"github:cardano-scaling/hydra/9f1027e0fdff6765f5233f19c4639fdaa3558bfa#packages.x86_64-linux.hydra-tui": "Missing zw3rk artifact",
"github:input-output-hk/cardano-ogmios/fbeed0e6c4b9506faef3329f8ca802660a36dd94#packages.x86_64-linux.ogmios": "Not writing modified lock file",
"github:IntersectMBO/cardano-cli/0cf8f2ed173cc0bd2a818a137593a3c064895a3a#packages.x86_64-linux.\"cardano-cli:exe:cardano-cli\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-cli/24dd2e18f3dd19133d4bd1c01e144f20a0b6dcbf#packages.x86_64-linux.\"cardano-cli:exe:cardano-cli\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-cli/553092f22f8215d5fd9a73f312a0a04ae81f6a2b#packages.x86_64-linux.\"cardano-cli:exe:cardano-cli\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-cli/55d627015935f01e6ab43d9ad63e21b5f5338dd3#packages.x86_64-linux.\"cardano-cli:exe:cardano-cli\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-cli/59d9717b010340837bcf1d5eb6b678d6ba5df36f#packages.x86_64-linux.\"cardano-cli:exe:cardano-cli\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-cli/66415fff7ad86b3de645325a57f12114fd8915b3#packages.x86_64-linux.\"cardano-cli:exe:cardano-cli\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-cli/79264057deda984ca955c36e56c15a5c28e9307a#packages.x86_64-linux.\"cardano-cli:exe:cardano-cli\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-cli/7a2c3514aa06a3fe72eb75dace02687884e192d6#packages.x86_64-linux.\"cardano-cli:exe:cardano-cli\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-cli/a38c1c6db06f65fdb95271e7549c97960fcabc74#packages.x86_64-linux.\"cardano-cli:exe:cardano-cli\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-cli/d00a7d45fcb7750288679f9c2a4ce8915a75aed9#packages.x86_64-linux.\"cardano-cli:exe:cardano-cli\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-cli/d4eaabdcde1f650fee17ec6b31906c0f2913f0d8#packages.x86_64-linux.\"cardano-cli:exe:cardano-cli\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-cli/de368c88fdd0d96fa7390b0287bf0379711fe8f0#packages.x86_64-linux.\"cardano-cli:exe:cardano-cli\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-cli/e198717d59768cee7a0283d2dc4b673f943576d7#packages.x86_64-linux.\"cardano-cli:exe:cardano-cli\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-cli/e62ea2f94a7e1bc3211f9a538093d20718a3c76b#packages.x86_64-linux.\"cardano-cli:exe:cardano-cli\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-cli/f6ddefa53d9911ea666b96a8827c54c19cf06b10#packages.x86_64-linux.\"cardano-cli:exe:cardano-cli\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-db-sync/0ea93d223100db816ea5bf4468f789cadcd98b0e#packages.x86_64-linux.cardano-db-sync": "Newer nix version incompat; flake.nix does not exist",
"github:IntersectMBO/cardano-db-sync/0ea93d223100db816ea5bf4468f789cadcd98b0e#packages.x86_64-linux.\"cardano-db-sync:exe:cardano-db-sync\"": "Newer nix version incompat; flake.nix does not exist",
"github:IntersectMBO/cardano-db-sync/0ea93d223100db816ea5bf4468f789cadcd98b0e#packages.x86_64-linux.cardano-db-tool": "Newer nix version incompat; flake.nix does not exist",
"github:IntersectMBO/cardano-db-sync/0ea93d223100db816ea5bf4468f789cadcd98b0e#packages.x86_64-linux.\"cardano-db-tool:exe:cardano-db-tool\"": "Newer nix version incompat; flake.nix does not exist",
"github:IntersectMBO/cardano-db-sync/0ea93d223100db816ea5bf4468f789cadcd98b0e#packages.x86_64-linux.cardano-smash-server": "Newer nix version incompat; flake.nix does not exist",
"github:IntersectMBO/cardano-db-sync/0ea93d223100db816ea5bf4468f789cadcd98b0e#packages.x86_64-linux.\"cardano-smash-server:exe:cardano-smash-server\"": "Newer nix version incompat; flake.nix does not exist",
"github:IntersectMBO/cardano-db-sync/1f26396e91a0124a545201d4c6519d015b24fa6e#packages.x86_64-linux.\"cardano-db-sync:exe:cardano-db-sync\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-db-sync/1f26396e91a0124a545201d4c6519d015b24fa6e#packages.x86_64-linux.\"cardano-db-tool:exe:cardano-db-tool\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-db-sync/1f26396e91a0124a545201d4c6519d015b24fa6e#packages.x86_64-linux.\"cardano-smash-server:exe:cardano-smash-server\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-db-sync/1f26396e91a0124a545201d4c6519d015b24fa6e#packages.x86_64-linux.cardano-smash-server-no-basic-auth": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-db-sync/4a1e378febaacda67aae410deee6812e2a1e3d8e#packages.x86_64-linux.\"cardano-db-sync:exe:cardano-db-sync\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-db-sync/4a1e378febaacda67aae410deee6812e2a1e3d8e#packages.x86_64-linux.\"cardano-db-tool:exe:cardano-db-tool\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-db-sync/4a1e378febaacda67aae410deee6812e2a1e3d8e#packages.x86_64-linux.\"cardano-smash-server:exe:cardano-smash-server\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-db-sync/4a1e378febaacda67aae410deee6812e2a1e3d8e#packages.x86_64-linux.cardano-smash-server-no-basic-auth": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-db-sync/790dc0ed32f92816a477d4229060fca2f8533479#packages.x86_64-linux.\"cardano-db-sync:exe:cardano-db-sync\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-db-sync/790dc0ed32f92816a477d4229060fca2f8533479#packages.x86_64-linux.\"cardano-db-tool:exe:cardano-db-tool\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-db-sync/790dc0ed32f92816a477d4229060fca2f8533479#packages.x86_64-linux.\"cardano-smash-server:exe:cardano-smash-server\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-db-sync/790dc0ed32f92816a477d4229060fca2f8533479#packages.x86_64-linux.cardano-smash-server-no-basic-auth": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-db-sync/935e1fe5d5d11d694e9b6c54030a592b85742671#packages.x86_64-linux.cardano-db-sync": "Not writing modified lock file",
"github:IntersectMBO/cardano-db-sync/935e1fe5d5d11d694e9b6c54030a592b85742671#packages.x86_64-linux.\"cardano-db-sync:exe:cardano-db-sync\"": "Not writing modified lock file",
"github:IntersectMBO/cardano-db-sync/935e1fe5d5d11d694e9b6c54030a592b85742671#packages.x86_64-linux.cardano-db-tool": "Not writing modified lock file",
"github:IntersectMBO/cardano-db-sync/935e1fe5d5d11d694e9b6c54030a592b85742671#packages.x86_64-linux.\"cardano-db-tool:exe:cardano-db-tool\"": "Not writing modified lock file",
"github:IntersectMBO/cardano-db-sync/935e1fe5d5d11d694e9b6c54030a592b85742671#packages.x86_64-linux.cardano-smash-server": "Not writing modified lock file",
"github:IntersectMBO/cardano-db-sync/935e1fe5d5d11d694e9b6c54030a592b85742671#packages.x86_64-linux.\"cardano-smash-server:exe:cardano-smash-server\"": "Not writing modified lock file",
"github:IntersectMBO/cardano-db-sync/b3519b7e5380476c5ed0126c89ccba1355a2fc12#packages.x86_64-linux.\"cardano-db-sync:exe:cardano-db-sync\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-db-sync/b3519b7e5380476c5ed0126c89ccba1355a2fc12#packages.x86_64-linux.\"cardano-db-tool:exe:cardano-db-tool\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-db-sync/b3519b7e5380476c5ed0126c89ccba1355a2fc12#packages.x86_64-linux.\"cardano-smash-server:exe:cardano-smash-server\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-db-sync/b3519b7e5380476c5ed0126c89ccba1355a2fc12#packages.x86_64-linux.cardano-smash-server-no-basic-auth": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-db-sync/ba77cf22ae86974482e4eabc0dadb8af5f2e891a#packages.x86_64-linux.\"cardano-db-sync:exe:cardano-db-sync\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-db-sync/ba77cf22ae86974482e4eabc0dadb8af5f2e891a#packages.x86_64-linux.\"cardano-db-tool:exe:cardano-db-tool\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-db-sync/ba77cf22ae86974482e4eabc0dadb8af5f2e891a#packages.x86_64-linux.\"cardano-smash-server:exe:cardano-smash-server\"": "Missing zw3rk artifact",
"github:IntersectMBO/cardano-db-sync/ba77cf22ae86974482e4eabc0dadb8af5f2e891a#packages.x86_64-linux.cardano-smash-server-no-basic-auth": "Missing zw3rk artifact"
}
19 changes: 18 additions & 1 deletion packages.cr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,12 @@ class CAPkgs
end

def update_packages
exclusions = load_exclusions

({} of String => CAPkgs::Package).tap do |valid|
each_package do |pkg|
next if exclusions.includes?(pkg.flake_url)

pkg.mkdirs
pkg.nix_eval &&
pkg.nix_build &&
Expand All @@ -32,6 +36,12 @@ class CAPkgs
end
end

def load_exclusions
return [] of String unless File.exists?("exclusions.json")
data = JSON.parse(File.read("exclusions.json")).as_h.keys
data
end

def each_package(&block : Package -> Nil)
packages =
config.systems.map do |system|
Expand Down Expand Up @@ -131,7 +141,13 @@ class CAPkgs
curl_result = sh("curl", "--netrc", "-L", "-s", "--fail-with-body", releases_url)
raise "Couldn't fetch releases for '#{releases_url}'" unless curl_result.success?

tag_names = Array(GithubRelease).from_json(curl_result.stdout).map { |release| release.tag_name }
releases = Array(GithubRelease).from_json(curl_result.stdout)

# If running locally with required privileges, draft releases may be
# included causing an exception if not filtered.
releases.reject!(&.draft)

tag_names = releases.map { |release| release.tag_name }
refs_tags = tag_names.map do |tag_name|
# Once we record a tag, prevent it from updating
if File.file?(dest) && (found = JSON.parse(File.read(dest))[tag_name]?)
Expand Down Expand Up @@ -264,6 +280,7 @@ class CAPkgs
struct GithubRelease
include JSON::Serializable
property tag_name : String
property draft : Bool = false
end

struct Repo
Expand Down
Loading