diff --git a/.arg b/.arg index 455a198..5a4fbd5 100644 --- a/.arg +++ b/.arg @@ -1,3 +1,4 @@ -DOCKERHUB_USER=earthly +CR_HOST=ghcr.io +CR_ORG=earthbuild IMAGE_NAME=dind -EARTHLY_REPO_VERSION=0b28ef80785fd88df1ec7e674475b02e046d5b36 +EARTHLY_REPO_VERSION=bc955577ae9cb81dd5d54fd89f417e845e6da3e4 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e2d3d45..231659a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,2 @@ -# PRs require approval from the earthly core team -* @earthly/core +# PRs require approval from the EarthBuild admin team +* @EarthBuild/fork-admins diff --git a/.github/renovate.json5 b/.github/renovate.json5 index a10c498..6b797af 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -1,51 +1,48 @@ { - $schema: "https://docs.renovatebot.com/renovate-schema.json", + $schema: 'https://docs.renovatebot.com/renovate-schema.json', extends: [ - "config:recommended" + 'config:recommended', + 'security:openssf-scorecard' ], + forkProcessing: 'enabled', configMigration: true, prHourlyLimit: 1, - schedule: [ // utc timezone - 'after 6am on monday', - ], + schedule: [], enabledManagers: [ 'dockerfile', 'custom.regex', 'github-actions', ], dockerfile: { - fileMatch: [ - 'Earthfile', + managerFilePatterns: [ + '/Earthfile/', ], }, customManagers: [ { - // Earthly Version in GH Actions customType: 'regex', - fileMatch: [ - '.*?\\.yml', + managerFilePatterns: [ + '/.*?\\.yml/', ], matchStrings: [ - '- uses: earthly/actions/setup-earthly@.*?[\\s\\n]+.*?[\\s\\n]+with:[\\s\\n]*version: (?.*?)[\\s\\n/]+', + '- uses: earthbuild/actions-setup@.*?[\\s\\n]+.*?[\\s\\n]+with:[\\s\\n]*version: (?.*?)[\\s\\n/]+', ], - depNameTemplate: 'earthly/earthly', + depNameTemplate: 'earthbuild/earthbuild', datasourceTemplate: 'github-releases', }, { - // ARG/LET in Earthfile that have a Renovate comment customType: 'regex', - fileMatch: [ - 'Earthfile', + managerFilePatterns: [ + '/Earthfile/', ], matchStrings: [ '\\s*#\\s*renovate:\\s*datasource=(?.*?)\\s+depName=(?.*?)[\\s\\n]+(versioning=(?.*?)\\s+)?(ARG|LET)\\s+.*?(_VERSION|_VER|version)=(?.*?)($|\\s|\\n)', ], }, { - // Updating the Repology alpine version customType: 'regex', - fileMatch: [ - 'Earthfile', + managerFilePatterns: [ + '/Earthfile/', ], matchStrings: [ '# renovate: datasource=repology depName=alpine_(?\\d+_\\d+)', @@ -57,26 +54,27 @@ autoReplaceStringTemplate: '# renovate: datasource=repology depName=alpine_{{{newMajor}}}_{{{newMinor}}}', }, { - // Updating the commit hash of earthly/earthly customType: 'regex', - fileMatch: ['^\\.arg$'], + managerFilePatterns: [ + '/^\\.arg$/', + ], matchStrings: [ 'EARTHLY_REPO_VERSION=(?.*?)($|\\s|\\n)', ], currentValueTemplate: 'main', - depNameTemplate: 'earthly/earthly', - packageNameTemplate: 'https://github.com/earthly/earthly', + depNameTemplate: 'earthbuild/earthbuild', + packageNameTemplate: 'https://github.com/earthbuild/earthbuild', datasourceTemplate: 'git-refs', }, { customType: 'regex', - fileMatch: [ - 'docs/dockerhub.md$', + managerFilePatterns: [ + '/^README.md$/', ], matchStrings: [ '\\*\\s+\\`(?.+?)\\`\\n', ], - depNameTemplate: 'earthly/dind', + depNameTemplate: 'earthbuild/dind', datasourceTemplate: 'docker', }, ], @@ -84,14 +82,8 @@ 'renovate', ], packageRules: [ - // set the group name for alpine dind dependencies { groupName: "{{{ replace 'os\\/(.*?)$' '$1' packageFileDir}}}-dind-image", - matchPackagePatterns: [ - 'alpine', - 'alpine.*?/docker', - 'docker/docker', - ], matchManagers: [ 'custom.regex', ], @@ -103,46 +95,75 @@ matchFileNames: [ 'os/**/Earthfile', ], + matchPackageNames: [ + '/alpine/', + '/alpine.*?/docker/', + '/docker/docker/', + ], + }, + { + // Lock docker to v28.1.1 for Ubuntu 20.04. + matchFileNames: [ + 'os/ubuntu-20.04/Earthfile', + ], + matchPackageNames: [ + 'docker/docker', + ], + allowedVersions: '28.1.1', + }, + { + // Lock docker to v25.0.2 for Ubuntu 23.04. + matchFileNames: [ + 'os/ubuntu-23.04/Earthfile', + ], + matchPackageNames: [ + 'docker/docker', + ], + allowedVersions: '25.0.2', }, { - // let all package updates auto merge - matchPackagePatterns: [ '*' ], automerge: true, + matchPackageNames: [ + '*', + ], }, { - // rule to update earthly/dind:alpine-* images groupName: 'docs-dind-images', - matchPackageNames: ['earthly/dind'], - matchCurrentVersion: "/^alpine-.*/", - allowedVersions: "/^alpine-.*/", - versioning: "regex:^(?.*)-(?\\d+\\.\\d+)-docker-(?\\d+)\\.(?\\d+)\\.(?\\d+)-r(?\\d+)$", + matchPackageNames: [ + 'earthbuild/dind', + ], + matchCurrentVersion: '/^alpine-.*/', + allowedVersions: '/^alpine-.*/', + versioning: 'regex:^(?.*)-(?\\d+\\.\\d+)-docker-(?\\d+)\\.(?\\d+)\\.(?\\d+)-r(?\\d+)$', }, { - // rule to update earthly/dind:ubuntu:20.04-* images groupName: 'docs-dind-images', - matchPackageNames: ['earthly/dind'], - matchCurrentVersion: "/^ubuntu-20\\.04-.*/", - allowedVersions: "/^ubuntu-20\\.04-.*/", - versioning: "regex:^(?.*)-(?\\d+\\.\\d+)-docker-(?\\d+)\\.(?\\d+)\\.(?\\d+)-(?\\d+)$", + matchPackageNames: [ + 'earthbuild/dind', + ], + matchCurrentVersion: '/^ubuntu-20\\.04-.*/', + allowedVersions: '/^ubuntu-20\\.04-.*/', + versioning: 'regex:^(?.*)-(?\\d+\\.\\d+)-docker-(?\\d+)\\.(?\\d+)\\.(?\\d+)-(?\\d+)$', }, { - // rule to update earthly/dind:ubuntu:23.04-* images groupName: 'docs-dind-images', - matchPackageNames: ['earthly/dind'], - matchCurrentVersion: "/^ubuntu-23\\.04-.*/", - allowedVersions: "/^ubuntu-23\\.04-.*/", - versioning: "regex:^(?.*)-(?\\d+\\.\\d+)-docker-(?\\d+)\\.(?\\d+)\\.(?\\d+)-(?\\d+)$", + matchPackageNames: [ + 'earthbuild/dind', + ], + matchCurrentVersion: '/^ubuntu-23\\.04-.*/', + allowedVersions: '/^ubuntu-23\\.04-.*/', + versioning: 'regex:^(?.*)-(?\\d+\\.\\d+)-docker-(?\\d+)\\.(?\\d+)\\.(?\\d+)-(?\\d+)$', }, { - // rule to update earthly/dind:ubuntu:24.04-* images groupName: 'docs-dind-images', - matchPackageNames: ['earthly/dind'], - matchCurrentVersion: "/^ubuntu-24\\.04-.*/", - allowedVersions: "/^ubuntu-24\\.04-.*/", - versioning: "regex:^(?.*)-(?\\d+\\.\\d+)-docker-(?\\d+)\\.(?\\d+)\\.(?\\d+)-(?\\d+)$", + matchPackageNames: [ + 'earthbuild/dind', + ], + matchCurrentVersion: '/^ubuntu-24\\.04-.*/', + allowedVersions: '/^ubuntu-24\\.04-.*/', + versioning: 'regex:^(?.*)-(?\\d+\\.\\d+)-docker-(?\\d+)\\.(?\\d+)\\.(?\\d+)-(?\\d+)$', }, { - // create PRs for multiple docker versions in case we're more than one major version behind matchPackageNames: [ 'docker/docker', ], diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2363f68..b263a83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,35 +5,51 @@ on: # push: # branches: [ "main" ] pull_request: - branches: [ "main" ] + branches: ["main"] concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true - + jobs: test: - name: test + name: test (${{ matrix.os }}) + strategy: + + matrix: + os: [alpine, ubuntu-20.04, ubuntu-23.04, ubuntu-24.04] runs-on: ubuntu-latest + permissions: + contents: read + packages: write env: FORCE_COLOR: 1 EARTHLY_CONVERSION_PARALLELISM: "5" - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" - EARTHLY_INSTALL_ID: "earthly-dind-githubactions" + EARTHLY_INSTALL_ID: "earthbuild-dind-githubactions" steps: - - uses: earthly/actions/setup-earthly@v1 + - uses: actions/checkout@v5 + # The dind (common+alpine-kind-test and common+ubuntu-kind-test) detects + # the host's IPv6 capability and then requires IPv6 NAT to create networks + # for kind. So we load it here. + # Ref: https://github.com/moby/moby/pull/47062 + - name: Load kernel module IPv6 NAT + run: sudo modprobe ip6table_nat + - name: Set up Docker Hub mirrors + run: | + sudo mkdir -p /etc/docker + echo '{"registry-mirrors": ["https://mirror.gcr.io", "https://public.ecr.aws"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + - name: Set up QEMU + run: | + sudo apt-get update + sudo apt-get install -y qemu-user-static binfmt-support + sudo update-binfmts --display + - uses: earthbuild/actions-setup@main with: version: v0.8.15 - - uses: actions/checkout@v4 - - name: Docker login (non fork only) - run: |- - docker login --username "${{ secrets.DOCKERHUB_USERNAME }}" --password "${{ secrets.DOCKERHUB_TOKEN }}" - docker login registry-1.docker.io.mirror.corp.earthly.dev --username "${{ secrets.DOCKERHUB_MIRROR_USERNAME }}" --password "${{ secrets.DOCKERHUB_MIRROR_PASSWORD }}" - if: github.event.pull_request.head.repo.full_name == github.repository - - name: Configure Earthly to use mirror (non fork only) - run: |- - earthly config global.buildkit_additional_config "'[registry.\"docker.io\"] - mirrors = [\"registry-1.docker.io.mirror.corp.earthly.dev\"]'" + - name: Log in to GitHub Container Registry (non fork only) + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin if: github.event.pull_request.head.repo.full_name == github.repository - name: Run tests - run: earthly --ci -P --push --org earthly-technologies --satellite dind +test + run: earthly --ci -P --push +test --OS=${{ matrix.os }} + diff --git a/.github/workflows/dockerhub-description.yml b/.github/workflows/dockerhub-description.yml new file mode 100644 index 0000000..a122eaf --- /dev/null +++ b/.github/workflows/dockerhub-description.yml @@ -0,0 +1,21 @@ +name: Update Docker Hub description + +on: + push: + branches: [ main ] + paths: [ README.md ] + +jobs: + docker-hub-description: + runs-on: ubuntu-24.04-arm + steps: + - uses: actions/checkout@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Update DockerHub description + uses: peter-evans/dockerhub-description@v4.0.2 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: earthbuild/dind + short-description: The `dind` image is designed for EarthBuild targets \ No newline at end of file diff --git a/.github/workflows/earthly-count.yml b/.github/workflows/earthly-count.yml new file mode 100644 index 0000000..8107b0b --- /dev/null +++ b/.github/workflows/earthly-count.yml @@ -0,0 +1,185 @@ +name: Track Earthly to Earthbuild Progress + +on: + pull_request: + types: [opened, synchronize] + +jobs: + count-earthly: + runs-on: ubuntu-24.04-arm + permissions: + contents: read + pull-requests: write + + steps: + - name: Checkout PR branch + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 + with: + fetch-depth: 0 + + - name: Count occurrences in PR branch + id: count_pr + run: | + # Count total occurrences + total_count=$(git grep -i "earthly" 2>/dev/null | wc -l || echo "0") + echo "total_count=$total_count" >> $GITHUB_OUTPUT + + # Count by file type + go_count=$(git grep -i "earthly" -- "*.go" 2>/dev/null | wc -l || echo "0") + md_count=$(git grep -i "earthly" -- "*.md" 2>/dev/null | wc -l || echo "0") + earthfile_count=$(($(git grep -i "earthly" -- "Earthfile" 2>/dev/null | wc -l || echo "0") + $(git grep -i "earthly" -- "*.earth" 2>/dev/null | wc -l || echo "0"))) + + echo "go_count=$go_count" >> $GITHUB_OUTPUT + echo "md_count=$md_count" >> $GITHUB_OUTPUT + echo "earthfile_count=$earthfile_count" >> $GITHUB_OUTPUT + + echo "PR branch - Total: $total_count (Go: $go_count, MD: $md_count, Earthfiles: $earthfile_count)" + + - name: Checkout main branch + run: | + git checkout origin/main + + - name: Count occurrences in main branch + id: count_main + run: | + # Count total occurrences + total_count=$(git grep -i "earthly" 2>/dev/null | wc -l || echo "0") + echo "main_total_count=$total_count" >> $GITHUB_OUTPUT + + # Count by file type + go_count=$(git grep -i "earthly" -- "*.go" 2>/dev/null | wc -l || echo "0") + md_count=$(git grep -i "earthly" -- "*.md" 2>/dev/null | wc -l || echo "0") + earthfile_count=$(($(git grep -i "earthly" -- "Earthfile" 2>/dev/null | wc -l || echo "0") + $(git grep -i "earthly" -- "*.earth" 2>/dev/null | wc -l || echo "0"))) + + echo "main_go_count=$go_count" >> $GITHUB_OUTPUT + echo "main_md_count=$md_count" >> $GITHUB_OUTPUT + echo "main_earthfile_count=$earthfile_count" >> $GITHUB_OUTPUT + + echo "Main branch - Total: $total_count (Go: $go_count, MD: $md_count, Earthfiles: $earthfile_count)" + + - name: Calculate difference + id: calculate + run: | + pr_count=${{ steps.count_pr.outputs.total_count }} + main_count=${{ steps.count_main.outputs.main_total_count }} + difference=$((main_count - pr_count)) + + # Calculate percentage with proper formatting + if [ $main_count -gt 0 ]; then + # Use awk for better decimal handling + percentage=$(awk "BEGIN {printf \"%.2f\", $difference * 100 / $main_count}") + else + percentage="0.00" + fi + + echo "difference=$difference" >> $GITHUB_OUTPUT + echo "percentage=$percentage" >> $GITHUB_OUTPUT + echo "pr_count=$pr_count" >> $GITHUB_OUTPUT + echo "main_count=$main_count" >> $GITHUB_OUTPUT + + # Calculate differences by type + go_diff=$((${{ steps.count_main.outputs.main_go_count }} - ${{ steps.count_pr.outputs.go_count }})) + md_diff=$((${{ steps.count_main.outputs.main_md_count }} - ${{ steps.count_pr.outputs.md_count }})) + earthfile_diff=$((${{ steps.count_main.outputs.main_earthfile_count }} - ${{ steps.count_pr.outputs.earthfile_count }})) + + echo "go_diff=$go_diff" >> $GITHUB_OUTPUT + echo "md_diff=$md_diff" >> $GITHUB_OUTPUT + echo "earthfile_diff=$earthfile_diff" >> $GITHUB_OUTPUT + + - name: Comment on PR + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd #v8.0.0 + with: + script: | + const prCount = ${{ steps.calculate.outputs.pr_count }}; + const mainCount = ${{ steps.calculate.outputs.main_count }}; + const difference = ${{ steps.calculate.outputs.difference }}; + const percentage = '${{ steps.calculate.outputs.percentage }}'; + + // File type differences + const goDiff = ${{ steps.calculate.outputs.go_diff }}; + const mdDiff = ${{ steps.calculate.outputs.md_diff }}; + const earthfileDiff = ${{ steps.calculate.outputs.earthfile_diff }}; + + let emoji = '📊'; + let message = ''; + + if (difference > 0) { + emoji = '🎉'; + message = `Great progress! You've reduced "earthly" occurrences by **${difference}** (${percentage}%)`; + } else if (difference < 0) { + emoji = '⚠️'; + message = `Warning: "earthly" occurrences have increased by **${Math.abs(difference)}** (${Math.abs(parseFloat(percentage))}%)`; + } else { + emoji = '➖'; + message = 'No change in "earthly" occurrences'; + } + + // Build detailed breakdown + let breakdown = ''; + if (goDiff !== 0 || mdDiff !== 0 || earthfileDiff !== 0) { + breakdown = ` + + ### 📁 Changes by file type: + | File Type | Change | + |-----------|--------| + | Go files (.go) | ${goDiff > 0 ? '✅ -' + goDiff : goDiff < 0 ? '❌ +' + Math.abs(goDiff) : '➖ No change'} | + | Documentation (.md) | ${mdDiff > 0 ? '✅ -' + mdDiff : mdDiff < 0 ? '❌ +' + Math.abs(mdDiff) : '➖ No change'} | + | Earthfiles | ${earthfileDiff > 0 ? '✅ -' + earthfileDiff : earthfileDiff < 0 ? '❌ +' + Math.abs(earthfileDiff) : '➖ No change'} |`; + } + + const body = `## ${emoji} Are we earthbuild yet? + + ${message} + + ### 📈 Overall Progress + | Branch | Total Count | + |--------|-------------| + | main | ${mainCount} | + | This PR | ${prCount} | + | **Difference** | **${difference > 0 ? '-' : '+'}${Math.abs(difference)}** ${difference !== 0 ? `(${Math.abs(parseFloat(percentage))}%)` : ''} | + ${breakdown} + + --- + *Keep up the great work migrating from Earthly to Earthbuild!* 🚀 + +
+ 💡 Tips for finding more occurrences + + Run locally to see detailed breakdown: + \`\`\`bash + ./.github/scripts/count-earthly.sh + \`\`\` + + **Note that the goal is not to reach 0.** + There is anticipated to be at least _some_ occurences of \`earthly\` in the source code due to backwards compatibility with config files and language constructs. +
`; + + // Find existing comment + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + const botComment = comments.find(comment => + comment.user.type === 'Bot' && + comment.body.includes('Are we earthbuild yet?') + ); + + if (botComment) { + // Update existing comment + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: body + }); + } else { + // Create new comment + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: body + }); + } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 63bdd51..7690b4b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,25 +9,40 @@ jobs: name: release after merging ${{ github.head_ref }} if: github.event.pull_request.merged == true && startsWith(github.head_ref, 'renovate/') && endsWith(github.head_ref, '-dind-image') runs-on: ubuntu-latest + permissions: + contents: read + packages: write env: FORCE_COLOR: 1 EARTHLY_CONVERSION_PARALLELISM: "5" - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" - EARTHLY_INSTALL_ID: "earthly-dind-githubactions" + EARTHLY_INSTALL_ID: "earthbuild-dind-githubactions" steps: - - uses: earthly/actions/setup-earthly@v1 + - uses: actions/checkout@v5 + # The dind (common+alpine-kind-test and common+ubuntu-kind-test) detects + # the host's IPv6 capability and then requires IPv6 NAT to create networks + # for kind. So we load it here. + # Ref: https://github.com/moby/moby/pull/47062 + - name: Load kernel module IPv6 NAT + run: sudo modprobe ip6table_nat + - name: Set up Docker Hub mirrors + run: | + sudo mkdir -p /etc/docker + echo '{"registry-mirrors": ["https://mirror.gcr.io", "https://public.ecr.aws"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + - name: Set up QEMU + run: | + sudo apt-get update + sudo apt-get install -y qemu-user-static binfmt-support + sudo update-binfmts --display + - uses: earthbuild/actions-setup@main with: version: v0.8.15 - - uses: actions/checkout@v4 - - name: Docker login (non fork only) + - name: Log in to container registries (non fork only) run: |- - docker login --username "${{ secrets.DOCKERHUB_USERNAME }}" --password "${{ secrets.DOCKERHUB_TOKEN }}" - docker login registry-1.docker.io.mirror.corp.earthly.dev --username "${{ secrets.DOCKERHUB_MIRROR_USERNAME }}" --password "${{ secrets.DOCKERHUB_MIRROR_PASSWORD }}" - if: github.event.pull_request.head.repo.full_name == github.repository - - name: Configure Earthly to use mirror (non fork only) - run: |- - earthly config global.buildkit_additional_config "'[registry.\"docker.io\"] - mirrors = [\"registry-1.docker.io.mirror.corp.earthly.dev\"]'" + docker login --username "${{ vars.DOCKERHUB_USERNAME }}" --password "${{ secrets.DOCKERHUB_TOKEN }}" + docker login ghcr.io --username ${{ github.actor }} --password "${{ secrets.GITHUB_TOKEN }}" if: github.event.pull_request.head.repo.full_name == github.repository - name: Build & Push image - run: earthly --ci -P --push --org earthly-technologies --satellite dind +release --RENOVATE_BRANCH=${{github.head_ref}} + run: | + earthly --ci -P --push +release --RENOVATE_BRANCH=${{github.head_ref}} # default:ghcr.io + earthly --ci -P --push +release --RENOVATE_BRANCH=${{github.head_ref}} --CR_HOST=docker.io diff --git a/.github/workflows/update_readme.yml b/.github/workflows/update_readme.yml deleted file mode 100644 index 7e9cd15..0000000 --- a/.github/workflows/update_readme.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Update Docker Hub README - -on: - push: - branches: [ main ] - paths: [ docs/dockerhub.md ] - -jobs: - update-docker-hub-readmes: - name: update dockerhub readmes - runs-on: ubuntu-latest - env: - # Used in our github action as the token - TODO: look to change it into an input - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@v4 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: Update DockerHub description for earthly/dind - uses: peter-evans/dockerhub-description@v4.0.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - repository: earthly/dind - readme-filepath: ./docs/dockerhub.md - short-description: Earthly dind (Docker In Docker) Images diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a83ba7d..0db4c44 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,12 +4,68 @@ Please refer to the [CNCF Community Code of Conduct v1.0](https://github.com/cncf/foundation/blob/main/code-of-conduct.md) -## CLA +## How Images are Built -### Individual +In this repository, we maintain the OS & Docker versions that warrants releasing a new version of the image. +However, the installations of docker and other dependencies are done via an installation script that is currently maintained in [earthbuild/earthbuild](https://github.com/earthbuild/earthbuild). -All contributions must indicate agreement to the [Earthly Contributor License Agreement](https://gist.github.com/vladaionescu/ed990fa149a38a53ac74b64155bc6766) by logging into GitHub via the CLA assistant and signing the provided CLA. The CLA assistant will automatically notify the PRs that require CLA signing. +### Dependencies -### Entity +Dependencies are maintained by Renovate and will be merged automatically (provided required checks pass), primarily +dependencies that will trigger new versions of the dind images such as the docker or the os (alpine) versions. -If you are an entity, please use the [Earthly Contributor License Agreement form](https://earthly.dev/cla-form) in addition to requiring your individual contributors to sign all contributions. +## Repo structure + +```bash +. +├── Earthfile // Targets that apply to all images (e.g. +test) +├── common +│ └── Earthfile // A library of common helper targets +└── os // Each directory contains an Earthfile with targets to maintain the specific os (e.g. +test, +build) + ├── alpine + │ └── Earthfile + ├── ubuntu-20.04 + │ └── Earthfile + ├── ubuntu-23.04 + │ └── Earthfile + └── ubuntu-24.04 + └── Earthfile +``` + +## Testing + +Images are tested by running remote test targets that are maintained in [earthbuild/earthbuild](https://github.com/earthbuild/earthbuild/tree/main/tests/with-docker). This is because these tests also help test [WITH DOCKER](https://docs.earthly.dev/docs/earthfile#with-docker) command in earthly cli. + +Temporary images are built, pushed, and pulled as part of the test cycle. + +### How to run tests + +* Test a specific image os: + +```bash +earthly --push -P ./os/+test-build +``` + +* Test all images: +```bash +earthly --push -P +test +``` + +#### Community members + +Community members do not have permissions to push a built image and run the tests against it. However, they can easily set a different container registry repository by changing the `CR_HOST` (default: ghcr.io) and `CR_ORG` ARG values in [.arg](.arg) to a private container registry repository or by passing the args in the earthly command, e.g. `earthly --push -P +test --CR_HOST= --CR_ORG=`. + +## Deployment + +When the relevant dependencies are updated by Renovate, new images/tags will be pushed automatically to the container registries - [ghcr.io/earthbuild/dind](https://ghcr.io/earthbuild/dind) and [earthbuild/dind](https://hub.docker.com/r/earthbuild/dind). + +## Contributing + +* Please report bugs as [GitHub issues](https://github.com/earthbuild/dind/issues). +* Join us on [Slack](https://earthly.dev/slack)! +* Questions via GitHub issues are welcome! +* PRs welcome! But please give a heads-up in a GitHub issue before starting work. If there is no GitHub issue for what you want to do, please create one. + +## Licensing + +Earthly is licensed under the Mozilla Public License Version 2.0. See [LICENSE](./LICENSE). \ No newline at end of file diff --git a/Earthfile b/Earthfile index 4f9bc27..ba76935 100644 --- a/Earthfile +++ b/Earthfile @@ -2,9 +2,10 @@ VERSION --wildcard-builds 0.8 PROJECT earthly-technologies/core -# test runs tests for all defined dind images in this repo +# test runs tests for for the given OS image (os/*/Earthfile) in this repo test: - BUILD --pass-args ./os/*+test-build + ARG --required OS + BUILD --pass-args ./os/$OS+test-build # release expects to get a renovate branch in the form of renovate/-dind-image, extracts the os and then kicks off its +release target # this is meant to be run by a github workflow diff --git a/README.md b/README.md index 11c67ab..69086ac 100644 --- a/README.md +++ b/README.md @@ -1,84 +1,24 @@ -# Earthly dind (Docker In Docker) Images -[![Release](https://github.com/earthly/dind/actions/workflows/release.yml/badge.svg)](https://github.com/earthly/dind/actions/workflows/release.yml) -![Docker Pulls](https://img.shields.io/docker/pulls/earthly/dind) +[![Release](https://github.com/earthbuild/dind/actions/workflows/release.yml/badge.svg)](https://github.com/earthbuild/dind/actions/workflows/release.yml) ![Docker Pulls](https://img.shields.io/docker/pulls/earthbuild/dind) +# EarthBuild Docker In Docker (dind) Images -Earthly's official [earthly/dind](https://hub.docker.com/repository/docker/earthly/dind/general) docker images. -For information on how to use these images, please refer to [docker in earthly](https://docs.earthly.dev/docs/guides/docker-in-earthly). +The `dind` (docker-in-docker) image is designed for EarthBuild targets that use the `WITH DOCKER` command. -## Supported Distributions - -There are currently 3 supported dind distributions available: -- `alpine` -- `ubuntu:20.04` -- `ubuntu:23.04` -- `ubuntu:24.04` - -Other distributions and/or base images can be used with our [dind+INSTALL](https://docs.earthly.dev/docs/guides/docker-in-earthly#performance) [FUNCTION](https://docs.earthly.dev/docs/guides/functions). - -## How Images are Built - -In this repository, we maintain the OS & Docker versions that warrants releasing a new version of the image. -However, the installations of docker and other dependencies are done via an installation script that is currently maintained in [earthly/earthly](https://github.com/earthly/earthly). - -### Dependencies - -Dependencies are maintained by Renovate and will be merged automatically (provided required checks pass), primarly -dependencies that will trigger new versions of the dind images such as the docker or the os (alpine) versions. - -## Repo structure - -```bash -. -├── Earthfile // Targets that apply to all images (e.g. +test) -├── common -│ └── Earthfile // A library of common helper targets -└── os // Each directory contains an Earthfile with targets to maintain the specific os (e.g. +test, +build) - ├── alpine - │ └── Earthfile - ├── ubuntu-20.04 - │ └── Earthfile - └── ubuntu-23.04 - └── Earthfile - └── ubuntu-24.04 - └── Earthfile -``` - -## Testing +For information on how to use these images, please refer to [docker in EarthBuild](https://docs.earthly.dev/docs/guides/docker-in-earthly). -Images are tested by running remote test targets that are maintained in [earthly/earthly](https://github.com/earthly/earthly/tree/main/tests/with-docker). This is because these tests also help test [WITH DOCKER](https://docs.earthly.dev/docs/earthfile#with-docker) command in earthly cli. - -Temporary images are built, pushed, and pulled as part of the test cycle. - -### How to run tests - -* Test a specific image os: - -```bash -earthly --push -P ./os/+test-build -``` - -* Test all images: -```bash -earthly --push -P +test -``` - -#### Community members - -Community members do not have permissions to push a built image and run the tests against it. However, they can easily set a different dockerhub repository by changing the `DOCKERHUB_USER` ARG value in [.arg](.arg) to a private repository or by passing the arg in the earthly command, e.g. `earthly --push -P +test --DOCKERHUB_USER=`. - -## Deployment - -When the relevant dependencies are updates by Renovate, new images/tags will be pushed automatically to the docker registry. +## Supported Distributions -## Contributing +This image supports the following Linux distributions: +* alpine +* ubuntu:20.04 +* ubuntu:23.04 +* ubuntu:24.04 -* Please report bugs as [GitHub issues](https://github.com/earthly/dind/issues). -* Join us on [Slack](https://earthly.dev/slack)! -* Questions via GitHub issues are welcome! -* PRs welcome! But please give a heads-up in a GitHub issue before starting work. If there is no GitHub issue for what you want to do, please create one. -* Check the [contributing page](./CONTRIBUTING.md) for more details. +For which the current latest tags (respectively) are: +* `alpine-3.22-docker-28.3.3-r2` +* `ubuntu-20.04-docker-28.1.1-1` +* `ubuntu-23.04-docker-25.0.2-1` +* `ubuntu-24.04-docker-28.4.0-1` -## Licensing +For other available tags, please check out [ghcr.io/earthbuild/dind](https://github.com/earthbuild/dind/pkgs/container/dind/versions?filters%5Bversion_type%5D=tagged) or [earthbuild/dind](https://hub.docker.com/r/earthbuild/dind/tags). -Earthly is licensed under the Mozilla Public License Version 2.0. See [LICENSE](./LICENSE). diff --git a/common/Earthfile b/common/Earthfile index 9a22e2e..8b415d7 100644 --- a/common/Earthfile +++ b/common/Earthfile @@ -6,14 +6,14 @@ FROM alpine # EARTHLY_REPO_VERSION specifies a branch and/or commit of earthly/earthly (it defaults to the earthly cli version if left empty) ARG EARTHLY_REPO_VERSION -IMPORT github.com/earthly/earthly/buildkitd:$EARTHLY_REPO_VERSION AS earthly -IMPORT github.com/earthly/earthly/tests/with-docker:$EARTHLY_REPO_VERSION AS with-docker -IMPORT github.com/earthly/earthly/tests/with-docker-compose:$EARTHLY_REPO_VERSION AS with-docker-compose -IMPORT github.com/earthly/earthly/tests/with-docker-kind:$EARTHLY_REPO_VERSION AS with-docker-kind -IMPORT github.com/earthly/earthly/tests/dind-auto-install:$EARTHLY_REPO_VERSION AS dind-auto-install +IMPORT github.com/EarthBuild/earthbuild/buildkitd:$EARTHLY_REPO_VERSION AS earthly +IMPORT github.com/EarthBuild/earthbuild/tests/with-docker:$EARTHLY_REPO_VERSION AS with-docker +IMPORT github.com/EarthBuild/earthbuild/tests/with-docker-compose:$EARTHLY_REPO_VERSION AS with-docker-compose +IMPORT github.com/EarthBuild/earthbuild/tests/with-docker-kind:$EARTHLY_REPO_VERSION AS with-docker-kind +IMPORT github.com/EarthBuild/earthbuild/tests/dind-auto-install:$EARTHLY_REPO_VERSION AS dind-auto-install -# DOCKERHUB_USER is the organization name in docker hub (default: earthly) -ARG --global DOCKERHUB_USER +# CR_ORG is the organization name in a container registry (default: earthbuild) +ARG --global CR_ORG # IMAGE_NAME is the image repository in docker hub (default: dind) ARG --global IMAGE_NAME @@ -26,14 +26,22 @@ build: # DOCKER_VERSION is the version of docker to use, e.g. 20.10.14 ARG --required DOCKER_VERSION FROM $OS_IMAGE:$OS_VERSION + # Ubuntu 23.04 has reached EOL + IF [ "$OS_VERSION" = "23.04" ] + RUN sed -i \ + -e 's/archive.ubuntu.com\/ubuntu/old-releases.ubuntu.com\/ubuntu/g' \ + -e 's/ports.ubuntu.com\/ubuntu-ports/old-releases.ubuntu.com\/ubuntu/g' \ + -e 's/security.ubuntu.com\/ubuntu/old-releases.ubuntu.com\/ubuntu/g' \ + /etc/apt/sources.list + END COPY earthly+export-docker-script/docker-auto-install.sh /usr/local/bin/docker-auto-install.sh RUN docker-auto-install.sh LET DOCKER_VERSION_TAG=$DOCKER_VERSION IF [ "$OS_IMAGE" = "alpine" ] RUN apk add iptables-legacy # required for older kernels END - # DOCKERHUB_USER is the organization name in docker hub (default: earthly) - ARG DOCKERHUB_USER + # CR_ORG is the organization name in container registry (default: earthbuild) + ARG CR_ORG # IMAGE_NAME is the image repository in docker hub (default: dind) ARG IMAGE_NAME COPY --dir --pass-args +get-image-info/image-info . @@ -41,7 +49,8 @@ build: LET image_tag=$(cat image-info/tag) RUN rm -rf image-info ENV OTEL_TRACES_EXPORTER=none # disabled for speed improvement; see https://github.com/earthly/earthly/issues/4066 - SAVE IMAGE --push $image_name:$image_tag + ARG --required CR_HOST + SAVE IMAGE --push $CR_HOST/$image_name:$image_tag # get-image-info generates the image name and tag as a saved artifact so it can be used by multiple targets get-image-info: @@ -59,11 +68,12 @@ get-image-info: RUN if echo $DOCKER_VERSION_TAG | grep "[^0-9.-]"; then echo "DOCKER_VERSION_TAG looks bad; got $DOCKER_VERSION_TAG" && exit 1; fi END LET TAG=$OS_IMAGE-$OS_VERSION-docker-$DOCKER_VERSION_TAG - # DOCKERHUB_USER is the organization name in docker hub (default: earthly) - ARG --required DOCKERHUB_USER + # CR_ORG is the organization name in container registry (default: earthbuild) + ARG --required CR_ORG # IMAGE_NAME is the image repository in docker hub (default: dind) ARG --required IMAGE_NAME - LET image_full_name=$DOCKERHUB_USER/$IMAGE_NAME + # image_full_name excludes container registry host + LET image_full_name=$CR_ORG/$IMAGE_NAME ARG SUFFIX IF [ -n "$SUFFIX" ] SET TAG="$TAG-$SUFFIX" @@ -75,7 +85,7 @@ get-image-info: # test runs tests against the specified image (DIND_IMAGE) or otherwise uses the image specified in earthly/earthly repo test: - # DIND_IMAGE is the full docker image name & tag to run the tests against. The image must exist in the remote docker registry + # DIND_IMAGE is the full container image name & tag to run the tests against. The image must exist in the remote container registry ARG DIND_IMAGE BUILD --pass-args with-docker+all --DIND_IMAGE=$DIND_IMAGE BUILD --pass-args dind-auto-install+test --BASE_IMAGE=$DIND_IMAGE @@ -95,11 +105,12 @@ build-and-test: COPY --dir --pass-args +get-image-info/image-info . LET image_name=$(cat image-info/name) LET image_tag=$(cat image-info/tag) + ARG --required CR_HOST WAIT # EARTHLY_PUSH is a builtin arg that helps determine if the tests should run ARG EARTHLY_PUSH IF [ "$EARTHLY_PUSH" = "true" ] - BUILD --pass-args ../os/$DIR_PATH+test --DIND_IMAGE=$image_name:$image_tag + BUILD --pass-args ../os/$DIR_PATH+test --DIND_IMAGE=$CR_HOST/$image_name:$image_tag END END @@ -114,12 +125,13 @@ push-new-tag: exit 1 END END - FROM $DOCKERHUB_USER/$IMAGE_NAME:$TAG_WITH_DATE + ARG --required CR_HOST + FROM $CR_HOST/$CR_ORG/$IMAGE_NAME:$TAG_WITH_DATE LET new_tag="$(echo ${TAG_WITH_DATE%-*})" IF [ "$new_tag" = "$TAG_WITH_DATE" ] RUN --no-cache echo "failed to remove date from tag $TAG_WITH_DATE" && exit 1 END - SAVE IMAGE --push $DOCKERHUB_USER/$IMAGE_NAME:$new_tag + SAVE IMAGE --push $CR_HOST/$CR_ORG/$IMAGE_NAME:$new_tag # push-new-tag-multi-platform builds push-new-tag using both amd64 & arm64 platforms. push-new-tag-multi-platform: diff --git a/docs/dockerhub.md b/docs/dockerhub.md deleted file mode 100644 index 3532d0c..0000000 --- a/docs/dockerhub.md +++ /dev/null @@ -1,54 +0,0 @@ -The `dind` (docker-in-docker) image is designed for Earthfile targets that use the `WITH DOCKER` command. - -See the ["use-earthly-dind" best-practice](https://docs.earthly.dev/best-practices#use-earthly-dind) for details. - -## Tags - -This image supports 3 Linux distributions: -* alpine -* ubuntu:20.04 -* ubuntu:23.04 -* ubuntu:24.04 - -For which the current latest tags (respectively) are: -* `alpine-3.20-docker-26.1.5-r0` -* `ubuntu-20.04-docker-27.2.1-1` -* `ubuntu-23.04-docker-25.0.2-1` -* `ubuntu-24.04-docker-27.3.1-1` - -For other available tags, please check out https://hub.docker.com/r/earthly/dind/tags - -## Outdated Tags - -* `alpine` -* `ubuntu` - -## Note - -The outdated `ubuntu` image is incompatible with the earthly v0.7.14 (and fixed in v0.7.15). -Correspondingly the `alpine` image at one point was also incompatible with v0.7.14, but was updated with -a backwards-compatable fix. - -Users, however, are encouraged to pin to specific version tags moving forward. The unversioned tags will be left as-is -to help backwards-breaking changes. - -To ease this transition, one can make use of an `IF` command that depends on the `EARTHLY_VERSION` builtin argument: - -``` -VERSION 0.8 - -dind: - FROM earthly/dind:alpine - ARG EARTHLY_VERSION - ARG SMALLEST_VERSION="$(echo -e "$EARTHLY_VERSION\nv0.7.14" | sort -V | head -n 1)" - IF [ "$SMALLEST_VERSION" = "v0.7.14" ] - # earthly is at v0.7.14 or newer, and must use the more recent dind:alpine-3.19-docker-25.0.2-r0 image - FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 - END - -test: - FROM +dind - WITH DOCKER - RUN docker --version # old versions of earthly will get 20.10.14, and newer will get 23.0.6 - END -``` diff --git a/os/alpine/Earthfile b/os/alpine/Earthfile index 4e28a73..5e21fc0 100644 --- a/os/alpine/Earthfile +++ b/os/alpine/Earthfile @@ -9,9 +9,9 @@ IMPORT ../../common AS common ARG --global OS_IMAGE=alpine # renovate: datasource=docker depName=alpine -ARG --global OS_VERSION=3.20 -# renovate: datasource=repology depName=alpine_3_20/docker versioning=loose -ARG --global DOCKER_VERSION=26.1.5-r0 +ARG --global OS_VERSION=3.22 +# renovate: datasource=repology depName=alpine_3_22/docker versioning=loose +ARG --global DOCKER_VERSION=28.3.3-r2 # DIR_PATH is set to that common targets can call os specific targets. It should match the directory name this Earthfile is located in ARG --global DIR_PATH=$OS_IMAGE diff --git a/os/ubuntu-20.04/Earthfile b/os/ubuntu-20.04/Earthfile index a6ec062..3978dad 100644 --- a/os/ubuntu-20.04/Earthfile +++ b/os/ubuntu-20.04/Earthfile @@ -10,7 +10,7 @@ ARG --global OS_IMAGE=ubuntu ARG --global OS_VERSION=20.04 # renovate: datasource=github-releases depName=docker/docker -LET docker_package_version=27.2.1 +LET docker_package_version=28.1.1 ARG --global DOCKER_VERSION=5:$docker_package_version-1~ubuntu.$OS_VERSION~focal # DIR_PATH is set to that common targets can call os specific targets. It should match the directory name this Earthfile is located in diff --git a/os/ubuntu-24.04/Earthfile b/os/ubuntu-24.04/Earthfile index fcfbcb0..d0c0738 100644 --- a/os/ubuntu-24.04/Earthfile +++ b/os/ubuntu-24.04/Earthfile @@ -10,7 +10,7 @@ ARG --global OS_IMAGE=ubuntu ARG --global OS_VERSION=24.04 # renovate: datasource=github-releases depName=docker/docker -LET docker_package_version=27.3.1 +LET docker_package_version=28.4.0 ARG --global DOCKER_VERSION=5:$docker_package_version-1~ubuntu.$OS_VERSION~noble # DIR_PATH is set to that common targets can call os specific targets. It should match the directory name this Earthfile is located in