diff --git a/.circleci/config.yml b/.circleci/config.yml index a35fd0c680..f2955446ab 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,7 +24,7 @@ jobs: name: Run tests command: |- ./build/linux/amd64/earthly --ci -P \ - +smoke-test --DOCKERHUB_MIRROR_AUTH_FROM_CLOUD_SECRETS=true && echo "circleci tests passed" + +smoke-test && echo "circleci tests passed" workflows: version: 2 circle-ci: diff --git a/.gitbook.yaml b/.gitbook.yaml index 979da6f831..4f2ede525f 100644 --- a/.gitbook.yaml +++ b/.gitbook.yaml @@ -13,8 +13,6 @@ redirects: guides/auth: guides/auth.md guides/cache: caching/managing-cache.md docs/guides/cache: caching/managing-cache.md - docs/guides/cloud-secrets: cloud/cloud-secrets.md - guides/cloud-secrets: cloud/cloud-secrets.md guides/debugging: guides/debugging.md guides/docker-in-earthly: guides/docker-in-earthly.md guides/integration: guides/integration.md @@ -78,4 +76,3 @@ redirects: guides/configuring-registries: guides/auth.md guides/shared-cache: ./caching/caching-via-registry.md docs/remote-caching: ./caching/caching-via-registry.md - earthly-cloud/satellites: cloud/satellites.md diff --git a/.github/actions/stage2-setup/action.yml b/.github/actions/stage2-setup/action.yml index 75bff634a5..104106d8c9 100644 --- a/.github/actions/stage2-setup/action.yml +++ b/.github/actions/stage2-setup/action.yml @@ -21,12 +21,6 @@ inputs: description: "Whether to use earthly-next" required: false default: 'false' - EARTHLY_ORG: - description: "The Earthly organization" - required: false - EARTHLY_TOKEN: - description: "Earthly token (legacy - not used with GCR mirror)" - required: false GHCR_USERNAME: description: "GHCR username (legacy - now uses github.actor)" required: false diff --git a/.github/workflows/build-earthly.yml b/.github/workflows/build-earthly.yml index cbde75cf1c..6b97a91f39 100644 --- a/.github/workflows/build-earthly.yml +++ b/.github/workflows/build-earthly.yml @@ -34,7 +34,6 @@ jobs: permissions: write-all env: FORCE_COLOR: 1 - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" EARTHLY_INSTALL_ID: "earthly-githubactions" # Used in our github action as the token - TODO: look to change it into an input GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-docker-ubuntu.yml b/.github/workflows/ci-docker-ubuntu.yml index 5a947108df..caca4ce5ab 100644 --- a/.github/workflows/ci-docker-ubuntu.yml +++ b/.github/workflows/ci-docker-ubuntu.yml @@ -264,75 +264,20 @@ jobs: # EXTRA_ARGS: "--auto-skip" secrets: inherit - docker-tests-test-account: - if: ${{ !failure() && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) }} - needs: build-earthly - uses: ./.github/workflows/reusable-test.yml - with: - TEST_TARGET: "./tests/account+test" - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "docker" - SUDO: "" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - # EXTRA_ARGS: "--auto-skip" - secrets: inherit - - docker-test-registry-command-dockerhub: - if: ${{ !failure() && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) }} - needs: build-earthly - uses: ./.github/workflows/reusable-test.yml - with: - TEST_TARGET: "./tests/registry-command+test-dockerhub" - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "docker" - SUDO: "" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - # EXTRA_ARGS: "--auto-skip" - secrets: inherit - - docker-test-registry-command-multi: - if: ${{ !failure() && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) }} - needs: build-earthly - uses: ./.github/workflows/reusable-test.yml - with: - TEST_TARGET: "./tests/registry-command+test-multi" - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "docker" - SUDO: "" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - # EXTRA_ARGS: "--auto-skip" - secrets: inherit - - docker-test-web-command: - if: ${{ !failure() && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) }} - needs: build-earthly - uses: ./.github/workflows/reusable-test.yml - with: - TEST_TARGET: "./tests/web+test" - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "docker" - SUDO: "" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - # EXTRA_ARGS: "--auto-skip" - secrets: inherit - - docker-test-oidc-command: - if: ${{ !failure() && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) }} - needs: build-earthly - uses: ./.github/workflows/reusable-test.yml - with: - TEST_TARGET: "./tests/oidc+test" - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "docker" - SUDO: "" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - # EXTRA_ARGS: "--auto-skip" - secrets: inherit + # TODO: The oidc tests are ignored until we have an AWS account to test with. + # docker-test-oidc-command: + # if: ${{ !failure() && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) }} + # needs: build-earthly + # uses: ./.github/workflows/reusable-test.yml + # with: + # TEST_TARGET: "./tests/oidc+test" + # BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + # RUNS_ON: "ubuntu-latest" + # BINARY: "docker" + # SUDO: "" + # SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} + # # EXTRA_ARGS: "--auto-skip" + # secrets: inherit docker-tests-qemu: needs: build-earthly @@ -413,18 +358,6 @@ jobs: SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} secrets: inherit - docker-secret-integrations: - needs: build-earthly - if: ${{ !failure() }} - uses: ./.github/workflows/reusable-secrets-integrations.yml - with: - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "docker" - SUDO: "" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - secrets: inherit - docker-bootstrap-integrations: needs: build-earthly if: ${{ !failure() }} diff --git a/.github/workflows/ci-lint-docs.yml b/.github/workflows/ci-lint-docs.yml index 4e39c8cbd1..4ce3c1b9be 100644 --- a/.github/workflows/ci-lint-docs.yml +++ b/.github/workflows/ci-lint-docs.yml @@ -16,7 +16,6 @@ jobs: runs-on: "ubuntu-latest" env: FORCE_COLOR: 1 - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" EARTHLY_INSTALL_ID: "earthly-githubactions" DOCKERHUB_MIRROR_USERNAME: "${{ secrets.DOCKERHUB_MIRROR_USERNAME }}" DOCKERHUB_MIRROR_PASSWORD: "${{ secrets.DOCKERHUB_MIRROR_PASSWORD }}" diff --git a/.github/workflows/ci-podman-ubuntu.yml b/.github/workflows/ci-podman-ubuntu.yml index a12f720a04..4204c4da74 100644 --- a/.github/workflows/ci-podman-ubuntu.yml +++ b/.github/workflows/ci-podman-ubuntu.yml @@ -236,109 +236,19 @@ jobs: SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} secrets: inherit - podman-test-earthly-mirror-was-setup: - if: ${{ !failure() && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) }} - needs: build-earthly - uses: ./.github/workflows/reusable-test.yml - with: - TEST_TARGET: "./tests+test-earthly-mirror-was-setup" - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "podman" - SUDO: "sudo -E" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - secrets: inherit - - podman-tests-test-account: - if: ${{ !failure() && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) }} - needs: build-earthly - uses: ./.github/workflows/reusable-test.yml - with: - TEST_TARGET: "./tests/account+test" - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "podman" - SUDO: "sudo -E" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - secrets: inherit - - podman-test-registry-command-dockerhub: - if: ${{ !failure() && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) }} - needs: build-earthly - uses: ./.github/workflows/reusable-test.yml - with: - TEST_TARGET: "./tests/registry-command+test-dockerhub" - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "podman" - SUDO: "sudo -E" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - secrets: inherit - - podman-test-registry-command-ecr: - if: ${{ !failure() && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) }} - needs: build-earthly - uses: ./.github/workflows/reusable-test.yml - with: - TEST_TARGET: "./tests/registry-command+test-ecr" - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "podman" - SUDO: "sudo -E" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - secrets: inherit - - podman-test-registry-command-gcp: - if: ${{ !failure() && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) }} - needs: build-earthly - uses: ./.github/workflows/reusable-test.yml - with: - TEST_TARGET: "./tests/registry-command+test-gcp" - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "podman" - SUDO: "sudo -E" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - secrets: inherit - - podman-test-registry-command-multi: - if: ${{ !failure() && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) }} - needs: build-earthly - uses: ./.github/workflows/reusable-test.yml - with: - TEST_TARGET: "./tests/registry-command+test-multi" - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "podman" - SUDO: "sudo -E" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - secrets: inherit - - podman-test-web-command: - if: ${{ !failure() && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) }} - needs: build-earthly - uses: ./.github/workflows/reusable-test.yml - with: - TEST_TARGET: "./tests/web+test" - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "podman" - SUDO: "sudo -E" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - secrets: inherit - - podman-test-oidc-command: - if: ${{ !failure() && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) }} - needs: build-earthly - uses: ./.github/workflows/reusable-test.yml - with: - TEST_TARGET: "./tests/oidc+test" - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "podman" - SUDO: "sudo -E" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - secrets: inherit + # TODO: Commented out until we have an AWS account to run tests in + # podman-test-oidc-command: + # if: ${{ !failure() && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) }} + # needs: build-earthly + # uses: ./.github/workflows/reusable-test.yml + # with: + # TEST_TARGET: "./tests/oidc+test" + # BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + # RUNS_ON: "ubuntu-latest" + # BINARY: "podman" + # SUDO: "sudo -E" + # SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} + # secrets: inherit podman-tests-qemu: needs: build-earthly @@ -427,19 +337,6 @@ jobs: # SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} # secrets: inherit # -# podman-secret-integrations: -# # TODO: Figure out how to run multiple Podman instances in parallel with different ports for buildkitd -# needs: build-earthly -# if: ${{ !failure() }} -# uses: ./.github/workflows/reusable-secrets-integrations.yml -# with: -# BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" -# RUNS_ON: "ubuntu-latest" -# BINARY: "podman" -# SUDO: "sudo -E" -# SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} -# secrets: inherit -# # podman-bootstrap-integrations: # # TODO: Figure out how to run multiple Podman instances in parallel with different ports for buildkitd # needs: build-earthly diff --git a/.github/workflows/ci-staging-deploy.yml b/.github/workflows/ci-staging-deploy.yml index 03446ac6ed..782746e223 100644 --- a/.github/workflows/ci-staging-deploy.yml +++ b/.github/workflows/ci-staging-deploy.yml @@ -29,7 +29,6 @@ jobs: runs-on: ubuntu-latest env: FORCE_COLOR: 1 - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" EARTHLY_INSTALL_ID: "earthly-githubactions" GITHUB_USER: "earthly" EARTHLY_REPO: "earthly-staging" diff --git a/.github/workflows/docs-checks-links.yml b/.github/workflows/docs-checks-links.yml index 27b06345fe..2aa6707402 100644 --- a/.github/workflows/docs-checks-links.yml +++ b/.github/workflows/docs-checks-links.yml @@ -15,7 +15,6 @@ jobs: runs-on: "ubuntu-latest" env: FORCE_COLOR: 1 - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" EARTHLY_INSTALL_ID: "earthly-githubactions" DOCKERHUB_MIRROR_USERNAME: "${{ secrets.DOCKERHUB_MIRROR_USERNAME }}" DOCKERHUB_MIRROR_PASSWORD: "${{ secrets.DOCKERHUB_MIRROR_PASSWORD }}" diff --git a/.github/workflows/release-merge-docs.yml b/.github/workflows/release-merge-docs.yml index ad8a0dbd82..8ae89882e6 100644 --- a/.github/workflows/release-merge-docs.yml +++ b/.github/workflows/release-merge-docs.yml @@ -15,7 +15,6 @@ jobs: runs-on: "ubuntu-latest" env: FORCE_COLOR: 1 - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" EARTHLY_INSTALL_ID: "earthly-githubactions" DOCKERHUB_MIRROR_USERNAME: "${{ secrets.DOCKERHUB_MIRROR_USERNAME }}" DOCKERHUB_MIRROR_PASSWORD: "${{ secrets.DOCKERHUB_MIRROR_PASSWORD }}" diff --git a/.github/workflows/reusable-bootstrap-integrations.yml b/.github/workflows/reusable-bootstrap-integrations.yml index ebbdb46b65..e3f9221f8d 100644 --- a/.github/workflows/reusable-bootstrap-integrations.yml +++ b/.github/workflows/reusable-bootstrap-integrations.yml @@ -24,7 +24,6 @@ jobs: secret-integration: env: FORCE_COLOR: 1 - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" EARTHLY_INSTALL_ID: "earthly-githubactions" DOCKERHUB_MIRROR_USERNAME: "${{ secrets.DOCKERHUB_MIRROR_USERNAME }}" DOCKERHUB_MIRROR_PASSWORD: "${{ secrets.DOCKERHUB_MIRROR_PASSWORD }}" @@ -40,7 +39,6 @@ jobs: with: GHCR_USERNAME: ${{ github.actor }} GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" BUILT_EARTHLY_PATH: "${{ inputs.BUILT_EARTHLY_PATH }}" BINARY: "${{ inputs.BINARY }}" SUDO: "${{ inputs.SUDO }}" diff --git a/.github/workflows/reusable-docker-build-integrations.yml b/.github/workflows/reusable-docker-build-integrations.yml index 3151a8a191..9270149bee 100644 --- a/.github/workflows/reusable-docker-build-integrations.yml +++ b/.github/workflows/reusable-docker-build-integrations.yml @@ -29,7 +29,6 @@ jobs: runs-on: ${{inputs.RUNS_ON}} env: FORCE_COLOR: 1 - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" EARTHLY_ORG: "${{inputs.EARTHLY_ORG}}" EARTHLY_INSTALL_ID: "earthly-githubactions" DOCKERHUB_MIRROR_USERNAME: "${{ secrets.DOCKERHUB_MIRROR_USERNAME }}" @@ -44,7 +43,6 @@ jobs: with: GHCR_USERNAME: ${{ github.actor }} GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" BUILT_EARTHLY_PATH: "${{ inputs.BUILT_EARTHLY_PATH }}" BINARY: "${{ inputs.BINARY }}" SUDO: "${{ inputs.SUDO }}" diff --git a/.github/workflows/reusable-earthbuild-image-tests.yml b/.github/workflows/reusable-earthbuild-image-tests.yml index 96f0296f86..460cc4abe3 100644 --- a/.github/workflows/reusable-earthbuild-image-tests.yml +++ b/.github/workflows/reusable-earthbuild-image-tests.yml @@ -34,7 +34,6 @@ jobs: runs-on: ${{inputs.RUNS_ON}} env: FORCE_COLOR: 1 - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" EARTHLY_INSTALL_ID: "earthly-githubactions" # Used in our github action as the token - TODO: look to change it into an input GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -46,7 +45,6 @@ jobs: with: GHCR_USERNAME: ${{ github.actor }} GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" BUILT_EARTHLY_PATH: "${{ inputs.BUILT_EARTHLY_PATH }}" BINARY: "${{ inputs.BINARY }}" SUDO: "${{ inputs.SUDO }}" diff --git a/.github/workflows/reusable-example.yml b/.github/workflows/reusable-example.yml index cfa946cd3a..2de3c42a94 100644 --- a/.github/workflows/reusable-example.yml +++ b/.github/workflows/reusable-example.yml @@ -37,7 +37,6 @@ jobs: runs-on: ${{inputs.RUNS_ON}} env: FORCE_COLOR: 1 - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" EARTHLY_INSTALL_ID: "earthly-githubactions" EARTHLY_ORG: "${{inputs.EARTHLY_ORG}}" DOCKERHUB_MIRROR_USERNAME: "${{ secrets.DOCKERHUB_MIRROR_USERNAME }}" @@ -52,7 +51,6 @@ jobs: with: GHCR_USERNAME: ${{ github.actor }} GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" BUILT_EARTHLY_PATH: "${{ inputs.BUILT_EARTHLY_PATH }}" BINARY: "${{ inputs.BINARY }}" USE_QEMU: "${{ inputs.USE_QEMU }}" diff --git a/.github/workflows/reusable-export-test.yml b/.github/workflows/reusable-export-test.yml index c0f8abd845..f46670840d 100644 --- a/.github/workflows/reusable-export-test.yml +++ b/.github/workflows/reusable-export-test.yml @@ -27,7 +27,6 @@ jobs: if: ${{ !inputs.SKIP_JOB && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) }} env: FORCE_COLOR: 1 - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" EARTHLY_INSTALL_ID: "earthly-githubactions" # Used in our github action as the token - TODO: look to change it into an input GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -39,7 +38,6 @@ jobs: with: GHCR_USERNAME: ${{ github.actor }} GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" BUILT_EARTHLY_PATH: "${{ inputs.BUILT_EARTHLY_PATH }}" BINARY: "${{ inputs.BINARY }}" SUDO: "${{ inputs.SUDO }}" diff --git a/.github/workflows/reusable-git-metadata-test.yml b/.github/workflows/reusable-git-metadata-test.yml index a952ebdda6..fd16b42051 100644 --- a/.github/workflows/reusable-git-metadata-test.yml +++ b/.github/workflows/reusable-git-metadata-test.yml @@ -31,7 +31,6 @@ jobs: runs-on: ${{inputs.RUNS_ON}} env: FORCE_COLOR: 1 - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" EARTHLY_INSTALL_ID: "earthly-githubactions" EARTHLY_ORG: "${{inputs.EARTHLY_ORG}}" DOCKERHUB_MIRROR_USERNAME: "${{ secrets.DOCKERHUB_MIRROR_USERNAME }}" @@ -46,7 +45,6 @@ jobs: with: GHCR_USERNAME: ${{ github.actor }} GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" BUILT_EARTHLY_PATH: "${{ inputs.BUILT_EARTHLY_PATH }}" BINARY: "${{ inputs.BINARY }}" SUDO: "${{ inputs.SUDO }}" diff --git a/.github/workflows/reusable-misc-tests-1.yml b/.github/workflows/reusable-misc-tests-1.yml index 87bfcbea9e..189dd079ea 100644 --- a/.github/workflows/reusable-misc-tests-1.yml +++ b/.github/workflows/reusable-misc-tests-1.yml @@ -34,7 +34,6 @@ jobs: runs-on: ${{inputs.RUNS_ON}} env: FORCE_COLOR: 1 - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" EARTHLY_INSTALL_ID: "earthly-githubactions" # Used in our github action as the token - TODO: look to change it into an input GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -46,7 +45,6 @@ jobs: with: GHCR_USERNAME: ${{ github.actor }} GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" BUILT_EARTHLY_PATH: "${{ inputs.BUILT_EARTHLY_PATH }}" BINARY: "${{ inputs.BINARY }}" SUDO: "${{ inputs.SUDO }}" diff --git a/.github/workflows/reusable-misc-tests-2.yml b/.github/workflows/reusable-misc-tests-2.yml index f0892649f4..71744bc854 100644 --- a/.github/workflows/reusable-misc-tests-2.yml +++ b/.github/workflows/reusable-misc-tests-2.yml @@ -37,7 +37,6 @@ jobs: packages: read env: FORCE_COLOR: 1 - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" EARTHLY_INSTALL_ID: "earthly-githubactions" # Used in our github action as the token - TODO: look to change it into an input GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -49,7 +48,6 @@ jobs: with: GHCR_USERNAME: ${{ github.actor }} GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" BUILT_EARTHLY_PATH: "${{ inputs.BUILT_EARTHLY_PATH }}" BINARY: "${{ inputs.BINARY }}" SUDO: "${{ inputs.SUDO }}" diff --git a/.github/workflows/reusable-push-integrations.yml b/.github/workflows/reusable-push-integrations.yml index 5e27ee0629..a68156a5b2 100644 --- a/.github/workflows/reusable-push-integrations.yml +++ b/.github/workflows/reusable-push-integrations.yml @@ -33,7 +33,6 @@ jobs: runs-on: ${{inputs.RUNS_ON}} env: FORCE_COLOR: 1 - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" EARTHLY_INSTALL_ID: "earthly-githubactions" # Used in our github action as the token - TODO: look to change it into an input GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -45,7 +44,6 @@ jobs: with: GHCR_USERNAME: ${{ github.actor }} GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" BUILT_EARTHLY_PATH: "${{ inputs.BUILT_EARTHLY_PATH }}" BINARY: "${{ inputs.BINARY }}" SUDO: "${{ inputs.SUDO }}" diff --git a/.github/workflows/reusable-race-test.yml b/.github/workflows/reusable-race-test.yml index c8b8f2edf0..14fbcd8810 100644 --- a/.github/workflows/reusable-race-test.yml +++ b/.github/workflows/reusable-race-test.yml @@ -43,7 +43,6 @@ jobs: runs-on: ${{inputs.RUNS_ON}} env: FORCE_COLOR: 1 - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" EARTHLY_INSTALL_ID: "earthly-githubactions" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: @@ -54,7 +53,6 @@ jobs: with: GHCR_USERNAME: ${{ github.actor }} GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" BUILT_EARTHLY_PATH: "${{ inputs.BUILT_EARTHLY_PATH }}" BINARY: "${{ inputs.BINARY }}" SUDO: "${{ inputs.SUDO }}" diff --git a/.github/workflows/reusable-repo-auth-tests.yml b/.github/workflows/reusable-repo-auth-tests.yml index df037e6218..7adccc1bc8 100644 --- a/.github/workflows/reusable-repo-auth-tests.yml +++ b/.github/workflows/reusable-repo-auth-tests.yml @@ -39,7 +39,6 @@ jobs: - 2222:22 env: FORCE_COLOR: 1 - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" EARTHLY_INSTALL_ID: "earthly-githubactions" SSH_PORT: "2222" # Used in our github action as the token - TODO: look to change it into an input @@ -52,7 +51,6 @@ jobs: with: GHCR_USERNAME: ${{ github.actor }} GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" BUILT_EARTHLY_PATH: "${{ inputs.BUILT_EARTHLY_PATH }}" BINARY: "${{ inputs.BINARY }}" SUDO: "${{ inputs.SUDO }}" diff --git a/.github/workflows/reusable-test-local.yml b/.github/workflows/reusable-test-local.yml index c274053228..4a9f4b4103 100644 --- a/.github/workflows/reusable-test-local.yml +++ b/.github/workflows/reusable-test-local.yml @@ -38,7 +38,6 @@ jobs: runs-on: ${{inputs.RUNS_ON}} env: FORCE_COLOR: 1 - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" EARTHLY_INSTALL_ID: "earthly-githubactions" EARTHLY_ORG: "${{inputs.EARTHLY_ORG}}" DOCKERHUB_MIRROR_USERNAME: "${{ secrets.DOCKERHUB_MIRROR_USERNAME }}" @@ -55,7 +54,6 @@ jobs: DOCKERHUB_MIRROR_PASSWORD: "${{ secrets.DOCKERHUB_MIRROR_PASSWORD }}" DOCKERHUB_USERNAME: "${{ secrets.DOCKERHUB_USERNAME }}" DOCKERHUB_PASSWORD: "${{ secrets.DOCKERHUB_TOKEN }}" - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" BUILT_EARTHLY_PATH: "${{ inputs.BUILT_EARTHLY_PATH }}" BINARY: "${{ inputs.BINARY }}" SUDO: "${{ inputs.SUDO }}" diff --git a/.github/workflows/reusable-test.yml b/.github/workflows/reusable-test.yml index 3f7cc6565a..af7276906a 100644 --- a/.github/workflows/reusable-test.yml +++ b/.github/workflows/reusable-test.yml @@ -43,7 +43,6 @@ jobs: runs-on: ${{inputs.RUNS_ON}} env: FORCE_COLOR: 1 - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" EARTHLY_INSTALL_ID: "earthly-githubactions" EARTHLY_ORG: "${{inputs.EARTHLY_ORG}}" # Used in our github action as the token - TODO: look to change it into an input @@ -56,7 +55,6 @@ jobs: with: GHCR_USERNAME: ${{ github.actor }} GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" BUILT_EARTHLY_PATH: "${{ inputs.BUILT_EARTHLY_PATH }}" BINARY: "${{ inputs.BINARY }}" USE_QEMU: "${{ inputs.USE_QEMU }}" diff --git a/.github/workflows/reusable-wait-block-main.yml b/.github/workflows/reusable-wait-block-main.yml index 647c776666..a9fd4f8c62 100644 --- a/.github/workflows/reusable-wait-block-main.yml +++ b/.github/workflows/reusable-wait-block-main.yml @@ -33,7 +33,6 @@ jobs: runs-on: ${{inputs.RUNS_ON}} env: FORCE_COLOR: 1 - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" EARTHLY_INSTALL_ID: "earthly-githubactions" # Used in our github action as the token - TODO: look to change it into an input GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -47,7 +46,6 @@ jobs: DOCKERHUB_MIRROR_PASSWORD: "${{ secrets.DOCKERHUB_MIRROR_PASSWORD }}" DOCKERHUB_USERNAME: "${{ secrets.DOCKERHUB_USERNAME }}" DOCKERHUB_PASSWORD: "${{ secrets.DOCKERHUB_TOKEN }}" - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" BUILT_EARTHLY_PATH: "${{ inputs.BUILT_EARTHLY_PATH }}" BINARY: "${{ inputs.BINARY }}" SUDO: "${{ inputs.SUDO }}" diff --git a/.github/workflows/reusable-wait-block-target.yml b/.github/workflows/reusable-wait-block-target.yml index 46256bdd2f..a7d5a69c95 100644 --- a/.github/workflows/reusable-wait-block-target.yml +++ b/.github/workflows/reusable-wait-block-target.yml @@ -40,7 +40,6 @@ jobs: runs-on: ${{inputs.RUNS_ON}} env: FORCE_COLOR: 1 - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" EARTHLY_INSTALL_ID: "earthly-githubactions" # Used in our github action as the token - TODO: look to change it into an input GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -54,7 +53,6 @@ jobs: DOCKERHUB_MIRROR_PASSWORD: "${{ secrets.DOCKERHUB_MIRROR_PASSWORD }}" DOCKERHUB_USERNAME: "${{ secrets.DOCKERHUB_USERNAME }}" DOCKERHUB_PASSWORD: "${{ secrets.DOCKERHUB_TOKEN }}" - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" BUILT_EARTHLY_PATH: "${{ inputs.BUILT_EARTHLY_PATH }}" BINARY: "${{ inputs.BINARY }}" SUDO: "${{ inputs.SUDO }}" diff --git a/Earthfile b/Earthfile index bdf9d7c407..57a3f6271c 100644 --- a/Earthfile +++ b/Earthfile @@ -244,7 +244,6 @@ unit-test: ARG DOCKERHUB_MIRROR_INSECURE=false ARG DOCKERHUB_MIRROR_HTTP=false ARG DOCKERHUB_MIRROR_AUTH=false - ARG DOCKERHUB_MIRROR_AUTH_FROM_CLOUD_SECRETS=false IF [ -n "$DOCKERHUB_MIRROR" ] RUN mkdir -p /etc/docker @@ -254,14 +253,7 @@ unit-test: END RUN echo "}" >> /etc/docker/daemon.json END - IF [ "$DOCKERHUB_MIRROR_AUTH_FROM_CLOUD_SECRETS" = "true" ] - RUN if [ "$DOCKERHUB_MIRROR_AUTH" = "true" ]; then echo "ERROR: DOCKERHUB_MIRROR_AUTH_FROM_CLOUD_SECRETS and DOCKERHUB_MIRROR_AUTH are mutually exclusive" && exit 1; fi - WITH DOCKER - RUN --secret DOCKERHUB_MIRROR_USER=dockerhub-mirror/user \ - --secret DOCKERHUB_MIRROR_PASS=dockerhub-mirror/pass \ - USE_EARTHLY_MIRROR=true ./not-a-unit-test.sh - END - ELSE IF [ "$DOCKERHUB_MIRROR_AUTH" = "true" ] + IF [ "$DOCKERHUB_MIRROR_AUTH" = "true" ] WITH DOCKER RUN --secret DOCKERHUB_MIRROR_USER \ --secret DOCKERHUB_MIRROR_PASS \ @@ -496,18 +488,13 @@ earthly-integration-test-base: ARG DOCKERHUB_MIRROR_INSECURE=false ARG DOCKERHUB_MIRROR_HTTP=false ARG DOCKERHUB_MIRROR_AUTH=false - ARG DOCKERHUB_MIRROR_AUTH_FROM_CLOUD_SECRETS=false # DOCKERHUB_AUTH will login to docker hub (and pull from docker hub rather than a mirror) ARG DOCKERHUB_AUTH=false COPY setup-registry.sh . - # TODO: Check this - IF [ "$DOCKERHUB_MIRROR_AUTH_FROM_CLOUD_SECRETS" = "true" ] - RUN if [ "$DOCKERHUB_MIRROR_AUTH" = "true" ]; then echo "ERROR: DOCKERHUB_MIRROR_AUTH_FROM_CLOUD_SECRETS and DOCKERHUB_MIRROR_AUTH are mutually exclusive" && exit 1; fi - RUN --secret DOCKERHUB_MIRROR_USER=dockerhub-mirror/user --secret DOCKERHUB_MIRROR_PASS=dockerhub-mirror/pass USE_EARTHLY_MIRROR=true ./setup-registry.sh - ELSE IF [ "$DOCKERHUB_MIRROR_AUTH" = "true" ] + IF [ "$DOCKERHUB_MIRROR_AUTH" = "true" ] RUN --secret DOCKERHUB_MIRROR_USER --secret DOCKERHUB_MIRROR_PASS ./setup-registry.sh ELSE IF [ "$DOCKERHUB_AUTH" = "true" ] RUN --secret DOCKERHUB_USER --secret DOCKERHUB_PASS ./setup-registry.sh @@ -834,7 +821,6 @@ examples-1: BUILD ./examples/cutoff-optimization+run BUILD ./examples/import+build BUILD ./examples/secrets+base - BUILD ./examples/cloud-secrets+base examples-2: BUILD ./examples/readme/go1+all @@ -861,7 +847,6 @@ examples-3: BUILD ./examples/typescript-node+docker BUILD ./examples/bazel+run BUILD ./examples/bazel+image - BUILD ./examples/aws-sso+base BUILD ./examples/mkdocs+build BUILD ./examples/zig+docker diff --git a/FORK_CI_GUIDE.md b/FORK_CI_GUIDE.md index e07b648825..d07665403f 100644 --- a/FORK_CI_GUIDE.md +++ b/FORK_CI_GUIDE.md @@ -25,9 +25,6 @@ The CI workflows depend on several external GitHub repositories and actions. The The CI workflows require numerous secrets. These need to be created in the `earthbuild/earthly` repository settings (`Settings -> Secrets and variables -> Actions`). -- **`EARTHLY_TOKEN`**: For interacting with Earthly Cloud (e.g., for remote caching). - - **Action Required**: Create an account on [Earthly Cloud](https://cloud.earthly.dev) for the `earthbuild` organization and generate a token. - - **`DOCKERHUB_USERNAME`** and **`DOCKERHUB_TOKEN`**: For pushing and pulling images from Docker Hub. - **Action Required**: Create a Docker Hub organization/account for `earthbuild` and create an access token. diff --git a/buildkitd/docker-auto-install.sh b/buildkitd/docker-auto-install.sh index c2460077c1..a8b8c0a50f 100755 --- a/buildkitd/docker-auto-install.sh +++ b/buildkitd/docker-auto-install.sh @@ -159,12 +159,12 @@ install_dockerd_debian_like() { apt-transport-https \ ca-certificates \ curl \ - gnupg-agent \ - software-properties-common + gnupg-agent VERSION="$(. /etc/os-release && echo "$VERSION_ID")" case "$distro" in ubuntu) + apt-get install -y software-properties-common MAJOR="$(echo "$VERSION" | awk -F. '{print $1}')" if [ "$MAJOR" -ge "22" ]; then install_docker_apt_repo_new @@ -174,6 +174,9 @@ install_dockerd_debian_like() { ;; debian) + if [ "$VERSION" -le "13" ]; then + apt-get install -y software-properties-common + fi if [ "$VERSION" -ge "12" ]; then install_docker_apt_repo_new else @@ -182,6 +185,7 @@ install_dockerd_debian_like() { ;; *) + apt-get install -y software-properties-common install_docker_apt_repo_old ;; esac diff --git a/docs/README.md b/docs/README.md index e82f097837..6958fd47ae 100644 --- a/docs/README.md +++ b/docs/README.md @@ -28,23 +28,9 @@ Earthly's key features/benefits are: * **♻️ Reuse, Don't Repeat** Never have to write the same code in multiple builds again. With Earthly, you can reuse targets, artifacts, and images across multiple Earthfiles, even ones in other repositories, in a single line. Earthly is cache-aware, based on the individual hashes of each file, and has shared caching capabilities. So you can create a vast and efficient build hierarchy that only executes the minimum required steps. -## Earthly Cloud - -Earthly Cloud is a cloud-based build automation platform that allows you to run your Earthly builds in the cloud, and is compatible with any CI. Earthly Cloud gives teams repeatable pipelines that run exactly the same in CI as on your laptop; has an automatic and instantly available build cache that makes builds faster; and is super simple to use. - -Earthly is better when you're logged in to Earthly Cloud, and Earthly Cloud has a generous free tier that includes additional capabilities like: - * Sharing cache with your team - * Remote build runners - * Shared logs - * Shared secrets - -To get started, visit the [Earthly Cloud sign up](https://cloud.earthly.dev/login) page. - ## Installation -The best way to install Earthly is by [visiting Earthly Cloud and signing up for free](https://cloud.earthly.dev/login). - -If you prefer not to create an online account, you can also install and use Earthly locally without an account. See the [installation instructions](https://earthly.dev/get-earthly). +The best way to install EarthBuild is by folowing the [installation instructions](https://earthly.dev/get-earthly). For a full list of installation options see the [alternative installation page](./alt-installation/alt-installation.md). @@ -57,11 +43,9 @@ A high-level overview is available on [the Earthly GitHub page](https://github.c ## Quick Links * [Earthly GitHub page](https://github.com/earthly/earthly) -* [Earthly Cloud Login](https://cloud.earthly.dev/login) * [Earthly basics](./basics/basics.md) * [Earthfile reference](./earthfile/earthfile.md) * [Earthly command reference](./earthly-command/earthly-command.md) * [Configuration reference](./earthly-config/earthly-config.md) * [Earthfile examples](./examples/examples.md) * [Best practices](./guides/best-practices.md) -* [Earthly Cloud documentation](./cloud/overview.md) diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 68f67d03f1..a5088f2719 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -74,15 +74,3 @@ * [Bitbucket Pipelines](ci-integration/guides/bitbucket-pipelines-integration.md) * [Woodpecker CI](ci-integration/guides/woodpecker-integration.md) * [Kubernetes](ci-integration/guides/kubernetes.md) - -## ☁️ Earthly Cloud - -* [Overview](cloud/overview.md) -* [Managing permissions](cloud/managing-permissions.md) -* [Cloud secrets](cloud/cloud-secrets.md) -* [Earthly Satellites](cloud/satellites.md) - * [Managing Satellites](cloud/satellites/managing.md) - * [Using Satellites](cloud/satellites/using.md) - * [Self-Hosted Satellites](cloud/satellites/self-hosted.md) - * [GitHub runners](cloud/satellites/gha-runners.md) - * [Best Practices](cloud/satellites/best-practices.md) diff --git a/docs/basics/part-8a-using-earthly-in-your-current-ci.md b/docs/basics/part-8a-using-earthly-in-your-current-ci.md index 62d7758910..557334bd1e 100644 --- a/docs/basics/part-8a-using-earthly-in-your-current-ci.md +++ b/docs/basics/part-8a-using-earthly-in-your-current-ci.md @@ -36,7 +36,6 @@ jobs: env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} FORCE_COLOR: 1 steps: - uses: earthly/actions/setup-earthly@v1 diff --git a/docs/ci-integration/guides/bitbucket-pipelines-integration.md b/docs/ci-integration/guides/bitbucket-pipelines-integration.md index 3113bbc76f..8a76e933b3 100644 --- a/docs/ci-integration/guides/bitbucket-pipelines-integration.md +++ b/docs/ci-integration/guides/bitbucket-pipelines-integration.md @@ -2,27 +2,4 @@ Bitbucket Pipelines run in a shared Docker environment and do not support running Earthly builds directly due to [restrictions](https://jira.atlassian.com/browse/BCLOUD-21419) that Bitbucket has put in place. -You can however, run Earthly builds on Bitbucket pipelines via [remote runners](../../remote-runners.md) such as [Earthly Satellites](../../cloud/satellites.md). Because Bitbucket Pipelines run as containers you can also use the official Earthly Docker image. Here is an example of a Bitbucket Pipeline build. This example assumes your Earthfile has a `+build` target defined. - -```yml -# ./bitbucket-pipelines.yml - -image: earthly/earthly:v0.8.13 - -pipelines: - default: - - step: - name: "Set Earthly token" - script: - - export EARTHLY_TOKEN=$EARTHLY_TOKEN - - step: - name: "Docker login" - script: - - docker login --username "$DOCKERHUB_USERNAME" --password "$DOCKERHUB_TOKEN" - - step: - name: "Build" - script: - - earthly --ci --push --sat $EARTHLY_SAT --org $EARTHLY_ORG +build -``` - -For a complete guide on CI integration see the [CI integration guide](../overview.md). +You can however, run Earthly builds on Bitbucket pipelines via [remote buildkit](../../ci-integration/remote-buildkit.md). Because Bitbucket Pipelines run as containers you can also use the official Earthly Docker image. diff --git a/docs/ci-integration/guides/google-cloud-build.md b/docs/ci-integration/guides/google-cloud-build.md index 1535728204..9d6ef7086a 100644 --- a/docs/ci-integration/guides/google-cloud-build.md +++ b/docs/ci-integration/guides/google-cloud-build.md @@ -54,9 +54,7 @@ Click "Add Key", and then "Create New Key". Choose "JSON" as the key format, and ![The list view of available keys for a Google Cloud service account](img/google-cloud-build-5.png) -Stash the key in your secret management utility of choice. You'll need to make this key available to your build at runtime. For the rest of our example, we will be using Earthly's [Cloud Secrets](../../cloud/cloud-secrets.md). - -Often, external secrets management requires some kind of bootstrapping secret (or additional integration) to allow you to access the rest of the secrets in your store. Earthly is no different. We will keep our `EARTHLY_TOKEN` in [Googles Secret Manager](https://cloud.google.com/build/docs/securing-builds/use-secrets) for ease of use. +Stash the key in your secret management utility of choice. You'll need to make this key available to your build at runtime. {% hint style='danger' %} ##### Note @@ -74,7 +72,7 @@ It is also possible to perform these steps via the CLI; the steps are [also deta {% hint style='danger' %} ##### Note -This example is not production ready, and is intended to showcase configuration needed to get Earthly off the ground. If you run into any issues, or need help, [don't hesitate to reach out](https://github.com/earthly/earthly/issues/new)! +This example is not production ready, and is intended to showcase configuration needed to get Earthly off the ground. If you run into any issues, or need help, [don't hesitate to reach out](https://github.com/EarthBuild/earthbuild/issues/new)! {% endhint %} @@ -112,23 +110,3 @@ Running this build will use the [`cloudbuild.yaml`](https://github.com/earthly/c - --push - +build ``` - -[The second step](https://github.com/earthly/ci-example-project/blob/ea44992b020b52cb5a46920d5d11d4b8389ce19d/cloudbuild.yaml#L8-L13) runs a sample, Google Cloud Build only example to show how you would use an external service account to do things that normally requires credentials. - -```yaml - - id: 'gcp-test' - name: 'earthly/earthly:v0.8.13' - args: - - +gcp-cloudbuild - secretEnv: - - 'EARTHLY_TOKEN' -``` - -The secret environment variable bootstraps the Earthly secret store, and we can load it from Google's Secret Store like this: - -```yaml -availableSecrets: - secretManager: - - versionName: projects/earthly-jupyterlab/secrets/EARTHLY_TOKEN/versions/2 - env: 'EARTHLY_TOKEN' -``` diff --git a/docs/ci-integration/guides/kubernetes.md b/docs/ci-integration/guides/kubernetes.md index 464841cd4c..455f548569 100644 --- a/docs/ci-integration/guides/kubernetes.md +++ b/docs/ci-integration/guides/kubernetes.md @@ -2,7 +2,7 @@ {% hint style='info' %} ##### Note -This guide is related to self-hosting a remote BuildKit, however, Self-Hosted Satellites **beta** are now available. Self-Hosted Satellites provide more features, have better security, and are easier to deploy than remote BuildKit. Check out the [Self-Hosted Satellites Guide](../../cloud/satellites/self-hosted.md) for more details and instructions to deploy in Kubernetes or AWS EC2. +This guide is related to self-hosting a remote BuildKit, however, Self-Hosted Satellites **beta** are now available. Self-Hosted Satellites provide more features, have better security, and are easier to deploy than remote BuildKit. {% endhint %} diff --git a/docs/ci-integration/use-earthly-ci-image.md b/docs/ci-integration/use-earthly-ci-image.md index f95204f7a0..6d8e870dc6 100644 --- a/docs/ci-integration/use-earthly-ci-image.md +++ b/docs/ci-integration/use-earthly-ci-image.md @@ -27,9 +27,7 @@ Note that using the `earthly` binary as the entrypoint will not start up BuildKi An alternative option is to use the `earthly/earthly` image in conjunction with a remote BuildKit Daemon. You may use the environment variable `BUILDKIT_HOST` to specify the hostname of the remote BuildKit Daemon. When this environment variable is set, the `earthly/earthly` image will not attempt to start BuildKit and will instead use the remote BuildKit Daemon. -You may also use the `earthly/earthly` image to run a build against an Earthly Satellite. To achieve this you can pass along an `EARTHLY_TOKEN` environment variable, along with the command-line flags `--sat` and `--org`, to point the build to a specific satellite. - -For more details on using remote execution, [see our guide on remote BuildKit](./remote-buildkit.md) or the [introduction to Satellites](../cloud/satellites.md). +For more details on using remote execution, [see our guide on remote BuildKit](./remote-buildkit.md). #### Mounting the source code @@ -53,7 +51,7 @@ docker run --privileged --rm earthly/earthly:v0.8.13 github.com/foo/bar:my-branc #### `NO_BUILDKIT` Environment Variable -As the embedded BuildKit daemon requires `--privileged`, for some operations you may be able to use the `NO_BUILDKIT=1` environment variable to disable the embedded BuildKit daemon. This is especially useful when running against a remote BuildKit (like a Satellite), or when not performing a build as part of the command (like when using `earthly account`). +As the embedded BuildKit daemon requires `--privileged`, for some operations you may be able to use the `NO_BUILDKIT=1` environment variable to disable the embedded BuildKit daemon. This is especially useful when running against a remote BuildKit. ## An important note about running the image diff --git a/docs/cloud/managing-permissions.md b/docs/cloud/managing-permissions.md index 2cc79c2931..389202f958 100644 --- a/docs/cloud/managing-permissions.md +++ b/docs/cloud/managing-permissions.md @@ -1,57 +1,3 @@ -# Managing permissions - -This page covers managing permissions for Earthly Cloud products, such as Earthly Cloud Secrets, and Earthly Satellites. - -## Overview - -Earthly Cloud's permissions model has two security boundaries: - -* Earthly orgs -* Earthly projects - -Users may be invited to a specific organization, and optionally to specific projects within the organization. - -Earthly orgs may contain the following shared resources: - -* Satellites -* Earthly projects - -Earthly projects, in turn, may contain the following resources: - -* Secrets -* Build history, including build logs - -## Earthly org access levels - -Within an Earthly org, users may be granted one of the following access levels: - -* `read`: Can view the org, projects, and user membership. Can view, inspect, wake and build on satellites. Can also stream and share logs. -* `read+secrets`: Same as read, but can also view and use secrets. -* `write`: Everything in `read+secrets`, plus the ability to create and modify satellites, projects, and secrets. -* `admin`: Can manage the org, including adding and removing users, and managing projects, secrets and satellites. - -Having a certain level of access for a given org automatically grants the same level of access to all projects within that org. - -### Managing access to an Earthly org - -To grant access to an Earthly org, you must invite the user to the org. This can be done by running: - -```bash -earthly org invite --permission -``` - -If the user is already part of the org, you can change their access level by running: - -```bash -earthly org member update --permission -``` - -If you want to revoke access to an Earthly org, you can do so by running: - -```bash -earthly org member rm -``` - ## Earthly project access levels Within an Earthly project, users may be granted one of the following access levels: diff --git a/docs/cloud/oidc.md b/docs/cloud/oidc.md index 1f66d86271..381ae9cde3 100644 --- a/docs/cloud/oidc.md +++ b/docs/cloud/oidc.md @@ -1,12 +1,12 @@ # OpenID Connect (OIDC) Authentication Earthly can support cases where you might require access to a 3rd-party cloud provider as part of your build, without storing secrets in your CI or accessing credentials from your local environment. -This is especially useful in CI where otherwise, authentication requires MFA(multi-factor authentication). +This is especially useful in CI where otherwise, authentication requires MFA(multi-factor authentication). The OIDC protocol allows you to access the provider without storing credentials in your local environment or CI. ## Introduction -This page covers how to set up OIDC with cloud providers. +This page covers how to set up OIDC with cloud providers. At the moment the only AWS is supported. ## Cloud Providers @@ -81,4 +81,4 @@ aws: RUN --aws --oidc=$OIDC aws s3 ls ``` -For more information on the `RUN --aws --oidc` flags, see [here](../earthfile/earthfile.md#--oidc-oidc-spec-experimental) +For more information on the `RUN --aws --oidc` flags, see [here](../earthfile/earthfile.md#--oidc-oidc-spec-experimental) diff --git a/docs/cloud/overview.md b/docs/cloud/overview.md index cd89ee5086..f12d8a9e6f 100644 --- a/docs/cloud/overview.md +++ b/docs/cloud/overview.md @@ -2,82 +2,12 @@ Earthly Cloud is a collection of features that enrich the Earthly experience via cloud-based services. These include: -* [Earthly Satellites](./satellites.md): Cloud-based BuildKit instances managed by the Earthly team. * [Earthly Cloud Secrets](./cloud-secrets.md): A secret management system that allows you to store secrets in a cloud-based service and use them across builds. -* [Auto-skip](../caching/caching-in-earthfiles.md#auto-skip): A feature that allows you to skip large parts of a build in certain situations. -* **Log sharing**: The ability to share build logs with coworkers. -* [OIDC Authentication](./oidc.md): The ability to authenticate to 3rd-party cloud services without storing long-term credentials. - -## Sign up for Earthly Cloud for free! - -*Get 6,000 build minutes/month on Satellites as part of Earthly Cloud's no time limit free tier.* ***[Sign up today](https://cloud.earthly.dev/login).*** - -## Getting started - -### Creating an account - -To get started with Earthly Cloud, you'll need to register an Earthly account. You can do so by visiting [Earthly Cloud Sign up page](https://cloud.earthly.dev/login), or by using the CLI as described below. - -```bash -earthly account register --email -``` - -An email will be sent to you containing a verification token. Next run: - -```bash -earthly account register --email --token -``` - -This command will prompt you to set a password, and to optionally register a public-key for password-less authentication. - -### Creating or joining an Earthly org - -An Earthly org allows you to share projects, secrets, and satellites with colleagues. To view the orgs you belong to, run: - -```bash -earthly org ls -``` - -To create an Earthly org you can run: - -```bash -earthly org create -``` - -To select the org you would like to use, run: - -```bash -earthly org select -``` - -To invite another user to join your org, run: -```bash -earthly org invite -``` - -You can join an Earthly org by following the steps outlined in the invitation email sent to you by an Earthly admin. +* [OIDC Authentication](./oidc.md): The ability to authenticate to 3rd-party cloud services without storing long-term credentials. ### Creating a project - To use certain features, Earthly Cloud Secrets, you will additionally need to create an Earthly Project. You can create a project by using the CLI as described below. - ```bash earthly project create ``` - -## Logging in from a CI - -To be able to use certain Earthly features, such as Cloud Secrets, or Satellites from your CI, you will need to log into Earthly. The easiest way to do that is to create an Earthly authentication token by running - -```bash -earthly account create-token [--write] -``` - -This token can then be exported as an environment variable in the CI of choice. - -```bash -EARTHLY_TOKEN=... -``` - -Which will then force Earthly to use that token when accessing secrets or satellites. diff --git a/docs/cloud/satellites.md b/docs/cloud/satellites.md deleted file mode 100644 index c3d01cf1c2..0000000000 --- a/docs/cloud/satellites.md +++ /dev/null @@ -1,173 +0,0 @@ -# Earthly Satellites - -Earthly Satellites are [remote runners](../remote-runners.md) that work seamlessly with Earthly, using persistent cache to improve build times. -Satellites can be either [fully managed](https://earthly.dev/earthly-satellites) by Earthly Cloud or [self-hosted](./satellites/self-hosted.md) in your own environment. - -## Get started with Earthly Cloud Satellites for free! - -Fully managed Satellites are included with [Earthly Cloud](https://docs.earthly.dev/earthly-cloud/overview). Earthly Cloud is a SaaS build automation platform with consistent builds, ridiculous speed, and a next-gen developer experience that works seamlessly with any CI. *Get 6,000 build minutes/month as part of Earthly Cloud's no time limit free tier.* ***[Sign up today](https://cloud.earthly.dev/login).*** - -## Benefits - -Typical use cases for Earthly Satellites include: - -* **Speeding up CI builds** in sandboxed CI environments such as GitHub Actions, GitLab, CircleCI, and others. Most CI build times are improved by 2-20X with Satellites. -* **Sharing compute and cache with coworkers** or with the CI. -* **Executing cross-platform builds natively**. For example, executing builds on x86 architecture natively when you are working from an Apple Silicon machine (Apple M1/M2) and vice versa, arm64 builds from an x86 machine. -* **Benefiting from high-bandwidth internet access** from the satellite, allowing for fast downloads of dependencies and pushes for deployments. This is particularly useful if you are in a location with slow internet. -* **Using Earthly in restricted environments**, where privileged access or docker-in-docker are not supported. - -## How Earthly Satellites work - -### On your laptop - -* You kick off the build from the command line, and Earthly uses a remote satellite for execution. -* The source files used are the ones you have locally in the current directory. -* The build logs from the satellite are streamed back to your terminal in real time, so you can see the progress of the build. -* The outputs of the build - images and artifacts - are downloaded back to your local machine upon success. -* Everything looks and feels as if it is executing on your computer in your terminal. -* In reality, the execution takes place in the cloud with high parallelism and a lot of caching. - -### In your CI of choice - -* The CI starts a build and invokes Earthly. -* Earthly starts the build on a remote satellite, executing each step in isolated containers. -* The same cache is used between runs on the same satellite, so parts that haven’t changed do not repeat. -* Logs are streamed back to the CI in real time. -* Any images, artifacts, or deployments that need to be pushed as part of the build are pushed directly from the satellite. -* Build pass/fail is returned as an exit code, so your CI can report the status accordingly. - -## Getting started - -### 1. Sign up for Earthly Cloud (free) - -Earthly Satellites is part of Earthly Cloud. You can use it for free as part of our free tier. Get started with Earthly Cloud by visiting the [sign up](https://cloud.earthly.dev/login) page, and get 6,000 build minutes/month for free. - -### 2. Launch a new satellite - -Satellites are launched in one of the following two ways, depending on which kind of satellite you intend on creating. - -#### Earthly Cloud - -To launch a new managed Satellite on Earthly Cloud, run: - -```bash -earthly sat launch -``` - -The Satellite name can be any arbitrary string. - -If you are part of multiple Earthly organizations, you may want to first select the org under which you would like to launch the satellite: - -```bash -earthly org select -earthly sat launch -``` - -Once the satellite is created it will be automatically selected for use as part of your builds. The selection takes place by Earthly adding some information in your Earthly config file (usually located under `~/.earthly/config.yml`). - -#### Self-Hosted - -Self-Hosted Satellites are instead launched by running the satellite container directly. See the [self-hosted guide](./satellites/self-hosted.md) for instructions. - -### 3. Run a build - -To execute a build using the newly created satellite, simply run Earthly like you always have. For example: - -```bash -earthly +my-target -``` - -Because the satellite has been automatically selected in the step above, the build will be executed on it. - -To go back to using your local machine for builds, you may "unselect" the satellite by running: - -```bash -earthly sat unselect -``` - -You can always go back to using the satellite by running: - -```bash -earthly sat select -``` - -Or, you can use a satellite only for a specific build, even if it is not selected: - -```bash -earthly --sat +my-target -``` - -Conversely, if a satellite is currently selected, but you want to execute a build on your local machine, you can use the `--no-sat` flag: - -```bash -earthly --no-sat +my-target -``` - -For more information on using satellites, see the [Using satellites page](./satellites/using.md). - -### 4. Invite your team - -A final step is to invite your team to use the satellite. This can be done by running: - -```bash -earthly org invite -``` - -Once a user has been invited, you can forward them a link to the page [Using Satellites](./satellites/using.md) for them to get started. - -## Managing Satellites - -For more information on managing satellites, see the [Managing Satellites page](./satellites/managing.md). - -## Satellite specs - -When using Cloud Satellites, the size and architecture can be specified at launch time using the `--size` and `--platform` flags. -For the full list of supported options, please see the [Pricing Page](https://earthly.dev/pricing). - -## Using Satellites in CI - -A key benefit of using satellites in a CI environment is that the cache is shared between runs. This results in significant speedups in CIs that would otherwise have to start from scratch each time. - -{% hint style='danger' %} -##### Note - -If a satellite is shared between multiple CI pipelines, it is possible that it becomes overloaded by too many parallel builds. For best performance, you can create a dedicated satellite for each CI pipeline. See the [best practices guide](./satellites/best-practices.md) for more details. -{% endhint %} - -To get started with using Earthly Satellites in CI, you can create a login token for access. - -First, run - -```bash -earthly account create-token -``` - -to create your login token. - -Copy and paste the value into an environment variable called `EARTHLY_TOKEN` in your CI environment. - -Then as part of your CI script, simply select your satellite using one of these supported methods - -* Selection command: `earthly sat select ` -* Setellite flag: `earthly --sat my-satellite +build` -* Environment variable: `EARTHLY_SATELLITE=my-satellite` - -before running your Earthly targets. - -Note that when using [Self-Hosted Satelites](./satellites/self-hosted.md), your CI runner must be able to access the satellite on the network where it is hosted. - -{% hint style='danger' %} -##### Registry Login - -It's best to avoid using an image registry like Dockerhub without authentication, since the IP address from the satellite easily become rate-limited. -A simple `docker login` command before running your build should be used to pass registry credentials to your satellite. -See our [Docker authentication](../guides/auth.md) guide for more details. - -{% endhint %} - -## Known limitations - -* Pull-through cache is currently not supported - -If you run into any issues please let us know either via [Slack](https://earthly.dev/slack), [GitHub issues](https://github.com/earthly/cloud-issues/issues) or by [emailing support](mailto:support+satellite@earthly.dev). diff --git a/docs/cloud/satellites/best-practices.md b/docs/cloud/satellites/best-practices.md deleted file mode 100644 index 9aaca1029a..0000000000 --- a/docs/cloud/satellites/best-practices.md +++ /dev/null @@ -1,3 +0,0 @@ -# Satellites Best Practices - -EarthBuild does not support Satellites. Please use a [remote buildkit](../../ci-integration/remote-buildkit.md) runner which is essentially the same thing. diff --git a/docs/cloud/satellites/gha-runners.md b/docs/cloud/satellites/gha-runners.md deleted file mode 100644 index b5d8ca19a6..0000000000 --- a/docs/cloud/satellites/gha-runners.md +++ /dev/null @@ -1,160 +0,0 @@ -# Satellites as GitHub Actions runners - -{% hint style='warning' %} -This feature is experimental. - -Not recommended for production usage yet, since it might introduce breaking changes in the future. - -Feedback is welcome and much appreciated! - -{% endhint %} - -Earthly satellites are now bundled with a GitHub Actions runner, so they can directly pull jobs from GitHub Actions without the need of an intermediate runner. - -These runners come with the Earthly CLI preinstalled and configured to use the satellite BuildKit instance, so GitHub Actions jobs will share the same satellite cache as the traditional satellite builds. - -## Creating a GitHub Actions integration - -Satellite-based GitHub Actions runners can be enabled for a particular repository or for all repositories of a GitHub organization at once. - -The integration process requires you to be a GitHub organization or repository admin, and to provide us with a GitHub token, so we can: -- register a webhook in your GitHub repository/organization to receive the events generated from your GitHub Actions jobs -- create GitHub self-hosted runners on demand, to process your repository/organization jobs - -Follow the next steps to create such integrations: - -### 1. Create a GitHub token - -- Go to [GitHub's new token page](https://github.com/settings/tokens/new) to create a new GitHub classic token. - -- Give a name to your token that clearly shows its purpose, for example: - - ![token name](./gha/token-name.png) - -- Set the token as non-expiring (notice that the integration won't work after the token expires) - - ![token expiration](./gha/token-expiration.png) - -- Check the following scopes: - - For organization integrations: `admin:org`,`admin:org_hook` - - Alternatively, for repository integrations: `repo`,`admin:repo_hook` - -- Click "generate token" - - ![token generate](./gha/token-generate.png) - -- Copy the token value to use it in the following step - -{% hint style='info' %} -Alternatively, if you prefer creating a fine-grained token, make sure to set the following permissions for it: org: `organization_hooks:write`, `organization_self_hosted_runners:write`, repo: `repository_hooks:write`, `administration:write` -and an expiration time long enough, since the integration won't work after the token expires. -{% endhint %} - -### 2. Register the integration via CLI -Create the integration using the `earthly gha add` CLI command, passing the token created in the previous step. - -#### Organization integration -``` -earthly gha add \ - --org \ - --gh-org \ - --gh-token -``` - -#### Single repository integration -``` -earthly gha add \ - --org \ - --gh-org \ - --gh-repo \ - --gh-token -``` - -### 3. Configure your satellites - -This feature needs to be enabled during satellite creation to be able to use it. - -#### Self-hosted satellites -To enable the GH runner for a self-hosted satellite, set this environment entry when launching it: -``` --e RUNNER_GHA_ENABLED=true -``` -also note that the satellite container must have access to the docker daemon in order to run the GitHub Actions jobs in containers: -``` --v /var/run/docker.sock:/var/run/docker.sock -``` - -##### Example -```shell -docker run --privileged \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v satellite-cache:/tmp/earthly:rw \ - -p 8372:8372 \ - -e EARTHLY_TOKEN= \ - -e EARTHLY_ORG= \ - -e SATELLITE_NAME= \ - -e SATELLITE_HOST= \ - -e RUNNER_GHA_ENABLED=true \ - earthly/satellite:v0.8.13 -``` -{% hint style='info' %} -**Required version:** Use at least `earthly/satellite:v0.8.13` -{% endhint %} - -##### Logs -You should see a log message like this, when the GitHub Actions runner is enabled: -``` -{...,"msg":"starting GitHub Actions job polling loop",...} -``` - -### 4. Configure your GitHub Actions jobs -In order to make a job run into the satellite, you'll need to reference the satellite name in its [runs-on](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idruns-on) label as follows: - -``` -runs-on: [earthly-satellite#] -``` - -#### Example -The following example runs the `+build` target in the satellite. Given that the GH runner is configured to use the satellite BuildKit instance, the persistent satellite cache is implicitly used here. -```yml -earthly-job: - runs-on: [earthly-satellite#my-gha-satellite] - env: - FORCE_COLOR: 1 - EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" - steps: - - uses: actions/checkout@v2 - - name: Earthly build - run: earthly -ci +build -``` - -{% hint style='warning' %} -Make sure you have an [EARTHLY_TOKEN](https://docs.earthly.dev/docs/earthly-command#earthly-account-create-token) available in your [GitHub Actions secrets](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions) store, and add it to the job environment, as shown in the previous example. Future versions will remove this requirement. - -{% endhint %} - -## Listing registered integrations -List the integrations of your Earthly organization with the `earthly gha ls` CLI command: - -``` -earthly gha ls \ - --org -``` - -## Removing an integration -Remove an integration using the `earthly gha remove` CLI command: - -### Organization integration -``` -earthly gha remove \ - --org \ - --gh-org -``` - -### Single repository integration -``` -earthly gha remove \ - --org \ - --gh-org \ - --gh-repo -``` diff --git a/docs/cloud/satellites/gha/token-copy.png b/docs/cloud/satellites/gha/token-copy.png deleted file mode 100644 index 7e22005575..0000000000 Binary files a/docs/cloud/satellites/gha/token-copy.png and /dev/null differ diff --git a/docs/cloud/satellites/gha/token-expiration.png b/docs/cloud/satellites/gha/token-expiration.png deleted file mode 100644 index 2ad7c43e44..0000000000 Binary files a/docs/cloud/satellites/gha/token-expiration.png and /dev/null differ diff --git a/docs/cloud/satellites/gha/token-generate.png b/docs/cloud/satellites/gha/token-generate.png deleted file mode 100644 index 42f21b3272..0000000000 Binary files a/docs/cloud/satellites/gha/token-generate.png and /dev/null differ diff --git a/docs/cloud/satellites/gha/token-name.png b/docs/cloud/satellites/gha/token-name.png deleted file mode 100644 index 3161f2edba..0000000000 Binary files a/docs/cloud/satellites/gha/token-name.png and /dev/null differ diff --git a/docs/cloud/satellites/managing.md b/docs/cloud/satellites/managing.md deleted file mode 100644 index 7fb28d3e44..0000000000 --- a/docs/cloud/satellites/managing.md +++ /dev/null @@ -1,3 +0,0 @@ -# Managing Satellites - -EarthBuild does not support Satellites. Please use a [remote buildkit](../../ci-integration/remote-buildkit.md) runner which is essentially the same thing. diff --git a/docs/cloud/satellites/self-hosted.md b/docs/cloud/satellites/self-hosted.md deleted file mode 100644 index a95cbe8583..0000000000 --- a/docs/cloud/satellites/self-hosted.md +++ /dev/null @@ -1,3 +0,0 @@ -# Self-Hosted Satellites - -EarthBuild does not support Satellites. Please use a [remote buildkit](../../ci-integration/remote-buildkit.md) runner which is essentially the same thing. diff --git a/docs/cloud/satellites/using.md b/docs/cloud/satellites/using.md deleted file mode 100644 index 7fb28d3e44..0000000000 --- a/docs/cloud/satellites/using.md +++ /dev/null @@ -1,3 +0,0 @@ -# Managing Satellites - -EarthBuild does not support Satellites. Please use a [remote buildkit](../../ci-integration/remote-buildkit.md) runner which is essentially the same thing. diff --git a/docs/docker-images/all-in-one.md b/docs/docker-images/all-in-one.md index fabe28e42f..7438c19a7e 100644 --- a/docs/docker-images/all-in-one.md +++ b/docs/docker-images/all-in-one.md @@ -41,33 +41,6 @@ Here's a quick breakdown: - `-v earthly-tmp:/tmp/earthly:rw` mounts (and creates, if necessary) the `earthly-tmp` Docker volume into the containers `/tmp/earthly`. This is used as a temporary/working directory for `buildkitd` during builds. - `+for-linux` is the target to be invoked. All arguments specified after the image tag will be passed to `earthly`. -### Usage with Satellites and No Local Code - -This example utilizes an [Earthly Satellite](https://docs.earthly.dev/earthly-cloud/satellites) to perform builds. The code to be built is downloaded directly from GitHub. - -```bash -docker run -t -e NO_BUILDKIT=1 -e EARTHLY_TOKEN= earthly/earthly:v0.8.13 --ci --org --sat github.com/earthly/earthly+for-linux -``` - -Here's what this does: - -- `-e EARTHLY_TOKEN=` passes along an Earthly token such that Earthly can access satellites. This token can be created via `earthly account create-token`. -- `--org ` specifies the organization that the satellite belongs to. -- `--sat ` specifies the satellite to use. -- `github.com/earthly/earthly+for-linux` specifies the target to build. This target is located on GitHub, and will be pulled from the Satellite. - -### Usage for non-build commands - -This example shows how to use the Earthly container to run non-build commands. This is useful for running commands like `earthly account`, or `earthly secret`. - -```bash -docker run -t -e NO_BUILDKIT=1 -e EARTHLY_TOKEN= earthly/earthly:v0.8.13 account list-tokens -``` - -```bash -docker run -t -e NO_BUILDKIT=1 -e EARTHLY_TOKEN= earthly/earthly:v0.8.13 secret get foo -``` - ## Using This Image ### Requirements diff --git a/docs/earthfile/earthfile.md b/docs/earthfile/earthfile.md index 8857285778..9b56a07f54 100644 --- a/docs/earthfile/earthfile.md +++ b/docs/earthfile/earthfile.md @@ -254,7 +254,7 @@ earthly +release-short --SECRET_ID="" It is also possible to mount a secret as a file with `RUN --mount type=secret,id=secret-id,target=/path/of/secret,chmod=0400`. See `--mount` below. -For more information on how to use secrets see the [Secrets guide](../guides/secrets.md). See also the [Cloud secrets guide](../cloud/cloud-secrets.md). +For more information on how to use secrets see the [Secrets guide](../guides/secrets.md). ##### `--network=none` diff --git a/docs/earthly-command/earthly-command.md b/docs/earthly-command/earthly-command.md index 16724bbed5..f6e6e37880 100644 --- a/docs/earthly-command/earthly-command.md +++ b/docs/earthly-command/earthly-command.md @@ -163,12 +163,6 @@ On Mac systems, this setting defaults to `/run/host-services/ssh-auth.sock` to m For more information see the [Authentication page](../guides/auth.md). -##### `--auth-token ` - -Also available as an env var setting: `EARTHLY_TOKEN=`. - -Force Earthly account login to authenticate with supplied token. - ##### `--verbose` Also available as an env var setting: `EARTHLY_VERBOSE=1`. @@ -521,492 +515,6 @@ git: user: git ``` -## earthly account - -Contains sub-commands for registering and administration an Earthly account. - -### earthly account register - -#### Synopsis - - -* ``` - # Register an account using your email - earthly [options] account register --email - - # Complete account registration - earthly [options] account register --email --token [--password ] [--public-key ] [--accept-terms-conditions-privacy] - ``` - -#### Description - -Register for an Earthly account. Registration is done in two steps: first run the register command with only the --email argument, this will then send an email to the -supplied email address with a registration token (which is used to verify your email address), second re-run the register command with both the --email and --token arguments -to complete the registration process. - -#### Options - -##### `--email ` - -Pass in an email address for registering your Earthly account. An email will be sent containing your registration token. - -##### `--token ` - -Pass in token for email verification. Retrieve the token from your email and register it with the `--email` option. - -##### `--password ` - -Specify your password on the command line instead of interactively being asked. - -##### `--public-key ` - -Path to public key to register. - -##### `--accept-terms-of-service-privacy` - -Accept the Terms & Conditions, and Privacy Policy. - -### earthly account login - -#### Synopsis - -* ``` - # login using registered public keys or check who you are logged in as - earthly [options] account login - - # Login with email and input password interactively - earthly [options] account login --email - - # Login with email and password - earthly [options] account login --email --password - - # Login with your tokem - earthly [options] account login --token - ``` - -#### Description - -Login to an existing Earthly account. If no email or token is given, earthly will attempt to login using [registered public keys](../public-key-auth/public-key-auth.md). - -#### Options - -##### `--email ` - -Pass in email address connected with your Earthly account. - -##### `--token ` - -Pass in your authentication token - -##### `--password ` - -Pass in the password for your Earthly account. If not provided you will be interactively asked. - -### earthly account logout - -#### Synopsis - -* ``` - earthly [options] account logout - ``` - -#### Description - -Removes cached login information from `~/.earthly/auth.token`. - -### earthly account list-keys - -#### Synopsis - -* ``` - earthly [options] account list-keys - ``` - -#### Description - -Lists all public keys that are authorized to login to the current Earthly account. - -### earthly account add-key - -#### Synopsis - -* ``` - earthly [options] account add-key [] - ``` - -#### Description - -Authorize a new public key to login to the current Earthly account. If `key` is omitted, an interactive prompt is displayed to select a public key to add. - -### earthly account remove-key - -#### Synopsis - -* ``` - earthly [options] account remove-key - ``` - -#### Description - -Removes an authorized public key from accessing the current Earthly account. - -### earthly account list-tokens - -#### Synopsis - -* ``` - earthly [options] account list-tokens - ``` - -#### Description - -List account tokens associated with the current Earthly account. A token is useful for environments where the ssh-agent is not accessible (e.g. a CI system). - -### earthly account create-token - -#### Synopsis - -* ``` - earthly [options] account create-token [--write] [--expiry ] [--overwrite] - ``` - -#### Description - -Creates a new authentication token. A read-only token is created by default, If the `--write` flag is specified the token will have read+write access. -The token will never expire unless a different date is supplied via the `--expiry` flag. -If the token by the same name already exists, it will not be overwritten unless the `--overwrite` flag is specified. - -{% hint style='info' %} -It is then possible to `export EARTHLY_TOKEN=...`, which will force earthly to use this token for all authentication (overriding any other currently-logged in sessions). -{% endhint %} - -#### Options - -##### `--write` - -Grant write permissions in addition to read permissions - -##### `--expiry` - -Set token expiry date in the form YYYY-MM-DD or never - -##### `--overwrite` - -Overwrite the token if it already exists - -### earthly account remove-token - -#### Synopsis - -* ``` - earthly [options] account remove-token - ``` - -#### Description - -Removes a token from the current Earthly account. - -### earthly account reset - -#### Synopsis - -* ``` - earthly [options] account reset --email [--token ] - ``` - -#### Description - -Reset the password associated with the provided email. The command should first be run without a token, which will cause a token to be emailed to you. Once the command is re-run with the provided token, it will prompt you for a new password. - -#### Options - -##### `--email ` - -Email address for which to reset the password. - -##### `--token ` - -Authentication token with with to rerun the command with your email to reset your password. Once run you will be prompted for a new password. - -## earthly org - -Contains sub-commands for creating and managing Earthly organizations. - -### earthly org create - -#### Synopsis - -* ``` - earthly [options] org create - ``` - -#### Description - -Create a new organization, which can be used to share secrets between different user accounts. - -### earthly org list - -#### Synopsis - -* ``` - earthly [options] org list - - earthly [options] org ls - ``` - -#### Description - -List all organizations the current account is a member, or administrator of. - -### earthly org list-permissions - -#### Synopsis - -* ``` - earthly [options] org list-permissions - ``` - -#### Description - -List all accounts and the paths they have permission to access under a particular organization. - -### earthly org invite - -#### Synopsis - -* ``` - earthly [options] org [--org ] invite [--name ] [--permission ] [--message ] - ``` - -#### Description - -Invites a user into an organization; `` can either be a top-level org access by granting permission on `//`, or finer-grained access can be granted to a subpath e.g. `//path/to/share/`. -By default users are granted read-only access unless the `--write` flag is given. - -#### Subcommands - -##### `accept` - -Accept an invitation to join an organization - -##### `ls | list` - -List all sent invitations (both pending and accepted) - -#### Options - -##### `--permission` - -The access level the new organization member will have. Can be one of: read, write, or admin. - -##### `--message` - -An optional message to send with the invitation email - -### earthly org revoke - -#### Synopsis - -* ``` - earthly [options] org revoke [, ...] - ``` - -#### Description - -Revokes a previously invited user from an organization. - -### earthly org member - -#### Synopsis - -* ``` - earthly [options] org [--org ] members (ls|update|rm) - ``` - -#### Description - -Manage organization members - -#### Subcommands - -##### `ls` - -List organization members and their permission level - -##### `update` - -Update an organization member's permissions. - -###### `--permission` - -Flag for `update` subcommand. Can be one of: read, write, or admin. - -##### `rm` - -Remove a user from the organization - -### earthly org select - -#### Synopsis - -* ``` - earthly [options] org select - ``` - -#### Description - -Selects an existing Earthly org to be the default. Analogous to the `EARTHLY_ORG` environment variable, or the `--org` flag available on some commands. When multiple organizations are specified, the precedence order is the following: - -1. `--org` argument -2. `EARTHLY_ORG` environment variable -3. The configuration setting controlled by this command - -### earthly org unselect - -#### Synopsis - -* ``` - earthly [options] org unselect - ``` - -#### Description - -Removes the configuration option specifying a default organization. - -## earthly secrets - -#### Synopsis - -Alias `earthly secret` - -* ``` - earthly [options] secrets [--org , --project ] (set|get|ls|rm|migrate|permission) - ``` -Contains sub-commands for creating and managing Earthly secrets. - -#### Description - -Contains sub-commands for creating and managing Earthly secrets. - -#### Options - -##### `--org` - -The organization to which the project belongs. - -##### `--project` - -The organization project in which to store secrets. - -### earthly secrets set - -#### Synopsis - -* ``` - earthly [options] secrets set - earthly [options] secrets set --file - ``` - -#### Description - -Stores a secret in the secrets store. - -#### Options - -##### `--file` - -Stores secret from file to the path. - -##### `--stdin` - -Stores secret read from stdin to the path. - -### earthly secrets get - -#### Synopsis - -* ``` - earthly [options] secrets get [-n] - ``` - -#### Description - -Retrieve a secret from the secrets store. If `-n` is given, no newline is printed after the contents of the secret. - -#### Options - -##### `--n` - -Disables newline at the end of the stored secret. - -### earthly secrets ls - -#### Synopsis - -* ``` - earthly [options] secrets ls [] - ``` - -#### Description - -List secrets the current account has access to. - -### earthly secrets rm - -#### Synopsis - -* ``` - earthly [options] secrets rm - ``` - -#### Description - -Removes a secret from the secrets store. - -### earthly secrets migrate - -#### Synopsis - -* ``` - earthly [options] secrets --org --project migrate - ``` - -#### Description - -Migrate existing secrets into the new project-based structure. - -#### Options - -##### `--dry-run` - -Output what the command will do without actually doing it. - -### earthly secrets permission - -#### Synopsis - -* ``` - earthly [options] secrets permission (ls|set|rm) - ``` - -#### Description - -Manage user-level secret permissions. - -#### Subcommands - -##### `ls` - -List any user secret permissions. - -##### `rm` - -Remove a user secret permission. - -##### `set` - -Create or update a user secret permission. - ## earthly registry #### Synopsis @@ -1223,12 +731,6 @@ The provider to use when logging into the web ui. The command `earthly docker-build` builds a docker image from a Dockerfile instead of an Earthfile. The `` is the path where the Dockerfile build context exists. By default, it is assumed that a file named Dockerfile exists in that directory. -Just like a regular build, `docker-build` can be used with a satellite. For example: -```shell -earthly docker-build --sat my-satellite --tag my-image:latest . -``` -For more information see the [Using Satellites guide](../cloud/satellites/using.md). - Additionally, all other build options are supported when using `docker-build`. For more information see [build-options](#build-options). #### Examples diff --git a/docs/guides/best-practices.md b/docs/guides/best-practices.md index a7b3ca688b..fe8d40cff0 100644 --- a/docs/guides/best-practices.md +++ b/docs/guides/best-practices.md @@ -395,7 +395,7 @@ RUN --push --secret GITHUB_TOKEN github-release upload ... ### Use `--secret`, not `ARG`s to pass secrets to the build -If a build requires the usage of secrets, it is strongly recommended that you use the builtin secrets constructs, such as `earthly --secret`, [Earthly Cloud Secrets](../cloud/cloud-secrets.md), and `RUN --secret`. +If a build requires the usage of secrets, it is strongly recommended that you use the builtin secrets constructs, such as `earthly --secret`, [Earthly Cloud Secrets](../cloud/cloud-secrets.md), `RUN --secret`. Using `ARG`s for passing secrets is strongly discouraged, as the secrets will be leaked in build logs, the build cache and the possibly in published images. diff --git a/docs/guides/cloud-providers/aws.md b/docs/guides/cloud-providers/aws.md index 37d0a4721c..79d015166d 100644 --- a/docs/guides/cloud-providers/aws.md +++ b/docs/guides/cloud-providers/aws.md @@ -2,7 +2,7 @@ ## Introduction -It is common for builds to be able to access AWS resources (For example, one might want to upload artifacts to S3). +It is common for builds to be able to access AWS resources (For example, one might want to upload artifacts to S3). Earthly provides two ways to easily authenticate to AWS in order to access resources. ## Authentication Methods diff --git a/docs/install/install.md b/docs/install/install.md index da3b98fbac..e89f73b41b 100644 --- a/docs/install/install.md +++ b/docs/install/install.md @@ -1,15 +1,3 @@ # Install -## Log In and Install - -To install the Earthly CLI on your machine, [head over to Earthly Cloud and get started for free](https://cloud.earthly.dev/login). - -The logged-in experience gives you access to the following additional features: - -* Logs sharing -* [Earthly Cloud Secrets](../cloud/cloud-secrets.md) -* [Earthly Satellites](../cloud/satellites.md) (6,000 minutes/month free) - -## Install without Logging In - -If you prefer to install Earthly without logging in, head over to the [Get Earthly page](https://earthly.dev/get-earthly). +To install the Earthly CLI on your machine, head over to the [Get Earthly page](https://earthly.dev/get-earthly). diff --git a/docs/lang-guides/featured-guides.md b/docs/lang-guides/featured-guides.md index d0fc261f8a..033227698c 100644 --- a/docs/lang-guides/featured-guides.md +++ b/docs/lang-guides/featured-guides.md @@ -1,5 +1,3 @@ # Featured guides Earthly works with any programming language, as shown in our extensive collection of [examples](../examples/examples.md). However, we are now working on a series of [language-specific libraries](../earthly-lib/earthly-lib.md) that make Earthly that much easier to use with various languages and frameworks. To that end, we are starting with [Rust](./rust.md). - -If you are just starting out, the [onboarding tutorial in Earthly Cloud](https://cloud.earthly.dev/login) is a great way to get introduced to Earthly. diff --git a/examples/cloud-secrets/Earthfile b/examples/cloud-secrets/Earthfile deleted file mode 100644 index 51a16fa9d3..0000000000 --- a/examples/cloud-secrets/Earthfile +++ /dev/null @@ -1,28 +0,0 @@ -VERSION 0.8 -PROJECT earthly-technologies/example -FROM alpine:latest - -cloud-secrets: - # This example assumes you have created a project using a command similar to: - # earthly projects --org earthly-technologies --project example - # (where earthly-technologies has been replaced by your organization name) - # - # Next, a secret has been set by running: - # earthly secrets --org earthly-technologies --project example set token abc123 - # - # You can verify it was set by running: - # earthly secrets --org earthly-technologies --project example ls - # which should output: - # token - - # You can also view the secret by running: - # earthly secrets --org earthly-technologies --project example get token - # which should output: - # abc123 - - # secrets are referenced in an Earthfile by first defining the project name (using the - # `PROJECT org/project-name` command); this is done right after the `VERSION` command. - # Then the secret can be referenced via RUN: - RUN --secret MY_SECRET=token test -n "$MY_SECRET" - RUN --secret token test -n "$token" - RUN --mount=type=secret,id=token,mode=0600,target=/root/secret-token ls -la /root/secret-token diff --git a/examples/secrets/Earthfile b/examples/secrets/Earthfile index 7848f1a708..48294e71ee 100644 --- a/examples/secrets/Earthfile +++ b/examples/secrets/Earthfile @@ -14,6 +14,3 @@ secrets: # the secret path can be omitted if the ID and the environment variable are matching RUN --secret MY_SECRET test -n "$MY_SECRET" RUN --mount=type=secret,id=MY_SECRET,target=/root/secret ls /root/secret - -# Note: it is also possible to store secrets in the cloud, which can be shared between users of -# an organization. See the ../cloud-secrets/Earthfile for details. diff --git a/scripts/tests/auth/setup.sh b/scripts/tests/auth/setup.sh index 58ff6302a7..0adb763068 100644 --- a/scripts/tests/auth/setup.sh +++ b/scripts/tests/auth/setup.sh @@ -10,12 +10,6 @@ earthly=${earthly:=earthly} earthly=$(realpath "$earthly") echo "running tests with $earthly" -# ensure earthly login works (and print out who gets logged in) -"$earthly" account login - -# these tests require the EARTHLY_TOKEN not be set -unset EARTHLY_TOKEN - # make sure ssh-agent is not running test -z "${SSH_AUTH_SOCK:-}" diff --git a/scripts/tests/backwards-compatability.sh b/scripts/tests/backwards-compatability.sh index 5dba99a78a..fd853b4211 100755 --- a/scripts/tests/backwards-compatability.sh +++ b/scripts/tests/backwards-compatability.sh @@ -36,17 +36,11 @@ PATH="$(realpath ../acbtest):$PATH" # which would mess with the secrets data being fetched) date +%s > /tmp/last-earthly-prerelease-check -set +x # dont remove or the token will be leaked -test -n "$EARTHLY_TOKEN" || (echo "error: EARTHLY_TOKEN is not set" && exit 1) -set -x - EARTHLY_INSTALLATION_NAME="earthly-integration" export EARTHLY_INSTALLATION_NAME rm -rf "$HOME/.earthly.integration/" echo "$earthly" -# ensure earthly login works (and print out who gets logged in) -"$earthly" account login # start buildkitd container "$earthly" bootstrap diff --git a/scripts/tests/export.sh b/scripts/tests/export.sh index 3390b0918c..5a585bfbee 100755 --- a/scripts/tests/export.sh +++ b/scripts/tests/export.sh @@ -17,17 +17,11 @@ PATH="$(realpath "$(dirname "$0")/../acbtest"):$PATH" # which would mess with the secrets data being fetched) date +%s > /tmp/last-earthly-prerelease-check -set +x # dont remove or the token will be leaked -test -n "$EARTHLY_TOKEN" || (echo "error: EARTHLY_TOKEN is not set" && exit 1) -set -x - EARTHLY_INSTALLATION_NAME="earthly-integration" export EARTHLY_INSTALLATION_NAME rm -rf "$HOME/.earthly.integration/" echo "$earthly" -# ensure earthly login works (and print out who gets logged in) -"$earthly" account login # Test 1: export without anything echo ==== Running test 1 ==== @@ -190,21 +184,22 @@ if "$frontend" inspect earthly-export-test-6:test_linux_arm64 >/dev/null 2>&1 ; exit 1 fi -# Test 7: remote cache on target with only BUILDs -echo ==== Running test 7 ==== -rm -rf /tmp/earthly-export-test-7 -mkdir /tmp/earthly-export-test-7 -cd /tmp/earthly-export-test-7 -cat >> Earthfile <> Earthfile < /tmp/last-earthly-prerelease-check - -set +x # dont remove or the token will be leaked -test -n "$EARTHLY_TOKEN" || (echo "error: EARTHLY_TOKEN is not set" && exit 1) -set -x - -EARTHLY_INSTALLATION_NAME="earthly.integration" -export EARTHLY_INSTALLATION_NAME -rm -rf "$HOME/.earthly.integration/" - -# ensure earthly login works (and print out who gets logged in) -"$earthly" account login - -# test logout has no effect when EARTHLY_TOKEN is set -if GITHUB_ACTIONS="" NO_COLOR=0 "$earthly" account logout > output 2>&1; then - echo "earthly account logout should have failed" - exit 1 -fi -diff output <(echo "Error: account logout has no effect when --auth-token (or the EARTHLY_TOKEN environment variable) is set") - -# fetch shared secret key (this step assumes your personal user has access to the /earthly-technologies/ secrets org -echo "fetching manitou-id_rsa" -ID_RSA=$("$earthly" secrets --org earthly-technologies --project core get -n secrets-integration-manitou-id_rsa) - -# now that we grabbed the manitou credentials, unset our token, to ensure that we're only testing using manitou's credentials -unset EARTHLY_TOKEN -"$earthly" account logout - -echo starting new instance of ssh-agent, and loading credentials -eval "$(ssh-agent)" - -# grab first 6chars of md5sum of key to help sanity check that the same key is consistently used -set +x # make sure we don't print the key here -md5sum=$(echo -n "$ID_RSA" | md5sum | awk '{ print $1 }' | head -c6) - -echo "Adding key (with md5sum $md5sum...) into ssh-agent" -echo "$ID_RSA" | ssh-add - - -echo testing that key was correctly loaded into ssh-agent -ssh-add -l | acbgrep manitou - -echo testing that the ssh-agent only contains a single key -test "$(ssh-add -l | wc -l)" = "1" - -echo "testing earthly account login works (and is using the earthly-manitou account)" -"$earthly" account login 2>&1 | acbgrep 'Logged in as "other-service+earthly-manitou@earthly.dev" using ssh auth' - -mkdir -p /tmp/earthtest -cat << EOF > /tmp/earthtest/Earthfile -VERSION 0.7 -PROJECT manitou-org/earthly-core-integration-test -FROM alpine:3.18 -test-local-secret: - WORKDIR /test - RUN --mount=type=secret,target=/tmp/test_file,id=my_secret test "\$(cat /tmp/test_file)" = "my-local-value" -test-server-secret: - WORKDIR /test - RUN --mount=type=secret,target=/tmp/test_file,id=my_test_file test "\$(cat /tmp/test_file)" = "secret-value" -EOF - -# set and test get returns the correct value -"$earthly" secrets --org manitou-org --project earthly-core-integration-test set my_test_file "secret-value" -"$earthly" secrets --org manitou-org --project earthly-core-integration-test get my_test_file | acbgrep 'secret-value' - -# test earthly will prompt if value is missing -/usr/bin/expect -c ' -spawn '"$earthly"' secrets --org manitou-org --project earthly-core-integration-test set my_test_file -expect "secret value: " -send "its my secret value\n" -expect eof -' -"$earthly" secrets --org manitou-org --project earthly-core-integration-test get my_test_file | acbgrep 'its my secret value' - -# test set --stdin works -echo -e "hello\nworld" | "$earthly" secrets --org manitou-org --project earthly-core-integration-test set --stdin my_test_file -# note "echo -e "hello\nworld" | md5sum" -> 0f723ae7f9bf07744445e93ac5595156 -"$earthly" secrets --org manitou-org --project earthly-core-integration-test get -n my_test_file -"$earthly" secrets --org manitou-org --project earthly-core-integration-test get -n my_test_file | md5sum | acbgrep '0f723ae7f9bf07744445e93ac5595156' - -# test set --file works -"$earthly" secrets --org manitou-org --project earthly-core-integration-test set --file <(echo -e "foo\nbar") my_test_file -# note "echo -e "foo\nbar" | md5sum" -> f47c75614087a8dd938ba4acff252494 -"$earthly" secrets --org manitou-org --project earthly-core-integration-test get -n my_test_file | md5sum | acbgrep 'f47c75614087a8dd938ba4acff252494' - - -# restore the "secret-value", which the org selection test requires -"$earthly" secrets --org manitou-org --project earthly-core-integration-test set my_test_file "secret-value" - -# test selecting org -"$earthly" org select manitou-org -"$earthly" org ls | acbgrep '^\* \+manitou-org' - -# test secrets with org selected in config file -"$earthly" secrets --project earthly-core-integration-test get my_test_file | acbgrep 'secret-value' -"$earthly" secrets --project earthly-core-integration-test set my_other_file "super-secret-value" -"$earthly" secrets --project earthly-core-integration-test get my_other_file | acbgrep 'super-secret-value' -"$earthly" secrets --project earthly-core-integration-test ls | acbgrep '^my_test_file$' - -# test secrets with personal org -"$earthly" org select user:other-service+earthly-manitou@earthly.dev -"$earthly" secrets set super/secret hello -"$earthly" secrets get super/secret | acbgrep 'hello' -"$earthly" secrets get /user/super/secret | acbgrep 'hello' -"$earthly" secrets ls | acbgrep '^super/secret$' -"$earthly" secrets ls /user | acbgrep '^super/secret$' - -echo "=== test 1 ===" -# test RUN --mount can reference a secret from the command line -"$earthly" --no-cache --secret my_secret=my-local-value /tmp/earthtest+test-local-secret - -echo "=== test 2 ===" -# test RUN --mount can reference a secret from the server that is only specified in the Earthfile -"$earthly" --no-cache /tmp/earthtest+test-server-secret - -echo "=== test 3 ===" -# Test earthly will display a message containing the name of the secret that was not found -set +e -"$earthly" --no-cache /tmp/earthtest+test-local-secret > output 2>&1 -exit_code="$?" -set -e -cat output -test "$exit_code" != "0" -acbgrep 'unable to lookup secret "my_secret": not found' output -acbgrep 'Help: Make sure to set the project at the top of the Earthfile' output -echo "=== All tests have passed ===" diff --git a/tests/Earthfile b/tests/Earthfile index a4fb1df968..20d3e987f0 100644 --- a/tests/Earthfile +++ b/tests/Earthfile @@ -211,9 +211,9 @@ ga-no-qemu-group11: BUILD --pass-args ./autoskip+test-group3 ga-no-qemu-group12: - BUILD --pass-args ./warn-if-not-logged-in+test + # TODO: fixme + # BUILD --pass-args ./warn-if-not-logged-in+test BUILD --pass-args ./with-docker-validate-labels+all - BUILD --pass-args +test-aws-oidc BUILD --pass-args +run-no-cache-save-artifact ga-no-qemu-slow: @@ -245,16 +245,6 @@ ga-no-qemu: BUILD +ga-no-qemu-group12 BUILD +ga-no-qemu-slow -# Note that this target is split up under github action workflows -# since they are flaky and having the ability to restart them individually -# saves a lot of time -tests-that-require-earthly-technologies-account-access: - BUILD --pass-args +test-earthly-mirror-was-setup - BUILD --pass-args ./account+test - BUILD --pass-args ./registry-command+test - BUILD --pass-args ./web+test - BUILD --pass-args ./oidc+test - # tests that only run on linux amd64 # Note: this target is used to validate the USERPLATFORM user arg, # and should not be used to programmatically detect if this should be @@ -1623,19 +1613,20 @@ test-aws-flag-envs: RUN cat earthly.output | acbgrep "AWS_REGION=us-west-1" RUN cat earthly.output | acbgrep "AWS_SECRET_ACCESS_KEY=aws-secret-key" -# test-aws-oidc tests expected errors for misusing of oidc flag -# for happy path test go to /tests/oidc -test-aws-oidc: - DO +RUN_EARTHLY --earthfile=aws-flag.earth --target=+oidc --should_fail=true --output_contains="RUN --aws-oidc requires the --run-with-aws-oidc feature flag" - DO +RUN_EARTHLY --earthfile=aws-flag.earth --extra_args="--allow-privileged" --target=+oidc-with-docker --should_fail=true --output_contains="RUN --aws-oidc requires the --run-with-aws-oidc feature flag" - # enable flag - RUN sed -i "1s/VERSION \(.*\)/VERSION --run-with-aws-oidc \1/" Earthfile - # empty oidc flag - DO +RUN_EARTHLY --target=+oidc --should_fail=true --output_contains="role-arn must be specified" - DO +RUN_EARTHLY --extra_args="--allow-privileged" --target=+oidc-with-docker --should_fail=true --output_contains="role-arn must be specified" - # invalid oidc flag - DO +RUN_EARTHLY --target=+oidc --extra_args="--build-arg OIDC=\"foo=bar\"" --should_fail=true --output_contains="invalid value for oidc flag: 1 error(s) decoding" - DO +RUN_EARTHLY --extra_args="--allow-privileged" --target=+oidc-with-docker --extra_args="--build-arg OIDC=\"foo=bar\"" --should_fail=true --output_contains="invalid value for oidc flag: 1 error(s) decoding" +# TODO: reinstate test once we have AWS access. +# # test-aws-oidc tests expected errors for misusing of oidc flag +# # for happy path test go to /tests/oidc +# test-aws-oidc: +# DO +RUN_EARTHLY --earthfile=aws-flag.earth --target=+oidc --should_fail=true --output_contains="RUN --aws-oidc requires the --run-with-aws-oidc feature flag" +# DO +RUN_EARTHLY --earthfile=aws-flag.earth --extra_args="--allow-privileged" --target=+oidc-with-docker --should_fail=true --output_contains="RUN --aws-oidc requires the --run-with-aws-oidc feature flag" +# # enable flag +# RUN sed -i "1s/VERSION \(.*\)/VERSION --run-with-aws-oidc \1/" Earthfile +# # empty oidc flag +# DO +RUN_EARTHLY --target=+oidc --should_fail=true --output_contains="role-arn must be specified" +# DO +RUN_EARTHLY --extra_args="--allow-privileged" --target=+oidc-with-docker --should_fail=true --output_contains="role-arn must be specified" +# # invalid oidc flag +# DO +RUN_EARTHLY --target=+oidc --extra_args="--build-arg OIDC=\"foo=bar\"" --should_fail=true --output_contains="invalid value for oidc flag: 1 error(s) decoding" +# DO +RUN_EARTHLY --extra_args="--allow-privileged" --target=+oidc-with-docker --extra_args="--build-arg OIDC=\"foo=bar\"" --should_fail=true --output_contains="invalid value for oidc flag: 1 error(s) decoding" test-aws-flag-configs: RUN mkdir -p /root/.aws diff --git a/tests/account/Earthfile b/tests/account/Earthfile deleted file mode 100644 index 2023d90770..0000000000 --- a/tests/account/Earthfile +++ /dev/null @@ -1,19 +0,0 @@ -VERSION 0.8 -PROJECT earthly-technologies/core - -FROM --pass-args ..+base - -IMPORT .. AS tests - -WORKDIR /test - -test: - BUILD +test-login - -test-login: - COPY test-login.sh . - RUN \ - --secret USER1_TOKEN=test-user1/token \ - --secret USER2_TOKEN=test-user2/token \ - --secret USER2_SSH_KEY=test-user2/ssh-key \ - ./test-login.sh diff --git a/tests/account/test-login.sh b/tests/account/test-login.sh deleted file mode 100755 index 3075ba97d8..0000000000 --- a/tests/account/test-login.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh -set -eo pipefail # DONT add a set -x or you will leak the key - -acbtest -n "$USER1_TOKEN" -acbtest -n "$USER2_TOKEN" -acbtest -n "$USER2_SSH_KEY" -acbtest -z "$SSH_AUTH_SOCK" - -echo "== it should login to user1 with token ==" -EARTHLY_TOKEN="$USER1_TOKEN" earthly account login 2>&1 | acbgrep 'Logged in as "other-service.earthly-user1@earthly.dev" using token auth' - -echo "== it should stay logged in as user1 even though EARTHLY_TOKEN is no longer set ==" -earthly account login 2>&1 | acbgrep 'Logged in as "other-service.earthly-user1@earthly.dev" using cached jwt auth' - -echo "== it should stay logged in as user1 since the cached jwt is used (even though user2's ssh key is available via ssh keys) ==" -eval "$(ssh-agent)" -echo "$USER2_SSH_KEY" | ssh-add - -ssh-add -l | acbgrep '(ED25519)' - -earthly account login 2>&1 | acbgrep 'Logged in as "other-service.earthly-user1@earthly.dev" using cached jwt auth' - -ssh-add -D # remove the key - -echo "== forcing a logout should allow us to change users ==" -earthly account logout -EARTHLY_TOKEN="$USER2_TOKEN" earthly account login 2>&1 | acbgrep 'Logged in as "other-service.earthly-user2@earthly.dev" using token auth' - -echo "== it should stay logged in as user2 ==" -earthly account login 2>&1 | acbgrep 'Logged in as "other-service.earthly-user2@earthly.dev" using cached jwt auth' - -echo "== it should be able to login as user2 with ssh ==" -earthly account logout -echo "$USER2_SSH_KEY" | ssh-add - -earthly account login 2>&1 | acbgrep 'Logged in as "other-service.earthly-user2@earthly.dev" using ssh auth' - -echo "== using token param should behave similarly to EARTHLY_TOKEN env ==" -earthly account login --token "$USER2_TOKEN" 2>&1 | acbgrep 'Logged in as "other-service.earthly-user2@earthly.dev" using token auth' - -echo "== same as above but first ensure we're logged out ==" -earthly account logout -rm -vf ~/.earthly/auth.* -earthly account login --token "$USER2_TOKEN" 2>&1 | acbgrep 'Logged in as "other-service.earthly-user2@earthly.dev" using token auth' - -echo "== ensure auth files are recreated ==" -acbtest -f ~/.earthly/auth.credentials -acbtest -f ~/.earthly/auth.jwt diff --git a/tests/aws-flag.earth b/tests/aws-flag.earth index fcd73d6763..c20d95dc5d 100644 --- a/tests/aws-flag.earth +++ b/tests/aws-flag.earth @@ -6,16 +6,3 @@ FROM alpine basic: RUN --aws env | grep AWS - -oidc: - ARG OIDC="" - RUN --aws --oidc=$OIDC echo this should not succeed - -oidc-with-docker: - FROM earthly/dind:alpine-3.20-docker-26.1.3-r1 - - ARG OIDC="" - - WITH DOCKER - RUN --aws --oidc=$OIDC echo this should not succeed - END diff --git a/tests/cloud-push-pull/Earthfile b/tests/cloud-push-pull/Earthfile index 93f7dbda5a..73e37a58fd 100644 --- a/tests/cloud-push-pull/Earthfile +++ b/tests/cloud-push-pull/Earthfile @@ -4,8 +4,9 @@ FROM --pass-args ..+base WORKDIR /test all: - BUILD +google-artifact-registry - BUILD +google-container-repository + # TODO: fixme by setting up GCP login + # BUILD +google-artifact-registry + # BUILD +google-container-repository BUILD +azure-container-registry BUILD +amazon-elastic-container-registry @@ -57,26 +58,27 @@ google-container-repository: azure-container-registry: COPY azure-container-registry.earth ./Earthfile - # Note that we dont have to install a cred helper here, also the cred helper wouldnt help here anyways - RUN --secret AZ_USERNAME=azure/ci-cd-username \ - --secret AZ_PASSWORD=azure/ci-cd-password \ - (test -n "$AZ_USERNAME" || (echo "ERROR: AZ_USERNAME not set"; exit 1)) && \ - (test -n "$AZ_PASSWORD" || (echo "ERROR: AZ_PASSWORD not set"; exit 1)) && \ - docker login earthlyintegrationtest.azurecr.io --username "$AZ_USERNAME" --password "$AZ_PASSWORD" - - RUN --privileged \ - --entrypoint \ - --mount=type=tmpfs,target=/tmp/earthly \ - -- --ci --push +push - - RUN --privileged \ - --entrypoint \ - --mount=type=tmpfs,target=/tmp/earthly \ - -- -P +pull - +# TODO: Sort out azure login. +# # Note that we dont have to install a cred helper here, also the cred helper wouldn't help here anyway +# RUN --secret AZ_USERNAME=azure/ci-cd-username \ +# --secret AZ_PASSWORD=azure/ci-cd-password \ +# (test -n "$AZ_USERNAME" || (echo "ERROR: AZ_USERNAME not set"; exit 1)) && \ +# (test -n "$AZ_PASSWORD" || (echo "ERROR: AZ_PASSWORD not set"; exit 1)) && \ +# docker login earthlyintegrationtest.azurecr.io --username "$AZ_USERNAME" --password "$AZ_PASSWORD" + +# RUN --privileged \ +# --entrypoint \ +# --mount=type=tmpfs,target=/tmp/earthly \ +# -- --ci --push +push + +# RUN --privileged \ +# --entrypoint \ +# --mount=type=tmpfs,target=/tmp/earthly \ +# -- -P +pull + +# TODO: Sort out a login for aws: AWS_ACCESS_KEY_ID and friends. amazon-elastic-container-registry: - RUN apk add go aws-cli - RUN go install github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login@latest + RUN apk add go aws-cli docker-credential-ecr-login COPY amazon-elastic-container-registry.earth ./Earthfile @@ -88,27 +90,27 @@ amazon-elastic-container-registry: # echo "{\"credHelpers\":{\"$ACCT_ID.dkr.ecr.us-west-2.amazonaws.com\": \"ecr-login\"}}" > /root/.docker/config.json - # Do a manual login, since the credential helper doesn't work with a pull-through cache. - RUN --secret AWS_ACCESS_KEY_ID=aws/ci-cd-access-key \ - --secret AWS_SECRET_ACCESS_KEY=aws/ci-cd-access-secret \ - --secret ACCT_ID=aws/account-id \ - --privileged \ - aws ecr get-login-password | docker login --username AWS --password-stdin 404851345508.dkr.ecr.us-west-2.amazonaws.com - - # Pass through Account ID this way to avoid checking it into GitHub; - # its not strictly secret this way but its secret enough - RUN --secret AWS_ACCESS_KEY_ID=aws/ci-cd-access-key \ - --secret AWS_SECRET_ACCESS_KEY=aws/ci-cd-access-secret \ - --secret ACCT_ID=aws/account-id \ - --privileged \ - --entrypoint \ - --mount=type=tmpfs,target=/tmp/earthly \ - -- --build-arg ACCT_ID --ci --push +push - - RUN --secret AWS_ACCESS_KEY_ID=aws/ci-cd-access-key \ - --secret AWS_SECRET_ACCESS_KEY=aws/ci-cd-access-secret \ - --secret ACCT_ID=aws/account-id \ - --privileged \ - --entrypoint \ - --mount=type=tmpfs,target=/tmp/earthly \ - -- --build-arg ACCT_ID -P +pull + # # Do a manual login, since the credential helper doesn't work with a pull-through cache. + # RUN --secret AWS_ACCESS_KEY_ID=aws/ci-cd-access-key \ + # --secret AWS_SECRET_ACCESS_KEY=aws/ci-cd-access-secret \ + # --secret ACCT_ID=aws/account-id \ + # --privileged \ + # aws ecr get-login-password | docker login --username AWS --password-stdin 404851345508.dkr.ecr.us-west-2.amazonaws.com + + # # Pass through Account ID this way to avoid checking it into GitHub; + # # its not strictly secret this way but its secret enough + # RUN --secret AWS_ACCESS_KEY_ID=aws/ci-cd-access-key \ + # --secret AWS_SECRET_ACCESS_KEY=aws/ci-cd-access-secret \ + # --secret ACCT_ID=aws/account-id \ + # --privileged \ + # --entrypoint \ + # --mount=type=tmpfs,target=/tmp/earthly \ + # -- --build-arg ACCT_ID --ci --push +push + + # RUN --secret AWS_ACCESS_KEY_ID=aws/ci-cd-access-key \ + # --secret AWS_SECRET_ACCESS_KEY=aws/ci-cd-access-secret \ + # --secret ACCT_ID=aws/account-id \ + # --privileged \ + # --entrypoint \ + # --mount=type=tmpfs,target=/tmp/earthly \ + # -- --build-arg ACCT_ID -P +pull diff --git a/tests/dind-auto-install/Earthfile b/tests/dind-auto-install/Earthfile index d34c98852d..ac00e1cf83 100644 --- a/tests/dind-auto-install/Earthfile +++ b/tests/dind-auto-install/Earthfile @@ -11,9 +11,9 @@ all: --BASE_IMAGE=ubuntu:latest \ --BASE_IMAGE=amazonlinux:1 \ --BASE_IMAGE=amazonlinux:2 \ - --BASE_IMAGE=earthly/dind:alpine-3.19-docker-25.0.2-r0 \ - --BASE_IMAGE=earthly/dind:ubuntu-20.04-docker-24.0.5-1 \ - --BASE_IMAGE=earthly/dind:ubuntu-23.04-docker-24.0.5-1 + --BASE_IMAGE=earthbuild/dind:alpine-3.22-docker-28.3.3-r1 \ + --BASE_IMAGE=earthbuild/dind:ubuntu-20.04-docker-28.1.1-1 \ + --BASE_IMAGE=earthbuild/dind:ubuntu-24.04-docker-28.3.3-1 test: ARG --required BASE_IMAGE diff --git a/tests/docker2earth/Dockerfile2 b/tests/docker2earth/Dockerfile2 index b1216e2794..881133bf8e 100644 --- a/tests/docker2earth/Dockerfile2 +++ b/tests/docker2earth/Dockerfile2 @@ -1,10 +1,10 @@ -FROM golang:1.16 +FROM golang:1.24 WORKDIR /go/src/github.com/alexellis/href-counter/ COPY app.go . RUN go mod init RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o app . -FROM alpine:latest as greet +FROM alpine:latest AS greet WORKDIR /root/ RUN echo greetings > /root/hello.txt diff --git a/tests/docker2earth/go.mod b/tests/docker2earth/go.mod new file mode 100644 index 0000000000..df61465bfe --- /dev/null +++ b/tests/docker2earth/go.mod @@ -0,0 +1,5 @@ +module github.com/alexellis/href-counter + +go 1.24.5 + +require golang.org/x/net v0.42.0 // indirect diff --git a/tests/docker2earth/go.sum b/tests/docker2earth/go.sum new file mode 100644 index 0000000000..4af2a26b8f --- /dev/null +++ b/tests/docker2earth/go.sum @@ -0,0 +1,2 @@ +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= diff --git a/tests/integration-base/Earthfile b/tests/integration-base/Earthfile index 2b4c4ca127..728329588b 100644 --- a/tests/integration-base/Earthfile +++ b/tests/integration-base/Earthfile @@ -6,7 +6,6 @@ test-base: ARG DOCKERHUB_MIRROR_INSECURE=false ARG DOCKERHUB_MIRROR_HTTP=false ARG DOCKERHUB_MIRROR_AUTH=false - ARG DOCKERHUB_MIRROR_AUTH_FROM_CLOUD_SECRETS=false ARG DOCKERHUB_AUTH=false ARG BUILDKIT_PROJECT @@ -15,7 +14,6 @@ test-base: --DOCKERHUB_MIRROR_INSECURE=$DOCKERHUB_MIRROR_INSECURE \ --DOCKERHUB_MIRROR_HTTP=$DOCKERHUB_MIRROR_HTTP \ --DOCKERHUB_MIRROR_AUTH=$DOCKERHUB_MIRROR_AUTH \ - --DOCKERHUB_MIRROR_AUTH_FROM_CLOUD_SECRETS=$DOCKERHUB_MIRROR_AUTH_FROM_CLOUD_SECRETS \ --DOCKERHUB_AUTH=$DOCKERHUB_AUTH \ --BUILDKIT_PROJECT=$BUILDKIT_PROJECT diff --git a/tests/oidc/aws.earth b/tests/oidc/aws.earth index 367c1617f5..186f206410 100644 --- a/tests/oidc/aws.earth +++ b/tests/oidc/aws.earth @@ -25,7 +25,7 @@ oidc-with-docker: ARG --required ROLE_ARN LET OIDC="role-arn=$ROLE_ARN,session-name=earthly-ci-test-session,region=us-west-2" - + WITH DOCKER RUN --aws --oidc=$OIDC export result=$(env |grep AWS_ |wc -l); \ test $result -eq $expected || (echo "expected $expected env vars for AWS but got $result" && exit 1) diff --git a/tests/oidc/test-aws.sh b/tests/oidc/test-aws.sh old mode 100755 new mode 100644 diff --git a/tests/registry-command/Earthfile b/tests/registry-command/Earthfile deleted file mode 100644 index 53f0d9418d..0000000000 --- a/tests/registry-command/Earthfile +++ /dev/null @@ -1,70 +0,0 @@ -VERSION 0.8 -PROJECT earthly-technologies/core -FROM --pass-args ..+base - -IMPORT .. AS tests - -RUN apk add bash -WORKDIR /test - -test-dockerhub: - COPY lock.sh unlock.sh \ - test-dockerhub-project.sh test-dockerhub-user.sh \ - . - RUN --secret EARTHLY_TOKEN=fake-user-write-token \ - --secret AWS_ACCESS_KEY_ID=aws/ci-cd-access-key \ - --secret AWS_SECRET_ACCESS_KEY=aws/ci-cd-access-secret \ - ./lock.sh && ( \ - ./test-dockerhub-user.sh && \ - ./test-dockerhub-project.sh && \ - true) || ( ./unlock.sh && echo "dockerhub test failed"; exit 1) && ./unlock.sh - -test-ecr: - COPY lock.sh unlock.sh \ - test-ecr.sh \ - test-ecr-project.sh test-ecr-user.sh \ - . - - ENV EARTHLY_EXEC_CMD=/test/test-ecr.sh - RUN --secret EARTHLY_TOKEN=fake-user-write-token \ - --secret AWS_ACCESS_KEY_ID=aws/ci-cd-access-key \ - --secret AWS_SECRET_ACCESS_KEY=aws/ci-cd-access-secret \ - --mount=type=tmpfs,target=/tmp/earthly \ - --privileged \ - --entrypoint \ - --mount=type=tmpfs,target=/tmp/earthly - -test-gcp: - COPY lock.sh unlock.sh \ - test-gcp.sh \ - test-gcp-project.sh test-gcp-user.sh \ - . - - ENV EARTHLY_EXEC_CMD=/test/test-gcp.sh - RUN --secret EARTHLY_TOKEN=fake-user-write-token \ - --secret GCP_KEY=gcp/ci-cd-key \ - --mount=type=tmpfs,target=/tmp/earthly \ - --privileged \ - --entrypoint \ - --mount=type=tmpfs,target=/tmp/earthly - -test-multi: - COPY lock.sh unlock.sh \ - test-multi.sh \ - . - - ENV EARTHLY_EXEC_CMD=/test/test-multi.sh - RUN --secret EARTHLY_TOKEN=fake-user-write-token \ - --secret GCP_KEY=gcp/ci-cd-key \ - --secret AWS_ACCESS_KEY_ID=aws/ci-cd-access-key \ - --secret AWS_SECRET_ACCESS_KEY=aws/ci-cd-access-secret \ - --mount=type=tmpfs,target=/tmp/earthly \ - --privileged \ - --entrypoint \ - --mount=type=tmpfs,target=/tmp/earthly - -test: - BUILD +test-dockerhub - BUILD +test-ecr - BUILD +test-gcp - BUILD +test-multi diff --git a/tests/registry-command/lock.sh b/tests/registry-command/lock.sh deleted file mode 100755 index f09f7dd00f..0000000000 --- a/tests/registry-command/lock.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -set -ex - -ORG="ryan-test" -PROJECT="registry-command-test-project" - -oldlockvalue="" -while true; do - lock="$(earthly secrets --org "$ORG" --project "$PROJECT" get lock || true)" - if [ -z "$lock" ]; then - echo "no lock exists; proceeding to lock it" - break - fi - if [ "$lock" = "$oldlockvalue" ]; then - echo "lock value hasn't changed; forcing it open" - earthly secrets --org "$ORG" --project "$PROJECT" rm lock || true - sleep $[ ( $RANDOM % 5 ) + 1 ]s - continue - fi - oldlockvalue="$lock" - # TODO implement a secrets ls --long, which would show a "date created/modified" column - # then if the lock is older than 1 minute, we would consider it abandoned, delete it, and create - # a new lock. For now, we will simply sleep for 60 seconds (which should be enough time for the test to pass) - duration=$[ ( $RANDOM % 30 ) + 180 ] - echo "lock exists; sleeping for $duration seconds" - sleep "$duration" -done - -id="$(uuidgen)" -test -n "$id" - -earthly secrets --org "$ORG" --project "$PROJECT" set lock "$id" - -sleep 1 - -lock="$(earthly secrets --org "$ORG" --project "$PROJECT" get lock)" -if [ "$lock" != "$id" ]; then - echo "failed to lock" - exec ./lock.sh # try again -fi - -echo "$id" > /tmp/registry-command-lock -echo locked diff --git a/tests/registry-command/test-dockerhub-project.sh b/tests/registry-command/test-dockerhub-project.sh deleted file mode 100755 index 9e2234e140..0000000000 --- a/tests/registry-command/test-dockerhub-project.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh -set -ex - -# WARNING -- RACE-CONDITION: this test is not thread-safe (since it makes use of a shared project's secrets) -# the lock.sh and unlock.sh scripts must first be run - -ORG="ryan-test" -PROJECT="registry-command-test-project" - -clearprojectsecrets() { - earthly secrets --org "$ORG" --project "$PROJECT" ls /user/std/registry | xargs -r -n 1 earthly secrets --org "$ORG" --project "$PROJECT" rm -} - -# clear out secrets from previous test -clearprojectsecrets - -# test dockerhub credentials do not exist -earthly registry --org "$ORG" --project "$PROJECT" list | grep -v registry-1.docker.io - -# set dockerhub credentials -earthly registry --org "$ORG" --project "$PROJECT" setup --username myprojecttest --password keepitsecret - -# test dockerhub credentials exist -earthly registry --org "$ORG" --project "$PROJECT" list | grep registry-1.docker.io - -# test username and password were correctly stored in underlying std secret -test "$(earthly secrets --org "$ORG" --project "$PROJECT" get std/registry/registry-1.docker.io/username)" = "myprojecttest" -test "$(earthly secrets --org "$ORG" --project "$PROJECT" get std/registry/registry-1.docker.io/password)" = "keepitsecret" - -# test a different host -echo -n keepitsecret2 | earthly registry --org "$ORG" --project "$PROJECT" setup --username myprojecttest2 --password-stdin corp-registry.earthly.dev - -# both dockerhub and corp-registry should exist -earthly registry --org "$ORG" --project "$PROJECT" list | grep registry-1.docker.io -earthly registry --org "$ORG" --project "$PROJECT" list | grep corp-registry.earthly.dev - -# test username and password were correctly stored in underlying std secret -test "$(earthly secrets --org "$ORG" --project "$PROJECT" get std/registry/registry-1.docker.io/username)" = "myprojecttest" -test "$(earthly secrets --org "$ORG" --project "$PROJECT" get std/registry/registry-1.docker.io/password)" = "keepitsecret" -test "$(earthly secrets --org "$ORG" --project "$PROJECT" get std/registry/corp-registry.earthly.dev/username)" = "myprojecttest2" -test "$(earthly secrets --org "$ORG" --project "$PROJECT" get std/registry/corp-registry.earthly.dev/password)" = "keepitsecret2" - -earthly registry --org "$ORG" --project "$PROJECT" remove -earthly registry --org "$ORG" --project "$PROJECT" list | grep -v registry-1.docker.io - -clearprojectsecrets diff --git a/tests/registry-command/test-dockerhub-user.sh b/tests/registry-command/test-dockerhub-user.sh deleted file mode 100755 index 6c3c3acbc8..0000000000 --- a/tests/registry-command/test-dockerhub-user.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/sh -set -ex - -# WARNING -- RACE-CONDITION: this test is not thread-safe (since it makes use of a shared user's secrets) -# the lock.sh and unlock.sh scripts must first be run - -clearusersecrets() { - earthly secrets ls /user/std/ | xargs -r -n 1 earthly secrets rm -} - -# clear out secrets from previous test -clearusersecrets - -# test dockerhub credentials do not exist -earthly registry list | grep -v registry-1.docker.io - -# set dockerhub credentials -earthly registry setup --username mytest --password keepitsafe - -# test dockerhub credentials exist -earthly registry list | grep registry-1.docker.io - -# test username and password were correctly stored in underlying std secret -test "$(earthly secrets get /user/std/registry/registry-1.docker.io/username)" = "mytest" -test "$(earthly secrets get /user/std/registry/registry-1.docker.io/password)" = "keepitsafe" - -# set dockerhub credentials via stdin -echo -n "fromstdin" | earthly registry setup --username mytest2 --password-stdin - -# test username and password were correctly stored in underlying std secret -test "$(earthly secrets get /user/std/registry/registry-1.docker.io/username)" = "mytest2" -test "$(earthly secrets get /user/std/registry/registry-1.docker.io/password)" = "fromstdin" - -# test no extra newline was stored; note that "echo -n fromstdin | md5sum" = 4b1fb3bf88ee25da648fefd5af81c921 -earthly secrets get -n /user/std/registry/registry-1.docker.io/password | md5sum | grep 4b1fb3bf88ee25da648fefd5af81c921 - -# set dockerhub credentials via tty -/usr/bin/expect -c ' -spawn earthly registry setup -expect "username: " -send "mytest3\n" -expect "password: " -send "fromexpect\n" -expect eof -' - -# test username and password were correctly stored in underlying std secret -test "$(earthly secrets get /user/std/registry/registry-1.docker.io/username)" = "mytest3" -test "$(earthly secrets get /user/std/registry/registry-1.docker.io/password)" = "fromexpect" - -# test no extra newline was stored; note that "echo -n fromexpect | md5sum" = bd62328338f2f6a8cb8adf2e3712afad -earthly secrets get -n /user/std/registry/registry-1.docker.io/password | md5sum | grep bd62328338f2f6a8cb8adf2e3712afad - -# set dockerhub credentials via tty -/usr/bin/expect -c ' -spawn earthly registry setup --username mytest4 -expect "password: " -send "fromexpect2\n" -expect eof -' - -# test username and password were correctly stored in underlying std secret -test "$(earthly secrets get /user/std/registry/registry-1.docker.io/username)" = "mytest4" -test "$(earthly secrets get /user/std/registry/registry-1.docker.io/password)" = "fromexpect2" - -# test no extra newline was stored; note that "echo -n fromexpect2 | md5sum" = d581f3b642ece7e7b559b8a73c60aeae -earthly secrets get -n /user/std/registry/registry-1.docker.io/password | md5sum | grep d581f3b642ece7e7b559b8a73c60aeae - -earthly registry remove -earthly registry list | grep -v registry-1.docker.io - -# clear out secrets (just in case project-based registry accidentally uses user-based) -clearusersecrets diff --git a/tests/registry-command/test-ecr-project.sh b/tests/registry-command/test-ecr-project.sh deleted file mode 100755 index 08f2194ddb..0000000000 --- a/tests/registry-command/test-ecr-project.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/sh -set -ex - -# WARNING -- RACE-CONDITION: this test is not thread-safe (since it makes use of a shared user's secrets) -# the lock.sh and unlock.sh scripts must first be run - -ORG="ryan-test" -PROJECT="registry-command-test-project" - -clearprojectsecrets() { - earthly secrets --org "$ORG" --project "$PROJECT" ls std/ | xargs -r -n 1 earthly secrets --org "$ORG" --project "$PROJECT" rm -} - -test -n "$earthly_config" # set by earthly-entrypoint.sh -test -n "$ECR_REGISTRY_HOST" - -# clear out secrets from previous test -clearprojectsecrets - -# test credentials do not exist -earthly registry list | grep -v "$ECR_REGISTRY_HOST" # just in case -earthly registry --org "$ORG" --project "$PROJECT" list | grep -v "$ECR_REGISTRY_HOST" - -# set credentials -set +x # don't remove, or keys will be leaked -test -n "$AWS_ACCESS_KEY_ID" || (echo "AWS_ACCESS_KEY_ID is empty" && exit 1) -test -n "$AWS_SECRET_ACCESS_KEY" || (echo "AWS_SECRET_ACCESS_KEY is empty" && exit 1) -set -x -earthly registry --org "$ORG" --project "$PROJECT" setup --cred-helper=ecr-login "$ECR_REGISTRY_HOST" - -# test credentials exist -earthly registry --org "$ORG" --project "$PROJECT" list | grep "$ECR_REGISTRY_HOST" - -uuid="$(uuidgen)" - -cat > Earthfile < /some-data - SAVE IMAGE --push $ECR_REGISTRY_HOST/integration-test:latest -EOF - -# --no-output is required for earthly-in-earthly; however a --push to ecr will still occur -earthly --config "$earthly_config" --verbose +pull -earthly --config "$earthly_config" --no-output --push --verbose +push - -earthly registry --org "$ORG" --project "$PROJECT" remove "$ECR_REGISTRY_HOST" -earthly registry --org "$ORG" --project "$PROJECT" list | grep -v $ECR_REGISTRY_HOST - -# clear out secrets (just in case project-based registry accidentally uses user-based) -clearprojectsecrets diff --git a/tests/registry-command/test-ecr-user.sh b/tests/registry-command/test-ecr-user.sh deleted file mode 100755 index a0c4513536..0000000000 --- a/tests/registry-command/test-ecr-user.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/sh -set -ex - -# WARNING -- RACE-CONDITION: this test is not thread-safe (since it makes use of a shared user's secrets) -# the lock.sh and unlock.sh scripts must first be run - -clearusersecrets() { - earthly secrets ls /user/std/ | xargs -r -n 1 earthly secrets rm -} - -test -n "$earthly_config" # set by earthly-entrypoint.sh -test -n "$ECR_REGISTRY_HOST" - -# clear out secrets from previous test -clearusersecrets - -# test credentials do not exist -earthly registry list | grep -v $ECR_REGISTRY_HOST - -# set ecr credentials -set +x # don't remove, or keys will be leaked -test -n "$AWS_ACCESS_KEY_ID" || (echo "AWS_ACCESS_KEY_ID is empty" && exit 1) -test -n "$AWS_SECRET_ACCESS_KEY" || (echo "AWS_SECRET_ACCESS_KEY is empty" && exit 1) -set -x -earthly registry setup --cred-helper=ecr-login "$ECR_REGISTRY_HOST" -echo "done setting up cred helper (and secrets)" - -earthly registry list | grep "$ECR_REGISTRY_HOST" - -uuid="$(uuidgen)" - -cat > Earthfile < /some-data - SAVE IMAGE --push $ECR_REGISTRY_HOST/integration-test:latest -EOF - -# --no-output is required for earthly-in-earthly; however a --push to ecr will still occur -earthly --config "$earthly_config" --verbose +pull -earthly --config "$earthly_config" --no-output --push --verbose +push - -earthly registry remove "$ECR_REGISTRY_HOST" -earthly registry list | grep -v $ECR_REGISTRY_HOST - -# clear out secrets (just in case project-based registry accidentally uses user-based) -clearusersecrets diff --git a/tests/registry-command/test-ecr.sh b/tests/registry-command/test-ecr.sh deleted file mode 100755 index 102eda478f..0000000000 --- a/tests/registry-command/test-ecr.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh -set -ex -./lock.sh - -finish() { - status="$?" - ./unlock.sh - if [ "$status" = "0" ]; then - echo "$0 passed" - else - echo "$0 failed with $status" - fi -} -trap finish EXIT - -export ECR_REGISTRY_HOST="404851345508.dkr.ecr.us-west-2.amazonaws.com" - -./test-ecr-user.sh -./test-ecr-project.sh diff --git a/tests/registry-command/test-gcp-project.sh b/tests/registry-command/test-gcp-project.sh deleted file mode 100755 index db636f73f1..0000000000 --- a/tests/registry-command/test-gcp-project.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/sh -set -ex - -# WARNING -- RACE-CONDITION: this test is not thread-safe (since it makes use of a shared user's secrets) -# the lock.sh and unlock.sh scripts must first be run - -ORG="ryan-test" -PROJECT="registry-command-test-project" - -clearprojectsecrets() { - earthly secrets --org "$ORG" --project "$PROJECT" ls std/ | xargs -r -n 1 earthly secrets --org "$ORG" --project "$PROJECT" rm -} - -test -n "$earthly_config" # set by earthly-entrypoint.sh - -# clear out secrets from previous test -clearprojectsecrets - -# test credentials do not exist -earthly registry list | grep -v "$GCP_SERVER" # just in case -earthly registry --org "$ORG" --project "$PROJECT" list | grep -v "$GCP_SERVER" - -# set credentials -set +x # don't remove, or keys will be leaked -test -n "$GCP_KEY" || (echo "GCP_KEY is empty" && exit 1) -echo "$GCP_KEY" | earthly registry --org "$ORG" --project "$PROJECT" setup --cred-helper=gcloud --gcp-service-account-key-stdin "$GCP_SERVER" -set -x - -# test credentials exist -earthly registry --org "$ORG" --project "$PROJECT" list | grep "$GCP_SERVER" - -uuid="$(uuidgen)" - -cat > Earthfile < /some-data - SAVE IMAGE --push $GCP_FULL_ADDRESS/$IMAGE:latest -EOF - -# --no-output is required for earthly-in-earthly; however a --push to gcp will still occur -earthly --config "$earthly_config" --verbose +pull -earthly --config "$earthly_config" --no-output --push --verbose +push - -earthly registry --org "$ORG" --project "$PROJECT" remove "$GCP_SERVER" -earthly registry --org "$ORG" --project "$PROJECT" list | grep -v $GCP_SERVER - -# clear out secrets (just in case project-based registry accidentally uses user-based) -clearprojectsecrets diff --git a/tests/registry-command/test-gcp-user.sh b/tests/registry-command/test-gcp-user.sh deleted file mode 100755 index 50e8f8fea7..0000000000 --- a/tests/registry-command/test-gcp-user.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh -set -ex - -# WARNING -- RACE-CONDITION: this test is not thread-safe (since it makes use of a shared user's secrets) -# the lock.sh and unlock.sh scripts must first be run - -clearusersecrets() { - earthly secrets ls /user/std/ | xargs -r -n 1 earthly secrets rm -} - -test -n "$earthly_config" # set by earthly-entrypoint.sh - -# clear out secrets from previous test -clearusersecrets - -# test credentials do not exist -earthly registry list | grep -v "$GCP_SERVER" - -# set credentials -set +x # don't remove, or keys will be leaked -test -n "$GCP_KEY" || (echo "GCP_KEY is empty" && exit 1) -export GCP_SERVICE_ACCOUNT_KEY="$GCP_KEY" # registry setup reads from this env -set -x -earthly registry setup --cred-helper=gcloud "$GCP_SERVER" - -# test credentials exist -earthly registry list | grep "$GCP_SERVER" - -uuid="$(uuidgen)" - -cat > Earthfile < /some-data - SAVE IMAGE --push $GCP_FULL_ADDRESS/$IMAGE:latest -EOF - -# --no-output is required for earthly-in-earthly; however a --push to gcp will still occur -earthly --config "$earthly_config" --verbose +pull -earthly --config "$earthly_config" --no-output --push --verbose +push - -earthly registry remove "$GCP_SERVER" -earthly registry list | grep -v $GCP_SERVER - -# clear out secrets (just in case project-based registry accidentally uses user-based) -clearusersecrets diff --git a/tests/registry-command/test-gcp.sh b/tests/registry-command/test-gcp.sh deleted file mode 100755 index 9324832244..0000000000 --- a/tests/registry-command/test-gcp.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh -set -ex -./lock.sh - -finish() { - status="$?" - ./unlock.sh - if [ "$status" = "0" ]; then - echo "$0 passed" - else - echo "$0 failed with $status" - fi -} -trap finish EXIT - -# Test Google artifact registry -export GCP_SERVER="us-west1-docker.pkg.dev" -export GCP_FULL_ADDRESS="$GCP_SERVER/ci-cd-302220" -export IMAGE="integration-test/test" -./test-gcp-user.sh -./test-gcp-project.sh - -# Test Google container registry -export GCP_SERVER="gcr.io" -export GCP_FULL_ADDRESS="$GCP_SERVER/ci-cd-302220" -export IMAGE="test" -./test-gcp-user.sh -./test-gcp-project.sh diff --git a/tests/registry-command/test-multi.sh b/tests/registry-command/test-multi.sh deleted file mode 100755 index 67f59f6ae7..0000000000 --- a/tests/registry-command/test-multi.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/sh -set -ex - -# WARNING -- RACE-CONDITION: this test is not thread-safe (since it makes use of a shared user's secrets) -# the lock.sh and unlock.sh scripts must first be run - -./lock.sh - -finish() { - status="$?" - ./unlock.sh - if [ "$status" = "0" ]; then - echo "$0 passed" - else - echo "$0 failed with $status" - fi -} -trap finish EXIT - - -# ECR details -export ECR_REGISTRY_HOST="404851345508.dkr.ecr.us-west-2.amazonaws.com" - -# Google artifact registry details -export GCP_SERVER="us-west1-docker.pkg.dev" -export GCP_FULL_ADDRESS="$GCP_SERVER/ci-cd-302220" -export IMAGE="integration-test/test" - -clearusersecrets() { - earthly secrets ls /user/std/ | xargs -r -n 1 earthly secrets rm -} - -# clear out secrets from previous test -clearusersecrets - - -echo "Setting up ECR credentials" -set +x # don't remove, or keys will be leaked -test -n "$AWS_ACCESS_KEY_ID" || (echo "AWS_ACCESS_KEY_ID is empty" && exit 1) -test -n "$AWS_SECRET_ACCESS_KEY" || (echo "AWS_SECRET_ACCESS_KEY is empty" && exit 1) -set -x -earthly registry setup --cred-helper=ecr-login "$ECR_REGISTRY_HOST" - -echo "Setting up GCP credentials" -set +x # don't remove, or keys will be leaked -test -n "$GCP_KEY" || (echo "GCP_KEY is empty" && exit 1) -export GCP_SERVICE_ACCOUNT_KEY="$GCP_KEY" # registry setup reads from this env -set -x -earthly registry setup --cred-helper=gcloud "$GCP_SERVER" - - -echo "done setting up cred helper (and secrets)" - -earthly registry list | grep "$ECR_REGISTRY_HOST" -earthly registry list | grep "$GCP_SERVER" - -cat > Earthfile < /etc/.earthly/config.yml