feat: merge tonistiigi upstream (586307ad452fadb42e28be16d2b995b2eb6755e9)

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+# PRs require approval from the EarthBuild admin team
+* @EarthBuild/fork-admins

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    open-pull-requests-limit: 10
+    directory: "/"
+    schedule:
+      interval: "daily"
+    labels:
+      - "dependencies"
+      - "bot"

.github/workflows/ci.yml

@@ -5,35 +5,36 @@ concurrency:
   cancel-in-progress: true
 
 on:
+  workflow_dispatch:
   schedule:
     - cron: '0 8 */6 * *' # every 6 days
   push:
     branches:
-      - main
-      - gh_test_ci
+      - master
   pull_request:
-    branches:
-      - main
+
+env:
+  DESTDIR: ./bin
+  GO_VERSION: "1.23"
 
 jobs:
   validate:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         target:
-          - lint
+          - lint-golangci-cross
+          - lint-gopls-cross
+          - validate-generated-files
           - validate-gomod
           - validate-shfmt
     steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       -
         name: Run
-        uses: docker/bake-action@v3
+        uses: docker/bake-action@v6
         with:
           targets: ${{ matrix.target }}
 
@@ -45,85 +46,145 @@ jobs:
           - build
           - cross
     steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       -
         name: Run
-        uses: docker/bake-action@v3
+        uses: docker/bake-action@v6
         with:
           targets: ${{ matrix.target }}
 
   test:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go_version:
-          - "1.19"
-          - "1.20"
+        go:
+          - "1.23"
+          - "1.22"
+          - "1.21"
+        mode:
+          - "root"
+          - "noroot"
     env:
-      GO_VERSION: ${{ matrix.go_version }}
+      GO_VERSION: ${{ matrix.go }}
     steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       -
         name: Test
-        uses: docker/bake-action@v3
+        uses: docker/bake-action@v6
+        with:
+          targets: test-${{ matrix.mode }}
+      -
+        name: Upload coverage
+        uses: codecov/codecov-action@v5
+        # skip the step for EarthBuild
+        if: false
         with:
-          targets: test
+          directory: ${{ env.DESTDIR }}/coverage
+          flags: unit,${{ matrix.mode }},go-${{ matrix.go }}
+          token: ${{ secrets.CODECOV_TOKEN }}
 
-  test-macos:
-    runs-on: macos-latest
+  test-os:
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
+        os:
+          - macos-latest
+          - windows-latest
         go:
-          - "1.19"
-          - "1.20"
+          - "1.23"
+          - "1.22"
+          - "1.21"
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       -
         name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go }}
       -
         name: Test
         run: |
-          go test ./...
+          go test -coverprofile="coverage.txt" -covermode="atomic" ./...
+          go tool cover -func="coverage.txt"
+      -
+        name: Upload coverage
+        # skip the step for EarthBuild
+        if: false
+        uses: codecov/codecov-action@v5
+        with:
+          files: ./coverage.txt
+          env_vars: RUNNER_OS
+          flags: unit,go-${{ matrix.go }}
+          token: ${{ secrets.CODECOV_TOKEN }}
 
-  test-freebsd-amd64:
-    # TODO(jhorsts): macOS latest (v10.15) used to contain vagrant but not anymore.
-    # The comment can be removed when the latest fsutil is merged into this repo.
-    if: false 
-    runs-on: macos-latest
-    timeout-minutes: 60
-    env:
-      VAGRANT_VAGRANTFILE: hack/Vagrantfile.freebsd13
+  test-bsd-amd64:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - freebsd
+          - netbsd
+          - openbsd
     steps:
+      -
+        name: Prepare
+        run: |
+          echo "VAGRANT_FILE=hack/Vagrantfile.${{ matrix.os }}" >> $GITHUB_ENV
+
+          # Sets semver Go version to be able to download tarball during vagrant setup
+          goVersion=$(curl --silent "https://go.dev/dl/?mode=json&include=all" | jq -r '.[].files[].version' | uniq | sed -e 's/go//' | sort -V | grep $GO_VERSION | tail -1)
+          echo "GO_VERSION=$goVersion" >> $GITHUB_ENV
       -
         name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       -
         name: Cache Vagrant boxes
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.vagrant.d/boxes
-          key: ${{ runner.os }}-vagrant-${{ hashFiles('hack/Vagrantfile.freebsd13') }}
+          key: ${{ runner.os }}-vagrant-${{ matrix.os }}-${{ hashFiles(env.VAGRANT_FILE) }}
           restore-keys: |
-            ${{ runner.os }}-vagrant-
+            ${{ runner.os }}-vagrant-${{ matrix.os }}-
+      -
+        name: Install vagrant
+        run: |
+          set -x
+          wget -O - https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          sudo apt-get update
+          sudo apt-get install -y libvirt-dev libvirt-daemon libvirt-daemon-system vagrant vagrant-libvirt ruby-libvirt
+          sudo systemctl enable --now libvirtd
+          sudo chmod a+rw /var/run/libvirt/libvirt-sock
+          vagrant plugin install vagrant-libvirt
+          vagrant --version
       -
         name: Set up vagrant
-        run: vagrant up
+        run: |
+          ln -sf ${{ env.VAGRANT_FILE }} Vagrantfile
+          vagrant up --no-tty
       -
         name: Test
-        run: vagrant ssh -- "cd /vagrant; go test -buildvcs=false ./..."
+        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08  # v3.0.2
+        with:
+          timeout_minutes: 20
+          max_attempts: 5
+          command: |
+            vagrant ssh -- "cd /vagrant; go test -buildvcs=false -coverprofile=coverage.txt -covermode=atomic ./..."
+            vagrant ssh -c "sudo cat /vagrant/coverage.txt" > coverage.txt
+      -
+        name: Upload coverage
+        # skip the step for EarthBuild
+        if: false
+        uses: codecov/codecov-action@v5
+        with:
+          files: ./coverage.txt
+          flags: unit,${{ matrix.os }}
+          token: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -1,9 +1,5 @@
 # if you want to ignore files created by your editor/tools, consider using a
 # global .gitignore or .git/info/exclude see https://help.github.com/articles/ignoring-files
-.*
-!.github
-!.gitignore
-!.travis.yml
-*.prof
+bin/
 # support running go modules in vendor mode for local development
 vendor/

.golangci.yml

@@ -1,7 +1,5 @@
 run:
-  timeout: 10m
-  skip-files:
-    - ".*\\.pb\\.go$"
+  timeout: 30m
 
 linters:
   enable:
@@ -14,3 +12,21 @@ linters:
     - staticcheck
     - typecheck
   disable-all: true
+
+linters-settings:
+  depguard:
+    rules:
+      main:
+        deny:
+          # The io/ioutil package has been deprecated.
+          # https://go.dev/doc/go1.16#ioutil
+          - pkg: "io/ioutil"
+            desc: The io/ioutil package has been deprecated.
+
+issues:
+  exclude-files:
+    - ".*\\.pb\\.go$"
+
+  # show all
+  max-issues-per-linter: 0
+  max-same-issues: 0

Dockerfile

@@ -1,7 +1,9 @@
-#syntax=docker/dockerfile:1
-ARG GO_VERSION=1.20
+# syntax=docker/dockerfile:1
 
-FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.1.0 AS xx
+ARG GO_VERSION=1.23
+ARG XX_VERSION=1.6.1
+
+FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
 
 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS base
 RUN apk add --no-cache git
@@ -18,13 +20,19 @@ FROM base AS test
 ARG TESTFLAGS
 RUN --mount=target=. --mount=target=/go/pkg/mod,type=cache \
     --mount=target=/root/.cache,type=cache \
-    CGO_ENABLED=0 xx-go test -test.v ${TESTFLAGS} ./...
+    CGO_ENABLED=0 xx-go test -v -coverprofile=/tmp/coverage.txt -covermode=atomic ${TESTFLAGS} ./...
 
 FROM base AS test-noroot
 RUN mkdir /go/pkg && chmod 0777 /go/pkg
 USER 1000:1000
 RUN --mount=target=. \
     --mount=target=/tmp/.cache,type=cache \
-    CGO_ENABLED=0 GOCACHE=/tmp/gocache xx-go test -test.v ./...
+    CGO_ENABLED=0 GOCACHE=/tmp/gocache xx-go test -v -coverprofile=/tmp/coverage.txt -covermode=atomic ./...
+
+FROM scratch AS test-coverage
+COPY --from=test /tmp/coverage.txt /coverage-root.txt
+
+FROM scratch AS test-noroot-coverage
+COPY --from=test-noroot /tmp/coverage.txt /coverage-noroot.txt
 
 FROM build

bench/diffcopy.go

@@ -21,7 +21,11 @@ func diffCopy(proto bool, src, dest string) error {
 	}
 
 	eg.Go(func() error {
-		return fsutil.Send(ctx, s1, fsutil.NewFS(src, nil), nil, nil)
+		fs, err := fsutil.NewFS(src)
+		if err != nil {
+			panic(err)
+		}
+		return fsutil.Send(ctx, s1, fs, nil, nil)
 	})
 	eg.Go(func() error {
 		return fsutil.Receive(ctx, s2, dest, fsutil.ReceiveOpt{})

bench/go.mod

@@ -1,33 +1,30 @@
 module github.com/tonistiigi/fsutil/bench
 
-go 1.19
+go 1.21
 
 require (
-	github.com/containerd/continuity v0.4.1
-	github.com/docker/docker v20.10.18+incompatible
+	github.com/containerd/continuity v0.4.4
+	github.com/docker/docker v27.3.1+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/tonistiigi/fsutil v0.0.0-00010101000000-000000000000
-	golang.org/x/sync v0.1.0
+	golang.org/x/sync v0.8.0
 )
 
 require (
-	github.com/Microsoft/go-winio v0.5.2 // indirect
-	github.com/Microsoft/hcsshim v0.8.17 // indirect
-	github.com/containerd/cgroups v1.0.1 // indirect
-	github.com/containerd/containerd v1.5.2 // indirect
-	github.com/docker/go-units v0.4.0 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/klauspost/compress v1.11.13 // indirect
 	github.com/moby/patternmatcher v0.5.0 // indirect
-	github.com/moby/sys/mount v0.2.0 // indirect
-	github.com/moby/sys/mountinfo v0.4.1 // indirect
+	github.com/moby/sys/sequential v0.6.0 // indirect
+	github.com/moby/sys/user v0.3.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.1 // indirect
-	github.com/opencontainers/runc v1.0.0-rc93 // indirect
-	github.com/sirupsen/logrus v1.8.1 // indirect
-	go.opencensus.io v0.22.3 // indirect
-	golang.org/x/sys v0.1.0 // indirect
-	google.golang.org/protobuf v1.26.0 // indirect
+	github.com/planetscale/vtprotobuf v0.6.0 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	golang.org/x/sys v0.26.0 // indirect
+	google.golang.org/protobuf v1.35.1 // indirect
+	gotest.tools/v3 v3.0.3 // indirect
 )
 
 replace github.com/tonistiigi/fsutil => ../